tasker-engine 1.0.0

data/lib/tasker/orchestration/connection_pool_intelligence.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module Tasker
+  module Orchestration
+    # Rails-Framework-Aligned Connection Management
+    #
+    # Provides intelligent assessment of database connection pool health and
+    # recommends safe concurrency levels that work WITH Rails connection pool
+    # rather than around it.
+    #
+    # Key Design Principles:
+    # - CONSTANTS: Ruby/Rails optimization characteristics (safety thresholds, precision)
+    # - CONFIGURABLE: Environment-dependent pressure response factors
+    # - Conservative safety margins to prevent connection exhaustion
+    # - Comprehensive structured logging for observability
+    #
+    # @example Basic usage
+    #   health = ConnectionPoolIntelligence.assess_connection_health
+    #   concurrency = ConnectionPoolIntelligence.intelligent_concurrency_for_step_executor
+    #
+    # @example With custom configuration
+    #   config = Tasker.configuration.execution
+    #   config.connection_pressure_factors = { high: 0.3, critical: 0.1 }
+    #   concurrency = ConnectionPoolIntelligence.intelligent_concurrency_for_step_executor
+    class ConnectionPoolIntelligence
+      # ✅ CONSTANTS: Ruby/Rails optimization characteristics
+      # These are based on Rails connection pool behavior and should NOT be configurable
+
+      # Decimal places for utilization calculation (based on Rails pool stat precision)
+      CONNECTION_UTILIZATION_PRECISION = 3
+
+      # Pressure assessment thresholds (based on Rails connection pool characteristics)
+      PRESSURE_ASSESSMENT_THRESHOLDS = {
+        low: 0.0..0.5,
+        moderate: 0.5..0.7,
+        high: 0.7..0.85,
+        critical: 0.85..Float::INFINITY
+      }.freeze
+
+      # Conservative safety patterns (based on Rails connection pool behavior)
+      # Never use more than 60% of pool to prevent connection exhaustion
+      MAX_SAFE_CONNECTION_PERCENTAGE = 0.6
+
+      # Absolute minimum for system stability (based on Tasker orchestration needs)
+      EMERGENCY_FALLBACK_CONCURRENCY = 3
+
+      # Assess current database connection pool health
+      #
+      # Provides comprehensive analysis of Rails connection pool state including
+      # utilization metrics, pressure assessment, and concurrency recommendations.
+      #
+      # @return [Hash] Connection health assessment with structured metrics
+      #   @option return [Float] :pool_utilization Connection pool utilization ratio (0.0-1.0+)
+      #   @option return [Symbol] :connection_pressure Pressure level (:low, :moderate, :high, :critical)
+      #   @option return [Integer] :recommended_concurrency Safe concurrency level
+      #   @option return [Hash] :rails_pool_stats Raw Rails pool statistics
+      #   @option return [Symbol] :health_status Overall health status
+      #   @option return [Time] :assessment_timestamp When assessment was performed
+      def self.assess_connection_health
+        pool = ActiveRecord::Base.connection_pool
+        pool_stat = pool.stat
+
+        {
+          pool_utilization: calculate_utilization(pool_stat),
+          connection_pressure: assess_pressure(pool_stat),
+          recommended_concurrency: recommend_concurrency(pool_stat),
+          rails_pool_stats: pool_stat,
+          health_status: determine_health_status(pool_stat),
+          assessment_timestamp: Time.current
+        }
+      rescue StandardError => e
+        Rails.logger.error("ConnectionPoolIntelligence: Health assessment failed - #{e.class.name}: #{e.message}")
+
+        # Return safe fallback assessment
+        {
+          pool_utilization: 0.0,
+          connection_pressure: :unknown,
+          recommended_concurrency: EMERGENCY_FALLBACK_CONCURRENCY,
+          rails_pool_stats: {},
+          health_status: :unknown,
+          assessment_timestamp: Time.current,
+          assessment_error: e.message
+        }
+      end
+
+      # Calculate intelligent concurrency for StepExecutor integration
+      #
+      # Provides Rails-aware concurrency calculation that respects connection pool
+      # limits while applying configurable safety margins and pressure factors.
+      #
+      # @return [Integer] Safe concurrency level for step execution
+      def self.intelligent_concurrency_for_step_executor
+        health_data = assess_connection_health
+        config = Tasker.configuration.execution
+
+        # Get base recommendation from Rails pool analysis
+        base_recommendation = health_data[:recommended_concurrency]
+        safe_concurrency = apply_tasker_safety_margins(base_recommendation, health_data, config)
+
+        Rails.logger.debug do
+          "ConnectionPoolIntelligence: Dynamic concurrency=#{safe_concurrency}, " \
+            "pressure=#{health_data[:connection_pressure]}, " \
+            "pool_size=#{ActiveRecord::Base.connection_pool.size}, " \
+            "available=#{health_data[:rails_pool_stats][:available]}"
+        end
+
+        safe_concurrency
+      rescue StandardError => e
+        Rails.logger.error(
+          "ConnectionPoolIntelligence: Concurrency calculation failed - #{e.class.name}: #{e.message}, " \
+          "using fallback=#{EMERGENCY_FALLBACK_CONCURRENCY}"
+        )
+
+        EMERGENCY_FALLBACK_CONCURRENCY
+      end
+
+      private_class_method def self.calculate_utilization(pool_stat)
+        return 0.0 if pool_stat[:size].zero?
+
+        (pool_stat[:busy].to_f / pool_stat[:size]).round(CONNECTION_UTILIZATION_PRECISION)
+      end
+
+      private_class_method def self.assess_pressure(pool_stat)
+        utilization = calculate_utilization(pool_stat)
+
+        # Use CONSTANT thresholds for consistent pressure assessment
+        PRESSURE_ASSESSMENT_THRESHOLDS.each do |level, range|
+          return level if range.cover?(utilization)
+        end
+
+        :unknown
+      end
+
+      private_class_method def self.recommend_concurrency(pool_stat)
+        pressure = assess_pressure(pool_stat)
+
+        # ✅ CONFIGURABLE: Pressure response factors (environment-dependent)
+        pressure_config = Tasker.configuration.execution.connection_pressure_factors
+        factor = pressure_config[pressure] || 0.5
+
+        base_recommendation = [pool_stat[:available] * factor, 12].min.floor
+        [base_recommendation, EMERGENCY_FALLBACK_CONCURRENCY].max
+      end
+
+      private_class_method def self.determine_health_status(pool_stat)
+        pressure = assess_pressure(pool_stat)
+
+        case pressure
+        when :low, :moderate then :healthy
+        when :high then :degraded
+        when :critical then :critical
+        else :unknown
+        end
+      end
+
+      private_class_method def self.apply_tasker_safety_margins(base_recommendation, health_data, config)
+        # Use CONSTANT safety percentage with CONFIGURABLE bounds
+        max_safe = (health_data[:rails_pool_stats][:available] * MAX_SAFE_CONNECTION_PERCENTAGE).floor
+
+        # Apply configurable pressure adjustments based on connection pool state
+        pressure_adjusted = case health_data[:connection_pressure]
+                            when :moderate
+                              [base_recommendation, max_safe].min
+                            when :high
+                              [base_recommendation * 0.7, max_safe].min.floor
+                            when :critical, :unknown
+                              [EMERGENCY_FALLBACK_CONCURRENCY, max_safe].min
+                            else # :low pressure or any other state
+                              base_recommendation
+                            end
+
+        # Apply CONFIGURABLE absolute bounds from ExecutionConfig
+        pressure_adjusted.clamp(config.min_concurrent_steps, config.max_concurrent_steps_limit)
+      end
+    end
+  end
+end

data/lib/tasker/orchestration/coordinator.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require_relative '../concerns/idempotent_state_transitions'
+require_relative 'viable_step_discovery'
+require_relative 'step_executor'
+require_relative 'task_finalizer'
+require_relative 'task_initializer'
+require_relative 'step_sequence_factory'
+require_relative 'task_reenqueuer'
+
+module Tasker
+  module Orchestration
+    # Coordinator handles orchestration system initialization and monitoring
+    #
+    # This class ensures the orchestration system is properly initialized and provides
+    # observability into the system state. It does NOT handle task processing -
+    # that responsibility belongs to TaskHandler using the proven delegation pattern.
+    class Coordinator
+      include Tasker::Concerns::IdempotentStateTransitions
+
+      class << self
+        # Check if the orchestration system has been initialized
+        #
+        # @return [Boolean] True if initialized
+        def initialized?
+          @initialized == true
+        end
+
+        # Initialize the orchestration system
+        #
+        # Sets up all orchestration components, registers events, and connects subscribers.
+        # This method is idempotent and can be called multiple times safely.
+        def initialize!
+          return if @initialized
+
+          Rails.logger.info('Tasker::Orchestration::Coordinator: Initializing orchestration system')
+
+          # Initialize core orchestration components
+          setup_publisher
+          setup_event_subscriptions
+
+          # Initialize telemetry and monitoring
+          setup_telemetry_subscriber
+
+          @initialized = true
+          Rails.logger.info('Tasker::Orchestration::Coordinator: Orchestration system initialized successfully')
+        rescue StandardError => e
+          Rails.logger.error("Tasker::Orchestration::Coordinator: Failed to initialize: #{e.message}")
+          @initialized = false
+          raise
+        end
+
+        # Reset initialization state (primarily for testing)
+        def reset!
+          @initialized = false
+          @publisher = nil
+          Rails.logger.debug('Tasker::Orchestration::Coordinator: Initialization state reset')
+        end
+
+        # Get statistics about the workflow orchestration system
+        #
+        # @return [Hash] Statistics about component initialization and event subscriptions
+        def statistics
+          {
+            initialized: @initialized || false,
+            components: {
+              publisher: defined?(Tasker::Events::Publisher),
+              viable_step_discovery: defined?(Tasker::Orchestration::ViableStepDiscovery),
+              step_executor: defined?(Tasker::Orchestration::StepExecutor),
+              task_finalizer: defined?(Tasker::Orchestration::TaskFinalizer),
+              task_reenqueuer: defined?(Tasker::Orchestration::TaskReenqueuer)
+            },
+            publisher_instance: @publisher&.class&.name
+          }
+        end
+
+        # Get sequence for a task (utility method for orchestration components)
+        #
+        # @param task [Tasker::Task] The task
+        # @param task_handler [Object] The task handler
+        # @return [Tasker::Types::StepSequence] The step sequence
+        def get_sequence_for_task(task, task_handler)
+          Tasker::Orchestration::StepSequenceFactory.get_sequence(task, task_handler)
+        end
+
+        private
+
+        # Set up the main publisher
+        def setup_publisher
+          @publisher = Tasker::Events::Publisher.instance
+        end
+
+        # Set up event subscriptions for orchestration components
+        def setup_event_subscriptions
+          # Currently using direct method delegation instead of event subscriptions
+          # for the core workflow loop. Events are used for observability.
+          # Future enhancement: Add event-driven workflow subscriptions here
+        end
+
+        # Set up the telemetry subscriber for comprehensive observability
+        def setup_telemetry_subscriber
+          # Ensure telemetry subscriber is connected to the lifecycle events system
+          Tasker::Events::Publisher.instance.tap do |publisher|
+            Tasker::Events::Subscribers::TelemetrySubscriber.subscribe(publisher)
+          end
+          Rails.logger.debug('Tasker::Orchestration::Coordinator: TelemetrySubscriber connected successfully')
+
+          # Set up the metrics subscriber for automatic metrics collection via EventRouter
+          Tasker::Events::Publisher.instance.tap do |publisher|
+            Tasker::Events::Subscribers::MetricsSubscriber.subscribe(publisher)
+          end
+          Rails.logger.debug('Tasker::Orchestration::Coordinator: MetricsSubscriber connected successfully')
+        rescue StandardError => e
+          Rails.logger.error("Tasker::Orchestration::Coordinator: Failed to setup telemetry subscriber: #{e.message}")
+        end
+      end
+    end
+  end
+end

data/lib/tasker/orchestration/future_state_analyzer.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Tasker
+  module Orchestration
+    # Abstraction layer for analyzing Concurrent::Future states in our domain context
+    #
+    # This class encapsulates the complex state logic around Concurrent::Future objects,
+    # providing domain-specific methods that express our actual intentions rather than
+    # requiring callers to understand the intricacies of the Future state machine.
+    #
+    # The Concurrent::Future state machine has these states:
+    # - unscheduled: Created but not yet executed
+    # - pending: Scheduled but not yet started
+    # - executing: Currently running
+    # - fulfilled: Completed successfully
+    # - rejected: Completed with error
+    # - cancelled: Cancelled before completion
+    #
+    # Our domain needs are:
+    # - Should we cancel this future? (pending futures)
+    # - Should we wait for this future? (executing futures)
+    # - Can we ignore this future? (completed or unscheduled futures)
+    class FutureStateAnalyzer
+      # Initialize with a Concurrent::Future object
+      #
+      # @param future [Concurrent::Future] The future to analyze
+      def initialize(future)
+        @future = future
+      end
+
+      # Should this future be cancelled during cleanup?
+      #
+      # We cancel futures that are pending (scheduled but not started).
+      # This prevents them from starting when we're trying to clean up.
+      #
+      # @return [Boolean] true if the future should be cancelled
+      def should_cancel?
+        @future.pending?
+      end
+
+      # Should we wait for this future to complete during cleanup?
+      #
+      # We wait for futures that are actively executing to give them
+      # a chance to complete gracefully before forcing cleanup.
+      #
+      # A future is "executing" if:
+      # - It's incomplete (not finished)
+      # - It's scheduled (not unscheduled)
+      # - It's not pending (it has started)
+      #
+      # @return [Boolean] true if we should wait for this future
+      def should_wait_for_completion?
+        executing?
+      end
+
+      # Can we safely ignore this future during cleanup?
+      #
+      # We can ignore futures that are either:
+      # - Already completed (fulfilled, rejected, or cancelled)
+      # - Never scheduled (unscheduled)
+      #
+      # @return [Boolean] true if this future can be ignored
+      def can_ignore?
+        completed? || unscheduled?
+      end
+
+      # Is this future currently executing?
+      #
+      # A future is executing if it has been scheduled, started, but not completed.
+      #
+      # @return [Boolean] true if the future is actively executing
+      def executing?
+        @future.incomplete? && !@future.unscheduled? && !@future.pending?
+      end
+
+      # Is this future completed (in any completion state)?
+      #
+      # A future is completed if it's fulfilled, rejected, or cancelled.
+      #
+      # @return [Boolean] true if the future has completed
+      def completed?
+        @future.complete?
+      end
+
+      # Is this future unscheduled (never started)?
+      #
+      # An unscheduled future was created but never executed.
+      #
+      # @return [Boolean] true if the future is unscheduled
+      delegate :unscheduled?, to: :@future
+
+      # Is this future pending (scheduled but not started)?
+      #
+      # A pending future has been scheduled for execution but hasn't started yet.
+      #
+      # @return [Boolean] true if the future is pending
+      delegate :pending?, to: :@future
+
+      # Get a human-readable description of the future's state
+      #
+      # This is useful for logging and debugging.
+      #
+      # @return [String] A description of the current state
+      def state_description
+        return 'unscheduled' if unscheduled?
+        return 'pending' if pending?
+        return 'executing' if executing?
+        return 'fulfilled' if @future.fulfilled?
+        return 'rejected' if @future.rejected?
+        return 'cancelled' if @future.cancelled?
+
+        'unknown'
+      end
+
+      # Get detailed state information for debugging
+      #
+      # @return [Hash] Detailed state information
+      def debug_state
+        {
+          state_description: state_description,
+          should_cancel: should_cancel?,
+          should_wait: should_wait_for_completion?,
+          can_ignore: can_ignore?,
+          raw_states: {
+            pending: @future.pending?,
+            complete: @future.complete?,
+            incomplete: @future.incomplete?,
+            unscheduled: @future.unscheduled?,
+            fulfilled: @future.fulfilled?,
+            rejected: @future.rejected?,
+            cancelled: @future.cancelled?
+          }
+        }
+      end
+    end
+  end
+end

data/lib/tasker/orchestration/plugin_integration.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Tasker
+  module Orchestration
+    # PluginIntegration provides shared access to TaskHandler plugin functionality
+    #
+    # This module extracts the plugin integration patterns from TaskHandler
+    # and makes them available to orchestration components while preserving
+    # the extensibility that makes Tasker valuable.
+    #
+    # Key methods preserved:
+    # - get_step_handler: Access to step handler plugins
+    # - establish_step_dependencies_and_defaults: Extensibility hook
+    # - update_annotations: Extensibility hook
+    # - schema validation: Context validation hook
+    module PluginIntegration
+      # Get a step handler for a specific step
+      #
+      # This method integrates with the TaskHandler plugin system to get
+      # the appropriate handler for a step.
+      #
+      # @param step [Tasker::WorkflowStep] The step to get a handler for
+      # @param task_handler [Object] The task handler instance
+      # @return [Object] The step handler
+      # @raise [Tasker::ProceduralError] If no handler is registered for the step
+      def get_step_handler_from_task_handler(step, task_handler)
+        unless task_handler.step_handler_class_map[step.name]
+          raise(Tasker::ProceduralError,
+                "No registered class for #{step.name}")
+        end
+
+        handler_config = task_handler.step_handler_config_map[step.name]
+        handler_class = task_handler.step_handler_class_map[step.name].to_s.camelize.constantize
+
+        return handler_class.new if handler_config.nil?
+
+        handler_class.new(config: handler_config)
+      end
+
+      # Get the step sequence for a task using task handler
+      #
+      # This delegates to the task handler's get_sequence method which handles
+      # step template creation and dependency establishment.
+      #
+      # @param task [Tasker::Task] The task
+      # @param task_handler [Object] The task handler instance
+      # @return [Tasker::Types::StepSequence] The step sequence
+      def get_sequence_for_task(task, task_handler)
+        task_handler.get_sequence(task)
+      end
+
+      # Establish step dependencies and defaults using task handler hook
+      #
+      # @param task [Tasker::Task] The task being processed
+      # @param steps [Array<Tasker::WorkflowStep>] The steps to establish dependencies for
+      # @param task_handler [Object] The task handler instance
+      def establish_step_dependencies_and_defaults_via_handler(task, steps, task_handler)
+        # Call the task handler's hook method if it exists
+        return unless task_handler.respond_to?(:establish_step_dependencies_and_defaults)
+
+        task_handler.establish_step_dependencies_and_defaults(task, steps)
+      end
+
+      # Update annotations using task handler hook
+      #
+      # @param task [Tasker::Task] The task being processed
+      # @param sequence [Tasker::Types::StepSequence] The sequence of steps
+      # @param steps [Array<Tasker::WorkflowStep>] The processed steps
+      # @param task_handler [Object] The task handler instance
+      def update_annotations_via_handler(task, sequence, steps, task_handler)
+        return unless task_handler.respond_to?(:update_annotations)
+
+        task_handler.update_annotations(task, sequence, steps)
+      end
+
+      # Validate context using task handler schema
+      #
+      # @param context [Hash] The context to validate
+      # @param task_handler [Object] The task handler with schema
+      # @return [Array<String>] Validation errors, if any
+      def validate_context_via_handler(context, task_handler)
+        return [] unless task_handler.respond_to?(:schema) && task_handler.schema
+
+        data = context.to_hash.deep_symbolize_keys
+        JSON::Validator.fully_validate(task_handler.schema, data, strict: true, insert_defaults: true)
+      end
+
+      # Check if task handler supports concurrent processing
+      #
+      # This checks the task handler's concurrent processing configuration
+      # to determine the appropriate processing mode.
+      #
+      # @param task_handler [Object] The task handler instance
+      # @return [Boolean] True if concurrent processing is enabled
+      def supports_concurrent_processing?(_task_handler)
+        true
+      end
+
+      # Get task handler instance for a task
+      #
+      # @param task [Tasker::Task] The task
+      # @return [Object] The task handler instance
+      def get_task_handler_for_task(task)
+        handler_factory = Tasker::HandlerFactory.instance
+        handler_factory.get(
+          task.name,
+          namespace_name: task.named_task.task_namespace.name,
+          version: task.named_task.version
+        )
+      end
+
+      # Get task handler class for a task name
+      #
+      # This provides access to the handler class without instantiation.
+      #
+      # @param task_name [String] The task name
+      # @return [Class] The task handler class
+      def get_task_handler_class(task_name)
+        handler_factory = Tasker::HandlerFactory.instance
+        handler_factory.handler_class_for(task_name)
+      end
+    end
+  end
+end