tasker-rb 0.1.1

data/lib/tasker_core/worker/event_poller.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module TaskerCore
+  module Worker
+    # EventPoller polls for step execution events from Rust via FFI
+    #
+    # Solves the cross-thread communication issue between Rust and Ruby by
+    # actively polling the Rust worker for pending events. Events are forwarded
+    # to EventBridge for normal processing.
+    #
+    # The poller runs in a dedicated background thread, continuously checking
+    # for new events from the Rust worker at a configurable interval (default 10ms).
+    # This approach allows Ruby to control the thread context and safely process
+    # events without running into Ruby's Global Interpreter Lock (GIL) limitations.
+    #
+    # @example Starting the poller
+    #   poller = TaskerCore::Worker::EventPoller.instance
+    #   poller.start!
+    #   # => Spawns polling thread with 10ms interval
+    #
+    #   # Check if poller started successfully
+    #   if poller.active?
+    #     puts "Event poller is running"
+    #   end
+    #
+    # @example Checking poller status
+    #   poller.active?
+    #   # => true/false
+    #
+    #   if poller.active? && poller.polling_thread&.alive?
+    #     puts "Poller thread is healthy"
+    #   end
+    #
+    # @example Stopping the poller
+    #   poller.stop!
+    #   # => Gracefully stops polling thread (max 5 second wait)
+    #
+    #   # Poller will wait for current poll to complete
+    #   # and then cleanly shutdown
+    #
+    # Threading Model:
+    # - **Main Thread**: Ruby application, handler execution
+    # - **Polling Thread**: Dedicated background thread for event polling
+    # - **Rust Threads**: Rust worker runtime (separate from Ruby)
+    #
+    # The polling thread:
+    # 1. Calls `TaskerCore::FFI.poll_step_events` to check for events
+    # 2. If event found, forwards to EventBridge for processing
+    # 3. If no event, sleeps for POLL_INTERVAL
+    # 4. On error, logs and sleeps for 10x POLL_INTERVAL
+    #
+    # Why Polling is Necessary:
+    #
+    # Ruby's GIL prevents Rust from directly calling Ruby methods from Rust threads.
+    # Direct FFI callbacks from Rust to Ruby would either:
+    # - Cause segfaults (unsafe thread access)
+    # - Block indefinitely (GIL contention)
+    # - Require complex unsafe code
+    #
+    # Polling allows:
+    # - Ruby to control thread context
+    # - Safe event processing in Ruby thread
+    # - Simple, reliable cross-language communication
+    # - Predictable performance characteristics
+    #
+    # Performance Characteristics:
+    # - **Poll Interval**: 10ms (0.01 seconds) - configurable via POLL_INTERVAL
+    # - **Max Latency**: ~10ms from event generation to processing start
+    # - **CPU Usage**: Minimal (yields during sleep)
+    # - **Error Recovery**: Automatic with exponential backoff (10x interval)
+    #
+    # Event Flow:
+    # 1. Rust worker detects step ready for execution
+    # 2. Rust queues event in internal event queue
+    # 3. EventPoller calls poll_step_events via FFI
+    # 4. Rust returns next event (or nil if queue empty)
+    # 5. EventPoller forwards event to EventBridge
+    # 6. EventBridge publishes to Ruby subscribers
+    # 7. StepExecutionSubscriber executes handler
+    #
+    # @see TaskerCore::Worker::EventBridge For event processing
+    # @see TaskerCore::FFI For Rust FFI operations
+    # @see POLL_INTERVAL For polling frequency configuration
+    class EventPoller
+      include Singleton
+
+      attr_reader :logger, :active, :polling_thread
+
+      # Polling interval in seconds (10ms)
+      # Lower values reduce latency but increase CPU usage
+      # Higher values reduce CPU usage but increase latency
+      POLL_INTERVAL = 0.01
+
+      # TAS-67 Phase 2: Check for starvation warnings every N poll iterations
+      # At 10ms poll interval, 100 iterations = 1 second between starvation checks
+      STARVATION_CHECK_INTERVAL = 100
+
+      def initialize
+        @logger = TaskerCore::Logger.instance
+        @active = false
+        @polling_thread = nil
+        @poll_count = 0 # TAS-67 Phase 2: Counter for starvation check interval
+      end
+
+      # Start polling for events from Rust
+      def start!
+        return if @active
+
+        @active = true
+        logger.info 'Starting EventPoller - polling for step execution events from Rust'
+
+        @polling_thread = Thread.new do
+          poll_events_loop
+        end
+
+        logger.info '✅ EventPoller started successfully'
+      end
+
+      # Stop polling
+      def stop!
+        return unless @active
+
+        logger.info 'Stopping EventPoller...'
+        @active = false
+
+        if @polling_thread&.alive?
+          @polling_thread.join(5.0) # Wait up to 5 seconds for thread to finish
+          @polling_thread.kill if @polling_thread.alive? # Force kill if still running
+        end
+
+        @polling_thread = nil
+        logger.info '✅ EventPoller stopped'
+      end
+
+      # Check if poller is active
+      def active?
+        @active
+      end
+
+      # TAS-67 Phase 2: Get FFI dispatch channel metrics for monitoring
+      #
+      # Returns a hash with the following keys:
+      # - :pending_count - Number of events waiting for completion
+      # - :oldest_pending_age_ms - Age of the oldest pending event in milliseconds
+      # - :newest_pending_age_ms - Age of the newest pending event in milliseconds
+      # - :oldest_event_id - UUID of the oldest pending event (for debugging)
+      # - :starvation_detected - Boolean indicating if any events exceed starvation threshold
+      # - :starving_event_count - Number of events exceeding starvation threshold
+      #
+      # @return [Hash] FFI dispatch metrics, or empty hash if worker not running
+      # @example Check for starvation
+      #   metrics = EventPoller.instance.ffi_dispatch_metrics
+      #   if metrics[:starvation_detected]
+      #     logger.warn "Starvation detected: #{metrics[:starving_event_count]} events waiting"
+      #   end
+      def ffi_dispatch_metrics
+        TaskerCore::FFI.get_ffi_dispatch_metrics
+      rescue StandardError => e
+        logger.debug "Failed to get FFI dispatch metrics: #{e.message}"
+        {}
+      end
+
+      private
+
+      # Main polling loop - runs in dedicated thread
+      def poll_events_loop
+        logger.debug 'EventPoller: Starting poll loop'
+
+        while @active
+          begin
+            # TAS-67 Phase 2: Periodically check for starvation warnings
+            @poll_count += 1
+            check_starvation_periodically
+
+            # Poll for next event from Rust via FFI
+            event_data = TaskerCore::FFI.poll_step_events
+
+            if event_data.nil?
+              # No events available, sleep briefly
+              sleep(POLL_INTERVAL)
+              next
+            end
+
+            # Process the event through EventBridge
+            process_event(event_data)
+          rescue StandardError => e
+            logger.error "EventPoller error: #{e.message}"
+            logger.error e.backtrace.join("\n")
+
+            # Sleep longer on error to avoid tight error loops
+            sleep(POLL_INTERVAL * 10) if @active
+          end
+        end
+
+        logger.debug 'EventPoller: Poll loop terminated'
+      end
+
+      # TAS-67 Phase 2: Check for starvation warnings periodically
+      # This triggers Rust-side logging of any events exceeding the starvation threshold
+      def check_starvation_periodically
+        return unless (@poll_count % STARVATION_CHECK_INTERVAL).zero?
+
+        TaskerCore::FFI.check_starvation_warnings
+      rescue StandardError => e
+        # Don't let starvation check errors affect polling
+        logger.debug "Starvation check error (non-fatal): #{e.message}"
+      end
+
+      # Process a polled event through the EventBridge
+      def process_event(event_data)
+        logger.debug "EventPoller: Processing event from Rust: #{event_data}"
+
+        # Forward to EventBridge for normal processing
+        EventBridge.instance.publish_step_execution(event_data)
+
+        logger.debug 'EventPoller: Event forwarded to EventBridge successfully'
+      rescue StandardError => e
+        logger.error "EventPoller: Failed to process event: #{e.message}"
+        logger.error e.backtrace.join("\n")
+        raise
+      end
+    end
+  end
+end

data/lib/tasker_core/worker/in_process_domain_event_poller.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module TaskerCore
+  module Worker
+    # TAS-65 Phase 4.2: In-Process Domain Event Poller
+    #
+    # Polls for fast domain events (delivery_mode: fast) from the Rust in-process
+    # event bus via FFI. Ruby handlers can subscribe to specific event patterns
+    # for integration purposes (Sentry, DataDog, Slack, etc.).
+    #
+    # This follows the same polling pattern as EventPoller but for domain events
+    # rather than step execution events. Events flow through a broadcast channel
+    # from Rust to Ruby.
+    #
+    # Event Flow:
+    # 1. Step handler (Rust or Ruby) publishes domain event with delivery_mode: fast
+    # 2. EventRouter routes to InProcessEventBus
+    # 3. InProcessEventBus broadcasts to all subscribers (including FFI channel)
+    # 4. This poller calls poll_in_process_events via FFI
+    # 5. Events are dispatched to registered Ruby handlers based on pattern matching
+    #
+    # @example Starting the poller
+    #   poller = TaskerCore::Worker::InProcessDomainEventPoller.instance
+    #   poller.start!
+    #
+    # @example Subscribing to specific event patterns
+    #   poller.subscribe('payment.*') do |event|
+    #     puts "Payment event: #{event[:event_name]}"
+    #     # Forward to Sentry, DataDog, etc.
+    #   end
+    #
+    # @example Subscribing to all events
+    #   poller.subscribe('*') do |event|
+    #     MyMetricsCollector.record(event)
+    #   end
+    #
+    # @example Stopping the poller
+    #   poller.stop!
+    class InProcessDomainEventPoller
+      include Singleton
+
+      attr_reader :logger, :active, :polling_thread
+
+      # Polling interval in seconds (50ms - longer than step events since these are async notifications)
+      POLL_INTERVAL = 0.05
+
+      # Maximum events to poll per iteration
+      MAX_EVENTS_PER_POLL = 10
+
+      def initialize
+        @logger = TaskerCore::Logger.instance
+        @active = false
+        @polling_thread = nil
+        @subscribers = {} # pattern => array of handlers
+        @mutex = Mutex.new
+      end
+
+      # Subscribe a handler to an event pattern
+      #
+      # @param pattern [String] Event pattern (exact, wildcard 'payment.*', or global '*')
+      # @yield [event] Block called when matching event is received
+      # @yieldparam event [Hash] Domain event data
+      # @return [void]
+      #
+      # @example
+      #   poller.subscribe('payment.processed') { |event| puts event[:event_name] }
+      #   poller.subscribe('order.*') { |event| notify_slack(event) }
+      #   poller.subscribe('*') { |event| log_all_events(event) }
+      def subscribe(pattern, &handler)
+        raise ArgumentError, 'Block required for subscription' unless block_given?
+        raise ArgumentError, 'Pattern cannot be empty' if pattern.nil? || pattern.empty?
+
+        @mutex.synchronize do
+          @subscribers[pattern] ||= []
+          @subscribers[pattern] << handler
+        end
+
+        logger.info "InProcessDomainEventPoller: Subscribed to pattern '#{pattern}'"
+      end
+
+      # Unsubscribe all handlers for a pattern
+      #
+      # @param pattern [String] Event pattern to unsubscribe
+      # @return [void]
+      def unsubscribe(pattern)
+        @mutex.synchronize do
+          @subscribers.delete(pattern)
+        end
+
+        logger.info "InProcessDomainEventPoller: Unsubscribed from pattern '#{pattern}'"
+      end
+
+      # Clear all subscriptions
+      def clear_subscriptions!
+        @mutex.synchronize do
+          @subscribers.clear
+        end
+
+        logger.info 'InProcessDomainEventPoller: Cleared all subscriptions'
+      end
+
+      # Get count of registered subscribers
+      def subscriber_count
+        @mutex.synchronize do
+          @subscribers.values.flatten.size
+        end
+      end
+
+      # Start polling for in-process domain events
+      def start!
+        return if @active
+
+        @active = true
+        logger.info 'Starting InProcessDomainEventPoller - polling for fast domain events from Rust'
+
+        @polling_thread = Thread.new do
+          poll_events_loop
+        end
+
+        logger.info '✅ InProcessDomainEventPoller started successfully'
+      end
+
+      # Stop polling
+      def stop!
+        return unless @active
+
+        logger.info 'Stopping InProcessDomainEventPoller...'
+        @active = false
+
+        if @polling_thread&.alive?
+          @polling_thread.join(5.0) # Wait up to 5 seconds for thread to finish
+          @polling_thread.kill if @polling_thread.alive? # Force kill if still running
+        end
+
+        @polling_thread = nil
+        logger.info '✅ InProcessDomainEventPoller stopped'
+      end
+
+      # Check if poller is active
+      def active?
+        @active
+      end
+
+      # Get statistics about the poller
+      def stats
+        ffi_stats = begin
+          TaskerCore::FFI.in_process_event_stats
+        rescue StandardError => e
+          logger.warn "Failed to get FFI stats: #{e.message}"
+          { enabled: false, status: 'unknown' }
+        end
+
+        {
+          active: @active,
+          subscriber_count: subscriber_count,
+          patterns: @subscribers.keys,
+          ffi: ffi_stats
+        }
+      end
+
+      private
+
+      # Main polling loop - runs in dedicated thread
+      def poll_events_loop
+        logger.debug 'InProcessDomainEventPoller: Starting poll loop'
+
+        while @active
+          begin
+            # Poll for events from Rust via FFI
+            events = TaskerCore::FFI.poll_in_process_events(MAX_EVENTS_PER_POLL)
+
+            if events.nil? || events.empty?
+              # No events available, sleep briefly
+              sleep(POLL_INTERVAL)
+              next
+            end
+
+            # Process each event through pattern matching
+            events.each do |event_data|
+              dispatch_event(event_data)
+            end
+          rescue StandardError => e
+            logger.error "InProcessDomainEventPoller error: #{e.message}"
+            logger.error e.backtrace.first(5).join("\n")
+
+            # Sleep longer on error to avoid tight error loops
+            sleep(POLL_INTERVAL * 10) if @active
+          end
+        end
+
+        logger.debug 'InProcessDomainEventPoller: Poll loop terminated'
+      end
+
+      # Dispatch event to matching subscribers
+      def dispatch_event(event_data)
+        event_name = event_data[:event_name] || event_data['event_name']
+        return unless event_name
+
+        logger.debug "InProcessDomainEventPoller: Dispatching event '#{event_name}'"
+
+        # Parse business_payload if it's a JSON string
+        if event_data[:business_payload].is_a?(String)
+          begin
+            event_data[:business_payload] = JSON.parse(event_data[:business_payload])
+          rescue JSON::ParserError => e
+            logger.warn "Failed to parse business_payload JSON: #{e.message}"
+          end
+        end
+
+        # Parse error if it's a JSON string
+        if event_data.dig(:execution_result, :error).is_a?(String)
+          begin
+            event_data[:execution_result][:error] = JSON.parse(event_data[:execution_result][:error])
+          rescue JSON::ParserError => e
+            logger.warn "Failed to parse error JSON: #{e.message}"
+          end
+        end
+
+        # Find and invoke matching handlers
+        handlers = find_matching_handlers(event_name)
+
+        if handlers.empty?
+          logger.debug "InProcessDomainEventPoller: No handlers for event '#{event_name}'"
+          return
+        end
+
+        handlers.each do |handler|
+          invoke_handler_safely(handler, event_data, event_name)
+        end
+      end
+
+      # Find handlers that match the event name
+      def find_matching_handlers(event_name)
+        handlers = []
+
+        @mutex.synchronize do
+          @subscribers.each do |pattern, pattern_handlers|
+            handlers.concat(pattern_handlers) if matches_pattern?(event_name, pattern)
+          end
+        end
+
+        handlers
+      end
+
+      # Check if event name matches pattern
+      def matches_pattern?(event_name, pattern)
+        return true if pattern == '*' # Global wildcard
+
+        if pattern.end_with?('.*')
+          # Wildcard pattern: 'payment.*' matches 'payment.processed', 'payment.failed', etc.
+          prefix = pattern[0..-3] # Remove '.*'
+          event_name.start_with?("#{prefix}.")
+        else
+          # Exact match
+          event_name == pattern
+        end
+      end
+
+      # Safely invoke a handler (fire-and-forget semantics)
+      def invoke_handler_safely(handler, event_data, event_name)
+        handler.call(event_data)
+      rescue StandardError => e
+        # Fire-and-forget: log errors but don't propagate
+        logger.warn "InProcessDomainEventPoller: Handler failed for '#{event_name}': #{e.message}"
+        logger.warn e.backtrace.first(3).join("\n")
+      end
+    end
+  end
+end

data/lib/tasker_core.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require_relative 'tasker_core/version'
+require 'json'
+require 'faraday'
+require 'dry-events'
+require 'dry-struct'
+require 'dry-types'
+require 'dry-validation'
+require 'concurrent-ruby'
+require 'timeout'
+require 'dotenv'
+require 'active_support'
+require 'active_support/core_ext'
+
+# TaskerCore - High-performance workflow orchestration for Ruby
+#
+# This library provides a Ruby interface to the Rust-powered tasker-core orchestration
+# system, enabling developers to build complex, distributed workflows with automatic
+# retry logic, dependency management, and event-driven execution.
+#
+# The system consists of two main layers:
+# - **Rust Foundation**: High-performance orchestration, state management, and messaging
+# - **Ruby Business Logic**: Step handlers, task handlers, and custom workflow logic
+#
+# These layers communicate via FFI (Foreign Function Interface) and an event-driven
+# architecture, providing the performance of Rust with the expressiveness of Ruby.
+#
+# @example Getting started with TaskerCore
+#   # 1. Compile the Rust extension (first time only)
+#   bundle exec rake compile
+#
+#   # 2. Define a step handler for your business logic
+#   class ProcessPaymentHandler < TaskerCore::StepHandler::Base
+#     def call(context)
+#       # Access task context data
+#       amount = context.get_task_field('amount')
+#       currency = context.get_task_field('currency')
+#
+#       # Your business logic here
+#       payment_result = charge_payment(amount, currency)
+#
+#       # Return results
+#       success(result: { payment_id: payment_result.id, status: 'succeeded' })
+#     end
+#   end
+#
+#   # 3. Start the worker to process tasks
+#   TaskerCore::Worker::Bootstrap.start!(
+#     namespaces: ['payments']
+#   )
+#
+# @example Creating and submitting a task
+#   # Build a task request
+#   request = TaskerCore::Types::TaskRequest.build_test(
+#     namespace: "payments",
+#     name: "process_payment",
+#     context: { amount: 100.00, currency: "USD" }
+#   )
+#
+#   # Submit to orchestration system
+#   TaskerCore::FFI.submit_task(request)
+#
+# @example Using the API step handler for HTTP calls
+#   class FetchUserHandler < TaskerCore::StepHandler::Api
+#     def call(context)
+#       user_id = context.get_task_field('user_id')
+#
+#       # Automatic error classification and retry logic
+#       response = get("/users/#{user_id}")
+#
+#       success(result: response.body)
+#     end
+#   end
+#
+# Architecture Overview:
+#
+# 1. **Task State Machine**: 12 states managing overall workflow orchestration
+#    - Initial: Pending, Initializing
+#    - Active: EnqueuingSteps, StepsInProcess, EvaluatingResults
+#    - Waiting: WaitingForDependencies, WaitingForRetry, BlockedByFailures
+#    - Terminal: Complete, Error, Cancelled, ResolvedManually
+#
+# 2. **Step State Machine**: 9 states managing individual step execution
+#    - Pipeline: Pending, Enqueued, InProgress, EnqueuedForOrchestration
+#    - Terminal: Complete, Error, Cancelled, ResolvedManually, WaitingForRetry
+#
+# 3. **Event-Driven Communication**:
+#    - Rust → Ruby: Step execution events via EventPoller
+#    - Ruby → Rust: Step completion events via EventBridge
+#    - PostgreSQL LISTEN/NOTIFY for real-time coordination
+#
+# Load Order (Important):
+# 1. Rust native extension (tasker_rb) - provides FFI and base classes
+# 2. Ruby error classes - error hierarchy for classification
+# 3. Ruby models - data wrappers for FFI types
+# 4. Ruby handlers - step and task handler base classes
+# 5. Event system - EventBridge, EventPoller, Subscriber
+# 6. Bootstrap - worker initialization and lifecycle
+#
+# @see TaskerCore::StepHandler::Base For creating custom step handlers
+# @see TaskerCore::StepHandler::Api For HTTP-based step handlers
+# @see TaskerCore::Worker::Bootstrap For worker initialization
+# @see TaskerCore::Handlers For the public API namespace
+# @see TaskerCore::Errors For error classification and retry logic
+# @see https://github.com/tasker-systems/tasker-core Documentation and examples
+module TaskerCore
+end
+
+begin
+  Dotenv.load
+  # Load the compiled Rust extension first (provides base classes)
+  require_relative 'tasker_core/tasker_rb'
+rescue LoadError => e
+  raise LoadError, <<~MSG
+
+    ❌ Failed to load tasker-rb native extension!
+
+    This usually means the Rust extension hasn't been compiled yet.
+
+    To compile the extension:
+      cd #{File.dirname(__FILE__)}/../..
+      rake compile
+
+    Or if you're using this gem in a Rails application:
+      bundle exec rake tasker_core:compile
+
+    Original error: #{e.message}
+
+  MSG
+end
+
+# Load Ruby modules after Rust extension (they depend on Rust base classes)
+require_relative 'tasker_core/errors'
+require_relative 'tasker_core/logger'
+require_relative 'tasker_core/tracing' # TAS-29 Phase 6: Unified structured logging via FFI
+require_relative 'tasker_core/internal'
+require_relative 'tasker_core/template_discovery'
+require_relative 'tasker_core/test_environment'
+require_relative 'tasker_core/types'
+require_relative 'tasker_core/models'
+require_relative 'tasker_core/handlers'
+require_relative 'tasker_core/batch_processing/batch_worker_context'
+require_relative 'tasker_core/batch_processing/batch_aggregation_scenario'
+require_relative 'tasker_core/registry'
+require_relative 'tasker_core/subscriber'
+require_relative 'tasker_core/event_bridge'
+require_relative 'tasker_core/domain_events' # TAS-65: Domain event publisher infrastructure
+require_relative 'tasker_core/observability' # TAS-77: Observability via FFI
+require_relative 'tasker_core/worker/event_poller'
+require_relative 'tasker_core/worker/in_process_domain_event_poller' # TAS-65: Fast domain events
+require_relative 'tasker_core/bootstrap'
+
+module TaskerCore
+  module Worker
+  end
+end
+
+# Load test environment components if appropriate (before auto-boot)
+TaskerCore::TestEnvironment.load_if_test!