fractor 0.1.4 → 0.1.7

data/lib/fractor/supervisor.rb

@@ -1,28 +1,76 @@
 # frozen_string_literal: true
 
 require "etc"
+require "timeout"
+require_relative "signal_handler"
+require_relative "error_formatter"
 
 module Fractor
+  # Custom exception for shutdown signal handling
+  class ShutdownSignal < StandardError; end
+
   # Supervises multiple WrappedRactors, distributes work, and aggregates results.
   class Supervisor
-    attr_reader :work_queue, :workers, :results, :worker_pools
+    attr_reader :work_queue, :workers, :results, :worker_pools, :debug,
+                :error_reporter, :logger, :performance_monitor
 
     # Initializes the Supervisor.
     # - worker_pools: An array of worker pool configurations, each containing:
     #   - worker_class: The class inheriting from Fractor::Worker (e.g., MyWorker).
     #   - num_workers: The number of Ractors to spawn for this worker class.
     # - continuous_mode: Whether to run in continuous mode without expecting a fixed work count.
-    def initialize(worker_pools: [], continuous_mode: false)
-      @worker_pools = worker_pools.map do |pool_config|
+    # - debug: Enable verbose debugging output for all state changes.
+    # - logger: Optional logger instance for this Supervisor (defaults to Fractor.logger).
+    #          Provides isolation when multiple gems use Fractor in the same process.
+    # - tracer_enabled: Optional override for ExecutionTracer (nil uses global setting).
+    # - tracer_stream: Optional trace stream for this Supervisor (nil uses global setting).
+    # - enable_performance_monitoring: Enable performance monitoring (latency, throughput, etc.).
+    def initialize(worker_pools: [], continuous_mode: false, debug: false, logger: nil,
+                   tracer_enabled: nil, tracer_stream: nil, enable_performance_monitoring: false)
+      @debug = debug || ENV["FRACTOR_DEBUG"] == "1"
+      @logger = logger # Store instance-specific logger for isolation
+      @tracer_enabled = tracer_enabled
+      @tracer_stream = tracer_stream
+      @worker_pools = worker_pools.map.with_index do |pool_config, index|
         worker_class = pool_config[:worker_class]
         num_workers = pool_config[:num_workers] || detect_num_workers
 
-        raise ArgumentError, "#{worker_class} must inherit from Fractor::Worker" unless worker_class < Fractor::Worker
+        # Validate worker_class
+        unless worker_class.is_a?(Class)
+          raise ArgumentError,
+                "worker_class must be a Class (got #{worker_class.class}), in worker_pools[#{index}]\n\n" \
+                "Expected: { worker_class: MyWorker }\n" \
+                "Got:      { worker_class: #{worker_class.inspect} }\n\n" \
+                "Fix: Use the class itself, not a symbol or string.\n" \
+                "Example: { worker_class: MyWorker }  # Correct\n" \
+                "         { worker_class: 'MyWorker' } # Wrong - this is a string"
+        end
+
+        unless worker_class < Fractor::Worker
+          raise ArgumentError,
+                "#{worker_class} must inherit from Fractor::Worker, in worker_pools[#{index}]\n\n" \
+                "Your worker class must be defined as:\n" \
+                "  class #{worker_class} < Fractor::Worker\n" \
+                "    def process(work)\n" \
+                "      # ...\n" \
+                "    end\n" \
+                "  end\n\n" \
+                "Did you forget to inherit from Fractor::Worker?"
+        end
+
+        # Validate num_workers
+        unless num_workers.is_a?(Integer) && num_workers.positive?
+          raise ArgumentError,
+                "num_workers must be a positive integer (got #{num_workers.inspect}), in worker_pools[#{index}]\n\n" \
+                "Valid values: Integer >= 1\n" \
+                "Examples: { num_workers: 4 }  # Use 4 workers\n" \
+                "          { num_workers: Etc.nprocessors }  # Use available CPUs"
+        end
 
         {
           worker_class: worker_class,
           num_workers: num_workers,
-          workers: [] # Will hold the WrappedRactor instances
+          workers: [], # Will hold the WrappedRactor instances
         }
       end
 
@@ -34,16 +82,81 @@ module Fractor
       @continuous_mode = continuous_mode
       @running = false
       @work_callbacks = []
+      @wakeup_ractor = nil # Control ractor for unblocking select
+      @timer_thread = nil # Timer thread for periodic wakeup
+      @error_reporter = ErrorReporter.new # Track errors and statistics
+      @error_callbacks = [] # Custom error callbacks
+      @performance_monitor = nil # Performance monitor instance
+
+      # Initialize performance monitor if enabled
+      if enable_performance_monitoring
+        require_relative "performance_monitor"
+        @performance_monitor = PerformanceMonitor.new(self)
+        @performance_monitor.start
+      end
+
+      # Initialize work distribution manager (handles idle workers and work assignment)
+      @work_distribution_manager = WorkDistributionManager.new(
+        @work_queue,
+        @workers,
+        @ractors_map,
+        debug: @debug,
+        continuous_mode: @continuous_mode,
+        performance_monitor: @performance_monitor,
+      )
+
+      # Initialize shutdown handler (manages graceful shutdown)
+      @shutdown_handler = ShutdownHandler.new(
+        @workers,
+        @wakeup_ractor,
+        @timer_thread,
+        @performance_monitor,
+        debug: @debug,
+      )
+
+      # Initialize signal handler for graceful shutdown
+      @signal_handler = SignalHandler.new(
+        continuous_mode: @continuous_mode,
+        debug: @debug,
+        status_callback: -> { print_status },
+        shutdown_callback: ->(mode) { handle_shutdown_callback(mode) },
+      )
+
+      # Initialize error formatter for error messages
+      @error_formatter = ErrorFormatter.new
     end
 
     # Adds a single work item to the queue.
     # The item must be an instance of Fractor::Work or a subclass.
     def add_work_item(work)
-      raise ArgumentError, "#{work.class} must be an instance of Fractor::Work" unless work.is_a?(Fractor::Work)
+      unless work.is_a?(Fractor::Work)
+        raise ArgumentError,
+              "#{work.class} must be an instance of Fractor::Work.\n\n" \
+              "Received: #{work.inspect}\n\n" \
+              "To create a valid work item:\n" \
+              "  class MyWork < Fractor::Work\n" \
+              "    def initialize(data)\n" \
+              "      super({ value: data })\n" \
+              "    end\n" \
+              "  end\n\n" \
+              "  work = MyWork.new(42)\n" \
+              "  supervisor.add_work_item(work)"
+      end
 
       @work_queue << work
       @total_work_count += 1
-      return unless ENV["FRACTOR_DEBUG"]
+
+      # Trace work item queued
+      trace_work(:queued, work, queue_size: @work_queue.size)
+
+      # Distribute to idle workers if work_distribution_manager is available
+      # This ensures work added from callbacks gets picked up immediately
+      if @work_distribution_manager && @running
+        distributed = @work_distribution_manager.distribute_to_idle_workers
+        puts "Distributed work to #{distributed} idle workers after add_work_item" if @debug && distributed.positive?
+      end
+
+      return unless @debug
 
       puts "Work item added. Initial work count: #{@total_work_count}, Queue size: #{@work_queue.size}"
     end
@@ -62,14 +175,72 @@ module Fractor
       @work_callbacks << callback
     end
 
+    # Register a callback to handle errors
+    # The callback receives (error_result, worker_name, worker_class)
+    # Example: supervisor.on_error { |err, worker, klass| puts "Error in #{klass}: #{err.error}" }
+    def on_error(&callback)
+      @error_callbacks << callback
+    end
+
     # Starts the worker Ractors for all worker pools.
     def start_workers
+      # Capture debug flag for Ractor isolation (Ruby 4.0)
+      # Pass as parameter to avoid isolation error
+      debug_mode = @debug
+
+      # Check if running on Ruby 4.0
+      ruby_4_0 = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("4.0.0")
+
+      # Create a wakeup Ractor for unblocking Ractor.select
+      if ruby_4_0
+        # In Ruby 4.0, wakeup uses ports too
+        @wakeup_port = Ractor::Port.new
+        @wakeup_ractor = Ractor.new(@wakeup_port, debug_mode) do |port, debug|
+          puts "Wakeup Ractor started" if debug
+          loop do
+            msg = Ractor.receive
+            puts "Wakeup Ractor received: #{msg.inspect}" if debug
+            if %i[wakeup shutdown].include?(msg)
+              port << { type: :wakeup, message: msg }
+              break if msg == :shutdown
+            end
+          end
+          puts "Wakeup Ractor shutting down" if debug
+        end
+      else
+        @wakeup_ractor = Ractor.new(debug_mode) do |debug|
+          puts "Wakeup Ractor started" if debug
+          loop do
+            msg = Ractor.receive
+            puts "Wakeup Ractor received: #{msg.inspect}" if debug
+            if %i[wakeup shutdown].include?(msg)
+              Ractor.yield({ type: :wakeup, message: msg })
+              break if msg == :shutdown
+            end
+          end
+          puts "Wakeup Ractor shutting down" if debug
+        end
+      end
+
+      # Add wakeup ractor to the map with a special marker
+      @ractors_map[@wakeup_ractor] = :wakeup
+
       @worker_pools.each do |pool|
         worker_class = pool[:worker_class]
         num_workers = pool[:num_workers]
 
         pool[:workers] = (1..num_workers).map do |i|
-          wrapped_ractor = WrappedRactor.new("worker #{worker_class}:#{i}", worker_class)
+          # In Ruby 4.0, create a response port for each worker
+          response_port = if ruby_4_0
+                            Ractor::Port.new
+                          end
+
+          # Use the factory method to create the appropriate implementation
+          wrapped_ractor = WrappedRactor.create(
+            "worker #{worker_class}:#{i}",
+            worker_class,
+            response_port: response_port,
+          )
           wrapped_ractor.start # Start the underlying Ractor
           # Map the actual Ractor object to the WrappedRactor instance
           @ractors_map[wrapped_ractor.ractor] = wrapped_ractor if wrapped_ractor.ractor
@@ -80,40 +251,60 @@ module Fractor
       # Flatten all workers for easier access
       @workers = @worker_pools.flat_map { |pool| pool[:workers] }
       @ractors_map.compact! # Ensure map doesn't contain nil keys/values
-      return unless ENV["FRACTOR_DEBUG"]
+
+      # Update work distribution manager's workers reference
+      # This is critical because @workers was reassigned, and WorkDistributionManager
+      # needs the updated reference to properly track idle workers
+      @work_distribution_manager.update_workers(@workers)
+
+      # Mark all workers as idle initially so they can receive work
+      # This is critical for Ruby 4.0 where workers don't send :initialize messages
+      @workers.each do |worker|
+        @work_distribution_manager.mark_worker_idle(worker)
+      end
+
+      return unless @debug
 
       puts "Workers started: #{@workers.size} active across #{@worker_pools.size} pools."
+      puts "All workers marked as idle and ready for work."
     end
 
-    # Sets up a signal handler for graceful shutdown (Ctrl+C).
+    # Sets up signal handlers for graceful shutdown.
+    # Uses SignalHandler to manage signal handling logic.
     def setup_signal_handler
-      # Store instance variables in local variables for the signal handler
-      workers_ref = @workers
-
-      # Trap INT signal (Ctrl+C)
-      Signal.trap("INT") do
-        puts "\nCtrl+C received. Initiating immediate shutdown..." if ENV["FRACTOR_DEBUG"]
-
-        # Set running to false to break the main loop
-        @running = false
+      @signal_handler.setup
+    end
 
-        puts "Sending shutdown message to all Ractors..." if ENV["FRACTOR_DEBUG"]
+    # Callback for signal handler shutdown requests.
+    def handle_shutdown_callback(mode)
+      if mode == :graceful
+        stop
+      else
+        # Immediate shutdown - raise signal in current thread
+        Thread.current.raise(ShutdownSignal, "Interrupted by signal")
+      end
+    end
 
-        # Send shutdown message to each worker Ractor
-        workers_ref.each do |w|
-          w.send(:shutdown)
-          puts "Sent shutdown to Ractor: #{w.name}" if ENV["FRACTOR_DEBUG"]
-        rescue StandardError => e
-          puts "Error sending shutdown to Ractor #{w.name}: #{e.message}" if ENV["FRACTOR_DEBUG"]
-        end
+    # Prints current supervisor status
+    def print_status
+      status = @work_distribution_manager.status_summary
+      puts "\n=== Fractor Supervisor Status ==="
+      puts "Mode: #{@continuous_mode ? 'Continuous' : 'Batch'}"
+      puts "Running: #{@running}"
+      puts "Workers: #{@workers.size}"
+      puts "Idle workers: #{status[:idle]}"
+      puts "Queue size: #{@work_queue.size}"
+      puts "Results: #{@results.results.size}"
+      puts "Errors: #{@results.errors.size}"
+      puts "================================\n"
+    end
 
-        puts "Exiting now." if ENV["FRACTOR_DEBUG"]
-        exit!(1) # Use exit! to exit immediately without running at_exit handlers
-      rescue Exception => e
-        puts "Error in signal handler: #{e.class}: #{e.message}" if ENV["FRACTOR_DEBUG"]
-        puts e.backtrace.join("\n") if ENV["FRACTOR_DEBUG"]
-        exit!(1)
-      end
+    # Starts the supervisor (alias for run).
+    # Provides a consistent API with stop method.
+    #
+    # @see #run
+    def start
+      run
     end
 
     # Runs the main processing loop.
@@ -122,147 +313,242 @@ module Fractor
       start_workers
 
       @running = true
-      processed_count = 0
-
-      # Main loop: Process events until conditions are met for termination
-      while @running && (@continuous_mode || processed_count < @total_work_count)
-        processed_count = @results.results.size + @results.errors.size
-
-        if ENV["FRACTOR_DEBUG"]
-          if @continuous_mode
-            puts "Continuous mode: Waiting for Ractor results. Processed: #{processed_count}, Queue size: #{@work_queue.size}"
-          else
-            puts "Waiting for Ractor results. Processed: #{processed_count}/#{@total_work_count}, Queue size: #{@work_queue.size}"
-          end
-        end
-
-        # Get active Ractor objects from the map keys
-        active_ractors = @ractors_map.keys
-
-        # Check for new work from callbacks if in continuous mode and queue is empty
-        if @continuous_mode && @work_queue.empty? && !@work_callbacks.empty?
-          @work_callbacks.each do |callback|
-            new_work = callback.call
-            add_work_items(new_work) if new_work && !new_work.empty?
-          end
-        end
-
-        # Break if no active workers and queue is empty, but work remains (indicates potential issue)
-        if active_ractors.empty? && @work_queue.empty? && !@continuous_mode && processed_count < @total_work_count
-          puts "Warning: No active workers and queue is empty, but not all work is processed. Exiting loop." if ENV["FRACTOR_DEBUG"]
-          break
-        end
-
-        # In continuous mode, just wait if no active ractors but keep running
-        if active_ractors.empty?
-          break unless @continuous_mode
 
-          sleep(0.1) # Small delay to avoid CPU spinning
-          next
+      # Distribute any work that was added before run() was called
+      # This is critical for Ruby 4.0 where workers need explicit work distribution
+      if @work_distribution_manager
+        distributed = @work_distribution_manager.distribute_to_idle_workers
+        puts "Distributed initial work to #{distributed} idle workers (work_queue.size: #{@work_queue.size})" if @debug || true
+      end
 
-        end
+      # Start timer thread for continuous mode to periodically check work sources
+      start_timer_thread if @continuous_mode && !@work_callbacks.empty?
 
-        # Ractor.select blocks until a message is available from any active Ractor
-        ready_ractor_obj, message = Ractor.select(*active_ractors)
+      begin
+        # Run the main event loop through MainLoopHandler
+        @main_loop_handler = MainLoopHandler.create(self, debug: @debug)
+        @main_loop_handler.run_loop
+      rescue ShutdownSignal => e
+        puts "Shutdown signal caught: #{e.message}" if @debug
+        puts "Sending shutdown message to all Ractors..." if @debug
 
-        # Find the corresponding WrappedRactor instance
-        wrapped_ractor = @ractors_map[ready_ractor_obj]
-        unless wrapped_ractor
-          puts "Warning: Received message from unknown Ractor: #{ready_ractor_obj}. Ignoring." if ENV["FRACTOR_DEBUG"]
-          next
+        # Send shutdown message to each worker Ractor
+        @workers.each do |w|
+          w.send(:shutdown)
+          puts "Sent shutdown to Ractor: #{w.name}" if @debug
+        rescue StandardError => send_error
+          puts "Error sending shutdown to Ractor #{w.name}: #{send_error.message}" if @debug
         end
 
-        puts "Selected Ractor: #{wrapped_ractor.name}, Message Type: #{message[:type]}" if ENV["FRACTOR_DEBUG"]
-
-        # Process the received message
-        case message[:type]
-        when :initialize
-          puts "Ractor initialized: #{message[:processor]}" if ENV["FRACTOR_DEBUG"]
-          # Send work immediately upon initialization if available
-          send_next_work_if_available(wrapped_ractor)
-        when :result
-          # The message[:result] should be a WorkResult object
-          work_result = message[:result]
-          puts "Completed work: #{work_result.inspect} in Ractor: #{message[:processor]}" if ENV["FRACTOR_DEBUG"]
-          @results.add_result(work_result)
-          if ENV["FRACTOR_DEBUG"]
-            puts "Result processed. Total processed: #{@results.results.size + @results.errors.size}"
-            puts "Aggregated Results: #{@results.inspect}" unless @continuous_mode
-          end
-          # Send next piece of work
-          send_next_work_if_available(wrapped_ractor)
-        when :error
-          # The message[:result] should be a WorkResult object containing the error
-          error_result = message[:result]
-          puts "Error processing work #{error_result.work&.inspect} in Ractor: #{message[:processor]}: #{error_result.error}" if ENV["FRACTOR_DEBUG"]
-          @results.add_result(error_result) # Add error to aggregator
-          if ENV["FRACTOR_DEBUG"]
-            puts "Error handled. Total processed: #{@results.results.size + @results.errors.size}"
-            puts "Aggregated Results (including errors): #{@results.inspect}" unless @continuous_mode
-          end
-          # Send next piece of work even after an error
-          send_next_work_if_available(wrapped_ractor)
-        else
-          puts "Unknown message type received: #{message[:type]} from #{wrapped_ractor.name}" if ENV["FRACTOR_DEBUG"]
-        end
-        # Update processed count for the loop condition
-        processed_count = @results.results.size + @results.errors.size
+        puts "Exiting due to shutdown signal." if @debug
+        exit!(1) # Force exit immediately
       end
 
-      puts "Main loop finished." if ENV["FRACTOR_DEBUG"]
       return if @continuous_mode
 
-      return unless ENV["FRACTOR_DEBUG"]
+      return unless @debug
 
       puts "Final Aggregated Results: #{@results.inspect}"
     end
 
     # Stop the supervisor (for continuous mode)
     def stop
+      puts "Stopping supervisor..." if @debug
+
+      # Initiate shutdown in main loop handler first, so it continues
+      # processing shutdown acknowledgments even after @running = false
+      @main_loop_handler&.initiate_shutdown
+
       @running = false
-      puts "Stopping supervisor..." if ENV["FRACTOR_DEBUG"]
+
+      # Update shutdown handler with current references before shutdown
+      @shutdown_handler.instance_variable_set(:@workers, @workers)
+      @shutdown_handler.instance_variable_set(:@wakeup_ractor, @wakeup_ractor)
+      @shutdown_handler.instance_variable_set(:@timer_thread, @timer_thread)
+      @shutdown_handler.instance_variable_set(:@performance_monitor,
+                                              @performance_monitor)
+
+      # Send shutdown signals but don't wait for workers to close
+      # The caller (e.g., ContinuousServer) should wait for the main loop thread
+      @shutdown_handler.shutdown
     end
 
     private
 
+    # Start the timer thread for continuous mode.
+    # This thread periodically wakes up the main loop to check for new work.
+    #
+    # @return [void]
+    def start_timer_thread
+      @timer_thread = Thread.new do
+        while @running
+          sleep(0.1) # Check work sources every 100ms
+          if @wakeup_ractor && @running
+            begin
+              @wakeup_ractor.send(:wakeup)
+            rescue StandardError => e
+              puts "Timer thread error sending wakeup: #{e.message}" if @debug
+              break
+            end
+          end
+        end
+        puts "Timer thread shutting down" if @debug
+      end
+    end
+
+    # Format error context with rich information for debugging.
+    # Uses ErrorFormatter to generate formatted error messages.
+    #
+    # @param wrapped_ractor [WrappedRactor] The worker that encountered the error
+    # @param error_result [WorkResult] The error result
+    # @return [String] Formatted error message with context
+    def format_error_context(wrapped_ractor, error_result)
+      @error_formatter.format(wrapped_ractor, error_result)
+    end
+
+    # Trace a work event using instance-specific or global tracer configuration.
+    # This allows multiple Supervisors to have independent tracer settings.
+    # @param event [Symbol] The event type (:queued, :completed, :failed, etc.)
+    # @param work [Work] The work item
+    # @param context [Hash] Additional context (worker_name, worker_class, etc.)
+    def trace_work(event, work = nil, context = {})
+      # Check if instance-specific tracing is configured
+      if @tracer_enabled.nil? && @tracer_stream.nil?
+        # No instance config - use global ExecutionTracer
+        Fractor::ExecutionTracer.trace(event, work, context)
+        return
+      end
+
+      # Instance-specific tracing - do it here
+      enabled = @tracer_enabled.nil? ? ExecutionTracer.enabled? : @tracer_enabled
+      return unless enabled
+
+      stream = @tracer_stream || ExecutionTracer.trace_stream
+      timestamp = Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N")
+      thread_id = Thread.current.object_id
+
+      # Build trace line (simplified version of ExecutionTracer logic)
+      parts = [
+        "[TRACE]",
+        timestamp,
+        "[T#{thread_id}]",
+        event.to_s.upcase,
+      ]
+
+      if work
+        work_info = work.instance_of?(::Fractor::Work) ? "Work" : work.class.name
+        parts << "#{work_info}:#{work.object_id}"
+      end
+
+      if context[:worker_name]
+        parts << "worker=#{context[:worker_name]}"
+      end
+      if context[:worker_class]
+        parts << "class=#{context[:worker_class]}"
+      end
+      if context[:duration_ms]
+        parts << "duration=#{context[:duration_ms]}ms"
+      end
+      if context[:queue_size]
+        parts << "queue_size=#{context[:queue_size]}"
+      end
+
+      stream.puts(parts.join(" "))
+    end
+
     # Detects the number of available processors on the system.
     # Returns the number of processors, or 2 as a fallback if detection fails.
     def detect_num_workers
       num_processors = Etc.nprocessors
-      puts "Auto-detected #{num_processors} available processors" if ENV["FRACTOR_DEBUG"]
+      puts "Auto-detected #{num_processors} available processors" if @debug
       num_processors
     rescue StandardError => e
-      puts "Failed to detect processors: #{e.message}. Using default of 2 workers." if ENV["FRACTOR_DEBUG"]
+      puts "Failed to detect processors: #{e.message}. Using default of 2 workers." if @debug
       2
     end
 
-    # Helper method to send the next available work item to a specific Ractor.
-    def send_next_work_if_available(wrapped_ractor)
-      # Ensure the wrapped_ractor instance is valid and its underlying ractor is not closed
-      if wrapped_ractor && !wrapped_ractor.closed?
-        if !@work_queue.empty?
-          work_item = @work_queue.pop # Now directly a Work object
-
-          puts "Sending next work #{work_item.inspect} to Ractor: #{wrapped_ractor.name}" if ENV["FRACTOR_DEBUG"]
-          wrapped_ractor.send(work_item) # Send the Work object
-          puts "Work sent to #{wrapped_ractor.name}." if ENV["FRACTOR_DEBUG"]
-        else
-          puts "Work queue empty. Not sending new work to Ractor #{wrapped_ractor.name}." if ENV["FRACTOR_DEBUG"]
-          # In continuous mode, don't close workers as more work may come
-          unless @continuous_mode
-            # Consider closing the Ractor if the queue is empty and no more work is expected.
-            # wrapped_ractor.close
-            # @ractors_map.delete(wrapped_ractor.ractor)
-            # if ENV["FRACTOR_DEBUG"]
-            #   puts "Closed idle Ractor: #{wrapped_ractor.name}"
-            # end
-          end
-        end
-      else
-        puts "Attempted to send work to an invalid or closed Ractor: #{wrapped_ractor&.name || "unknown"}." if ENV["FRACTOR_DEBUG"]
-        # Remove from map if found but closed
-        @ractors_map.delete(wrapped_ractor.ractor) if wrapped_ractor && @ractors_map.key?(wrapped_ractor.ractor)
+    public
+
+    # ============================================
+    # DEBUGGING METHODS
+    # ============================================
+
+    # Inspect the current state of the work queue
+    # Returns a hash with queue information and items
+    def inspect_queue
+      items = []
+      # Queue doesn't have to_a, need to iterate
+      temp_queue = Queue.new
+      until @work_queue.empty?
+        item = @work_queue.pop
+        items << item
+        temp_queue.push(item)
+      end
+      # Restore the queue
+      until temp_queue.empty?
+        @work_queue.push(temp_queue.pop)
       end
+
+      {
+        size: @work_queue.size,
+        total_added: @total_work_count,
+        items: items.map do |work|
+          {
+            class: work.class.name,
+            input: work.input,
+            inspect: work.inspect,
+          }
+        end,
+      }
+    end
+
+    # Get current worker status
+    # Returns a hash with worker statistics
+    def workers_status
+      status = @work_distribution_manager.status_summary
+      idle_count = status[:idle]
+      busy_count = status[:busy]
+
+      {
+        total: @workers.size,
+        idle: idle_count,
+        busy: busy_count,
+        pools: @worker_pools.map do |pool|
+          {
+            worker_class: pool[:worker_class].name,
+            num_workers: pool[:num_workers],
+            workers: pool[:workers].map do |w|
+              {
+                name: w.name,
+                idle: @work_distribution_manager.idle_workers_list.include?(w),
+              }
+            end,
+          }
+        end,
+      }
+    end
+
+    # Enable debug mode for verbose output
+    def debug!
+      @debug = true
+    end
+
+    # Disable debug mode
+    def debug_off!
+      @debug = false
+    end
+
+    # Check if debug mode is enabled
+    def debug?
+      @debug
+    end
+
+    # Get performance metrics snapshot if performance monitoring is enabled
+    # Returns nil if performance monitoring is not enabled
+    def performance_metrics
+      return nil unless @performance_monitor
+
+      @performance_monitor.snapshot
     end
   end
 end