fractor 0.1.6 → 0.1.7

data/lib/fractor/supervisor.rb

@@ -2,6 +2,8 @@
 
 require "etc"
 require "timeout"
+require_relative "signal_handler"
+require_relative "error_formatter"
 
 module Fractor
   # Custom exception for shutdown signal handling
@@ -9,21 +11,60 @@ module Fractor
 
   # Supervises multiple WrappedRactors, distributes work, and aggregates results.
   class Supervisor
-    attr_reader :work_queue, :workers, :results, :worker_pools
+    attr_reader :work_queue, :workers, :results, :worker_pools, :debug,
+                :error_reporter, :logger, :performance_monitor
 
     # Initializes the Supervisor.
     # - worker_pools: An array of worker pool configurations, each containing:
     #   - worker_class: The class inheriting from Fractor::Worker (e.g., MyWorker).
     #   - num_workers: The number of Ractors to spawn for this worker class.
     # - continuous_mode: Whether to run in continuous mode without expecting a fixed work count.
-    def initialize(worker_pools: [], continuous_mode: false)
-      @worker_pools = worker_pools.map do |pool_config|
+    # - debug: Enable verbose debugging output for all state changes.
+    # - logger: Optional logger instance for this Supervisor (defaults to Fractor.logger).
+    #          Provides isolation when multiple gems use Fractor in the same process.
+    # - tracer_enabled: Optional override for ExecutionTracer (nil uses global setting).
+    # - tracer_stream: Optional trace stream for this Supervisor (nil uses global setting).
+    # - enable_performance_monitoring: Enable performance monitoring (latency, throughput, etc.).
+    def initialize(worker_pools: [], continuous_mode: false, debug: false, logger: nil,
+                   tracer_enabled: nil, tracer_stream: nil, enable_performance_monitoring: false)
+      @debug = debug || ENV["FRACTOR_DEBUG"] == "1"
+      @logger = logger # Store instance-specific logger for isolation
+      @tracer_enabled = tracer_enabled
+      @tracer_stream = tracer_stream
+      @worker_pools = worker_pools.map.with_index do |pool_config, index|
         worker_class = pool_config[:worker_class]
         num_workers = pool_config[:num_workers] || detect_num_workers
 
+        # Validate worker_class
+        unless worker_class.is_a?(Class)
+          raise ArgumentError,
+                "worker_class must be a Class (got #{worker_class.class}), in worker_pools[#{index}]\n\n" \
+                "Expected: { worker_class: MyWorker }\n" \
+                "Got:      { worker_class: #{worker_class.inspect} }\n\n" \
+                "Fix: Use the class itself, not a symbol or string.\n" \
+                "Example: { worker_class: MyWorker }  # Correct\n" \
+                "         { worker_class: 'MyWorker' } # Wrong - this is a string"
+        end
+
         unless worker_class < Fractor::Worker
           raise ArgumentError,
-                "#{worker_class} must inherit from Fractor::Worker"
+                "#{worker_class} must inherit from Fractor::Worker, in worker_pools[#{index}]\n\n" \
+                "Your worker class must be defined as:\n" \
+                "  class #{worker_class} < Fractor::Worker\n" \
+                "    def process(work)\n" \
+                "      # ...\n" \
+                "    end\n" \
+                "  end\n\n" \
+                "Did you forget to inherit from Fractor::Worker?"
+        end
+
+        # Validate num_workers
+        unless num_workers.is_a?(Integer) && num_workers.positive?
+          raise ArgumentError,
+                "num_workers must be a positive integer (got #{num_workers.inspect}), in worker_pools[#{index}]\n\n" \
+                "Valid values: Integer >= 1\n" \
+                "Examples: { num_workers: 4 }  # Use 4 workers\n" \
+                "          { num_workers: Etc.nprocessors }  # Use available CPUs"
         end
 
         {
@@ -43,7 +84,46 @@ module Fractor
       @work_callbacks = []
       @wakeup_ractor = nil # Control ractor for unblocking select
       @timer_thread = nil # Timer thread for periodic wakeup
-      @idle_workers = [] # Track workers waiting for work
+      @error_reporter = ErrorReporter.new # Track errors and statistics
+      @error_callbacks = [] # Custom error callbacks
+      @performance_monitor = nil # Performance monitor instance
+
+      # Initialize performance monitor if enabled
+      if enable_performance_monitoring
+        require_relative "performance_monitor"
+        @performance_monitor = PerformanceMonitor.new(self)
+        @performance_monitor.start
+      end
+
+      # Initialize work distribution manager (handles idle workers and work assignment)
+      @work_distribution_manager = WorkDistributionManager.new(
+        @work_queue,
+        @workers,
+        @ractors_map,
+        debug: @debug,
+        continuous_mode: @continuous_mode,
+        performance_monitor: @performance_monitor,
+      )
+
+      # Initialize shutdown handler (manages graceful shutdown)
+      @shutdown_handler = ShutdownHandler.new(
+        @workers,
+        @wakeup_ractor,
+        @timer_thread,
+        @performance_monitor,
+        debug: @debug,
+      )
+
+      # Initialize signal handler for graceful shutdown
+      @signal_handler = SignalHandler.new(
+        continuous_mode: @continuous_mode,
+        debug: @debug,
+        status_callback: -> { print_status },
+        shutdown_callback: ->(mode) { handle_shutdown_callback(mode) },
+      )
+
+      # Initialize error formatter for error messages
+      @error_formatter = ErrorFormatter.new
     end
 
     # Adds a single work item to the queue.
@@ -51,12 +131,32 @@ module Fractor
     def add_work_item(work)
       unless work.is_a?(Fractor::Work)
         raise ArgumentError,
-              "#{work.class} must be an instance of Fractor::Work"
+              "#{work.class} must be an instance of Fractor::Work.\n\n" \
+              "Received: #{work.inspect}\n\n" \
+              "To create a valid work item:\n" \
+              "  class MyWork < Fractor::Work\n" \
+              "    def initialize(data)\n" \
+              "      super({ value: data })\n" \
+              "    end\n" \
+              "  end\n\n" \
+              "  work = MyWork.new(42)\n" \
+              "  supervisor.add_work_item(work)"
       end
 
       @work_queue << work
       @total_work_count += 1
-      return unless ENV["FRACTOR_DEBUG"]
+
+      # Trace work item queued
+      trace_work(:queued, work, queue_size: @work_queue.size)
+
+      # Distribute to idle workers if work_distribution_manager is available
+      # This ensures work added from callbacks gets picked up immediately
+      if @work_distribution_manager && @running
+        distributed = @work_distribution_manager.distribute_to_idle_workers
+        puts "Distributed work to #{distributed} idle workers after add_work_item" if @debug && distributed.positive?
+      end
+
+      return unless @debug
 
       puts "Work item added. Initial work count: #{@total_work_count}, Queue size: #{@work_queue.size}"
     end
@@ -75,20 +175,51 @@ module Fractor
       @work_callbacks << callback
     end
 
+    # Register a callback to handle errors
+    # The callback receives (error_result, worker_name, worker_class)
+    # Example: supervisor.on_error { |err, worker, klass| puts "Error in #{klass}: #{err.error}" }
+    def on_error(&callback)
+      @error_callbacks << callback
+    end
+
     # Starts the worker Ractors for all worker pools.
     def start_workers
+      # Capture debug flag for Ractor isolation (Ruby 4.0)
+      # Pass as parameter to avoid isolation error
+      debug_mode = @debug
+
+      # Check if running on Ruby 4.0
+      ruby_4_0 = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("4.0.0")
+
       # Create a wakeup Ractor for unblocking Ractor.select
-      @wakeup_ractor = Ractor.new do
-        puts "Wakeup Ractor started" if ENV["FRACTOR_DEBUG"]
-        loop do
-          msg = Ractor.receive
-          puts "Wakeup Ractor received: #{msg.inspect}" if ENV["FRACTOR_DEBUG"]
-          if %i[wakeup shutdown].include?(msg)
-            Ractor.yield({ type: :wakeup, message: msg })
-            break if msg == :shutdown
+      if ruby_4_0
+        # In Ruby 4.0, wakeup uses ports too
+        @wakeup_port = Ractor::Port.new
+        @wakeup_ractor = Ractor.new(@wakeup_port, debug_mode) do |port, debug|
+          puts "Wakeup Ractor started" if debug
+          loop do
+            msg = Ractor.receive
+            puts "Wakeup Ractor received: #{msg.inspect}" if debug
+            if %i[wakeup shutdown].include?(msg)
+              port << { type: :wakeup, message: msg }
+              break if msg == :shutdown
+            end
           end
+          puts "Wakeup Ractor shutting down" if debug
+        end
+      else
+        @wakeup_ractor = Ractor.new(debug_mode) do |debug|
+          puts "Wakeup Ractor started" if debug
+          loop do
+            msg = Ractor.receive
+            puts "Wakeup Ractor received: #{msg.inspect}" if debug
+            if %i[wakeup shutdown].include?(msg)
+              Ractor.yield({ type: :wakeup, message: msg })
+              break if msg == :shutdown
+            end
+          end
+          puts "Wakeup Ractor shutting down" if debug
         end
-        puts "Wakeup Ractor shutting down" if ENV["FRACTOR_DEBUG"]
       end
 
       # Add wakeup ractor to the map with a special marker
@@ -99,7 +230,17 @@ module Fractor
         num_workers = pool[:num_workers]
 
         pool[:workers] = (1..num_workers).map do |i|
-          wrapped_ractor = WrappedRactor.new("worker #{worker_class}:#{i}", worker_class)
+          # In Ruby 4.0, create a response port for each worker
+          response_port = if ruby_4_0
+                            Ractor::Port.new
+                          end
+
+          # Use the factory method to create the appropriate implementation
+          wrapped_ractor = WrappedRactor.create(
+            "worker #{worker_class}:#{i}",
+            worker_class,
+            response_port: response_port,
+          )
           wrapped_ractor.start # Start the underlying Ractor
           # Map the actual Ractor object to the WrappedRactor instance
           @ractors_map[wrapped_ractor.ractor] = wrapped_ractor if wrapped_ractor.ractor
@@ -110,332 +251,304 @@ module Fractor
       # Flatten all workers for easier access
       @workers = @worker_pools.flat_map { |pool| pool[:workers] }
       @ractors_map.compact! # Ensure map doesn't contain nil keys/values
-      return unless ENV["FRACTOR_DEBUG"]
+
+      # Update work distribution manager's workers reference
+      # This is critical because @workers was reassigned, and WorkDistributionManager
+      # needs the updated reference to properly track idle workers
+      @work_distribution_manager.update_workers(@workers)
+
+      # Mark all workers as idle initially so they can receive work
+      # This is critical for Ruby 4.0 where workers don't send :initialize messages
+      @workers.each do |worker|
+        @work_distribution_manager.mark_worker_idle(worker)
+      end
+
+      return unless @debug
 
       puts "Workers started: #{@workers.size} active across #{@worker_pools.size} pools."
+      puts "All workers marked as idle and ready for work."
     end
 
     # Sets up signal handlers for graceful shutdown.
-    # Handles SIGINT (Ctrl+C), SIGTERM (systemd/docker), and platform-specific status signals.
+    # Uses SignalHandler to manage signal handling logic.
     def setup_signal_handler
-      # Universal signals (work on all platforms)
-      Signal.trap("INT") { handle_shutdown("SIGINT") }
-      Signal.trap("TERM") { handle_shutdown("SIGTERM") }
-
-      # Platform-specific status monitoring
-      setup_status_signal
+      @signal_handler.setup
     end
 
-    # Handles shutdown signal by mode (continuous vs batch)
-    def handle_shutdown(signal_name)
-      if @continuous_mode
-        puts "\n#{signal_name} received. Initiating graceful shutdown..." if ENV["FRACTOR_DEBUG"]
+    # Callback for signal handler shutdown requests.
+    def handle_shutdown_callback(mode)
+      if mode == :graceful
         stop
       else
-        puts "\n#{signal_name} received. Initiating immediate shutdown..." if ENV["FRACTOR_DEBUG"]
-        Thread.current.raise(ShutdownSignal, "Interrupted by #{signal_name}")
-      end
-    rescue Exception => e
-      puts "Error in signal handler: #{e.class}: #{e.message}" if ENV["FRACTOR_DEBUG"]
-      puts e.backtrace.join("\n") if ENV["FRACTOR_DEBUG"]
-      exit!(1)
-    end
-
-    # Sets up platform-specific status monitoring signal
-    def setup_status_signal
-      if Gem.win_platform?
-        # Windows: Try SIGBREAK (Ctrl+Break) if available
-        begin
-          Signal.trap("BREAK") { print_status }
-        rescue ArgumentError
-          # SIGBREAK not supported on this Ruby version/platform
-          # Status monitoring unavailable on Windows
-        end
-      else
-        # Unix/Linux/macOS: Use SIGUSR1
-        begin
-          Signal.trap("USR1") { print_status }
-        rescue ArgumentError
-          # SIGUSR1 not supported on this platform
-        end
+        # Immediate shutdown - raise signal in current thread
+        Thread.current.raise(ShutdownSignal, "Interrupted by signal")
       end
     end
 
     # Prints current supervisor status
     def print_status
+      status = @work_distribution_manager.status_summary
       puts "\n=== Fractor Supervisor Status ==="
       puts "Mode: #{@continuous_mode ? 'Continuous' : 'Batch'}"
       puts "Running: #{@running}"
       puts "Workers: #{@workers.size}"
-      puts "Idle workers: #{@idle_workers.size}"
+      puts "Idle workers: #{status[:idle]}"
       puts "Queue size: #{@work_queue.size}"
       puts "Results: #{@results.results.size}"
       puts "Errors: #{@results.errors.size}"
       puts "================================\n"
     end
 
+    # Starts the supervisor (alias for run).
+    # Provides a consistent API with stop method.
+    #
+    # @see #run
+    def start
+      run
+    end
+
     # Runs the main processing loop.
     def run
       setup_signal_handler
       start_workers
 
       @running = true
-      processed_count = 0
 
-      # Start timer thread for continuous mode to periodically check work sources
-      if @continuous_mode && !@work_callbacks.empty?
-        @timer_thread = Thread.new do
-          while @running
-            sleep(0.1) # Check work sources every 100ms
-            if @wakeup_ractor && @running
-              begin
-                @wakeup_ractor.send(:wakeup)
-              rescue StandardError => e
-                puts "Timer thread error sending wakeup: #{e.message}" if ENV["FRACTOR_DEBUG"]
-                break
-              end
-            end
-          end
-          puts "Timer thread shutting down" if ENV["FRACTOR_DEBUG"]
-        end
+      # Distribute any work that was added before run() was called
+      # This is critical for Ruby 4.0 where workers need explicit work distribution
+      if @work_distribution_manager
+        distributed = @work_distribution_manager.distribute_to_idle_workers
+        puts "Distributed initial work to #{distributed} idle workers (work_queue.size: #{@work_queue.size})" if @debug || true
       end
 
-      begin
-        # Main loop: Process events until conditions are met for termination
-        while @running && (@continuous_mode || processed_count < @total_work_count)
-          processed_count = @results.results.size + @results.errors.size
-
-          if ENV["FRACTOR_DEBUG"]
-            if @continuous_mode
-              puts "Continuous mode: Waiting for Ractor results. Processed: #{processed_count}, Queue size: #{@work_queue.size}"
-            else
-              puts "Waiting for Ractor results. Processed: #{processed_count}/#{@total_work_count}, Queue size: #{@work_queue.size}"
-            end
-          end
-
-          # Get active Ractor objects from the map keys
-          active_ractors = @ractors_map.keys
-
-          # Check for new work from callbacks if in continuous mode
-          if @continuous_mode && !@work_callbacks.empty?
-            @work_callbacks.each do |callback|
-              new_work = callback.call
-              if new_work && !new_work.empty?
-                add_work_items(new_work)
-                puts "Work source provided #{new_work.size} new items" if ENV["FRACTOR_DEBUG"]
-
-                # Try to send work to idle workers first
-                while !@work_queue.empty? && !@idle_workers.empty?
-                  worker = @idle_workers.shift
-                  if send_next_work_if_available(worker)
-                    puts "Sent work to idle worker #{worker.name}" if ENV["FRACTOR_DEBUG"]
-                  else
-                    # Worker couldn't accept work, don't re-add to idle list
-                  end
-                end
-              end
-            end
-          end
-
-          # Break if no active workers and queue is empty, but work remains (indicates potential issue)
-          if active_ractors.empty? && @work_queue.empty? && !@continuous_mode && processed_count < @total_work_count
-            puts "Warning: No active workers and queue is empty, but not all work is processed. Exiting loop." if ENV["FRACTOR_DEBUG"]
-            break
-          end
-
-          # In continuous mode, just wait if no active ractors but keep running
-          if active_ractors.empty?
-            break unless @continuous_mode
-
-            sleep(0.1) # Small delay to avoid CPU spinning
-            next
-          end
-
-          # Ractor.select blocks until a message is available from any active Ractor
-          # The wakeup ractor ensures we can unblock this call when needed
-          ready_ractor_obj, message = Ractor.select(*active_ractors)
-
-          # Check if this is the wakeup ractor
-          if ready_ractor_obj == @wakeup_ractor
-            puts "Wakeup signal received: #{message[:message]}" if ENV["FRACTOR_DEBUG"]
-            # Remove wakeup ractor from map if shutting down
-            if message[:message] == :shutdown
-              @ractors_map.delete(@wakeup_ractor)
-              @wakeup_ractor = nil
-            end
-            # Continue loop to check @running flag
-            next
-          end
-
-          # Find the corresponding WrappedRactor instance
-          wrapped_ractor = @ractors_map[ready_ractor_obj]
-          unless wrapped_ractor
-            puts "Warning: Received message from unknown Ractor: #{ready_ractor_obj}. Ignoring." if ENV["FRACTOR_DEBUG"]
-            next
-          end
-
-          puts "Selected Ractor: #{wrapped_ractor.name}, Message Type: #{message[:type]}" if ENV["FRACTOR_DEBUG"]
-
-          # Process the received message
-          case message[:type]
-          when :initialize
-            puts "Ractor initialized: #{message[:processor]}" if ENV["FRACTOR_DEBUG"]
-            # Send work immediately upon initialization if available
-            if send_next_work_if_available(wrapped_ractor)
-              # Work was sent
-            else
-              # No work available, mark worker as idle
-              @idle_workers << wrapped_ractor unless @idle_workers.include?(wrapped_ractor)
-              puts "Worker #{wrapped_ractor.name} marked as idle" if ENV["FRACTOR_DEBUG"]
-            end
-          when :shutdown
-            puts "Ractor #{wrapped_ractor.name} acknowledged shutdown" if ENV["FRACTOR_DEBUG"]
-            # Remove from active ractors
-            @ractors_map.delete(ready_ractor_obj)
-          when :result
-            # The message[:result] should be a WorkResult object
-            work_result = message[:result]
-            puts "Completed work: #{work_result.inspect} in Ractor: #{message[:processor]}" if ENV["FRACTOR_DEBUG"]
-            @results.add_result(work_result)
-            if ENV["FRACTOR_DEBUG"]
-              puts "Result processed. Total processed: #{@results.results.size + @results.errors.size}"
-              puts "Aggregated Results: #{@results.inspect}" unless @continuous_mode
-            end
-            # Send next piece of work
-            if send_next_work_if_available(wrapped_ractor)
-              # Work was sent
-            else
-              # No work available, mark worker as idle
-              @idle_workers << wrapped_ractor unless @idle_workers.include?(wrapped_ractor)
-              puts "Worker #{wrapped_ractor.name} marked as idle after completing work" if ENV["FRACTOR_DEBUG"]
-            end
-          when :error
-            # The message[:result] should be a WorkResult object containing the error
-            error_result = message[:result]
-            puts "Error processing work #{error_result.work&.inspect} in Ractor: #{message[:processor]}: #{error_result.error}" if ENV["FRACTOR_DEBUG"]
-            @results.add_result(error_result) # Add error to aggregator
-            if ENV["FRACTOR_DEBUG"]
-              puts "Error handled. Total processed: #{@results.results.size + @results.errors.size}"
-              puts "Aggregated Results (including errors): #{@results.inspect}" unless @continuous_mode
-            end
-            # Send next piece of work even after an error
-            if send_next_work_if_available(wrapped_ractor)
-              # Work was sent
-            else
-              # No work available, mark worker as idle
-              @idle_workers << wrapped_ractor unless @idle_workers.include?(wrapped_ractor)
-              puts "Worker #{wrapped_ractor.name} marked as idle after error" if ENV["FRACTOR_DEBUG"]
-            end
-          else
-            puts "Unknown message type received: #{message[:type]} from #{wrapped_ractor.name}" if ENV["FRACTOR_DEBUG"]
-          end
-          # Update processed count for the loop condition
-          processed_count = @results.results.size + @results.errors.size
-        end
+      # Start timer thread for continuous mode to periodically check work sources
+      start_timer_thread if @continuous_mode && !@work_callbacks.empty?
 
-        puts "Main loop finished." if ENV["FRACTOR_DEBUG"]
+      begin
+        # Run the main event loop through MainLoopHandler
+        @main_loop_handler = MainLoopHandler.create(self, debug: @debug)
+        @main_loop_handler.run_loop
       rescue ShutdownSignal => e
-        puts "Shutdown signal caught: #{e.message}" if ENV["FRACTOR_DEBUG"]
-        puts "Sending shutdown message to all Ractors..." if ENV["FRACTOR_DEBUG"]
+        puts "Shutdown signal caught: #{e.message}" if @debug
+        puts "Sending shutdown message to all Ractors..." if @debug
 
         # Send shutdown message to each worker Ractor
         @workers.each do |w|
           w.send(:shutdown)
-          puts "Sent shutdown to Ractor: #{w.name}" if ENV["FRACTOR_DEBUG"]
+          puts "Sent shutdown to Ractor: #{w.name}" if @debug
         rescue StandardError => send_error
-          puts "Error sending shutdown to Ractor #{w.name}: #{send_error.message}" if ENV["FRACTOR_DEBUG"]
+          puts "Error sending shutdown to Ractor #{w.name}: #{send_error.message}" if @debug
         end
 
-        puts "Exiting due to shutdown signal." if ENV["FRACTOR_DEBUG"]
+        puts "Exiting due to shutdown signal." if @debug
         exit!(1) # Force exit immediately
       end
 
       return if @continuous_mode
 
-      return unless ENV["FRACTOR_DEBUG"]
+      return unless @debug
 
       puts "Final Aggregated Results: #{@results.inspect}"
     end
 
     # Stop the supervisor (for continuous mode)
     def stop
+      puts "Stopping supervisor..." if @debug
+
+      # Initiate shutdown in main loop handler first, so it continues
+      # processing shutdown acknowledgments even after @running = false
+      @main_loop_handler&.initiate_shutdown
+
       @running = false
-      puts "Stopping supervisor..." if ENV["FRACTOR_DEBUG"]
 
-      # Wait for timer thread to finish if it exists
-      if @timer_thread&.alive?
-        @timer_thread.join(1) # Wait up to 1 second
-        puts "Timer thread stopped" if ENV["FRACTOR_DEBUG"]
-      end
+      # Update shutdown handler with current references before shutdown
+      @shutdown_handler.instance_variable_set(:@workers, @workers)
+      @shutdown_handler.instance_variable_set(:@wakeup_ractor, @wakeup_ractor)
+      @shutdown_handler.instance_variable_set(:@timer_thread, @timer_thread)
+      @shutdown_handler.instance_variable_set(:@performance_monitor,
+                                              @performance_monitor)
 
-      # Signal the wakeup ractor first to unblock Ractor.select
-      if @wakeup_ractor
-        begin
-          @wakeup_ractor.send(:shutdown)
-          puts "Sent shutdown signal to wakeup ractor" if ENV["FRACTOR_DEBUG"]
-        rescue StandardError => e
-          puts "Error sending shutdown to wakeup ractor: #{e.message}" if ENV["FRACTOR_DEBUG"]
-        end
-      end
+      # Send shutdown signals but don't wait for workers to close
+      # The caller (e.g., ContinuousServer) should wait for the main loop thread
+      @shutdown_handler.shutdown
+    end
 
-      # Send shutdown signal to all workers
-      @workers.each do |w|
-        begin
-          w.send(:shutdown)
-        rescue StandardError
-          nil
+    private
+
+    # Start the timer thread for continuous mode.
+    # This thread periodically wakes up the main loop to check for new work.
+    #
+    # @return [void]
+    def start_timer_thread
+      @timer_thread = Thread.new do
+        while @running
+          sleep(0.1) # Check work sources every 100ms
+          if @wakeup_ractor && @running
+            begin
+              @wakeup_ractor.send(:wakeup)
+            rescue StandardError => e
+              puts "Timer thread error sending wakeup: #{e.message}" if @debug
+              break
+            end
+          end
         end
-        puts "Sent shutdown signal to #{w.name}" if ENV["FRACTOR_DEBUG"]
+        puts "Timer thread shutting down" if @debug
       end
     end
 
-    private
+    # Format error context with rich information for debugging.
+    # Uses ErrorFormatter to generate formatted error messages.
+    #
+    # @param wrapped_ractor [WrappedRactor] The worker that encountered the error
+    # @param error_result [WorkResult] The error result
+    # @return [String] Formatted error message with context
+    def format_error_context(wrapped_ractor, error_result)
+      @error_formatter.format(wrapped_ractor, error_result)
+    end
+
+    # Trace a work event using instance-specific or global tracer configuration.
+    # This allows multiple Supervisors to have independent tracer settings.
+    # @param event [Symbol] The event type (:queued, :completed, :failed, etc.)
+    # @param work [Work] The work item
+    # @param context [Hash] Additional context (worker_name, worker_class, etc.)
+    def trace_work(event, work = nil, context = {})
+      # Check if instance-specific tracing is configured
+      if @tracer_enabled.nil? && @tracer_stream.nil?
+        # No instance config - use global ExecutionTracer
+        Fractor::ExecutionTracer.trace(event, work, context)
+        return
+      end
+
+      # Instance-specific tracing - do it here
+      enabled = @tracer_enabled.nil? ? ExecutionTracer.enabled? : @tracer_enabled
+      return unless enabled
+
+      stream = @tracer_stream || ExecutionTracer.trace_stream
+      timestamp = Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N")
+      thread_id = Thread.current.object_id
+
+      # Build trace line (simplified version of ExecutionTracer logic)
+      parts = [
+        "[TRACE]",
+        timestamp,
+        "[T#{thread_id}]",
+        event.to_s.upcase,
+      ]
+
+      if work
+        work_info = work.instance_of?(::Fractor::Work) ? "Work" : work.class.name
+        parts << "#{work_info}:#{work.object_id}"
+      end
+
+      if context[:worker_name]
+        parts << "worker=#{context[:worker_name]}"
+      end
+      if context[:worker_class]
+        parts << "class=#{context[:worker_class]}"
+      end
+      if context[:duration_ms]
+        parts << "duration=#{context[:duration_ms]}ms"
+      end
+      if context[:queue_size]
+        parts << "queue_size=#{context[:queue_size]}"
+      end
+
+      stream.puts(parts.join(" "))
+    end
 
     # Detects the number of available processors on the system.
     # Returns the number of processors, or 2 as a fallback if detection fails.
     def detect_num_workers
       num_processors = Etc.nprocessors
-      puts "Auto-detected #{num_processors} available processors" if ENV["FRACTOR_DEBUG"]
+      puts "Auto-detected #{num_processors} available processors" if @debug
       num_processors
     rescue StandardError => e
-      puts "Failed to detect processors: #{e.message}. Using default of 2 workers." if ENV["FRACTOR_DEBUG"]
+      puts "Failed to detect processors: #{e.message}. Using default of 2 workers." if @debug
       2
     end
 
-    # Helper method to send the next available work item to a specific Ractor.
-    # Returns true if work was sent, false otherwise.
-    def send_next_work_if_available(wrapped_ractor)
-      # Ensure the wrapped_ractor instance is valid and its underlying ractor is not closed
-      if wrapped_ractor && !wrapped_ractor.closed?
-        if @work_queue.empty?
-          puts "Work queue empty. Not sending new work to Ractor #{wrapped_ractor.name}." if ENV["FRACTOR_DEBUG"]
-          # In continuous mode, don't close workers as more work may come
-          unless @continuous_mode
-            # Consider closing the Ractor if the queue is empty and no more work is expected.
-            # wrapped_ractor.close
-            # @ractors_map.delete(wrapped_ractor.ractor)
-            # if ENV["FRACTOR_DEBUG"]
-            #   puts "Closed idle Ractor: #{wrapped_ractor.name}"
-            # end
-          end
-          false
-        else
-          work_item = @work_queue.pop # Now directly a Work object
+    public
+
+    # ============================================
+    # DEBUGGING METHODS
+    # ============================================
+
+    # Inspect the current state of the work queue
+    # Returns a hash with queue information and items
+    def inspect_queue
+      items = []
+      # Queue doesn't have to_a, need to iterate
+      temp_queue = Queue.new
+      until @work_queue.empty?
+        item = @work_queue.pop
+        items << item
+        temp_queue.push(item)
+      end
+      # Restore the queue
+      until temp_queue.empty?
+        @work_queue.push(temp_queue.pop)
+      end
 
-          puts "Sending next work #{work_item.inspect} to Ractor: #{wrapped_ractor.name}" if ENV["FRACTOR_DEBUG"]
-          wrapped_ractor.send(work_item) # Send the Work object
-          puts "Work sent to #{wrapped_ractor.name}." if ENV["FRACTOR_DEBUG"]
+      {
+        size: @work_queue.size,
+        total_added: @total_work_count,
+        items: items.map do |work|
+          {
+            class: work.class.name,
+            input: work.input,
+            inspect: work.inspect,
+          }
+        end,
+      }
+    end
 
-          # Remove from idle workers list since it's now busy
-          @idle_workers.delete(wrapped_ractor)
-          true
-        end
-      else
-        puts "Attempted to send work to an invalid or closed Ractor: #{wrapped_ractor&.name || 'unknown'}." if ENV["FRACTOR_DEBUG"]
-        # Remove from map if found but closed
-        @ractors_map.delete(wrapped_ractor.ractor) if wrapped_ractor && @ractors_map.key?(wrapped_ractor.ractor)
-        false
-      end
+    # Get current worker status
+    # Returns a hash with worker statistics
+    def workers_status
+      status = @work_distribution_manager.status_summary
+      idle_count = status[:idle]
+      busy_count = status[:busy]
+
+      {
+        total: @workers.size,
+        idle: idle_count,
+        busy: busy_count,
+        pools: @worker_pools.map do |pool|
+          {
+            worker_class: pool[:worker_class].name,
+            num_workers: pool[:num_workers],
+            workers: pool[:workers].map do |w|
+              {
+                name: w.name,
+                idle: @work_distribution_manager.idle_workers_list.include?(w),
+              }
+            end,
+          }
+        end,
+      }
+    end
+
+    # Enable debug mode for verbose output
+    def debug!
+      @debug = true
+    end
+
+    # Disable debug mode
+    def debug_off!
+      @debug = false
+    end
+
+    # Check if debug mode is enabled
+    def debug?
+      @debug
+    end
+
+    # Get performance metrics snapshot if performance monitoring is enabled
+    # Returns nil if performance monitoring is not enabled
+    def performance_metrics
+      return nil unless @performance_monitor
+
+      @performance_monitor.snapshot
     end
   end
 end