taski 0.8.3 → 0.9.0

data/lib/taski/execution/worker_pool.rb

@@ -4,100 +4,274 @@ require "etc"
 
 module Taski
   module Execution
-    # WorkerPool manages a pool of worker threads that execute tasks.
-    # It provides methods to start, stop, and enqueue tasks for execution.
-    #
-    # == Responsibilities
-    #
-    # - Manage worker thread lifecycle (start, shutdown)
-    # - Distribute tasks to worker threads via Queue
-    # - Execute tasks via callback provided by Executor
-    #
-    # == API
-    #
-    # - {#start} - Start all worker threads
-    # - {#enqueue} - Add a task to the execution queue
-    # - {#shutdown} - Gracefully shutdown all worker threads
-    # - {#execution_queue} - Access the underlying Queue (for testing)
+    def self.default_worker_count
+      Etc.nprocessors.clamp(2, 8)
+    end
+
+    # WorkerPool manages N threads, each with its own command Queue.
+    # Tasks are executed within Fibers on worker threads.
+    # When a Fiber yields [:need_dep, dep_class, method], the worker
+    # resolves the dependency via TaskWrapper#request_value:
     #
-    # == Thread Safety
+    # - :completed → resume Fiber immediately with the value
+    # - :wait → park the Fiber (it will be resumed later via the thread's queue)
+    # - :start → start the dependency as a nested Fiber on the same thread
     #
-    # WorkerPool uses Queue for thread-safe task distribution.
-    # The Queue handles synchronization between the main thread
-    # (which enqueues tasks) and worker threads (which pop tasks).
+    # Worker threads process these commands:
+    # - [:execute, task_class, wrapper]       → create and drive a new Fiber
+    # - [:execute_clean, task_class, wrapper] → run clean directly (no Fiber)
+    # - [:resume, fiber, value]               → resume a parked Fiber with a value
+    # - [:resume_error, fiber, error]         → resume a parked Fiber with an error
+    # - :shutdown                             → exit the worker loop
     class WorkerPool
-      attr_reader :execution_queue
+      attr_reader :worker_count
 
-      # @param registry [Registry] The task registry for thread tracking
-      # @param worker_count [Integer, nil] Number of worker threads (defaults to CPU count)
-      # @param on_execute [Proc] Callback to execute a task, receives (task_class, wrapper)
-      def initialize(registry:, worker_count: nil, &on_execute)
-        @worker_count = worker_count || default_worker_count
+      def initialize(registry:, execution_facade:, completion_queue:, worker_count: nil)
         @registry = registry
-        @on_execute = on_execute
-        @execution_queue = Queue.new
-        @workers = []
+        @execution_facade = execution_facade
+        @worker_count = worker_count || Execution.default_worker_count
+        @completion_queue = completion_queue
+        @threads = []
+        @thread_queues = []
+        @next_thread_index = 0
+        @fiber_contexts_mutex = Mutex.new
+        @fiber_contexts = {}
+        @task_start_times_mutex = Mutex.new
+        @task_start_times = {}
       end
 
-      # Start all worker threads.
       def start
         @worker_count.times do
-          worker = Thread.new { worker_loop }
-          @workers << worker
-          @registry.register_thread(worker)
+          queue = Queue.new
+          @thread_queues << queue
+          thread = Thread.new(queue) { |q| worker_loop(q) }
+          @threads << thread
+          @registry.register_thread(thread)
         end
       end
 
-      # Enqueue a task for execution.
-      #
-      # @param task_class [Class] The task class to execute
-      # @param wrapper [TaskWrapper] The task wrapper
+      # Round-robins across worker threads.
       def enqueue(task_class, wrapper)
-        @execution_queue.push({task_class: task_class, wrapper: wrapper})
-        debug_log("Enqueued: #{task_class}")
+        queue = @thread_queues[@next_thread_index % @worker_count]
+        @next_thread_index += 1
+        queue.push([:execute, task_class, wrapper])
+        Taski::Logging.debug(Taski::Logging::Events::WORKER_POOL_ENQUEUED, task: task_class.name, thread_index: (@next_thread_index - 1) % @worker_count)
       end
 
-      # Shutdown all worker threads gracefully.
-      def shutdown
-        enqueue_shutdown_signals
-        @workers.each(&:join)
+      # Clean tasks run directly without Fiber wrapping.
+      def enqueue_clean(task_class, wrapper)
+        queue = @thread_queues[@next_thread_index % @worker_count]
+        @next_thread_index += 1
+        queue.push([:execute_clean, task_class, wrapper])
       end
 
-      # Enqueue shutdown signals for all workers.
-      def enqueue_shutdown_signals
-        @worker_count.times { @execution_queue.push(:shutdown) }
+      def shutdown
+        @thread_queues.each { |q| q.push(:shutdown) }
+        @threads.each(&:join)
       end
 
       private
 
-      def default_worker_count
-        Etc.nprocessors.clamp(2, 8)
+      def worker_loop(queue)
+        loop do
+          cmd = queue.pop
+          break if cmd == :shutdown
+
+          case cmd[0]
+          when :execute
+            _, task_class, wrapper = cmd
+            drive_fiber(task_class, wrapper, queue)
+          when :resume
+            _, fiber, value = cmd
+            resume_fiber(fiber, value, queue)
+          when :resume_error
+            _, fiber, error = cmd
+            resume_fiber_with_error(fiber, error, queue)
+          when :execute_clean
+            _, task_class, wrapper = cmd
+            execute_clean_task(task_class, wrapper)
+          end
+        end
       end
 
-      def worker_loop
-        loop do
-          work_item = @execution_queue.pop
-          break if work_item == :shutdown
+      # Drive a new Fiber for a task. The caller MUST have already called
+      # wrapper.mark_running before enqueueing — drive_fiber never calls it.
+      def drive_fiber(task_class, wrapper, queue)
+        return if @registry.abort_requested?
 
-          task_class = work_item[:task_class]
-          wrapper = work_item[:wrapper]
+        fiber = Fiber.new do
+          setup_run_thread_locals
+          wrapper.task.run
+        end
 
-          debug_log("Worker executing: #{task_class}")
+        now = Time.now
+        @task_start_times_mutex.synchronize { @task_start_times[task_class] = now }
+        Taski::Logging.info(Taski::Logging::Events::TASK_STARTED, task: task_class.name)
+        @execution_facade.notify_task_updated(task_class, previous_state: :pending, current_state: :running, phase: :run, timestamp: now)
 
-          begin
-            @on_execute.call(task_class, wrapper)
-          rescue => e
-            # Log error but don't crash the worker thread.
-            # Task-level errors are handled in the execute callback.
-            # This catches unexpected errors in the callback itself.
-            warn "[WorkerPool] Unexpected error executing #{task_class}: #{e.message}"
+        start_output_capture(task_class)
+        drive_fiber_loop(fiber, task_class, wrapper, queue)
+      end
+
+      # Drive a Fiber forward by resuming it with resume_value.
+      # fiber.resume is called INSIDE this method so that exceptions
+      # are caught by the rescue and routed to fail_task.
+      def drive_fiber_loop(fiber, task_class, wrapper, queue, resume_value = nil)
+        result = fiber.resume(resume_value)
+
+        while fiber.alive?
+          if result.is_a?(Array) && result[0] == :need_dep
+            _, dep_class, method = result
+            handle_dependency(dep_class, method, fiber, task_class, wrapper, queue)
+            return # Fiber is either continuing or parked
+          else
+            break
           end
         end
+
+        complete_task(task_class, wrapper, result)
+      rescue => e
+        fail_task(task_class, wrapper, e)
+      end
+
+      def handle_dependency(dep_class, method, fiber, task_class, wrapper, queue)
+        dep_wrapper = @registry.create_wrapper(dep_class, execution_facade: @execution_facade)
+        status = dep_wrapper.request_value(method, queue, fiber)
+
+        case status[0]
+        when :completed
+          drive_fiber_loop(fiber, task_class, wrapper, queue, status[1])
+        when :failed
+          drive_fiber_loop(fiber, task_class, wrapper, queue, [:_taski_error, status[1]])
+        when :wait
+          store_fiber_context(fiber, task_class, wrapper)
+        when :start
+          store_fiber_context(fiber, task_class, wrapper)
+          start_dependency(dep_class, dep_wrapper, queue)
+        end
+      end
+
+      # Resume a parked Fiber from the thread queue.
+      # Restores fiber context before resuming since teardown_thread_locals
+      # cleared thread-local state when the fiber was parked.
+      def resume_fiber(fiber, value, queue)
+        context = get_fiber_context(fiber)
+        return unless context
+
+        task_class, wrapper = context
+        setup_run_thread_locals
+        start_output_capture(task_class)
+        drive_fiber_loop(fiber, task_class, wrapper, queue, value)
+      end
+
+      def resume_fiber_with_error(fiber, error, queue)
+        context = get_fiber_context(fiber)
+        return unless context
+
+        task_class, wrapper = context
+        setup_run_thread_locals
+        start_output_capture(task_class)
+        drive_fiber_loop(fiber, task_class, wrapper, queue, [:_taski_error, error])
+      end
+
+      # Start a dependency task as a new Fiber on this thread.
+      # The wrapper is already RUNNING (set atomically by request_value).
+      def start_dependency(dep_class, dep_wrapper, queue)
+        drive_fiber(dep_class, dep_wrapper, queue)
+      end
+
+      def complete_task(task_class, wrapper, result)
+        stop_output_capture
+        duration = task_duration_ms(task_class)
+        Taski::Logging.info(Taski::Logging::Events::TASK_COMPLETED, task: task_class.name, duration_ms: duration)
+        wrapper.mark_completed(result)
+        @completion_queue.push({task_class: task_class, wrapper: wrapper})
+        teardown_thread_locals
+      end
+
+      def fail_task(task_class, wrapper, error)
+        stop_output_capture
+        @registry.request_abort! if error.is_a?(Taski::TaskAbortException)
+        duration = task_duration_ms(task_class)
+        Taski::Logging.error(Taski::Logging::Events::TASK_FAILED, task: task_class.name, duration_ms: duration)
+        wrapper.mark_failed(error)
+        @completion_queue.push({task_class: task_class, wrapper: wrapper, error: error})
+        teardown_thread_locals
+      end
+
+      # Execute a clean task directly (no Fiber needed).
+      def execute_clean_task(task_class, wrapper)
+        return if @registry.abort_requested?
+
+        setup_clean_thread_locals
+        start_output_capture(task_class)
+        clean_start = Time.now
+        @execution_facade.notify_task_updated(task_class, previous_state: :pending, current_state: :running, phase: :clean, timestamp: clean_start)
+        Taski::Logging.debug(Taski::Logging::Events::TASK_CLEAN_STARTED, task: task_class.name)
+
+        result = wrapper.task.clean
+        duration = ((Time.now - clean_start) * 1000).round(1)
+        Taski::Logging.debug(Taski::Logging::Events::TASK_CLEAN_COMPLETED, task: task_class.name, duration_ms: duration)
+        wrapper.mark_clean_completed(result)
+        @completion_queue.push({task_class: task_class, wrapper: wrapper, clean: true})
+      rescue => e
+        @registry.request_abort! if e.is_a?(Taski::TaskAbortException)
+        duration = ((Time.now - clean_start) * 1000).round(1) if clean_start
+        Taski::Logging.warn(Taski::Logging::Events::TASK_CLEAN_FAILED, task: task_class.name, duration_ms: duration)
+        wrapper.mark_clean_failed(e)
+        @completion_queue.push({task_class: task_class, wrapper: wrapper, error: e, clean: true})
+      ensure
+        stop_output_capture
+        teardown_thread_locals
+      end
+
+      # Set up context for clean execution (no Fiber flag).
+      def setup_clean_thread_locals
+        Thread.current[:taski_current_phase] = :clean
+        ExecutionFacade.current = @execution_facade
+        Taski.set_current_registry(@registry)
+      end
+
+      def setup_run_thread_locals
+        Thread.current[:taski_fiber_context] = true
+        Thread.current[:taski_current_phase] = :run
+        ExecutionFacade.current = @execution_facade
+        Taski.set_current_registry(@registry)
+      end
+
+      def teardown_thread_locals
+        Thread.current[:taski_fiber_context] = nil
+        Thread.current[:taski_current_phase] = nil
+        ExecutionFacade.current = nil
+        Taski.clear_current_registry
+      end
+
+      def task_duration_ms(task_class)
+        start = @task_start_times_mutex.synchronize { @task_start_times.delete(task_class) }
+        return nil unless start
+        ((Time.now - start) * 1000).round(1)
+      end
+
+      def start_output_capture(task_class)
+        output_capture = @execution_facade.output_capture
+        output_capture&.start_capture(task_class)
+      end
+
+      def stop_output_capture
+        output_capture = @execution_facade.output_capture
+        output_capture&.stop_capture
+      end
+
+      def store_fiber_context(fiber, task_class, wrapper)
+        @fiber_contexts_mutex.synchronize do
+          @fiber_contexts[fiber.object_id] = [task_class, wrapper]
+        end
       end
 
-      def debug_log(message)
-        return unless ENV["TASKI_DEBUG"]
-        puts "[WorkerPool] #{message}"
+      def get_fiber_context(fiber)
+        @fiber_contexts_mutex.synchronize do
+          @fiber_contexts.delete(fiber.object_id)
+        end
       end
     end
   end

data/lib/taski/logging.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "json"
+require "monitor"
+require "time"
+module Taski
+  # Logging module provides structured logging support for debugging and monitoring.
+  # Logging is disabled by default and has zero overhead when not configured.
+  #
+  # @example Basic setup
+  #   require 'logger'
+  #   Taski.logger = Logger.new($stderr, level: Logger::INFO)
+  #
+  # @example JSON output for monitoring systems
+  #   Taski.logger = Logger.new('/var/log/taski.log')
+  module Logging
+    # Event type constants
+    module Events
+      # Execution lifecycle
+      EXECUTION_STARTED = "execution.started"
+      EXECUTION_COMPLETED = "execution.completed"
+
+      # Task lifecycle
+      TASK_STARTED = "task.started"
+      TASK_COMPLETED = "task.completed"
+      TASK_FAILED = "task.failed"
+      TASK_SKIPPED = "task.skipped"
+      TASK_ERROR_DETAIL = "task.error_detail"
+      TASK_OUTPUT = "task.output"
+
+      # Clean lifecycle
+      TASK_CLEAN_STARTED = "task.clean_started"
+      TASK_CLEAN_COMPLETED = "task.clean_completed"
+      TASK_CLEAN_FAILED = "task.clean_failed"
+
+      # Dependency resolution
+      DEPENDENCY_RESOLVED = "dependency.resolved"
+
+      # Internal components (debug-level)
+      WORKER_POOL_ENQUEUED = "worker_pool.enqueued"
+      EXECUTOR_TASK_COMPLETED = "executor.task_completed"
+      EXECUTOR_CLEAN_COMPLETED = "executor.clean_completed"
+      OUTPUT_ROUTER_START_CAPTURE = "output_router.start_capture"
+      OUTPUT_ROUTER_STOP_CAPTURE = "output_router.stop_capture"
+      OUTPUT_ROUTER_STOP_CAPTURE_UNREGISTERED = "output_router.stop_capture_unregistered"
+      OUTPUT_ROUTER_DRAIN_PIPE = "output_router.drain_pipe"
+      OUTPUT_ROUTER_STORE_LINES = "output_router.store_lines"
+      OBSERVER_ERROR = "observer.error"
+    end
+
+    # Log severity levels matching Ruby Logger
+    module Levels
+      DEBUG = 0
+      INFO = 1
+      WARN = 2
+      ERROR = 3
+    end
+
+    class << self
+      # Log a structured event. No-op if logger is nil.
+      #
+      # @param level [Integer] Log level (DEBUG, INFO, WARN, ERROR)
+      # @param event [String] Event type constant
+      # @param task [String, nil] Task class name
+      # @param data [Hash] Additional event data
+      def log(level, event, task: nil, **data)
+        logger = Taski.logger
+        return unless logger
+
+        entry = build_entry(event, task, data)
+        message = entry.to_json
+
+        case level
+        when Levels::DEBUG
+          logger.debug(message)
+        when Levels::INFO
+          logger.info(message)
+        when Levels::WARN
+          logger.warn(message)
+        when Levels::ERROR
+          logger.error(message)
+        end
+      end
+
+      # Convenience methods for each log level
+      def debug(event, **kwargs) = log(Levels::DEBUG, event, **kwargs)
+      def info(event, **kwargs) = log(Levels::INFO, event, **kwargs)
+      def warn(event, **kwargs) = log(Levels::WARN, event, **kwargs)
+      def error(event, **kwargs) = log(Levels::ERROR, event, **kwargs)
+
+      private
+
+      def build_entry(event, task, data)
+        entry = {
+          timestamp: Time.now.utc.iso8601(3),
+          event: event,
+          thread_id: Thread.current.object_id
+        }
+        entry[:task] = task if task
+        entry[:data] = data unless data.empty?
+        entry
+      end
+    end
+  end
+end