taski 0.4.2 → 0.5.0

data/lib/taski/execution/executor.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+require "monitor"
+require "etc"
+
+module Taski
+  module Execution
+    # Producer-Consumer pattern executor for parallel task execution.
+    #
+    # Architecture:
+    # - Main Thread: Manages all state, coordinates execution, handles events
+    # - Worker Threads: Execute tasks and send completion events
+    #
+    # Communication Queues:
+    # - Execution Queue (Main -> Worker): Tasks ready to execute
+    # - Completion Queue (Worker -> Main): Events from workers
+    class Executor
+      # Task execution states for the executor's internal tracking
+      STATE_PENDING = :pending
+      STATE_ENQUEUED = :enqueued
+      STATE_COMPLETED = :completed
+
+      class << self
+        # Execute a task and all its dependencies
+        # @param root_task_class [Class] The root task class to execute
+        # @param registry [Registry] The task registry
+        def execute(root_task_class, registry:)
+          new(registry: registry).execute(root_task_class)
+        end
+      end
+
+      def initialize(registry:, worker_count: nil)
+        @registry = registry
+        @worker_count = worker_count || default_worker_count
+        @execution_queue = Queue.new
+        @completion_queue = Queue.new
+        @workers = []
+
+        # State managed by main thread only
+        @dependencies = {}
+        @task_states = {}
+        @completed_tasks = Set.new
+      end
+
+      # Execute root task and all dependencies
+      # @param root_task_class [Class] The root task class to execute
+      def execute(root_task_class)
+        # Build dependency graph from static analysis
+        build_dependency_graph(root_task_class)
+
+        # Set up tree progress display with root task (before start)
+        setup_tree_progress(root_task_class)
+
+        # Start progress display automatically for tree progress
+        start_progress_display
+
+        # Start worker threads
+        start_workers
+
+        # Enqueue tasks with no dependencies
+        enqueue_ready_tasks
+
+        # Main event loop - continues until root task completes
+        run_main_loop(root_task_class)
+
+        # Shutdown workers
+        shutdown_workers
+
+        # Stop progress display
+        stop_progress_display
+      end
+
+      private
+
+      def default_worker_count
+        Etc.nprocessors.clamp(2, 8)
+      end
+
+      # Build dependency graph by traversing from root task
+      # Populates @dependencies and @task_states
+      def build_dependency_graph(root_task_class)
+        # @type var queue: Array[singleton(Taski::Task)]
+        queue = [root_task_class]
+
+        while (task_class = queue.shift)
+          next if @task_states.key?(task_class)
+
+          deps = task_class.cached_dependencies
+          @dependencies[task_class] = deps.dup
+          @task_states[task_class] = STATE_PENDING
+
+          deps.each { |dep| queue << dep }
+        end
+      end
+
+      # Enqueue tasks that have all dependencies completed
+      def enqueue_ready_tasks
+        @task_states.each_key do |task_class|
+          next unless @task_states[task_class] == STATE_PENDING
+          next unless ready_to_execute?(task_class)
+
+          enqueue_task(task_class)
+        end
+      end
+
+      # Check if a task is ready to execute
+      def ready_to_execute?(task_class)
+        task_deps = @dependencies[task_class] || Set.new
+        task_deps.subset?(@completed_tasks)
+      end
+
+      # Enqueue a single task for execution
+      def enqueue_task(task_class)
+        return if @registry.abort_requested?
+
+        @task_states[task_class] = STATE_ENQUEUED
+
+        wrapper = get_or_create_wrapper(task_class)
+        return unless wrapper.mark_running
+
+        Taski.progress_display&.register_task(task_class)
+        Taski.progress_display&.update_task(task_class, state: :running)
+
+        @execution_queue.push({task_class: task_class, wrapper: wrapper})
+
+        debug_log("Enqueued: #{task_class}")
+      end
+
+      # Get or create a task wrapper via Registry
+      def get_or_create_wrapper(task_class)
+        @registry.get_or_create(task_class) do
+          task_instance = task_class.allocate
+          task_instance.send(:initialize)
+          TaskWrapper.new(task_instance, registry: @registry)
+        end
+      end
+
+      # Start worker threads
+      def start_workers
+        @worker_count.times do
+          worker = Thread.new { worker_loop }
+          @workers << worker
+          @registry.register_thread(worker)
+        end
+      end
+
+      # Worker thread main loop
+      def worker_loop
+        loop do
+          work_item = @execution_queue.pop
+          break if work_item == :shutdown
+
+          task_class = work_item[:task_class]
+          wrapper = work_item[:wrapper]
+
+          debug_log("Worker executing: #{task_class}")
+
+          execute_task(task_class, wrapper)
+        end
+      end
+
+      # Execute a task and send completion event
+      def execute_task(task_class, wrapper)
+        return if @registry.abort_requested?
+
+        begin
+          result = execute_task_run(wrapper)
+          wrapper.mark_completed(result)
+          @completion_queue.push({task_class: task_class, wrapper: wrapper})
+        rescue Taski::TaskAbortException => e
+          @registry.request_abort!
+          wrapper.mark_failed(e)
+          @completion_queue.push({task_class: task_class, wrapper: wrapper, error: e})
+        rescue => e
+          wrapper.mark_failed(e)
+          @completion_queue.push({task_class: task_class, wrapper: wrapper, error: e})
+        end
+      end
+
+      # Execute task run method
+      # Note: Previously captured stdout for progress display, but this was removed
+      # due to thread-safety concerns with global $stdout mutation.
+      def execute_task_run(wrapper)
+        wrapper.task.run
+      end
+
+      # Main thread event loop - continues until root task completes
+      def run_main_loop(root_task_class)
+        until @completed_tasks.include?(root_task_class)
+          break if @registry.abort_requested? && no_running_tasks?
+
+          event = @completion_queue.pop
+          handle_completion(event)
+        end
+      end
+
+      def no_running_tasks?
+        @task_states.values.none? { |state| state == STATE_ENQUEUED }
+      end
+
+      # Handle task completion event
+      def handle_completion(event)
+        task_class = event[:task_class]
+
+        debug_log("Completed: #{task_class}")
+
+        @task_states[task_class] = STATE_COMPLETED
+        @completed_tasks.add(task_class)
+
+        # Enqueue newly ready tasks
+        enqueue_ready_tasks
+      end
+
+      # Shutdown worker threads
+      def shutdown_workers
+        @worker_count.times { @execution_queue.push(:shutdown) }
+        @workers.each(&:join)
+      end
+
+      def setup_tree_progress(root_task_class)
+        progress = Taski.progress_display
+        return unless progress.is_a?(TreeProgressDisplay)
+
+        progress.set_root_task(root_task_class)
+      end
+
+      def start_progress_display
+        progress = Taski.progress_display
+        return unless progress.is_a?(TreeProgressDisplay)
+
+        progress.start
+      end
+
+      def stop_progress_display
+        progress = Taski.progress_display
+        return unless progress.is_a?(TreeProgressDisplay)
+
+        progress.stop
+      end
+
+      def debug_log(message)
+        return unless ENV["TASKI_DEBUG"]
+        puts "[Executor] #{message}"
+      end
+    end
+  end
+end

data/lib/taski/execution/registry.rb

@@ -19,6 +19,12 @@ module Taski
         @tasks[task_class] ||= yield
       end
 
+      # @param task_class [Class] The task class
+      # @param wrapper [TaskWrapper] The wrapper instance to register
+      def register(task_class, wrapper)
+        @tasks[task_class] = wrapper
+      end
+
       # @param task_class [Class] The task class
       # @return [Object] The task instance
       # @raise [RuntimeError] If the task is not registered
@@ -65,7 +71,9 @@ module Taski
 
         wait_all
 
-        get_task(task_class).result
+        # @type var wrapper: Taski::Execution::TaskWrapper
+        wrapper = get_task(task_class)
+        wrapper.result
       end
     end
   end

data/lib/taski/execution/task_wrapper.rb

@@ -22,17 +22,19 @@ module Taski
       end
     end
 
+    # TaskWrapper manages the state and synchronization for a single task.
+    # In the Producer-Consumer pattern, TaskWrapper does NOT start threads.
+    # The Executor controls all scheduling and execution.
     class TaskWrapper
-      attr_reader :task, :result
+      attr_reader :task, :result, :error, :timing
 
       STATE_PENDING = :pending
       STATE_RUNNING = :running
       STATE_COMPLETED = :completed
 
-      def initialize(task, registry:, coordinator:)
+      def initialize(task, registry:)
         @task = task
         @registry = registry
-        @coordinator = coordinator
         @result = nil
         @clean_result = nil
         @error = nil
@@ -42,213 +44,190 @@ module Taski
         @state = STATE_PENDING
         @clean_state = STATE_PENDING
         @timing = nil
+      end
+
+      # @return [Symbol] Current state
+      def state
+        @monitor.synchronize { @state }
+      end
 
-        register_with_progress_display
+      # @return [Boolean] true if task is pending
+      def pending?
+        state == STATE_PENDING
       end
 
+      # @return [Boolean] true if task is completed
+      def completed?
+        state == STATE_COMPLETED
+      end
+
+      # Called by user code to get result. Triggers execution if needed.
       # @return [Object] The result of task execution
       def run
-        execute_task_if_needed
+        trigger_execution_and_wait
         raise @error if @error # steep:ignore
         @result
       end
 
+      # Called by user code to clean. Triggers clean execution if needed.
       # @return [Object] The result of cleanup
       def clean
-        execute_clean_if_needed
+        trigger_clean_and_wait
         @clean_result
       end
 
+      # Called by user code to get exported value. Triggers execution if needed.
       # @param method_name [Symbol] The name of the exported method
       # @return [Object] The exported value
       def get_exported_value(method_name)
-        execute_task_if_needed
+        trigger_execution_and_wait
         raise @error if @error # steep:ignore
         @task.public_send(method_name)
       end
 
-      private
-
-      def start_thread_with(&block)
-        thread = Thread.new(&block)
-        @registry.register_thread(thread)
-      end
-
-      # Thread-safe state machine that ensures operations are executed exactly once.
-      # Uses pattern matching for exhaustive state handling.
-      def execute_with_state_pattern(state_getter:, starter:, waiter:, pre_start_check: nil)
+      # Called by Executor to mark task as running
+      def mark_running
         @monitor.synchronize do
-          case state_getter.call
-          in STATE_PENDING
-            pre_start_check&.call
-            starter.call
-            waiter.call
-          in STATE_RUNNING
-            waiter.call
-          in STATE_COMPLETED
-            return
-          end
-        end
-      end
-
-      def execute_task_if_needed
-        execute_with_state_pattern(
-          state_getter: -> { @state },
-          starter: -> { start_async_execution },
-          waiter: -> { wait_for_completion },
-          pre_start_check: -> {
-            if @registry.abort_requested?
-              raise Taski::TaskAbortException, "Execution aborted - no new tasks will start"
-            end
-          }
-        )
-      end
-
-      def start_async_execution
-        @state = STATE_RUNNING
-        @timing = TaskTiming.start_now
-        update_progress(:running)
-        start_thread_with { execute_task }
-      end
-
-      def execute_task
-        if @registry.abort_requested?
-          @error = Taski::TaskAbortException.new("Execution aborted - no new tasks will start")
-          mark_completed
-          return
-        end
-
-        log_start
-        @coordinator.start_dependencies(@task.class)
-        wait_for_dependencies
-        @result = @task.run
-        mark_completed
-        log_completion
-      rescue Taski::TaskAbortException => e
-        @registry.request_abort!
-        @error = e
-        mark_completed
-      rescue => e
-        @error = e
-        mark_completed
-      end
-
-      def wait_for_dependencies
-        dependencies = @task.class.cached_dependencies
-        return if dependencies.empty?
-
-        dependencies.each do |dep_class|
-          dep_class.exported_methods.each do |method|
-            dep_class.public_send(method)
-          end
+          return false unless @state == STATE_PENDING
+          @state = STATE_RUNNING
+          @timing = TaskTiming.start_now
+          true
         end
       end
 
-      def execute_clean_if_needed
-        execute_with_state_pattern(
-          state_getter: -> { @clean_state },
-          starter: -> { start_async_clean },
-          waiter: -> { wait_for_clean_completion }
-        )
-      end
-
-      def start_async_clean
-        @clean_state = STATE_RUNNING
-        start_thread_with { execute_clean }
-      end
-
-      def execute_clean
-        log_clean_start
-        @clean_result = @task.clean
-        wait_for_clean_dependencies
-        mark_clean_completed
-        log_clean_completion
-      end
-
-      def wait_for_clean_dependencies
-        dependencies = @task.class.cached_dependencies
-        return if dependencies.empty?
-
-        wait_threads = dependencies.map do |dep_class|
-          Thread.new do
-            dep_class.public_send(:clean)
-          end
+      # Called by Executor after task.run completes successfully
+      # @param result [Object] The result of task execution
+      def mark_completed(result)
+        @timing = @timing&.with_end_now
+        @monitor.synchronize do
+          @result = result
+          @state = STATE_COMPLETED
+          @condition.broadcast
         end
-
-        wait_threads.each(&:join)
+        update_progress(:completed, duration: @timing&.duration_ms)
       end
 
-      def mark_completed
+      # Called by Executor when task.run raises an error
+      # @param error [Exception] The error that occurred
+      def mark_failed(error)
         @timing = @timing&.with_end_now
         @monitor.synchronize do
+          @error = error
           @state = STATE_COMPLETED
           @condition.broadcast
         end
-
-        if @error
-          update_progress(:failed, error: @error)
-        else
-          update_progress(:completed, duration: @timing&.duration_ms)
-        end
+        update_progress(:failed, error: error)
       end
 
-      def mark_clean_completed
+      # Called by Executor after clean completes
+      # @param result [Object] The result of cleanup
+      def mark_clean_completed(result)
         @monitor.synchronize do
+          @clean_result = result
           @clean_state = STATE_COMPLETED
           @clean_condition.broadcast
         end
       end
 
+      # Wait until task is completed
       def wait_for_completion
-        @condition.wait_until { @state == STATE_COMPLETED }
+        @monitor.synchronize do
+          @condition.wait_until { @state == STATE_COMPLETED }
+        end
       end
 
+      # Wait until clean is completed
       def wait_for_clean_completion
-        @clean_condition.wait_until { @clean_state == STATE_COMPLETED }
+        @monitor.synchronize do
+          @clean_condition.wait_until { @clean_state == STATE_COMPLETED }
+        end
       end
 
-      def debug_log(message)
-        return unless ENV["TASKI_DEBUG"]
-        puts message
+      def method_missing(method_name, *args, &block)
+        if @task.class.method_defined?(method_name)
+          get_exported_value(method_name)
+        else
+          super
+        end
       end
 
-      def log_start
-        debug_log("Invoking #{@task.class} wrapper in thread #{Thread.current.object_id}...")
+      def respond_to_missing?(method_name, include_private = false)
+        @task.class.method_defined?(method_name) || super
       end
 
-      def log_completion
-        debug_log("Wrapper #{@task.class} completed in thread #{Thread.current.object_id}.")
-      end
+      private
+
+      # Trigger execution via Executor and wait for completion
+      def trigger_execution_and_wait
+        should_execute = false
+        @monitor.synchronize do
+          case @state
+          when STATE_PENDING
+            check_abort!
+            should_execute = true
+          when STATE_RUNNING
+            @condition.wait_until { @state == STATE_COMPLETED }
+          when STATE_COMPLETED
+            # Already done
+          end
+        end
 
-      def log_clean_start
-        debug_log("Cleaning #{@task.class} in thread #{Thread.current.object_id}...")
+        if should_execute
+          # Execute outside the lock to avoid deadlock
+          Executor.execute(@task.class, registry: @registry)
+          # After Executor.execute returns, the task is completed
+        end
       end
 
-      def log_clean_completion
-        debug_log("Clean #{@task.class} completed in thread #{Thread.current.object_id}.")
+      # Trigger clean execution and wait for completion
+      def trigger_clean_and_wait
+        @monitor.synchronize do
+          case @clean_state
+          when STATE_PENDING
+            @clean_state = STATE_RUNNING
+            # Execute clean in a thread (clean doesn't use Producer-Consumer)
+            thread = Thread.new { execute_clean }
+            @registry.register_thread(thread)
+            @clean_condition.wait_until { @clean_state == STATE_COMPLETED }
+          when STATE_RUNNING
+            @clean_condition.wait_until { @clean_state == STATE_COMPLETED }
+          when STATE_COMPLETED
+            # Already done
+          end
+        end
       end
 
-      def register_with_progress_display
-        Taski.progress_display&.register_task(@task.class)
+      def execute_clean
+        debug_log("Cleaning #{@task.class}...")
+        result = @task.clean
+        wait_for_clean_dependencies
+        mark_clean_completed(result)
+        debug_log("Clean #{@task.class} completed.")
       end
 
-      # @param state [Symbol] The new state
-      # @param duration [Float, nil] Duration in milliseconds
-      # @param error [Exception, nil] Error object
-      def update_progress(state, duration: nil, error: nil)
-        Taski.progress_display&.update_task(@task.class, state: state, duration: duration, error: error)
+      def wait_for_clean_dependencies
+        dependencies = @task.class.cached_dependencies
+        return if dependencies.empty?
+
+        wait_threads = dependencies.map do |dep_class|
+          Thread.new { dep_class.clean }
+        end
+        wait_threads.each(&:join)
       end
 
-      def method_missing(method_name, *args, &block)
-        if @task.class.method_defined?(method_name)
-          get_exported_value(method_name)
-        else
-          super
+      def check_abort!
+        if @registry.abort_requested?
+          raise Taski::TaskAbortException, "Execution aborted - no new tasks will start"
         end
       end
 
-      def respond_to_missing?(method_name, include_private = false)
-        @task.class.method_defined?(method_name) || super
+      def update_progress(state, duration: nil, error: nil)
+        Taski.progress_display&.update_task(@task.class, state: state, duration: duration, error: error)
+      end
+
+      def debug_log(message)
+        return unless ENV["TASKI_DEBUG"]
+        puts "[TaskWrapper] #{message}"
       end
     end
   end