taski 0.5.0 → 0.7.0

data/lib/taski/execution/worker_pool.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module Taski
+  module Execution
+    # WorkerPool manages a pool of worker threads that execute tasks.
+    # It provides methods to start, stop, and enqueue tasks for execution.
+    #
+    # == Responsibilities
+    #
+    # - Manage worker thread lifecycle (start, shutdown)
+    # - Distribute tasks to worker threads via Queue
+    # - Execute tasks via callback provided by Executor
+    #
+    # == API
+    #
+    # - {#start} - Start all worker threads
+    # - {#enqueue} - Add a task to the execution queue
+    # - {#shutdown} - Gracefully shutdown all worker threads
+    # - {#execution_queue} - Access the underlying Queue (for testing)
+    #
+    # == Thread Safety
+    #
+    # WorkerPool uses Queue for thread-safe task distribution.
+    # The Queue handles synchronization between the main thread
+    # (which enqueues tasks) and worker threads (which pop tasks).
+    class WorkerPool
+      attr_reader :execution_queue
+
+      # @param registry [Registry] The task registry for thread tracking
+      # @param worker_count [Integer, nil] Number of worker threads (defaults to CPU count)
+      # @param on_execute [Proc] Callback to execute a task, receives (task_class, wrapper)
+      def initialize(registry:, worker_count: nil, &on_execute)
+        @worker_count = worker_count || default_worker_count
+        @registry = registry
+        @on_execute = on_execute
+        @execution_queue = Queue.new
+        @workers = []
+      end
+
+      # Start all worker threads.
+      def start
+        @worker_count.times do
+          worker = Thread.new { worker_loop }
+          @workers << worker
+          @registry.register_thread(worker)
+        end
+      end
+
+      # Enqueue a task for execution.
+      #
+      # @param task_class [Class] The task class to execute
+      # @param wrapper [TaskWrapper] The task wrapper
+      def enqueue(task_class, wrapper)
+        @execution_queue.push({task_class: task_class, wrapper: wrapper})
+        debug_log("Enqueued: #{task_class}")
+      end
+
+      # Shutdown all worker threads gracefully.
+      def shutdown
+        enqueue_shutdown_signals
+        @workers.each(&:join)
+      end
+
+      # Enqueue shutdown signals for all workers.
+      def enqueue_shutdown_signals
+        @worker_count.times { @execution_queue.push(:shutdown) }
+      end
+
+      private
+
+      def default_worker_count
+        Etc.nprocessors.clamp(2, 8)
+      end
+
+      def worker_loop
+        loop do
+          work_item = @execution_queue.pop
+          break if work_item == :shutdown
+
+          task_class = work_item[:task_class]
+          wrapper = work_item[:wrapper]
+
+          debug_log("Worker executing: #{task_class}")
+
+          begin
+            @on_execute.call(task_class, wrapper)
+          rescue => e
+            # Log error but don't crash the worker thread.
+            # Task-level errors are handled in the execute callback.
+            # This catches unexpected errors in the callback itself.
+            warn "[WorkerPool] Unexpected error executing #{task_class}: #{e.message}"
+          end
+        end
+      end
+
+      def debug_log(message)
+        return unless ENV["TASKI_DEBUG"]
+        puts "[WorkerPool] #{message}"
+      end
+    end
+  end
+end

data/lib/taski/section.rb

@@ -24,6 +24,9 @@ module Taski
         raise "Section #{self.class} does not have an implementation. Override 'impl' method."
       end
 
+      # Register runtime dependency for clean phase (before register_impl_selection)
+      register_runtime_dependency(implementation_class)
+
       # Register selected impl for progress display
       register_impl_selection(implementation_class)
 
@@ -43,11 +46,21 @@ module Taski
 
     private
 
+    # Register the selected implementation as a runtime dependency.
+    # This allows the clean phase to include the dynamically selected impl.
+    # Handles nil ExecutionContext gracefully.
+    #
+    # @param impl_class [Class] The selected implementation class
+    def register_runtime_dependency(impl_class)
+      context = Execution::ExecutionContext.current
+      context&.register_runtime_dependency(self.class, impl_class)
+    end
+
     def register_impl_selection(implementation_class)
-      progress = Taski.progress_display
-      return unless progress.is_a?(Execution::TreeProgressDisplay)
+      context = Execution::ExecutionContext.current
+      return unless context
 
-      progress.register_section_impl(self.class, implementation_class)
+      context.notify_section_impl_selected(self.class, implementation_class)
     end
 
     # @param implementation_class [Class] The implementation task class

data/lib/taski/task.rb

@@ -5,8 +5,34 @@ require_relative "execution/registry"
 require_relative "execution/task_wrapper"
 
 module Taski
+  # Base class for all tasks in the Taski framework.
+  # Tasks define units of work with dependencies and exported values.
+  #
+  # @example Defining a simple task
+  #   class MyTask < Taski::Task
+  #     exports :result
+  #
+  #     def run
+  #       @result = "completed"
+  #     end
+  #   end
   class Task
     class << self
+      ##
+      # Callback invoked when a subclass is created.
+      # Automatically creates a task-specific Error class for each subclass.
+      # @param subclass [Class] The newly created subclass.
+      def inherited(subclass)
+        super
+        # Create TaskClass::Error that inherits from Taski::TaskError
+        error_class = Class.new(Taski::TaskError)
+        subclass.const_set(:Error, error_class)
+      end
+
+      ##
+      # Declares exported methods that will be accessible after task execution.
+      # Creates instance reader and class accessor methods for each export.
+      # @param export_methods [Array<Symbol>] The method names to export.
       def exports(*export_methods)
         @exported_methods = export_methods
 
@@ -16,75 +42,128 @@ module Taski
         end
       end
 
+      ##
+      # Returns the list of exported method names.
+      # @return [Array<Symbol>] The exported method names.
       def exported_methods
         @exported_methods ||= []
       end
 
-      # Each call creates a fresh TaskWrapper instance for re-execution support.
+      ##
+      # Creates a fresh TaskWrapper instance for re-execution support.
       # Use class methods (e.g., MyTask.result) for cached single execution.
+      # @return [Execution::TaskWrapper] A new wrapper for this task.
       def new
-        fresh_registry = Execution::Registry.new
-        task_instance = allocate
-        task_instance.send(:initialize)
-        wrapper = Execution::TaskWrapper.new(
-          task_instance,
-          registry: fresh_registry
-        )
-        # Pre-register to prevent Executor from creating a duplicate wrapper
-        fresh_registry.register(self, wrapper)
-        wrapper
+        fresh_wrapper
       end
 
+      ##
+      # Returns cached static dependencies for this task class.
+      # Dependencies are analyzed from the run method body using static analysis.
+      # @return [Set<Class>] The set of task classes this task depends on.
       def cached_dependencies
         @dependencies_cache ||= StaticAnalysis::Analyzer.analyze(self)
       end
 
+      ##
+      # Clears the cached dependency analysis.
+      # Useful when task code has changed and dependencies need to be re-analyzed.
       def clear_dependency_cache
         @dependencies_cache = nil
       end
 
-      def run(context: {})
-        Taski.start_context(options: context, root_task: self)
+      ##
+      # Executes the task and all its dependencies.
+      # Creates a fresh registry each time for independent execution.
+      # @param args [Hash] User-defined arguments accessible via Taski.args.
+      # @param workers [Integer, nil] Number of worker threads for parallel execution.
+      #   Must be a positive integer or nil.
+      #   Use workers: 1 for sequential execution (useful for debugging).
+      # @raise [ArgumentError] If workers is not a positive integer or nil.
+      # @return [Object] The result of task execution.
+      def run(args: {}, workers: nil)
+        validate_workers!(workers)
+        Taski.start_args(options: args.merge(_workers: workers), root_task: self)
         validate_no_circular_dependencies!
-        cached_wrapper.run
+        fresh_wrapper.run
+      ensure
+        Taski.reset_args!
       end
 
-      def clean(context: {})
-        Taski.start_context(options: context, root_task: self)
+      ##
+      # Executes the clean phase for the task and all its dependencies.
+      # Clean is executed in reverse dependency order.
+      # Creates a fresh registry each time for independent execution.
+      # @param args [Hash] User-defined arguments accessible via Taski.args.
+      # @param workers [Integer, nil] Number of worker threads for parallel execution.
+      #   Must be a positive integer or nil.
+      # @raise [ArgumentError] If workers is not a positive integer or nil.
+      def clean(args: {}, workers: nil)
+        validate_workers!(workers)
+        Taski.start_args(options: args.merge(_workers: workers), root_task: self)
         validate_no_circular_dependencies!
-        cached_wrapper.clean
+        fresh_wrapper.clean
+      ensure
+        Taski.reset_args!
       end
 
-      def registry
-        Taski.global_registry
+      ##
+      # Execute run followed by clean in a single operation.
+      # If run fails, clean is still executed for resource release.
+      # Creates a fresh registry for both operations to share.
+      #
+      # @param args [Hash] User-defined arguments accessible via Taski.args.
+      # @param workers [Integer, nil] Number of worker threads for parallel execution.
+      #   Must be a positive integer or nil.
+      # @raise [ArgumentError] If workers is not a positive integer or nil.
+      # @return [Object] The result of task execution
+      def run_and_clean(args: {}, workers: nil)
+        validate_workers!(workers)
+        Taski.start_args(options: args.merge(_workers: workers), root_task: self)
+        validate_no_circular_dependencies!
+        fresh_wrapper.run_and_clean
+      ensure
+        Taski.reset_args!
       end
 
+      ##
+      # Resets the task state and progress display.
+      # Useful for testing or re-running tasks from scratch.
       def reset!
-        registry.reset!
-        Taski.reset_global_registry!
-        Taski.reset_context!
+        Taski.reset_args!
+        Taski.reset_progress_display!
         @circular_dependency_checked = false
       end
 
+      ##
+      # Renders a static tree representation of the task dependencies.
+      # @return [String] The rendered tree string.
       def tree
         Execution::TreeProgressDisplay.render_static_tree(self)
       end
 
       private
 
-      # Use allocate + initialize instead of new to avoid infinite loop
-      # since new is overridden to return TaskWrapper
-      def cached_wrapper
-        registry.get_or_create(self) do
-          task_instance = allocate
-          task_instance.send(:initialize)
-          Execution::TaskWrapper.new(
-            task_instance,
-            registry: registry
-          )
-        end
+      ##
+      # Creates a fresh TaskWrapper with its own registry.
+      # Used for class method execution (Task.run) where each call is independent.
+      # @return [Execution::TaskWrapper] A new wrapper with fresh registry.
+      def fresh_wrapper
+        fresh_registry = Execution::Registry.new
+        task_instance = allocate
+        task_instance.__send__(:initialize)
+        wrapper = Execution::TaskWrapper.new(
+          task_instance,
+          registry: fresh_registry,
+          execution_context: Execution::ExecutionContext.current
+        )
+        fresh_registry.register(self, wrapper)
+        wrapper
       end
 
+      ##
+      # Defines an instance reader method for an exported value.
+      # @param method [Symbol] The method name to define.
       def define_instance_reader(method)
         undef_method(method) if method_defined?(method)
 
@@ -94,16 +173,48 @@ module Taski
         end
       end
 
+      ##
+      # Defines a class accessor method for an exported value.
+      # When called inside an execution, returns cached value from registry.
+      # When called outside execution, creates fresh execution.
+      # @param method [Symbol] The method name to define.
       def define_class_accessor(method)
         singleton_class.undef_method(method) if singleton_class.method_defined?(method)
 
         define_singleton_method(method) do
-          Taski.start_context(options: {}, root_task: self)
-          validate_no_circular_dependencies!
-          cached_wrapper.get_exported_value(method)
+          # Check if running inside an execution with a registry
+          registry = Taski.current_registry
+          if registry
+            # Inside execution - get or create wrapper in current registry
+            # This handles both pre-registered dependencies and dynamic ones (like Section impl)
+            wrapper = registry.get_or_create(self) do
+              task_instance = allocate
+              task_instance.__send__(:initialize)
+              Execution::TaskWrapper.new(
+                task_instance,
+                registry: registry,
+                execution_context: Execution::ExecutionContext.current
+              )
+            end
+            wrapper.get_exported_value(method)
+          else
+            # Outside execution - fresh execution (top-level call)
+            args_was_nil = Taski.args.nil?
+            begin
+              Taski.start_args(options: {}, root_task: self) if args_was_nil
+              validate_no_circular_dependencies!
+              fresh_wrapper.get_exported_value(method)
+            ensure
+              # Only reset if we set args
+              Taski.reset_args! if args_was_nil
+            end
+          end
         end
       end
 
+      ##
+      # Validates that no circular dependencies exist in the task graph.
+      # @raise [Taski::CircularDependencyError] If circular dependencies are detected.
       def validate_no_circular_dependencies!
         return if @circular_dependency_checked
 
@@ -116,15 +227,90 @@ module Taski
 
         @circular_dependency_checked = true
       end
+
+      ##
+      # Validates the workers parameter.
+      # @param workers [Object] The workers parameter to validate.
+      # @raise [ArgumentError] If workers is not a positive integer or nil.
+      def validate_workers!(workers)
+        return if workers.nil?
+
+        unless workers.is_a?(Integer) && workers >= 1
+          raise ArgumentError, "workers must be a positive integer or nil, got: #{workers.inspect}"
+        end
+      end
     end
 
+    ##
+    # Executes the task's main logic.
+    # Subclasses must override this method to implement task behavior.
+    # @raise [NotImplementedError] If not overridden in a subclass.
     def run
       raise NotImplementedError, "Subclasses must implement the run method"
     end
 
+    ##
+    # Cleans up resources after task execution.
+    # Override in subclasses to implement cleanup logic.
     def clean
     end
 
+    # Override system() to capture subprocess output through the pipe-based architecture.
+    # Uses Kernel.system with :out option to redirect output to the task's pipe.
+    # If user provides :out or :err options, they are respected (no automatic redirection).
+    # @param args [Array] Command arguments (shell mode if single string, exec mode if array)
+    # @param opts [Hash] Options passed to Kernel.system
+    # @return [Boolean, nil] true if command succeeded, false if failed, nil if command not found
+    def system(*args, **opts)
+      write_io = $stdout.respond_to?(:current_write_io) ? $stdout.current_write_io : nil
+
+      if write_io && !opts.key?(:out)
+        # Redirect subprocess output to the task's pipe (stderr merged into stdout)
+        Kernel.system(*args, out: write_io, err: [:child, :out], **opts)
+      else
+        # No capture active or user provided custom :out, use normal system
+        Kernel.system(*args, **opts)
+      end
+    end
+
+    # Groups related output within a task for organized progress display.
+    # The group name is shown in the progress tree as a child of the task.
+    # Groups cannot be nested.
+    #
+    # @param name [String] The group name to display
+    # @yield The block to execute within the group
+    # @return [Object] The result of the block
+    # @raise Re-raises any exception from the block after marking group as failed
+    #
+    # @example
+    #   def run
+    #     group("Preparing") do
+    #       puts "Checking dependencies..."
+    #       puts "Validating config..."
+    #     end
+    #     group("Deploying") do
+    #       puts "Uploading files..."
+    #     end
+    #   end
+    def group(name)
+      context = Execution::ExecutionContext.current
+      context&.notify_group_started(self.class, name)
+      start_time = Time.now
+
+      begin
+        result = yield
+        duration_ms = ((Time.now - start_time) * 1000).round(0)
+        context&.notify_group_completed(self.class, name, duration: duration_ms)
+        result
+      rescue => e
+        duration_ms = ((Time.now - start_time) * 1000).round(0)
+        context&.notify_group_completed(self.class, name, duration: duration_ms, error: e)
+        raise
+      end
+    end
+
+    ##
+    # Resets the instance's exported values to nil.
     def reset!
       self.class.exported_methods.each do |method|
         instance_variable_set("@#{method}", nil)

data/lib/taski/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Taski
-  VERSION = "0.5.0"
+  VERSION = "0.7.0"
 end

data/lib/taski.rb

@@ -5,12 +5,13 @@ require_relative "taski/static_analysis/analyzer"
 require_relative "taski/static_analysis/visitor"
 require_relative "taski/static_analysis/dependency_graph"
 require_relative "taski/execution/registry"
+require_relative "taski/execution/execution_context"
 require_relative "taski/execution/task_wrapper"
+require_relative "taski/execution/scheduler"
+require_relative "taski/execution/worker_pool"
 require_relative "taski/execution/executor"
 require_relative "taski/execution/tree_progress_display"
-require_relative "taski/context"
-require_relative "taski/task"
-require_relative "taski/section"
+require_relative "taski/args"
 
 module Taski
   class TaskAbortException < StandardError
@@ -28,41 +29,125 @@ module Taski
     end
   end
 
-  @context_monitor = Monitor.new
+  # Represents a single task failure with its context
+  class TaskFailure
+    attr_reader :task_class, :error
 
-  # Get the current execution context
-  # @return [Context, nil] The current context or nil if no task is running
-  def self.context
-    @context_monitor.synchronize { @context }
+    # @param task_class [Class] The task class that failed
+    # @param error [Exception] The exception that was raised
+    def initialize(task_class:, error:)
+      @task_class = task_class
+      @error = error
+    end
   end
 
-  # Start a new execution context (internal use only)
-  # @api private
-  def self.start_context(options:, root_task:)
-    @context_monitor.synchronize do
-      return if @context
-      @context = Context.new(options: options, root_task: root_task)
+  # Mixin for exception classes to enable transparent rescue matching with AggregateError.
+  # When extended by an exception class, `rescue ThatError` will also match
+  # an AggregateError that contains ThatError.
+  #
+  # @note TaskError and all TaskClass::Error classes already extend this module.
+  #
+  # @example
+  #   begin
+  #     MyTask.value  # raises AggregateError containing MyTask::Error
+  #   rescue MyTask::Error => e
+  #     puts "MyTask failed: #{e.message}"
+  #   end
+  module AggregateAware
+    def ===(other)
+      return super unless other.is_a?(Taski::AggregateError)
+
+      other.includes?(self)
     end
   end
 
-  # Reset the execution context (internal use only)
-  # @api private
-  def self.reset_context!
-    @context_monitor.synchronize { @context = nil }
+  # Base class for task-specific error wrappers.
+  # Each Task subclass automatically gets a ::Error class that inherits from this.
+  # This allows rescuing errors by task class: rescue MyTask::Error => e
+  #
+  # @example Rescuing task-specific errors
+  #   begin
+  #     MyTask.value
+  #   rescue MyTask::Error => e
+  #     puts "MyTask failed: #{e.message}"
+  #   end
+  class TaskError < StandardError
+    extend AggregateAware
+
+    # @return [Exception] The original error that occurred in the task
+    attr_reader :cause
+
+    # @return [Class] The task class where the error occurred
+    attr_reader :task_class
+
+    # @param cause [Exception] The original error
+    # @param task_class [Class] The task class where the error occurred
+    def initialize(cause, task_class:)
+      @cause = cause
+      @task_class = task_class
+      super(cause.message)
+      set_backtrace(cause.backtrace)
+    end
+  end
+
+  # Raised when multiple tasks fail during parallel execution
+  class AggregateError < StandardError
+    attr_reader :errors
+
+    # @param errors [Array<TaskFailure>] List of task failures
+    def initialize(errors)
+      @errors = errors
+      super(build_message)
+    end
+
+    # Returns the first error for compatibility with exception chaining
+    # @return [Exception, nil] The first error or nil if no errors
+    def cause
+      errors.first&.error
+    end
+
+    # Check if this aggregate contains an error of the given type
+    # @param exception_class [Class] The exception class to check for
+    # @return [Boolean] true if any contained error is of the given type
+    def includes?(exception_class)
+      errors.any? { |f| f.error.is_a?(exception_class) }
+    end
+
+    private
+
+    def build_message
+      task_word = (errors.size == 1) ? "task" : "tasks"
+      "#{errors.size} #{task_word} failed:\n" +
+        errors.map { |f| "  - #{f.task_class.name}: #{f.error.message}" }.join("\n")
+    end
   end
 
-  def self.global_registry
-    @global_registry ||= Execution::Registry.new
+  @args_monitor = Monitor.new
+
+  # Get the current runtime arguments
+  # @return [Args, nil] The current args or nil if no task is running
+  def self.args
+    @args_monitor.synchronize { @args }
+  end
+
+  # Start new runtime arguments (internal use only)
+  # @api private
+  def self.start_args(options:, root_task:)
+    @args_monitor.synchronize do
+      return if @args
+      @args = Args.new(options: options, root_task: root_task)
+    end
   end
 
-  def self.reset_global_registry!
-    @global_registry = nil
+  # Reset the runtime arguments (internal use only)
+  # @api private
+  def self.reset_args!
+    @args_monitor.synchronize { @args = nil }
   end
 
   # Progress display is enabled by default (tree-style).
   # Environment variables:
   # - TASKI_PROGRESS_DISABLE=1: Disable progress display entirely
-  # - TASKI_FORCE_PROGRESS=1: Force enable even without TTY (for testing)
   def self.progress_display
     return nil if progress_disabled?
     @progress_display ||= Execution::TreeProgressDisplay.new
@@ -76,4 +161,34 @@ module Taski
     @progress_display&.stop
     @progress_display = nil
   end
+
+  # Get the worker count from the current args (set via Task.run(workers: n))
+  # @return [Integer, nil] The worker count or nil to use WorkerPool default
+  # @api private
+  def self.args_worker_count
+    args&.fetch(:_workers, nil)
+  end
+
+  # Get the current registry for this thread (used during dependency resolution)
+  # @return [Execution::Registry, nil] The current registry or nil
+  # @api private
+  def self.current_registry
+    Thread.current[:taski_current_registry]
+  end
+
+  # Set the current registry for this thread (internal use only)
+  # @api private
+  def self.set_current_registry(registry)
+    Thread.current[:taski_current_registry] = registry
+  end
+
+  # Clear the current registry for this thread (internal use only)
+  # @api private
+  def self.clear_current_registry
+    Thread.current[:taski_current_registry] = nil
+  end
 end
+
+# Load Task and Section after Taski module is defined (they depend on TaskError)
+require_relative "taski/task"
+require_relative "taski/section"