aidp 0.15.2 → 0.16.0

data/lib/aidp/concurrency/backoff.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "concurrent-ruby"
+
+module Aidp
+  module Concurrency
+    # Retry logic with exponential backoff and jitter.
+    #
+    # Replaces ad-hoc retry loops that use sleep() with a standardized,
+    # configurable retry mechanism that includes backoff strategies and jitter.
+    #
+    # @example Simple retry
+    #   Backoff.retry(max_attempts: 5) { call_external_api() }
+    #
+    # @example Custom backoff strategy
+    #   Backoff.retry(max_attempts: 10, base: 1.0, strategy: :linear) do
+    #     unstable_operation()
+    #   end
+    #
+    # @example With error filtering
+    #   Backoff.retry(max_attempts: 3, on: [Net::ReadTimeout, Errno::ECONNREFUSED]) do
+    #     http_request()
+    #   end
+    module Backoff
+      class << self
+        # Retry a block with exponential backoff and jitter.
+        #
+        # @param max_attempts [Integer] Maximum number of attempts (default: from config)
+        # @param base [Float] Base delay in seconds (default: from config)
+        # @param max_delay [Float] Maximum delay between retries (default: from config)
+        # @param jitter [Float] Jitter factor 0.0-1.0 (default: from config)
+        # @param strategy [Symbol] Backoff strategy :exponential, :linear, or :constant
+        # @param on [Array<Class>] Array of exception classes to retry (default: StandardError)
+        # @yield Block to retry
+        # @return [Object] Result of the block
+        # @raise [Concurrency::MaxAttemptsError] if all attempts fail
+        #
+        # @example
+        #   result = Backoff.retry(max_attempts: 5, base: 0.5, jitter: 0.2) do
+        #     api_client.fetch_data
+        #   end
+        def retry(max_attempts: nil, base: nil, max_delay: nil, jitter: nil,
+          strategy: :exponential, on: [StandardError], &block)
+          max_attempts ||= Concurrency.configuration.default_max_attempts
+          base ||= Concurrency.configuration.default_backoff_base
+          max_delay ||= Concurrency.configuration.default_backoff_max
+          jitter ||= Concurrency.configuration.default_jitter
+
+          raise ArgumentError, "Block required" unless block_given?
+          raise ArgumentError, "max_attempts must be >= 1" if max_attempts < 1
+
+          on = Array(on)
+          last_error = nil
+          attempt = 0
+
+          while attempt < max_attempts
+            attempt += 1
+
+            begin
+              result = block.call
+              log_retry_success(attempt) if attempt > 1
+              return result
+            rescue => e
+              last_error = e
+
+              # Re-raise if error is not in the retry list
+              raise unless on.any? { |klass| e.is_a?(klass) }
+
+              # Re-raise on last attempt
+              if attempt >= max_attempts
+                log_max_attempts_exceeded(attempt, e)
+                raise Concurrency::MaxAttemptsError, "Max attempts (#{max_attempts}) exceeded: #{e.class} - #{e.message}"
+              end
+
+              # Calculate delay and wait
+              delay = calculate_delay(attempt, strategy, base, max_delay, jitter)
+              log_retry_attempt(attempt, max_attempts, delay, e)
+              sleep(delay) if delay > 0
+            end
+          end
+
+          # Should never reach here, but just in case
+          raise Concurrency::MaxAttemptsError, "Max attempts (#{max_attempts}) exceeded: #{last_error&.class} - #{last_error&.message}"
+        end
+
+        # Calculate backoff delay for a given attempt.
+        #
+        # @param attempt [Integer] Current attempt number (1-indexed)
+        # @param strategy [Symbol] :exponential, :linear, or :constant
+        # @param base [Float] Base delay in seconds
+        # @param max_delay [Float] Maximum delay cap
+        # @param jitter [Float] Jitter factor 0.0-1.0
+        # @return [Float] Delay in seconds
+        def calculate_delay(attempt, strategy, base, max_delay, jitter)
+          delay = case strategy
+          when :exponential
+            base * (2**(attempt - 1))
+          when :linear
+            base * attempt
+          when :constant
+            base
+          else
+            raise ArgumentError, "Unknown strategy: #{strategy}"
+          end
+
+          # Cap at max_delay
+          delay = [delay, max_delay].min
+
+          # Add jitter: randomize between (1-jitter)*delay and delay
+          # e.g., with jitter=0.2, delay is reduced by 0-20%
+          if jitter > 0
+            jitter_amount = delay * jitter * rand
+            delay -= jitter_amount
+          end
+
+          delay
+        end
+
+        private
+
+        def log_retry_attempt(attempt, max_attempts, delay, error)
+          return unless Concurrency.configuration.log_retries
+
+          Concurrency.logger&.info(
+            "concurrency_retry",
+            "Retry attempt #{attempt}/#{max_attempts} after #{delay.round(2)}s: #{error.class} - #{error.message}"
+          )
+        end
+
+        def log_retry_success(attempt)
+          return unless Concurrency.configuration.log_retries
+
+          Concurrency.logger&.info(
+            "concurrency_retry",
+            "Retry succeeded on attempt #{attempt}"
+          )
+        end
+
+        def log_max_attempts_exceeded(attempt, error)
+          Concurrency.logger&.error(
+            "concurrency_retry",
+            "Max attempts (#{attempt}) exceeded: #{error.class} - #{error.message}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/aidp/concurrency/exec.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+require "concurrent-ruby"
+
+module Aidp
+  module Concurrency
+    # Centralized executor and thread pool management.
+    #
+    # Provides named, configured executors for different workload types
+    # (I/O-bound, CPU-bound, background tasks) with standardized error
+    # handling and instrumentation.
+    #
+    # @example Get a named pool
+    #   pool = Exec.pool(name: :io_pool, size: 10)
+    #   future = Concurrent::Promises.future_on(pool) { fetch_remote_data() }
+    #
+    # @example Execute a future
+    #   result = Exec.future { expensive_computation() }.value!
+    #
+    # @example Shutdown all pools
+    #   Exec.shutdown_all
+    module Exec
+      class << self
+        # Get or create a named thread pool.
+        #
+        # Pools are cached by name. Calling this method multiple times with the
+        # same name returns the same pool instance.
+        #
+        # @param name [Symbol] Pool name (e.g., :io_pool, :cpu_pool, :background)
+        # @param size [Integer] Pool size (default: based on pool type)
+        # @param type [Symbol] Pool type :fixed or :cached (default: :fixed)
+        # @return [Concurrent::ThreadPoolExecutor] The thread pool
+        #
+        # @example
+        #   io_pool = Exec.pool(name: :io, size: 20)
+        #   cpu_pool = Exec.pool(name: :cpu, size: 4)
+        def pool(name:, size: nil, type: :fixed)
+          @pools ||= Concurrent::Hash.new
+          @pools[name] ||= create_pool(name, size, type)
+        end
+
+        # Execute a block on a future using the default pool.
+        #
+        # @param executor [Concurrent::ExecutorService] Custom executor (optional)
+        # @yield Block to execute asynchronously
+        # @return [Concurrent::Promises::Future] The future
+        #
+        # @example
+        #   future = Exec.future { slow_operation() }
+        #   result = future.value! # Wait for result
+        def future(executor: nil, &block)
+          raise ArgumentError, "Block required" unless block_given?
+
+          executor ||= default_pool
+          Concurrent::Promises.future_on(executor, &block)
+        end
+
+        # Execute multiple futures in parallel and wait for all to complete.
+        #
+        # @param futures [Array<Concurrent::Promises::Future>] Futures to zip
+        # @return [Concurrent::Promises::Future] Future that resolves when all complete
+        #
+        # @example
+        #   futures = [
+        #     Exec.future { task1() },
+        #     Exec.future { task2() },
+        #     Exec.future { task3() }
+        #   ]
+        #   results = Exec.zip(*futures).value!
+        def zip(*futures)
+          Concurrent::Promises.zip(*futures)
+        end
+
+        # Get the default executor pool.
+        #
+        # @return [Concurrent::ThreadPoolExecutor]
+        def default_pool
+          @default_pool ||= pool(name: :default, size: default_pool_size)
+        end
+
+        # Shutdown a specific pool.
+        #
+        # @param name [Symbol] Pool name
+        # @param timeout [Float] Seconds to wait for shutdown
+        # @return [Boolean] true if shutdown cleanly
+        def shutdown_pool(name, timeout: 60)
+          @pools ||= Concurrent::Hash.new
+          pool = @pools.delete(name)
+          return true unless pool
+
+          pool.shutdown
+          pool.wait_for_termination(timeout)
+        end
+
+        # Shutdown all managed pools.
+        #
+        # @param timeout [Float] Seconds to wait for each pool
+        # @return [Hash] Map of pool name to shutdown success
+        def shutdown_all(timeout: 60)
+          @pools ||= Concurrent::Hash.new
+          results = {}
+
+          @pools.each_key do |name|
+            results[name] = shutdown_pool(name, timeout: timeout)
+          end
+
+          @default_pool&.shutdown
+          @default_pool&.wait_for_termination(timeout)
+
+          results
+        end
+
+        # Get statistics for all pools.
+        #
+        # @return [Hash] Map of pool name to stats
+        def stats
+          @pools ||= Concurrent::Hash.new
+          stats = {}
+
+          @pools.each do |name, pool|
+            stats[name] = pool_stats(pool)
+          end
+
+          stats[:default] = pool_stats(@default_pool) if @default_pool
+
+          stats
+        end
+
+        private
+
+        def create_pool(name, size, type)
+          size ||= default_size_for_pool(name, type)
+
+          pool = case type
+          when :fixed
+            Concurrent::FixedThreadPool.new(size)
+          when :cached
+            Concurrent::CachedThreadPool.new
+          else
+            raise ArgumentError, "Unknown pool type: #{type}"
+          end
+
+          log_pool_created(name, type, size)
+          pool
+        end
+
+        def default_size_for_pool(name, type)
+          return nil if type == :cached
+
+          case name
+          when :io, :io_pool
+            20 # I/O-bound: can have more threads
+          when :cpu, :cpu_pool
+            processor_count # CPU-bound: match CPU cores
+          when :background
+            5 # Background tasks: small pool
+          else
+            10 # Generic default
+          end
+        end
+
+        def default_pool_size
+          [processor_count * 2, 10].max
+        end
+
+        def processor_count
+          @processor_count ||= Concurrent.processor_count
+        end
+
+        def pool_stats(pool)
+          return nil unless pool
+
+          {
+            pool_size: pool.max_length,
+            queue_length: pool.queue_length,
+            active_threads: pool.length,
+            completed_tasks: pool.completed_task_count
+          }
+        rescue => e
+          {error: e.message}
+        end
+
+        def log_pool_created(name, type, size)
+          Concurrency.logger&.debug(
+            "concurrency_pool",
+            "Created #{type} pool :#{name} with size #{size || "dynamic"}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/aidp/concurrency/wait.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "concurrent-ruby"
+
+module Aidp
+  module Concurrency
+    # Deterministic condition waiting with timeouts and intervals.
+    #
+    # Replaces sleep-based polling loops with proper timeout enforcement
+    # and early exit on condition satisfaction.
+    #
+    # @example Wait for file to exist
+    #   Wait.until(timeout: 30, interval: 0.2) { File.exist?("/tmp/ready") }
+    #
+    # @example Wait for port to open
+    #   Wait.until(timeout: 60, interval: 1) do
+    #     TCPSocket.new("localhost", 8080).close rescue false
+    #   end
+    #
+    # @example Custom error message
+    #   Wait.until(timeout: 10, message: "Service failed to start") do
+    #     service_ready?
+    #   end
+    module Wait
+      class << self
+        # Wait until a condition becomes true, with timeout and interval polling.
+        #
+        # @param timeout [Float] Maximum seconds to wait (default: from config)
+        # @param interval [Float] Seconds between condition checks (default: from config)
+        # @param message [String] Custom error message on timeout
+        # @yield Block that returns truthy when condition is met
+        # @return [Object] The truthy value from the block
+        # @raise [Concurrency::TimeoutError] if timeout is reached before condition is met
+        #
+        # @example
+        #   result = Wait.until(timeout: 5, interval: 0.1) { expensive_check() }
+        def until(timeout: nil, interval: nil, message: nil, &block)
+          timeout ||= Concurrency.configuration.default_timeout
+          interval ||= Concurrency.configuration.default_interval
+          message ||= "Condition not met within #{timeout}s"
+
+          raise ArgumentError, "Block required" unless block_given?
+
+          start_time = monotonic_time
+          deadline = start_time + timeout
+          elapsed = 0.0
+
+          loop do
+            result = block.call
+            if result
+              log_wait_completion(elapsed) if should_log_wait?(elapsed)
+              return result
+            end
+
+            elapsed = monotonic_time - start_time
+            remaining = deadline - monotonic_time
+
+            if remaining <= 0
+              log_timeout(elapsed, message)
+              raise Concurrency::TimeoutError, message
+            end
+
+            # Sleep for interval or remaining time, whichever is shorter
+            sleep_duration = [interval, remaining].min
+            sleep(sleep_duration) if sleep_duration > 0
+          end
+        end
+
+        # Wait for a file to exist.
+        #
+        # @param path [String] File path to check
+        # @param timeout [Float] Maximum seconds to wait
+        # @param interval [Float] Seconds between checks
+        # @return [String] The file path if it exists
+        # @raise [Concurrency::TimeoutError] if file doesn't appear in time
+        def for_file(path, timeout: nil, interval: nil)
+          self.until(
+            timeout: timeout,
+            interval: interval,
+            message: "File not found: #{path} (waited #{timeout || Concurrency.configuration.default_timeout}s)"
+          ) { File.exist?(path) }
+          path
+        end
+
+        # Wait for a TCP port to be open.
+        #
+        # @param host [String] Hostname or IP
+        # @param port [Integer] Port number
+        # @param timeout [Float] Maximum seconds to wait
+        # @param interval [Float] Seconds between checks
+        # @return [Boolean] true if port is open
+        # @raise [Concurrency::TimeoutError] if port doesn't open in time
+        def for_port(host, port, timeout: nil, interval: nil)
+          require "socket"
+          self.until(
+            timeout: timeout,
+            interval: interval,
+            message: "Port #{host}:#{port} not open (waited #{timeout || Concurrency.configuration.default_timeout}s)"
+          ) do
+            socket = TCPSocket.new(host, port)
+            socket.close
+            true
+          rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ETIMEDOUT
+            false
+          end
+        end
+
+        # Wait for a process to exit.
+        #
+        # @param pid [Integer] Process ID
+        # @param timeout [Float] Maximum seconds to wait
+        # @param interval [Float] Seconds between checks
+        # @return [Process::Status] The process exit status
+        # @raise [Concurrency::TimeoutError] if process doesn't exit in time
+        def for_process_exit(pid, timeout: nil, interval: nil)
+          status = nil
+          self.until(
+            timeout: timeout,
+            interval: interval,
+            message: "Process #{pid} did not exit (waited #{timeout || Concurrency.configuration.default_timeout}s)"
+          ) do
+            _, status = Process.waitpid2(pid, Process::WNOHANG)
+            status # Returns truthy when process has exited
+          end
+          status
+        end
+
+        private
+
+        def monotonic_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        def should_log_wait?(elapsed)
+          elapsed >= Concurrency.configuration.log_long_waits_threshold
+        end
+
+        def log_wait_completion(elapsed)
+          Concurrency.logger&.info("concurrency_wait", "Long wait completed: #{elapsed.round(2)}s")
+        end
+
+        def log_timeout(elapsed, message)
+          Concurrency.logger&.warn("concurrency_timeout", "Wait timeout: #{message} (elapsed: #{elapsed.round(2)}s)")
+        end
+      end
+    end
+  end
+end

data/lib/aidp/concurrency.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "concurrent-ruby"
+require_relative "concurrency/wait"
+require_relative "concurrency/backoff"
+require_relative "concurrency/exec"
+
+module Aidp
+  # Concurrency utilities for deterministic waiting, retry/backoff, and executor management.
+  #
+  # This module provides standardized primitives to replace arbitrary sleep() calls with
+  # proper synchronization, timeouts, and event-based coordination using concurrent-ruby.
+  #
+  # @example Wait for a condition
+  #   Concurrency::Wait.until(timeout: 30) { File.exist?(path) }
+  #
+  # @example Retry with backoff
+  #   Concurrency::Backoff.retry(max_attempts: 5) { risky_call() }
+  #
+  # @example Get a named executor
+  #   pool = Concurrency::Exec.pool(name: :io_pool, size: 10)
+  #   future = Concurrent::Promises.future_on(pool) { fetch_data() }
+  module Concurrency
+    class Error < StandardError; end
+    class TimeoutError < Error; end
+    class MaxAttemptsError < Error; end
+
+    # Default configuration for executors and timeouts
+    class Configuration
+      attr_accessor :default_timeout, :default_interval, :default_max_attempts,
+        :default_backoff_base, :default_backoff_max, :default_jitter,
+        :log_long_waits_threshold, :log_retries
+
+      def initialize
+        @default_timeout = 30.0
+        @default_interval = 0.2
+        @default_max_attempts = 5
+        @default_backoff_base = 0.5
+        @default_backoff_max = 30.0
+        @default_jitter = 0.2
+        @log_long_waits_threshold = 5.0 # Log if wait takes > 5s
+        @log_retries = true
+      end
+    end
+
+    class << self
+      attr_writer :configuration
+
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield(configuration)
+      end
+
+      def logger
+        @logger ||= if defined?(Aidp.logger)
+          Aidp.logger
+        elsif defined?(Rails)
+          Rails.logger
+        else
+          require "logger"
+          Logger.new($stdout)
+        end
+      end
+
+      attr_writer :logger
+    end
+  end
+end

data/lib/aidp/config.rb

@@ -162,6 +162,11 @@ module Aidp
             interactive: true
           }
         }
+      },
+      skills: {
+        search_paths: [],
+        default_provider_filter: true,
+        enable_custom_skills: true
       }
     }.freeze
 
@@ -243,6 +248,15 @@ module Aidp
       providers_section.keys.map(&:to_s)
     end
 
+    # Get skills configuration
+    def self.skills_config(project_dir = Dir.pwd)
+      config = load_harness_config(project_dir)
+      skills_section = config[:skills] || config["skills"] || {}
+
+      # Convert string keys to symbols for consistency
+      symbolize_keys(skills_section)
+    end
+
     # Check if configuration file exists
     def self.config_exists?(project_dir = Dir.pwd)
       ConfigPaths.config_exists?(project_dir)
@@ -322,6 +336,12 @@ module Aidp
         end
       end
 
+      # Deep merge skills config
+      if config[:skills] || config["skills"]
+        skills_section = config[:skills] || config["skills"]
+        merged[:skills] = merged[:skills].merge(symbolize_keys(skills_section))
+      end
+
       merged
     end
 

data/lib/aidp/daemon/runner.rb

@@ -3,6 +3,7 @@
 require "socket"
 require_relative "process_manager"
 require_relative "../execute/async_work_loop_runner"
+require_relative "../concurrency"
 
 module Aidp
   module Daemon
@@ -35,18 +36,20 @@ module Aidp
 
         Process.detach(daemon_pid)
 
-        # Wait for daemon to start
-        sleep 0.5
+        # Wait for daemon to start (check if it's running)
+        begin
+          Aidp::Concurrency::Wait.until(timeout: 5, interval: 0.1) do
+            @process_manager.running?
+          end
 
-        if @process_manager.running?
           {
             success: true,
             message: "Daemon started in #{mode} mode",
             pid: daemon_pid,
             log_file: @process_manager.log_file_path
           }
-        else
-          {success: false, message: "Failed to start daemon"}
+        rescue Aidp::Concurrency::TimeoutError
+          {success: false, message: "Failed to start daemon (timeout)"}
         end
       end
 
@@ -189,7 +192,7 @@ module Aidp
             sleep(@options[:interval] || 60)
           rescue => e
             Aidp.logger.error("watch_error", "Watch cycle error: #{e.message}")
-            sleep 30 # Back off on error
+            sleep 30
           end
         end
 
@@ -199,8 +202,6 @@ module Aidp
       def run_work_loop_mode
         Aidp.logger.info("daemon_lifecycle", "Starting work loop mode")
 
-        # This would integrate with AsyncWorkLoopRunner
-        # For now, just log that we're running
         while @running
           Aidp.logger.debug("heartbeat", "Daemon running")
           sleep 10

data/lib/aidp/debug_mixin.rb

@@ -111,6 +111,7 @@ module Aidp
     # Log error with debug context
     def debug_error(error, context = {})
       return unless debug_basic?
+      return unless error # Handle nil error gracefully
 
       error_message = "💥 Error: #{error.class.name}: #{error.message}"
       debug_logger.error(component_name, error_message, error: error.class.name, **context)

data/lib/aidp/errors.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Aidp
+  # Error classes for AIDP
+  module Errors
+    class ConfigurationError < StandardError; end
+    class ProviderError < StandardError; end
+    class ValidationError < StandardError; end
+    class StateError < StandardError; end
+    class UserError < StandardError; end
+  end
+end