aidp 0.15.2 → 0.16.0

data/lib/aidp/workstream_executor.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require "concurrent-ruby"
+require "time"
+require_relative "worktree"
+require_relative "workstream_state"
+require_relative "harness/runner"
+require_relative "message_display"
+
+module Aidp
+  # Executes multiple workstreams in parallel using concurrent-ruby.
+  # Provides true parallel execution with process isolation and status tracking.
+  class WorkstreamExecutor
+    include Aidp::MessageDisplay
+
+    # Result from executing a workstream
+    WorkstreamResult = Struct.new(
+      :slug,
+      :status,
+      :exit_code,
+      :started_at,
+      :completed_at,
+      :duration,
+      :error,
+      keyword_init: true
+    )
+
+    def initialize(project_dir: Dir.pwd, max_concurrent: 3)
+      @project_dir = project_dir
+      @max_concurrent = max_concurrent
+      @results = Concurrent::Hash.new
+      @start_times = Concurrent::Hash.new
+    end
+
+    # Execute multiple workstreams in parallel
+    #
+    # @param slugs [Array<String>] Workstream slugs to execute
+    # @param options [Hash] Execution options
+    # @option options [Array<String>] :selected_steps Steps to execute
+    # @option options [Symbol] :workflow_type Workflow type (:execute, :analyze, etc.)
+    # @option options [Hash] :user_input User input for harness
+    # @return [Array<WorkstreamResult>] Results for each workstream
+    def execute_parallel(slugs, options = {})
+      validate_workstreams!(slugs)
+
+      display_message("🚀 Starting parallel execution of #{slugs.size} workstreams (max #{@max_concurrent} concurrent)", type: :info)
+
+      # Create thread pool with max concurrent limit
+      pool = Concurrent::FixedThreadPool.new(@max_concurrent)
+
+      # Create futures for each workstream
+      futures = slugs.map do |slug|
+        Concurrent::Future.execute(executor: pool) do
+          execute_workstream(slug, options)
+        end
+      end
+
+      # Wait for all futures to complete
+      results = futures.map(&:value)
+
+      # Shutdown pool gracefully
+      pool.shutdown
+      pool.wait_for_termination(30)
+
+      display_execution_summary(results)
+      results
+    end
+
+    # Execute all active workstreams in parallel
+    #
+    # @param options [Hash] Execution options (same as execute_parallel)
+    # @return [Array<WorkstreamResult>] Results for each workstream
+    def execute_all(options = {})
+      workstreams = Aidp::Worktree.list(project_dir: @project_dir)
+      active_slugs = workstreams.select { |ws| ws[:active] }.map { |ws| ws[:slug] }
+
+      if active_slugs.empty?
+        display_message("⚠️  No active workstreams found", type: :warn)
+        return []
+      end
+
+      execute_parallel(active_slugs, options)
+    end
+
+    # Execute a single workstream (used by futures in parallel execution)
+    #
+    # @param slug [String] Workstream slug
+    # @param options [Hash] Execution options
+    # @return [WorkstreamResult] Execution result
+    def execute_workstream(slug, options = {})
+      started_at = Time.now
+      @start_times[slug] = started_at
+
+      workstream = Aidp::Worktree.info(slug: slug, project_dir: @project_dir)
+      unless workstream
+        return WorkstreamResult.new(
+          slug: slug,
+          status: "error",
+          exit_code: 1,
+          started_at: started_at,
+          completed_at: Time.now,
+          duration: 0,
+          error: "Workstream not found"
+        )
+      end
+
+      display_message("▶️  [#{slug}] Starting execution in #{workstream[:path]}", type: :info)
+
+      # Update workstream state to active
+      Aidp::WorkstreamState.update(
+        slug: slug,
+        project_dir: @project_dir,
+        status: "active",
+        started_at: started_at.utc.iso8601
+      )
+
+      # Execute in forked process for true isolation
+      pid = fork do
+        # Change to workstream directory
+        Dir.chdir(workstream[:path])
+
+        # Execute harness
+        runner = Aidp::Harness::Runner.new(
+          workstream[:path],
+          options[:mode] || :execute,
+          options
+        )
+
+        result = runner.run
+
+        # Update state on completion
+        exit_code = (result[:status] == "completed") ? 0 : 1
+        final_status = (result[:status] == "completed") ? "completed" : "failed"
+
+        Aidp::WorkstreamState.update(
+          slug: slug,
+          project_dir: @project_dir,
+          status: final_status,
+          completed_at: Time.now.utc.iso8601
+        )
+
+        exit(exit_code)
+      rescue => e
+        # Update state on error
+        Aidp::WorkstreamState.update(
+          slug: slug,
+          project_dir: @project_dir,
+          status: "failed",
+          completed_at: Time.now.utc.iso8601
+        )
+
+        # Log error and exit
+        warn("Error in workstream #{slug}: #{e.message}")
+        warn(e.backtrace.first(5).join("\n"))
+        exit(1)
+      end
+
+      # Wait for child process
+      _pid, status = Process.wait2(pid)
+      completed_at = Time.now
+      duration = completed_at - started_at
+
+      # Build result
+      result_status = status.success? ? "completed" : "failed"
+      result = WorkstreamResult.new(
+        slug: slug,
+        status: result_status,
+        exit_code: status.exitstatus,
+        started_at: started_at,
+        completed_at: completed_at,
+        duration: duration,
+        error: status.success? ? nil : "Process exited with code #{status.exitstatus}"
+      )
+
+      @results[slug] = result
+
+      display_message("#{status.success? ? "✅" : "❌"} [#{slug}] #{result_status.capitalize} in #{format_duration(duration)}", type: status.success? ? :success : :error)
+
+      result
+    rescue => e
+      completed_at = Time.now
+      duration = completed_at - started_at
+
+      WorkstreamResult.new(
+        slug: slug,
+        status: "error",
+        exit_code: 1,
+        started_at: started_at,
+        completed_at: completed_at,
+        duration: duration,
+        error: e.message
+      )
+    end
+
+    private
+
+    # Validate that all workstreams exist
+    def validate_workstreams!(slugs)
+      invalid = slugs.reject do |slug|
+        Aidp::Worktree.exists?(slug: slug, project_dir: @project_dir)
+      end
+
+      unless invalid.empty?
+        raise ArgumentError, "Workstreams not found: #{invalid.join(", ")}"
+      end
+    end
+
+    # Display execution summary
+    def display_execution_summary(results)
+      completed = results.count { |r| r.status == "completed" }
+      failed = results.count { |r| r.status == "failed" || r.status == "error" }
+      total_duration = results.sum(&:duration)
+
+      display_message("\n" + "=" * 60, type: :muted)
+      display_message("📊 Execution Summary", type: :info)
+      display_message("Total: #{results.size} | Completed: #{completed} | Failed: #{failed}", type: :info)
+      display_message("Total Duration: #{format_duration(total_duration)}", type: :info)
+
+      if failed > 0
+        display_message("\n❌ Failed Workstreams:", type: :error)
+        results.select { |r| r.status != "completed" }.each do |result|
+          display_message("  - #{result.slug}: #{result.error}", type: :error)
+        end
+      end
+
+      display_message("=" * 60, type: :muted)
+    end
+
+    # Format duration in human-readable format
+    def format_duration(seconds)
+      if seconds < 60
+        "#{seconds.round(1)}s"
+      elsif seconds < 3600
+        minutes = (seconds / 60).floor
+        secs = (seconds % 60).round
+        "#{minutes}m #{secs}s"
+      else
+        hours = (seconds / 3600).floor
+        minutes = ((seconds % 3600) / 60).floor
+        "#{hours}h #{minutes}m"
+      end
+    end
+  end
+end

data/lib/aidp/workstream_state.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Aidp
+  # Manages per-workstream state (task, iterations, timestamps, event log)
+  # Stored under: .aidp/workstreams/<slug>/state.json and history.jsonl
+  module WorkstreamState
+    class Error < StandardError; end
+
+    class << self
+      def root_dir(project_dir)
+        File.join(project_dir, ".aidp", "workstreams")
+      end
+
+      def workstream_dir(slug, project_dir)
+        File.join(root_dir(project_dir), slug)
+      end
+
+      def state_file(slug, project_dir)
+        File.join(workstream_dir(slug, project_dir), "state.json")
+      end
+
+      def history_file(slug, project_dir)
+        File.join(workstream_dir(slug, project_dir), "history.jsonl")
+      end
+
+      # Per-worktree state files (mirrored for local inspection)
+      def worktree_state_file(slug, project_dir)
+        worktree_path = File.join(project_dir, ".worktrees", slug)
+        return nil unless Dir.exist?(worktree_path)
+        File.join(worktree_path, ".aidp", "workstreams", slug, "state.json")
+      end
+
+      def worktree_history_file(slug, project_dir)
+        worktree_path = File.join(project_dir, ".worktrees", slug)
+        return nil unless Dir.exist?(worktree_path)
+        File.join(worktree_path, ".aidp", "workstreams", slug, "history.jsonl")
+      end
+
+      # Initialize state for a new workstream
+      def init(slug:, project_dir:, task: nil)
+        dir = workstream_dir(slug, project_dir)
+        FileUtils.mkdir_p(dir)
+        now = Time.now.utc
+        state = {
+          slug: slug,
+          status: "active",
+          task: task,
+          started_at: now.iso8601,
+          updated_at: now.iso8601,
+          iterations: 0
+        }
+        write_json(state_file(slug, project_dir), state)
+        # Mirror to worktree if it exists
+        mirror_to_worktree(slug, project_dir, state)
+        append_event(slug: slug, project_dir: project_dir, type: "created", data: {task: task})
+        state
+      end
+
+      # Read current state (returns hash or nil)
+      def read(slug:, project_dir:)
+        file = state_file(slug, project_dir)
+        return nil unless File.exist?(file)
+        JSON.parse(File.read(file), symbolize_names: true)
+      rescue JSON::ParserError
+        nil
+      end
+
+      # Update selected attributes; updates updated_at automatically
+      def update(slug:, project_dir:, **attrs)
+        state = read(slug: slug, project_dir: project_dir) || init(slug: slug, project_dir: project_dir)
+        state.merge!(attrs.transform_keys(&:to_sym))
+        state[:updated_at] = Time.now.utc.iso8601
+        write_json(state_file(slug, project_dir), state)
+        # Mirror to worktree if it exists
+        mirror_to_worktree(slug, project_dir, state)
+        state
+      end
+
+      # Increment iteration counter and record event
+      def increment_iteration(slug:, project_dir:)
+        state = read(slug: slug, project_dir: project_dir) || init(slug: slug, project_dir: project_dir)
+        state[:iterations] = (state[:iterations] || 0) + 1
+        state[:updated_at] = Time.now.utc.iso8601
+        # Update status to active if paused (auto-resume on iteration)
+        state[:status] = "active" if state[:status] == "paused"
+        write_json(state_file(slug, project_dir), state)
+        # Mirror to worktree if it exists
+        mirror_to_worktree(slug, project_dir, state)
+        append_event(slug: slug, project_dir: project_dir, type: "iteration", data: {count: state[:iterations]})
+        state
+      end
+
+      # Append event to history.jsonl
+      def append_event(slug:, project_dir:, type:, data: {})
+        file = history_file(slug, project_dir)
+        FileUtils.mkdir_p(File.dirname(file))
+        event = {
+          timestamp: Time.now.utc.iso8601,
+          type: type,
+          data: data
+        }
+        File.open(file, "a") { |f| f.puts(JSON.generate(event)) }
+        # Mirror to worktree if it exists
+        wt_file = worktree_history_file(slug, project_dir)
+        if wt_file
+          FileUtils.mkdir_p(File.dirname(wt_file))
+          File.open(wt_file, "a") { |f| f.puts(JSON.generate(event)) }
+        end
+        event
+      end
+
+      # Read recent N events
+      def recent_events(slug:, project_dir:, limit: 5)
+        file = history_file(slug, project_dir)
+        return [] unless File.exist?(file)
+        lines = File.readlines(file, chomp: true)
+        lines.last(limit).map do |line|
+          JSON.parse(line, symbolize_names: true)
+        rescue JSON::ParserError
+          nil
+        end.compact
+      end
+
+      def elapsed_seconds(slug:, project_dir:)
+        state = read(slug: slug, project_dir: project_dir)
+        return 0 unless state && state[:started_at]
+        (Time.now.utc - Time.parse(state[:started_at])).to_i
+      end
+
+      # Check if workstream appears stalled (no activity for threshold seconds)
+      def stalled?(slug:, project_dir:, threshold_seconds: 3600)
+        state = read(slug: slug, project_dir: project_dir)
+        return false unless state && state[:updated_at]
+        return false if state[:status] != "active" # Only check active workstreams
+        (Time.now.utc - Time.parse(state[:updated_at])).to_i > threshold_seconds
+      end
+
+      # Auto-complete stalled workstreams
+      def auto_complete_stalled(slug:, project_dir:, threshold_seconds: 3600)
+        return unless stalled?(slug: slug, project_dir: project_dir, threshold_seconds: threshold_seconds)
+        complete(slug: slug, project_dir: project_dir)
+        append_event(slug: slug, project_dir: project_dir, type: "auto_completed", data: {reason: "stalled"})
+      end
+
+      def mark_removed(slug:, project_dir:)
+        state = read(slug: slug, project_dir: project_dir)
+        # Auto-complete if active when removing
+        if state && state[:status] == "active"
+          complete(slug: slug, project_dir: project_dir)
+        end
+        update(slug: slug, project_dir: project_dir, status: "removed")
+        append_event(slug: slug, project_dir: project_dir, type: "removed", data: {})
+      end
+
+      # Pause workstream (stop iteration without completion)
+      def pause(slug:, project_dir:)
+        state = read(slug: slug, project_dir: project_dir)
+        return {error: "Workstream not found"} unless state
+        return {error: "Already paused"} if state[:status] == "paused"
+
+        now = Time.now.utc.iso8601
+        update(slug: slug, project_dir: project_dir, status: "paused", paused_at: now)
+        append_event(slug: slug, project_dir: project_dir, type: "paused", data: {})
+        {status: "paused"}
+      end
+
+      # Resume workstream (return to active status)
+      def resume(slug:, project_dir:)
+        state = read(slug: slug, project_dir: project_dir)
+        return {error: "Workstream not found"} unless state
+        return {error: "Not paused"} unless state[:status] == "paused"
+
+        now = Time.now.utc.iso8601
+        update(slug: slug, project_dir: project_dir, status: "active", resumed_at: now)
+        append_event(slug: slug, project_dir: project_dir, type: "resumed", data: {})
+        {status: "active"}
+      end
+
+      # Mark workstream as completed
+      def complete(slug:, project_dir:)
+        state = read(slug: slug, project_dir: project_dir)
+        return {error: "Workstream not found"} unless state
+        return {error: "Already completed"} if state[:status] == "completed"
+
+        now = Time.now.utc.iso8601
+        update(slug: slug, project_dir: project_dir, status: "completed", completed_at: now)
+        append_event(slug: slug, project_dir: project_dir, type: "completed", data: {iterations: state[:iterations]})
+        {status: "completed"}
+      end
+
+      private
+
+      def write_json(path, obj)
+        File.write(path, JSON.pretty_generate(obj))
+      end
+
+      # Mirror state to worktree's .aidp directory for local visibility
+      def mirror_to_worktree(slug, project_dir, state)
+        wt_file = worktree_state_file(slug, project_dir)
+        return unless wt_file
+        FileUtils.mkdir_p(File.dirname(wt_file))
+        write_json(wt_file, state)
+      rescue
+        # Silently ignore mirroring errors to not disrupt main operation
+        nil
+      end
+    end
+  end
+end

data/lib/aidp/worktree.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require_relative "workstream_state"
+
+module Aidp
+  # Manages git worktree operations for parallel workstreams.
+  # Each workstream gets an isolated git worktree with its own branch,
+  # allowing multiple agents to work concurrently without conflicts.
+  module Worktree
+    class Error < StandardError; end
+    class NotInGitRepo < Error; end
+    class WorktreeExists < Error; end
+    class WorktreeNotFound < Error; end
+
+    class << self
+      # Create a new git worktree for a workstream
+      #
+      # @param slug [String] Short identifier for this workstream (e.g., "iss-123-fix-login")
+      # @param project_dir [String] Project root directory
+      # @param branch [String, nil] Branch name (defaults to "aidp/#{slug}")
+      # @param base_branch [String] Branch to create from (defaults to current branch)
+      # @return [Hash] Worktree info: { path:, branch:, slug: }
+      def create(slug:, project_dir: Dir.pwd, branch: nil, base_branch: nil, task: nil)
+        ensure_git_repo!(project_dir)
+
+        branch ||= "aidp/#{slug}"
+        worktree_path = worktree_path_for(slug, project_dir)
+
+        if Dir.exist?(worktree_path)
+          raise WorktreeExists, "Worktree already exists at #{worktree_path}"
+        end
+
+        # Create the worktree
+        cmd = ["git", "worktree", "add", "-b", branch, worktree_path]
+        cmd << base_branch if base_branch
+
+        Dir.chdir(project_dir) do
+          success = system(*cmd, out: File::NULL, err: File::NULL)
+          unless success
+            raise Error, "Failed to create worktree: #{$?.exitstatus}"
+          end
+        end
+
+        # Initialize .aidp directory in the worktree
+        ensure_aidp_dir(worktree_path)
+
+        # Register the worktree
+        register_worktree(slug, worktree_path, branch, project_dir)
+
+        # Initialize per-workstream state (task, counters, events)
+        Aidp::WorkstreamState.init(slug: slug, project_dir: project_dir, task: task)
+
+        {
+          slug: slug,
+          path: worktree_path,
+          branch: branch
+        }
+      end
+
+      # List all active worktrees for this project
+      #
+      # @param project_dir [String] Project root directory
+      # @return [Array<Hash>] Array of worktree info hashes
+      def list(project_dir: Dir.pwd)
+        registry = load_registry(project_dir)
+        registry.map do |slug, info|
+          {
+            slug: slug,
+            path: info["path"],
+            branch: info["branch"],
+            created_at: info["created_at"],
+            active: Dir.exist?(info["path"])
+          }
+        end
+      end
+
+      # Remove a worktree and optionally its branch
+      #
+      # @param slug [String] Workstream identifier
+      # @param project_dir [String] Project root directory
+      # @param delete_branch [Boolean] Whether to delete the git branch
+      def remove(slug:, project_dir: Dir.pwd, delete_branch: false)
+        registry = load_registry(project_dir)
+        info = registry[slug]
+
+        raise WorktreeNotFound, "Worktree '#{slug}' not found" unless info
+
+        worktree_path = info["path"]
+        branch = info["branch"]
+
+        # Remove the git worktree
+        if Dir.exist?(worktree_path)
+          Dir.chdir(project_dir) do
+            system("git", "worktree", "remove", worktree_path, "--force", out: File::NULL, err: File::NULL)
+          end
+        end
+
+        # Remove the branch if requested
+        if delete_branch
+          Dir.chdir(project_dir) do
+            system("git", "branch", "-D", branch, out: File::NULL, err: File::NULL)
+          end
+        end
+
+        # Mark state removed (if exists) then unregister
+        if Aidp::WorkstreamState.read(slug: slug, project_dir: project_dir)
+          Aidp::WorkstreamState.mark_removed(slug: slug, project_dir: project_dir)
+        end
+        # Unregister the worktree
+        unregister_worktree(slug, project_dir)
+
+        true
+      end
+
+      # Get info for a specific worktree
+      #
+      # @param slug [String] Workstream identifier
+      # @param project_dir [String] Project root directory
+      # @return [Hash, nil] Worktree info or nil if not found
+      def info(slug:, project_dir: Dir.pwd)
+        registry = load_registry(project_dir)
+        data = registry[slug]
+        return nil unless data
+
+        {
+          slug: slug,
+          path: data["path"],
+          branch: data["branch"],
+          created_at: data["created_at"],
+          active: Dir.exist?(data["path"])
+        }
+      end
+
+      # Check if a worktree exists
+      #
+      # @param slug [String] Workstream identifier
+      # @param project_dir [String] Project root directory
+      # @return [Boolean]
+      def exists?(slug:, project_dir: Dir.pwd)
+        !info(slug: slug, project_dir: project_dir).nil?
+      end
+
+      private
+
+      # Ensure we're in a git repository
+      def ensure_git_repo!(project_dir)
+        Dir.chdir(project_dir) do
+          unless system("git", "rev-parse", "--git-dir", out: File::NULL, err: File::NULL)
+            raise NotInGitRepo, "Not in a git repository: #{project_dir}"
+          end
+        end
+      end
+
+      # Get the worktree path for a slug
+      def worktree_path_for(slug, project_dir)
+        File.join(project_dir, ".worktrees", slug)
+      end
+
+      # Ensure the .aidp directory exists in a worktree
+      def ensure_aidp_dir(worktree_path)
+        aidp_dir = File.join(worktree_path, ".aidp")
+        FileUtils.mkdir_p(aidp_dir) unless Dir.exist?(aidp_dir)
+      end
+
+      # Load the worktree registry
+      def load_registry(project_dir)
+        registry_file = registry_file_path(project_dir)
+        return {} unless File.exist?(registry_file)
+
+        JSON.parse(File.read(registry_file))
+      rescue JSON::ParserError
+        {}
+      end
+
+      # Save the worktree registry
+      def save_registry(registry, project_dir)
+        registry_file = registry_file_path(project_dir)
+        FileUtils.mkdir_p(File.dirname(registry_file))
+        File.write(registry_file, JSON.pretty_generate(registry))
+      end
+
+      # Register a new worktree
+      def register_worktree(slug, path, branch, project_dir)
+        registry = load_registry(project_dir)
+        registry[slug] = {
+          "path" => path,
+          "branch" => branch,
+          "created_at" => Time.now.utc.iso8601
+        }
+        save_registry(registry, project_dir)
+      end
+
+      # Unregister a worktree
+      def unregister_worktree(slug, project_dir)
+        registry = load_registry(project_dir)
+        registry.delete(slug)
+        save_registry(registry, project_dir)
+      end
+
+      # Path to the worktree registry file
+      def registry_file_path(project_dir)
+        File.join(project_dir, ".aidp", "worktrees.json")
+      end
+    end
+  end
+end

data/lib/aidp.rb

@@ -8,6 +8,7 @@ require_relative "aidp/version"
 require_relative "aidp/config"
 require_relative "aidp/util"
 require_relative "aidp/message_display"
+require_relative "aidp/concurrency"
 require_relative "aidp/setup/wizard"
 require_relative "aidp/init"
 require_relative "aidp/watch"
@@ -69,6 +70,11 @@ require_relative "aidp/logger"
 require_relative "aidp/daemon/process_manager"
 require_relative "aidp/daemon/runner"
 
+# Workstream/worktree management
+require_relative "aidp/worktree"
+require_relative "aidp/workstream_state"
+require_relative "aidp/workstream_executor"
+
 # Harness mode
 require_relative "aidp/harness/configuration"
 require_relative "aidp/harness/config_schema"