aidp 0.10.0 → 0.11.0

data/lib/aidp/jobs/background_runner.rb

@@ -0,0 +1,278 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "yaml"
+require "fileutils"
+
+module Aidp
+  module Jobs
+    # Manages background execution of work loops
+    # Runs harness in daemon process and tracks job metadata
+    class BackgroundRunner
+      include Aidp::MessageDisplay
+
+      attr_reader :project_dir, :jobs_dir
+
+      def initialize(project_dir = Dir.pwd)
+        @project_dir = project_dir
+        @jobs_dir = File.join(project_dir, ".aidp", "jobs")
+        ensure_jobs_directory
+      end
+
+      # Start a background job
+      # Returns job_id
+      def start(mode, options = {})
+        job_id = generate_job_id
+        log_file = File.join(@jobs_dir, job_id, "output.log")
+        pid_file = File.join(@jobs_dir, job_id, "job.pid")
+
+        # Create job directory
+        FileUtils.mkdir_p(File.dirname(log_file))
+
+        # Fork and daemonize
+        pid = fork do
+          # Detach from parent process
+          Process.daemon(true)
+
+          # Redirect stdout/stderr to log file
+          $stdout.reopen(log_file, "a")
+          $stderr.reopen(log_file, "a")
+          $stdout.sync = true
+          $stderr.sync = true
+
+          # Write PID file
+          File.write(pid_file, Process.pid)
+
+          begin
+            # Run the harness
+            puts "[#{Time.now}] Starting #{mode} mode in background"
+            puts "[#{Time.now}] Job ID: #{job_id}"
+            puts "[#{Time.now}] PID: #{Process.pid}"
+
+            runner = Aidp::Harness::Runner.new(@project_dir, mode, options.merge(job_id: job_id))
+            result = runner.run
+
+            puts "[#{Time.now}] Job completed with status: #{result[:status]}"
+            mark_job_completed(job_id, result)
+          rescue => e
+            puts "[#{Time.now}] Job failed with error: #{e.message}"
+            puts e.backtrace.join("\n")
+            mark_job_failed(job_id, e)
+          ensure
+            # Clean up PID file
+            File.delete(pid_file) if File.exist?(pid_file)
+          end
+        end
+
+        # Wait for child to fork
+        Process.detach(pid)
+        sleep 0.1 # Give daemon time to write PID file
+
+        # Save job metadata in parent process
+        save_job_metadata(job_id, pid, mode, options)
+
+        job_id
+      end
+
+      # List all jobs
+      def list_jobs
+        return [] unless Dir.exist?(@jobs_dir)
+
+        Dir.glob(File.join(@jobs_dir, "*")).select { |d| File.directory?(d) }.map do |job_dir|
+          job_id = File.basename(job_dir)
+          load_job_metadata(job_id)
+        end.compact.sort_by { |job| job[:started_at] || Time.now }.reverse
+      end
+
+      # Get job status
+      def job_status(job_id)
+        metadata = load_job_metadata(job_id)
+        return nil unless metadata
+
+        # Check if process is still running
+        pid = metadata[:pid]
+        running = pid && process_running?(pid)
+
+        # Get checkpoint data
+        checkpoint = get_job_checkpoint(job_id)
+
+        {
+          job_id: job_id,
+          mode: metadata[:mode],
+          status: determine_job_status(metadata, running, checkpoint),
+          pid: pid,
+          running: running,
+          started_at: metadata[:started_at],
+          completed_at: metadata[:completed_at],
+          checkpoint: checkpoint,
+          log_file: File.join(@jobs_dir, job_id, "output.log")
+        }
+      end
+
+      # Stop a running job
+      def stop_job(job_id)
+        metadata = load_job_metadata(job_id)
+        return {success: false, message: "Job not found"} unless metadata
+
+        pid = metadata[:pid]
+        unless pid && process_running?(pid)
+          return {success: false, message: "Job is not running"}
+        end
+
+        begin
+          # Send TERM signal
+          Process.kill("TERM", pid)
+
+          # Wait for process to terminate (max 10 seconds)
+          10.times do
+            sleep 0.5
+            break unless process_running?(pid)
+          end
+
+          # Force kill if still running
+          if process_running?(pid)
+            Process.kill("KILL", pid)
+          end
+
+          mark_job_stopped(job_id)
+          {success: true, message: "Job stopped successfully"}
+        rescue Errno::ESRCH
+          # Process already dead
+          mark_job_stopped(job_id)
+          {success: true, message: "Job was already stopped"}
+        rescue => e
+          {success: false, message: "Failed to stop job: #{e.message}"}
+        end
+      end
+
+      # Get job logs
+      def job_logs(job_id, options = {})
+        log_file = File.join(@jobs_dir, job_id, "output.log")
+        return nil unless File.exist?(log_file)
+
+        if options[:tail]
+          lines = options[:lines] || 50
+          `tail -n #{lines} #{log_file}`
+        else
+          File.read(log_file)
+        end
+      end
+
+      # Follow job logs in real-time
+      def follow_job_logs(job_id)
+        log_file = File.join(@jobs_dir, job_id, "output.log")
+        return unless File.exist?(log_file)
+
+        # Use tail -f to follow logs
+        exec("tail", "-f", log_file)
+      end
+
+      private
+
+      def ensure_jobs_directory
+        FileUtils.mkdir_p(@jobs_dir) unless Dir.exist?(@jobs_dir)
+      end
+
+      def generate_job_id
+        timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
+        random = SecureRandom.hex(4)
+        "#{timestamp}_#{random}"
+      end
+
+      def save_job_metadata(job_id, pid, mode, options)
+        metadata_file = File.join(@jobs_dir, job_id, "metadata.yml")
+
+        metadata = {
+          job_id: job_id,
+          pid: pid,
+          mode: mode,
+          started_at: Time.now,
+          status: "running",
+          options: options.except(:prompt) # Don't save prompt object
+        }
+
+        File.write(metadata_file, metadata.to_yaml)
+      end
+
+      def load_job_metadata(job_id)
+        metadata_file = File.join(@jobs_dir, job_id, "metadata.yml")
+        return nil unless File.exist?(metadata_file)
+
+        YAML.load_file(metadata_file)
+      rescue
+        nil
+      end
+
+      def update_job_metadata(job_id, updates)
+        metadata = load_job_metadata(job_id)
+        return unless metadata
+
+        metadata.merge!(updates)
+        metadata_file = File.join(@jobs_dir, job_id, "metadata.yml")
+        File.write(metadata_file, metadata.to_yaml)
+      end
+
+      def mark_job_completed(job_id, result)
+        update_job_metadata(job_id, {
+          status: "completed",
+          completed_at: Time.now,
+          result: result
+        })
+      end
+
+      def mark_job_failed(job_id, error)
+        update_job_metadata(job_id, {
+          status: "failed",
+          completed_at: Time.now,
+          error: {
+            message: error.message,
+            class: error.class.name,
+            backtrace: error.backtrace&.first(10)
+          }
+        })
+      end
+
+      def mark_job_stopped(job_id)
+        update_job_metadata(job_id, {
+          status: "stopped",
+          completed_at: Time.now
+        })
+      end
+
+      def process_running?(pid)
+        return false unless pid
+
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH, Errno::EPERM
+        false
+      end
+
+      def get_job_checkpoint(job_id)
+        # Try to load checkpoint from project directory
+        checkpoint = Aidp::Execute::Checkpoint.new(@project_dir)
+        checkpoint.latest_checkpoint
+      rescue
+        nil
+      end
+
+      def determine_job_status(metadata, running, checkpoint)
+        return metadata[:status] if metadata[:status] && !%w[running].include?(metadata[:status])
+
+        if running
+          # Check if job appears stuck based on checkpoint
+          if checkpoint && checkpoint[:timestamp]
+            last_update = Time.parse(checkpoint[:timestamp])
+            age = Time.now - last_update
+
+            return "stuck" if age > 600 # No update in 10 minutes
+            return "running"
+          end
+          "running"
+        else
+          metadata[:status] || "completed"
+        end
+      end
+    end
+  end
+end

data/lib/aidp/message_display.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "tty-prompt"
+
+module Aidp
+  # Mixin providing a consistent display_message helper across classes.
+  # Usage:
+  #   include Aidp::MessageDisplay
+  #   display_message("Hello", type: :success)
+  # Supports color types: :error, :success, :warning, :info, :highlight, :muted
+  module MessageDisplay
+    COLOR_MAP = {
+      error: :red,
+      success: :green,
+      warning: :yellow,
+      warn: :yellow,
+      info: :blue,
+      highlight: :cyan,
+      muted: :bright_black
+    }.freeze
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    # Instance helper for displaying a colored message via TTY::Prompt
+    def display_message(message, type: :info)
+      prompt = message_display_prompt
+      prompt.say(message, color: COLOR_MAP.fetch(type, :white))
+    end
+
+    # Provide a memoized prompt per including instance (if it defines @prompt)
+    def message_display_prompt
+      if instance_variable_defined?(:@prompt) && @prompt
+        @prompt
+      else
+        @__message_display_prompt ||= TTY::Prompt.new
+      end
+    end
+
+    module ClassMethods
+      # Class-level display helper (stateless)
+      def display_message(message, type: :info)
+        TTY::Prompt.new.say(message, color: COLOR_MAP.fetch(type, :white))
+      end
+    end
+  end
+end

data/lib/aidp/provider_manager.rb

@@ -138,7 +138,7 @@ module Aidp
         case provider_type
         when "cursor"
           Aidp::Providers::Cursor.new(prompt: prompt)
-        when "anthropic"
+        when "anthropic", "claude"
           Aidp::Providers::Anthropic.new(prompt: prompt)
         when "gemini"
           Aidp::Providers::Gemini.new(prompt: prompt)
@@ -146,6 +146,8 @@ module Aidp
           Aidp::Providers::MacOSUI.new(prompt: prompt)
         when "github_copilot"
           Aidp::Providers::GithubCopilot.new(prompt: prompt)
+        when "codex"
+          Aidp::Providers::Codex.new(prompt: prompt)
         end
       end
     end

data/lib/aidp/providers/anthropic.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "json"
 require_relative "base"
 require_relative "../debug_mixin"
 
@@ -16,6 +17,10 @@ module Aidp
         "anthropic"
       end
 
+      def display_name
+        "Anthropic Claude CLI"
+      end
+
       def available?
         self.class.available?
       end
@@ -29,16 +34,46 @@ module Aidp
         debug_provider("claude", "Starting execution", {timeout: timeout_seconds})
         debug_log("📝 Sending prompt to claude...", level: :info)
 
+        # Check if streaming mode is enabled
+        streaming_enabled = ENV["AIDP_STREAMING"] == "1" || ENV["DEBUG"] == "1"
+
+        # Build command arguments with proper streaming support
+        args = ["--print"]
+        if streaming_enabled
+          # Claude CLI requires --verbose when using --print with --output-format=stream-json
+          args += ["--verbose", "--output-format=stream-json", "--include-partial-messages"]
+          display_message("📺 True streaming enabled - real-time chunks from Claude API", type: :info)
+        else
+          # Use text format for non-streaming (default behavior)
+          args += ["--output-format=text"]
+        end
+
         begin
-          # Use debug_execute_command for better debugging
-          result = debug_execute_command("claude", args: ["--print"], input: prompt, timeout: timeout_seconds)
+          # Use debug_execute_command with streaming support
+          result = debug_execute_command("claude", args: args, input: prompt, timeout: timeout_seconds, streaming: streaming_enabled)
 
           # Log the results
-          debug_command("claude", args: ["--print"], input: prompt, output: result.out, error: result.err, exit_code: result.exit_status)
+          debug_command("claude", args: args, input: prompt, output: result.out, error: result.err, exit_code: result.exit_status)
 
           if result.exit_status == 0
-            result.out
+            # Handle different output formats
+            if streaming_enabled && args.include?("--output-format=stream-json")
+              # Parse stream-json output and extract final content
+              parse_stream_json_output(result.out)
+            else
+              # Return text output as-is
+              result.out
+            end
           else
+            # Detect auth issues in stdout/stderr (Claude sometimes prints JSON with auth error to stdout)
+            combined = [result.out, result.err].compact.join("\n")
+            if combined.downcase.include?("oauth token has expired") || combined.downcase.include?("authentication_error")
+              error_message = "Authentication error from Claude CLI: token expired or invalid. Run 'claude /login' or refresh credentials."
+              debug_error(StandardError.new(error_message), {exit_code: result.exit_status, stdout: result.out, stderr: result.err})
+              # Raise a recognizable error for classifier
+              raise error_message
+            end
+
             debug_error(StandardError.new("claude failed"), {exit_code: result.exit_status, stderr: result.err})
             raise "claude failed with exit code #{result.exit_status}: #{result.err}"
           end
@@ -58,8 +93,8 @@ module Aidp
         # 4. Default timeout
 
         if ENV["AIDP_QUICK_MODE"]
-          display_message("⚡ Quick mode enabled - 2 minute timeout", type: :highlight)
-          return 120
+          display_message("⚡ Quick mode enabled - #{TIMEOUT_QUICK_MODE / 60} minute timeout", type: :highlight)
+          return TIMEOUT_QUICK_MODE
         end
 
         if ENV["AIDP_ANTHROPIC_TIMEOUT"]
@@ -73,9 +108,9 @@ module Aidp
           return step_timeout
         end
 
-        # Default timeout (5 minutes for interactive use)
-        display_message("📋 Using default timeout: 5 minutes", type: :info)
-        300
+        # Default timeout
+        display_message("📋 Using default timeout: #{TIMEOUT_DEFAULT / 60} minutes", type: :info)
+        TIMEOUT_DEFAULT
       end
 
       def get_adaptive_timeout
@@ -84,23 +119,71 @@ module Aidp
 
         case step_name
         when /REPOSITORY_ANALYSIS/
-          180  # 3 minutes - repository analysis can be quick
+          TIMEOUT_REPOSITORY_ANALYSIS
         when /ARCHITECTURE_ANALYSIS/
-          600  # 10 minutes - architecture analysis needs more time
+          TIMEOUT_ARCHITECTURE_ANALYSIS
         when /TEST_ANALYSIS/
-          300  # 5 minutes - test analysis is moderate
+          TIMEOUT_TEST_ANALYSIS
         when /FUNCTIONALITY_ANALYSIS/
-          600  # 10 minutes - functionality analysis is complex
+          TIMEOUT_FUNCTIONALITY_ANALYSIS
         when /DOCUMENTATION_ANALYSIS/
-          300  # 5 minutes - documentation analysis is moderate
+          TIMEOUT_DOCUMENTATION_ANALYSIS
         when /STATIC_ANALYSIS/
-          450  # 7.5 minutes - static analysis can be intensive
+          TIMEOUT_STATIC_ANALYSIS
         when /REFACTORING_RECOMMENDATIONS/
-          600  # 10 minutes - refactoring recommendations are complex
+          TIMEOUT_REFACTORING_RECOMMENDATIONS
         else
-          nil  # Use default
+          nil # Use default
         end
       end
+
+      # Parse stream-json output from Claude CLI
+      def parse_stream_json_output(output)
+        return output if output.nil? || output.empty?
+
+        # Stream-json output contains multiple JSON objects, one per line
+        # We want to extract the final content from the last complete message
+        lines = output.strip.split("\n")
+        content_parts = []
+
+        lines.each do |line|
+          next if line.strip.empty?
+
+          begin
+            json_obj = JSON.parse(line)
+
+            # Look for content in various possible structures
+            if json_obj["type"] == "content_block_delta" && json_obj["delta"] && json_obj["delta"]["text"]
+              content_parts << json_obj["delta"]["text"]
+            elsif json_obj["content"]&.is_a?(Array)
+              json_obj["content"].each do |content_item|
+                content_parts << content_item["text"] if content_item["text"]
+              end
+            elsif json_obj["message"] && json_obj["message"]["content"]
+              if json_obj["message"]["content"].is_a?(Array)
+                json_obj["message"]["content"].each do |content_item|
+                  content_parts << content_item["text"] if content_item["text"]
+                end
+              elsif json_obj["message"]["content"].is_a?(String)
+                content_parts << json_obj["message"]["content"]
+              end
+            end
+          rescue JSON::ParserError => e
+            debug_log("⚠️ Failed to parse JSON line: #{e.message}", level: :warn, data: {line: line})
+            # If JSON parsing fails, treat as plain text
+            content_parts << line
+          end
+        end
+
+        result = content_parts.join
+
+        # Fallback: if no content found in JSON, return original output
+        result.empty? ? output : result
+      rescue => e
+        debug_log("⚠️ Failed to parse stream-json output: #{e.message}", level: :warn)
+        # Return original output if parsing fails
+        output
+      end
     end
   end
 end

data/lib/aidp/providers/base.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
 require "tty-prompt"
+require "tty-spinner"
 
 module Aidp
   module Providers
     class Base
+      include Aidp::MessageDisplay
+
       # Activity indicator states
       ACTIVITY_STATES = {
         idle: "⏳",
@@ -17,6 +20,18 @@ module Aidp
       # Default timeout for stuck detection (2 minutes)
       DEFAULT_STUCK_TIMEOUT = 120
 
+      # Configurable timeout values (can be overridden via environment or config)
+      # These defaults provide reasonable values for different execution scenarios
+      TIMEOUT_QUICK_MODE = 120 # 2 minutes - for quick testing
+      TIMEOUT_DEFAULT = 300 # 5 minutes - standard interactive timeout
+      TIMEOUT_REPOSITORY_ANALYSIS = 180 # 3 minutes - repository analysis
+      TIMEOUT_ARCHITECTURE_ANALYSIS = 600 # 10 minutes - architecture analysis
+      TIMEOUT_TEST_ANALYSIS = 300 # 5 minutes - test analysis
+      TIMEOUT_FUNCTIONALITY_ANALYSIS = 600 # 10 minutes - functionality analysis
+      TIMEOUT_DOCUMENTATION_ANALYSIS = 300 # 5 minutes - documentation analysis
+      TIMEOUT_STATIC_ANALYSIS = 450 # 7.5 minutes - static analysis
+      TIMEOUT_REFACTORING_RECOMMENDATIONS = 600 # 10 minutes - refactoring
+
       attr_reader :activity_state, :last_activity_time, :start_time, :step_name
 
       def initialize(output: nil, prompt: TTY::Prompt.new)
@@ -48,6 +63,12 @@ module Aidp
         raise NotImplementedError, "#{self.class} must implement #name"
       end
 
+      # Human-friendly display name for UI
+      # Override in subclasses to provide a better display name
+      def display_name
+        name
+      end
+
       def send(prompt:, session: nil)
         raise NotImplementedError, "#{self.class} must implement #send"
       end
@@ -332,20 +353,30 @@ module Aidp
         (success_rate * 50) + ((1 - rate_limit_ratio) * 30) + (response_time_score * 0.2)
       end
 
-      private
+      protected
 
-      def display_message(message, type: :info)
-        color = case type
-        when :error then :red
-        when :success then :green
-        when :warning then :yellow
-        when :info then :blue
-        when :highlight then :cyan
-        when :muted then :bright_black
-        else :white
+      # Update spinner status with elapsed time
+      # This is a shared method used by all providers to display progress
+      def update_spinner_status(spinner, elapsed, provider_name)
+        minutes = (elapsed / 60).to_i
+        seconds = (elapsed % 60).to_i
+
+        if minutes > 0
+          spinner.update(title: "#{provider_name} is running... (#{minutes}m #{seconds}s)")
+        else
+          spinner.update(title: "#{provider_name} is running... (#{seconds}s)")
         end
-        @prompt.say(message, color: color)
       end
+
+      # Clean up activity display thread and spinner
+      # Used by providers to ensure proper cleanup in both success and error paths
+      def cleanup_activity_display(activity_display_thread, spinner)
+        activity_display_thread.kill if activity_display_thread&.alive?
+        activity_display_thread&.join(0.1) # Give it 100ms to finish
+        spinner&.stop
+      end
+
+      private
     end
   end
 end