aidp 0.26.0 → 0.28.0

data/lib/aidp/watch/ci_fix_processor.rb

@@ -11,6 +11,9 @@ require_relative "../harness/config_manager"
 require_relative "../execute/prompt_manager"
 require_relative "../harness/runner"
 require_relative "../harness/state_manager"
+require_relative "../worktree"
+require_relative "github_state_extractor"
+require_relative "ci_log_extractor"
 
 module Aidp
   module Watch
@@ -30,6 +33,7 @@ module Aidp
       def initialize(repository_client:, state_store:, provider_name: nil, project_dir: Dir.pwd, label_config: {}, verbose: false)
         @repository_client = repository_client
         @state_store = state_store
+        @state_extractor = GitHubStateExtractor.new(repository_client: repository_client)
         @provider_name = provider_name
         @project_dir = project_dir
         @verbose = verbose
@@ -41,9 +45,12 @@ module Aidp
       def process(pr)
         number = pr[:number]
 
-        # Check if already processed successfully
-        if @state_store.ci_fix_completed?(number)
+        Aidp.log_debug("ci_fix_processor", "process_started", pr_number: number, pr_title: pr[:title])
+
+        # Check if already processed successfully via GitHub comments
+        if @state_extractor.ci_fix_completed?(pr)
           display_message("ℹ️  CI fix for PR ##{number} already completed. Skipping.", type: :muted)
+          Aidp.log_debug("ci_fix_processor", "already_completed", pr_number: number)
           return
         end
 
@@ -53,9 +60,16 @@ module Aidp
         pr_data = @repository_client.fetch_pull_request(number)
         ci_status = @repository_client.fetch_ci_status(number)
 
+        Aidp.log_debug("ci_fix_processor", "ci_status_fetched",
+          pr_number: number,
+          ci_state: ci_status[:state],
+          check_count: ci_status[:checks]&.length || 0,
+          checks: ci_status[:checks]&.map { |c| {name: c[:name], status: c[:status], conclusion: c[:conclusion]} })
+
         # Check if there are failures
         if ci_status[:state] == "success"
           display_message("✅ CI is passing for PR ##{number}. No fixes needed.", type: :success)
+          Aidp.log_debug("ci_fix_processor", "ci_passing", pr_number: number)
           post_success_comment(pr_data)
           @state_store.record_ci_fix(number, {status: "no_failures", timestamp: Time.now.utc.iso8601})
           begin
@@ -68,14 +82,25 @@ module Aidp
 
         if ci_status[:state] == "pending"
           display_message("⏳ CI is still running for PR ##{number}. Skipping for now.", type: :muted)
+          Aidp.log_debug("ci_fix_processor", "ci_pending", pr_number: number)
           return
         end
 
         # Get failed checks
         failed_checks = ci_status[:checks].select { |check| check[:conclusion] == "failure" }
 
+        Aidp.log_debug("ci_fix_processor", "failed_checks_filtered",
+          pr_number: number,
+          total_checks: ci_status[:checks]&.length || 0,
+          failed_count: failed_checks.length,
+          failed_checks: failed_checks.map { |c| c[:name] })
+
         if failed_checks.empty?
           display_message("⚠️  No specific failed checks found for PR ##{number}.", type: :warn)
+          Aidp.log_debug("ci_fix_processor", "no_failed_checks",
+            pr_number: number,
+            ci_state: ci_status[:state],
+            all_checks: ci_status[:checks]&.map { |c| {name: c[:name], conclusion: c[:conclusion]} })
           return
         end
 
@@ -99,31 +124,36 @@ module Aidp
         display_message("❌ CI fix failed: #{e.message}", type: :error)
         Aidp.log_error("ci_fix_processor", "CI fix failed", pr: pr[:number], error: e.message, backtrace: e.backtrace&.first(10))
 
-        # Post error comment
-        error_comment = <<~COMMENT
-          #{COMMENT_HEADER}
-
-          ❌ Automated CI fix failed: #{e.message}
-
-          Please investigate the CI failures manually or retry by re-adding the `#{@ci_fix_label}` label.
-        COMMENT
-        begin
-          @repository_client.post_comment(pr[:number], error_comment)
-        rescue
-          nil
-        end
+        # Record failure state internally but DON'T post error to GitHub
+        # (per issue #280 - error messages should never appear on issues)
+        @state_store.record_ci_fix(pr[:number], {
+          status: "error",
+          error: e.message,
+          error_class: e.class.name,
+          timestamp: Time.now.utc.iso8601
+        })
       end
 
       private
 
       def analyze_and_fix(pr_data:, ci_status:, failed_checks:)
-        # Fetch logs for failed checks (if available)
+        # Extract concise failure information to reduce token usage
+        provider = detect_default_provider
+        provider_manager = Aidp::ProviderManager.get_provider(provider)
+        log_extractor = CiLogExtractor.new(provider_manager: provider_manager)
+
         failure_details = failed_checks.map do |check|
+          Aidp.log_debug("ci_fix_processor", "extracting_logs", check_name: check[:name])
+          extracted = log_extractor.extract_failure_info(
+            check: check,
+            check_run_url: check[:details_url]
+          )
+
           {
             name: check[:name],
-            conclusion: check[:conclusion],
-            output: check[:output],
-            details_url: check[:details_url]
+            summary: extracted[:summary],
+            details: extracted[:details],
+            extraction_method: extracted[:extraction_method]
           }
         end
 
@@ -131,14 +161,14 @@ module Aidp
         analysis = analyze_failures_with_ai(pr_data: pr_data, failures: failure_details)
 
         if analysis[:can_fix]
-          # Checkout the PR branch and apply fixes
-          checkout_pr_branch(pr_data)
+          # Setup worktree for the PR branch
+          working_dir = setup_pr_worktree(pr_data)
 
           # Apply the proposed fixes
-          apply_fixes(analysis[:fixes])
+          apply_fixes(analysis[:fixes], working_dir: working_dir)
 
           # Commit and push
-          if commit_and_push(pr_data, analysis)
+          if commit_and_push(pr_data, analysis, working_dir: working_dir)
             {success: true, analysis: analysis, commit_created: true}
           else
             {success: false, analysis: analysis, reason: "No changes to commit"}
@@ -152,7 +182,7 @@ module Aidp
 
       def analyze_failures_with_ai(pr_data:, failures:)
         provider_name = @provider_name || detect_default_provider
-        provider = Aidp::ProviderManager.get_provider(provider_name, use_harness: false)
+        provider = Aidp::ProviderManager.get_provider(provider_name)
 
         user_prompt = build_ci_analysis_prompt(pr_data: pr_data, failures: failures)
         full_prompt = "#{ci_fix_system_prompt}\n\n#{user_prompt}"
@@ -222,12 +252,20 @@ module Aidp
           #{pr_data[:body]}
 
           **Failed Checks:**
-          #{failures.map { |f| "- #{f[:name]}: #{f[:conclusion]}\n  Output: #{f[:output].inspect}" }.join("\n")}
+          #{failures.map { |f| format_failure_for_prompt(f) }.join("\n\n")}
 
           Please analyze these failures and propose fixes if possible.
         PROMPT
       end
 
+      def format_failure_for_prompt(failure)
+        output = "**Check: #{failure[:name]}**\n"
+        output += "Summary: #{failure[:summary]}\n" if failure[:summary]
+        output += "\nDetails:\n```\n#{failure[:details]}\n```" if failure[:details]
+        output += "\n(Logs extracted using: #{failure[:extraction_method]})" if failure[:extraction_method]
+        output
+      end
+
       def detect_default_provider
         config_manager = Aidp::Harness::ConfigManager.new(@project_dir)
         config_manager.default_provider || "anthropic"
@@ -261,26 +299,54 @@ module Aidp
         end
       end
 
-      def checkout_pr_branch(pr_data)
+      def setup_pr_worktree(pr_data)
         head_ref = pr_data[:head_ref]
+        pr_number = pr_data[:number]
+
+        # Check if a worktree already exists for this branch
+        existing = Aidp::Worktree.find_by_branch(branch: head_ref, project_dir: @project_dir)
+
+        if existing && existing[:active]
+          display_message("🔄 Reusing existing worktree for branch: #{head_ref}", type: :info)
+          Aidp.log_debug("ci_fix_processor", "worktree_reused", pr_number: pr_number, branch: head_ref, path: existing[:path])
+
+          # Pull latest changes in the worktree
+          Dir.chdir(existing[:path]) do
+            run_git(%w[fetch origin], allow_failure: true)
+            run_git(["checkout", head_ref])
+            run_git(%w[pull --ff-only], allow_failure: true)
+          end
 
+          return existing[:path]
+        end
+
+        # Create a new worktree for this PR
+        slug = "pr-#{pr_number}-ci-fix"
+        display_message("🌿 Creating worktree for PR ##{pr_number}: #{head_ref}", type: :info)
+
+        # Fetch the branch first
         Dir.chdir(@project_dir) do
-          # Fetch latest
           run_git(%w[fetch origin])
+        end
 
-          # Checkout the PR branch
-          run_git(["checkout", head_ref])
+        # Create worktree
+        result = Aidp::Worktree.create(
+          slug: slug,
+          project_dir: @project_dir,
+          branch: head_ref,
+          base_branch: nil # Branch already exists, no base needed
+        )
 
-          # Pull latest changes
-          run_git(%w[pull --ff-only], allow_failure: true)
-        end
+        worktree_path = result[:path]
+        Aidp.log_debug("ci_fix_processor", "worktree_created", pr_number: pr_number, branch: head_ref, path: worktree_path)
+        display_message("✅ Worktree created at #{worktree_path}", type: :success)
 
-        display_message("🌿 Checked out branch: #{head_ref}", type: :info)
+        worktree_path
       end
 
-      def apply_fixes(fixes)
+      def apply_fixes(fixes, working_dir:)
         fixes.each do |fix|
-          file_path = File.join(@project_dir, fix["file"])
+          file_path = File.join(working_dir, fix["file"])
 
           case fix["action"]
           when "create", "edit"
@@ -296,8 +362,8 @@ module Aidp
         end
       end
 
-      def commit_and_push(pr_data, analysis)
-        Dir.chdir(@project_dir) do
+      def commit_and_push(pr_data, analysis, working_dir:)
+        Dir.chdir(working_dir) do
           # Check if there are changes
           status_output = run_git(%w[status --porcelain])
           if status_output.strip.empty?

data/lib/aidp/watch/ci_log_extractor.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require "open3"
+require "tempfile"
+require "json"
+
+module Aidp
+  module Watch
+    # Intelligently extracts relevant failure information from CI logs
+    # to reduce token usage when analyzing failures with AI.
+    #
+    # Instead of feeding full CI output to AI, this creates a tailored
+    # extraction script for each failure type, runs it, and provides
+    # only the relevant excerpts.
+    class CiLogExtractor
+      # Maximum size for extracted log content (in characters)
+      MAX_EXTRACTED_SIZE = 10_000
+
+      def initialize(provider_manager:)
+        @provider_manager = provider_manager
+      end
+
+      # Extract relevant failure information from a CI check
+      #
+      # @param check [Hash] Failed check information
+      # @param check_run_url [String] URL to the GitHub Actions run (optional)
+      # @return [Hash] Extracted failure info with :summary, :details, :script_used
+      def extract_failure_info(check:, check_run_url: nil)
+        Aidp.log_debug("ci_log_extractor", "extract_start",
+          check_name: check[:name],
+          has_output: !check[:output].nil?)
+
+        # If we have structured output from the check, use it
+        if check[:output] && check[:output]["summary"]
+          return extract_from_structured_output(check)
+        end
+
+        # If we have a check run URL, fetch the logs
+        if check_run_url
+          raw_logs = fetch_check_logs(check_run_url)
+          return extract_from_raw_logs(check: check, raw_logs: raw_logs) if raw_logs
+        end
+
+        # Fallback: minimal information
+        {
+          summary: "Check '#{check[:name]}' failed",
+          details: check[:output]&.dig("text") || "No additional details available",
+          extraction_method: "fallback"
+        }
+      end
+
+      private
+
+      def extract_from_structured_output(check)
+        summary = check[:output]["summary"] || ""
+        text = check[:output]["text"] || ""
+
+        # If the output is already concise, return it directly
+        full_output = "#{summary}\n\n#{text}".strip
+        if full_output.length <= MAX_EXTRACTED_SIZE
+          return {
+            summary: summary,
+            details: text,
+            extraction_method: "structured"
+          }
+        end
+
+        # Output is too large, create extraction script
+        extraction_result = create_and_run_extraction_script(
+          check_name: check[:name],
+          raw_content: full_output
+        )
+
+        {
+          summary: summary,
+          details: extraction_result[:extracted_content],
+          extraction_method: "ai_script",
+          script_used: extraction_result[:script]
+        }
+      end
+
+      def extract_from_raw_logs(check:, raw_logs:)
+        # If logs are already concise, return them
+        if raw_logs.length <= MAX_EXTRACTED_SIZE
+          return {
+            summary: "Check '#{check[:name]}' failed",
+            details: raw_logs,
+            extraction_method: "raw"
+          }
+        end
+
+        # Logs are too large, create extraction script
+        extraction_result = create_and_run_extraction_script(
+          check_name: check[:name],
+          raw_content: raw_logs
+        )
+
+        {
+          summary: "Check '#{check[:name]}' failed",
+          details: extraction_result[:extracted_content],
+          extraction_method: "ai_script",
+          script_used: extraction_result[:script]
+        }
+      end
+
+      def create_and_run_extraction_script(check_name:, raw_content:)
+        Aidp.log_debug("ci_log_extractor", "creating_script",
+          check_name: check_name,
+          content_size: raw_content.length)
+
+        # Ask AI to create an extraction script
+        script = generate_extraction_script(
+          check_name: check_name,
+          sample_content: truncate_sample(raw_content)
+        )
+
+        # Run the script
+        extracted = run_extraction_script(script: script, input: raw_content)
+
+        {
+          extracted_content: extracted,
+          script: script
+        }
+      rescue => e
+        Aidp.log_warn("ci_log_extractor", "script_execution_failed",
+          error: e.message,
+          check_name: check_name)
+
+        # Fallback: simple head/tail extraction
+        {
+          extracted_content: simple_extract(raw_content),
+          script: nil
+        }
+      end
+
+      def generate_extraction_script(check_name:, sample_content:)
+        prompt = <<~PROMPT
+          Create a shell script that extracts relevant failure information from CI logs.
+
+          The script should:
+          1. Read from STDIN
+          2. Extract ONLY the relevant error messages, failed tests, and stack traces
+          3. Omit verbose output, successful tests, and build information
+          4. Keep the extracted output under #{MAX_EXTRACTED_SIZE} characters
+          5. Output the extracted content to STDOUT
+
+          Check type: #{check_name}
+
+          Sample of the log content (first ~2000 chars):
+          ```
+          #{sample_content}
+          ```
+
+          Requirements:
+          - Use standard Unix tools (grep, awk, sed, head, tail)
+          - Handle multi-line error messages
+          - Focus on actionable error information
+          - If it's a test failure, extract test names and failure messages
+          - If it's a linting error, extract file names, line numbers, and violations
+
+          Respond ONLY with the shell script code, no explanation.
+          The script must be a valid bash script that reads STDIN.
+
+          Example format:
+          #!/bin/bash
+          grep -A 5 "FAILED" | head -n 100
+        PROMPT
+
+        response = @provider_manager.send_message(prompt: prompt)
+        extract_script_from_response(response.to_s)
+      end
+
+      def extract_script_from_response(response)
+        # Remove markdown code fences if present
+        script = response.strip
+        script = script.gsub(/^```(?:bash|sh)?\n/, "")
+        script = script.gsub(/\n```$/, "")
+
+        # Ensure it starts with shebang
+        unless script.start_with?("#!/")
+          script = "#!/bin/bash\n#{script}"
+        end
+
+        script
+      end
+
+      def run_extraction_script(script:, input:)
+        Tempfile.create(["ci_extract", ".sh"]) do |script_file|
+          script_file.write(script)
+          script_file.flush
+          script_file.chmod(0o755)
+
+          stdout, stderr, status = Open3.capture3(
+            "bash", script_file.path,
+            stdin_data: input,
+            chdir: Dir.tmpdir
+          )
+
+          unless status.success?
+            Aidp.log_warn("ci_log_extractor", "script_failed",
+              exit_code: status.exitstatus,
+              stderr: stderr[0, 500])
+            return simple_extract(input)
+          end
+
+          # Ensure output is within size limit
+          truncate_to_size(stdout)
+        end
+      end
+
+      def fetch_check_logs(check_run_url)
+        # TODO: Implement fetching logs from GitHub Actions
+        # This would require parsing the check run URL and using gh CLI or API
+        # For now, return nil to use structured output
+        nil
+      end
+
+      def truncate_sample(content, size: 2000)
+        return content if content.length <= size
+        "#{content[0, size]}\n... [truncated]"
+      end
+
+      def simple_extract(content)
+        # Simple fallback: extract lines containing error keywords
+        lines = content.lines
+        error_lines = lines.select do |line|
+          line.match?(/error|fail|exception|assert/i)
+        end
+
+        # If we found error lines, return them with context
+        if error_lines.any?
+          # Get line numbers of errors and include surrounding context
+          result = []
+          lines.each_with_index do |line, i|
+            if line.match?(/error|fail|exception|assert/i)
+              # Include 2 lines before and 3 lines after
+              start_idx = [0, i - 2].max
+              end_idx = [lines.length - 1, i + 3].min
+              result.concat(lines[start_idx..end_idx])
+            end
+          end
+
+          truncate_to_size(result.uniq.join)
+        else
+          # No clear errors, return head and tail
+          head = lines.first(50).join
+          tail = lines.last(50).join
+          truncate_to_size("=== First 50 lines ===\n#{head}\n\n=== Last 50 lines ===\n#{tail}")
+        end
+      end
+
+      def truncate_to_size(content, size: MAX_EXTRACTED_SIZE)
+        return content if content.length <= size
+        "#{content[0, size - 20]}\n... [truncated]"
+      end
+    end
+  end
+end

data/lib/aidp/watch/github_state_extractor.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Aidp
+  module Watch
+    # Extracts state information from GitHub issues/PRs using labels and comments
+    # as the single source of truth, enabling multiple AIDP instances to coordinate
+    # without local state files.
+    class GitHubStateExtractor
+      # Pattern for detecting completion comments
+      COMPLETION_PATTERN = /✅ Implementation complete for #(\d+)/i
+
+      # Pattern for detecting detection comments
+      DETECTION_PATTERN = /aidp detected `([^`]+)` at ([^\n]+) and is working on it/i
+
+      # Pattern for detecting plan proposal comments
+      PLAN_PROPOSAL_PATTERN = /<!-- PLAN_SUMMARY_START -->/i
+
+      # Pattern for detecting in-progress label
+      IN_PROGRESS_LABEL = "aidp-in-progress"
+
+      def initialize(repository_client:)
+        @repository_client = repository_client
+      end
+
+      # Check if an issue/PR is currently being worked on by another instance
+      def in_progress?(item)
+        has_label?(item, IN_PROGRESS_LABEL)
+      end
+
+      # Check if build is already completed for an issue
+      # Looks for completion comment from AIDP
+      def build_completed?(issue)
+        return false unless issue[:comments]
+
+        issue[:comments].any? do |comment|
+          comment["body"]&.match?(COMPLETION_PATTERN)
+        end
+      end
+
+      # Check if a plan has been posted for an issue
+      def plan_posted?(issue)
+        return false unless issue[:comments]
+
+        issue[:comments].any? do |comment|
+          comment["body"]&.match?(PLAN_PROPOSAL_PATTERN)
+        end
+      end
+
+      # Extract the most recent plan data from comments
+      def extract_plan_data(issue)
+        return nil unless issue[:comments]
+
+        # Find the most recent plan proposal comment
+        plan_comment = issue[:comments].reverse.find do |comment|
+          comment["body"]&.match?(PLAN_PROPOSAL_PATTERN)
+        end
+
+        return nil unless plan_comment
+
+        body = plan_comment["body"]
+
+        {
+          summary: extract_section(body, "PLAN_SUMMARY"),
+          tasks: extract_tasks(body),
+          questions: extract_questions(body),
+          comment_body: body,
+          comment_hint: "## 🤖 AIDP Plan Proposal",
+          comment_id: plan_comment["id"],
+          posted_at: plan_comment["createdAt"] || Time.now.utc.iso8601
+        }
+      end
+
+      # Check if detection comment was already posted for this label
+      def detection_comment_posted?(item, label)
+        return false unless item[:comments]
+
+        item[:comments].any? do |comment|
+          next unless comment["body"]
+
+          match = comment["body"].match(DETECTION_PATTERN)
+          match && match[1] == label
+        end
+      end
+
+      # Check if review has been completed for a PR
+      def review_completed?(pr)
+        return false unless pr[:comments]
+
+        pr[:comments].any? do |comment|
+          comment["body"]&.match?(/🔍.*Review complete/i)
+        end
+      end
+
+      # Check if CI fix has been completed for a PR
+      def ci_fix_completed?(pr)
+        return false unless pr[:comments]
+
+        pr[:comments].any? do |comment|
+          body = comment["body"]
+          next false unless body
+
+          body.include?("✅") && (
+            body.match?(/CI fixes applied/i) ||
+            (body.match?(/CI check/i) && body.match?(/passed/i))
+          )
+        end
+      end
+
+      # Check if change request has been processed for a PR
+      def change_request_processed?(pr)
+        return false unless pr[:comments]
+
+        pr[:comments].any? do |comment|
+          body = comment["body"]
+          next false unless body
+
+          body.include?("✅") && body.match?(/Change requests? (?:addressed|applied|complete)/i)
+        end
+      end
+
+      # Extract linked issue number from PR description
+      # Looks for patterns like "Fixes #123", "Closes #456", "Resolves #789"
+      # Returns the issue number as an integer, or nil if not found
+      def extract_linked_issue(pr_body)
+        return nil unless pr_body
+
+        # Match common GitHub issue linking patterns
+        # https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue
+        match = pr_body.match(/(?:Fixes|Closes|Resolves|Close|Fix|Resolve)\s+#(\d+)/i)
+        match ? match[1].to_i : nil
+      end
+
+      private
+
+      def has_label?(item, label_name)
+        Array(item[:labels]).any? do |label|
+          name = label.is_a?(Hash) ? label["name"] : label.to_s
+          name.casecmp?(label_name)
+        end
+      end
+
+      def extract_section(body, section_name)
+        start_marker = "<!-- #{section_name}_START -->"
+        end_marker = "<!-- #{section_name}_END -->"
+
+        start_idx = body.index(start_marker)
+        end_idx = body.index(end_marker)
+
+        return nil unless start_idx && end_idx
+
+        content = body[(start_idx + start_marker.length)...end_idx]
+
+        # Remove markdown heading if present (e.g., "## Heading\n" or "### Heading\n")
+        # Strip lines starting with ## or ### to avoid ReDoS
+        content.lines.reject { |line| line.start_with?("##", "###") }.join.strip
+      end
+
+      def extract_tasks(body)
+        tasks_section = extract_section(body, "PLAN_TASKS")
+        return [] unless tasks_section
+
+        # Extract markdown list items (- followed by space and content)
+        tasks_section.scan(/^- (.+)$/).flatten
+      end
+
+      def extract_questions(body)
+        questions_section = extract_section(body, "CLARIFYING_QUESTIONS")
+        return [] unless questions_section
+
+        # Extract numbered list items (number, dot, space, content)
+        questions_section.scan(/^\d+\. (.+)$/).flatten
+      end
+    end
+  end
+end