aidp 0.32.0 → 0.34.0

data/lib/aidp/watch/change_request_processor.rb

@@ -13,6 +13,8 @@ require_relative "../harness/runner"
 require_relative "../harness/state_manager"
 require_relative "../harness/test_runner"
 require_relative "../worktree"
+require_relative "../pr_worktree_manager"
+require_relative "../worktree_branch_manager"
 require_relative "github_state_extractor"
 require_relative "implementation_verifier"
 
@@ -32,6 +34,9 @@ module Aidp
 
       attr_reader :change_request_label, :needs_input_label
 
+      # Expose state for testability
+      attr_accessor :project_dir, :worktree_branch_manager
+
       def initialize(repository_client:, state_store:, provider_name: nil, project_dir: Dir.pwd, label_config: {}, change_request_config: {}, safety_config: {}, verbose: false)
         @repository_client = repository_client
         @state_store = state_store
@@ -40,6 +45,10 @@ module Aidp
         @project_dir = project_dir
         @verbose = verbose
 
+        # Initialize worktree managers
+        @pr_worktree_manager = Aidp::PRWorktreeManager.new(project_dir: project_dir)
+        @worktree_branch_manager = Aidp::WorktreeBranchManager.new(project_dir: project_dir)
+
         # Initialize verifier
         @verifier = ImplementationVerifier.new(
           repository_client: repository_client,
@@ -58,9 +67,26 @@ module Aidp
           commit_message_prefix: "aidp: pr-change",
           require_comment_reference: true,
           max_diff_size: 2000,
-          allow_large_pr_worktree_bypass: true # Default to always using worktree for large PRs
+          large_pr_strategy: "create_worktree", # Options: create_worktree, manual, skip
+          worktree_strategy: "auto", # Options: auto, always_create, reuse_only
+          worktree_max_age: 7 * 24 * 60 * 60, # 7 days in seconds
+          worktree_cleanup_on_success: true,
+          worktree_cleanup_on_failure: false
         }.merge(symbolize_keys(change_request_config))
 
+        # Log the configuration, especially the large PR strategy
+        Aidp.log_debug(
+          "change_request_processor", "Initialized with config",
+          max_diff_size: @config[:max_diff_size],
+          large_pr_strategy: @config[:large_pr_strategy],
+          run_tests_before_push: @config[:run_tests_before_push],
+          enabled: @config[:enabled],
+          worktree_strategy: @config[:worktree_strategy],
+          worktree_max_age: @config[:worktree_max_age],
+          worktree_cleanup_on_success: @config[:worktree_cleanup_on_success],
+          worktree_cleanup_on_failure: @config[:worktree_cleanup_on_failure]
+        )
+
         # Load safety configuration
         @safety_config = safety_config
         @author_allowlist = Array(@safety_config[:author_allowlist] || @safety_config["author_allowlist"])
@@ -69,20 +95,34 @@ module Aidp
       def process(pr)
         number = pr[:number]
 
+        Aidp.log_debug(
+          "change_request_processor", "Starting change request processing",
+          pr_number: number, pr_title: pr[:title]
+        )
+
         unless @config[:enabled]
-          display_message("ℹ️  PR change requests are disabled in configuration. Skipping PR ##{number}.", type: :muted)
+          display_message(
+            "ℹ️  PR change requests are disabled in configuration. Skipping PR ##{number}.",
+            type: :muted
+          )
           return
         end
 
         # Check clarification round limit
         existing_data = @state_store.change_request_data(number)
         if existing_data && existing_data["clarification_count"].to_i >= MAX_CLARIFICATION_ROUNDS
-          display_message("⚠️  Max clarification rounds (#{MAX_CLARIFICATION_ROUNDS}) reached for PR ##{number}. Skipping.", type: :warn)
+          display_message(
+            "⚠️  Max clarification rounds (#{MAX_CLARIFICATION_ROUNDS}) reached for PR ##{number}. Skipping.",
+            type: :warn
+          )
           post_max_rounds_comment(pr)
           return
         end
 
-        display_message("📝 Processing change request for PR ##{number} (#{pr[:title]})", type: :info)
+        display_message(
+          "📝 Processing change request for PR ##{number} (#{pr[:title]})",
+          type: :info
+        )
 
         # Fetch PR details
         pr_data = @repository_client.fetch_pull_request(number)
@@ -92,29 +132,135 @@ module Aidp
         authorized_comments = filter_authorized_comments(comments, pr_data)
 
         if authorized_comments.empty?
-          display_message("ℹ️  No authorized comments found for PR ##{number}. Skipping.", type: :muted)
+          display_message(
+            "ℹ️  No authorized comments found for PR ##{number}. Skipping.",
+            type: :muted
+          )
           return
         end
 
-        # If max_diff_size is set, attempt to fetch and check diff
-        # But bypass restriction for worktree-based workflows
+        # Fetch diff to check size with enhanced strategy
         diff = @repository_client.fetch_pull_request_diff(number)
         diff_size = diff.lines.count
 
-        # Check if we want to use the worktree bypass
-        use_worktree_bypass = @config[:allow_large_pr_worktree_bypass] || @config[:allow_large_pr_worktree_bypass].nil?
+        # Enhanced diff size and worktree handling
+        large_pr = diff_size > @config[:max_diff_size]
+
+        if large_pr
+          # Comprehensive logging for large PR detection
+          Aidp.log_debug(
+            "change_request_processor", "Large PR detected",
+            pr_number: number,
+            diff_size: diff_size,
+            max_diff_size: @config[:max_diff_size],
+            large_pr_strategy: @config[:large_pr_strategy]
+          )
 
-        if diff_size > @config[:max_diff_size] && !use_worktree_bypass
-          display_message("⚠️  PR ##{number} diff too large (#{diff_size} lines > #{@config[:max_diff_size]}). Skipping.", type: :warn)
-          post_diff_too_large_comment(pr, diff_size)
-          return
-        end
+          display_message(
+            "⚠️  Large PR detected - applying enhanced worktree handling strategy.",
+            type: :info
+          )
 
-        # Log the diff size for observability
-        Aidp.log_debug("change_request_processor", "PR diff size", number: number, size: diff_size, max_allowed: @config[:max_diff_size], worktree_bypass: use_worktree_bypass)
+          # Handle different strategies for large PRs
+          case @config[:large_pr_strategy]
+          when "skip"
+            Aidp.log_debug(
+              "change_request_processor", "Skipping large PR processing",
+              pr_number: number
+            )
+            post_diff_too_large_comment(pr_data, diff_size)
+            return
+          when "manual"
+            post_diff_too_large_comment(pr_data, diff_size)
+            display_message("❌ Change request processing failed: Large PR requires manual processing. See comment for details.", type: :error)
+            @state_store.record_change_request(pr_data[:number], {
+              status: "manual_processing_required",
+              timestamp: Time.now.utc.iso8601,
+              diff_size: diff_size,
+              max_diff_size: @config[:max_diff_size]
+            })
+            raise "Large PR requires manual processing. See comment for details."
+          when "create_worktree"
+            # Use our enhanced WorktreeBranchManager to handle PR worktrees
+            begin
+              # Get PR branch information
+              head_ref = @worktree_branch_manager.get_pr_branch(number)
+
+              # Check if a PR-specific worktree already exists
+              Aidp.log_debug(
+                "change_request_processor", "Checking for existing PR worktree",
+                pr_number: number,
+                head_branch: head_ref
+              )
+
+              # Try to find existing worktree first
+              existing_worktree = @worktree_branch_manager.find_worktree(
+                branch: head_ref,
+                pr_number: number
+              )
+
+              # Create new worktree if none exists
+              if existing_worktree.nil?
+                Aidp.log_info(
+                  "change_request_processor", "Creating PR-specific worktree",
+                  pr_number: number,
+                  head_branch: head_ref,
+                  base_branch: pr_data[:base_ref],
+                  strategy: "create_worktree"
+                )
+
+                # Use find_or_create_pr_worktree for PR-specific handling
+                @worktree_branch_manager.find_or_create_pr_worktree(
+                  pr_number: number,
+                  head_branch: head_ref,
+                  base_branch: pr_data[:base_ref]
+                )
+              else
+                Aidp.log_debug(
+                  "change_request_processor", "Using existing PR worktree",
+                  pr_number: number,
+                  worktree_path: existing_worktree
+                )
+              end
+            rescue => e
+              Aidp.log_error(
+                "change_request_processor", "Large PR worktree handling failed",
+                pr_number: number,
+                error: e.message,
+                strategy: @config[:large_pr_strategy]
+              )
+
+              # Fallback error handling
+              post_diff_too_large_comment(pr_data, diff_size)
+              raise "Failed to handle large PR: #{e.message}"
+            end
+          else
+            # Default fallback
+            Aidp.log_warn(
+              "change_request_processor", "Unknown large_pr_strategy",
+              strategy: @config[:large_pr_strategy],
+              fallback: "skip"
+            )
+            post_diff_too_large_comment(pr_data, diff_size)
+            return
+          end
+
+          # Provide additional context via debug log
+          Aidp.log_info(
+            "change_request_processor", "Large PR worktree strategy applied",
+            pr_number: number,
+            diff_size: diff_size,
+            max_diff_size: @config[:max_diff_size],
+            strategy: @config[:large_pr_strategy]
+          )
+        end
 
         # Analyze change requests
-        analysis_result = analyze_change_requests(pr_data: pr_data, comments: authorized_comments, diff: diff)
+        analysis_result = analyze_change_requests(
+          pr_data: pr_data,
+          comments: authorized_comments,
+          diff: diff
+        )
 
         if analysis_result[:needs_clarification]
           handle_clarification_needed(pr: pr_data, analysis: analysis_result)
@@ -124,8 +270,17 @@ module Aidp
           handle_cannot_implement(pr: pr_data, analysis: analysis_result)
         end
       rescue => e
-        display_message("❌ Change request processing failed: #{e.message}", type: :error)
-        Aidp.log_error("change_request_processor", "Change request failed", pr: pr[:number], error: e.message, backtrace: e.backtrace&.first(10))
+        display_message(
+          "❌ Change request processing failed: #{e.message}",
+          type: :error
+        )
+        Aidp.log_error(
+          "change_request_processor", "Change request failed",
+          pr: pr[:number],
+          error: e.message,
+          backtrace: e.backtrace&.first(10),
+          error_class: e.class.name
+        )
 
         # Record failure state internally but DON'T post error to GitHub
         # (per issue #280 - error messages should never appear on issues)
@@ -172,19 +327,41 @@ module Aidp
         # Parse JSON response
         parsed = JSON.parse(json_content)
 
-        {
+        # Additional structured analysis
+        result = {
           can_implement: parsed["can_implement"],
           needs_clarification: parsed["needs_clarification"],
           clarifying_questions: parsed["clarifying_questions"] || [],
           reason: parsed["reason"],
-          changes: parsed["changes"] || []
+          changes: []
         }
+
+        # Enhanced change parsing
+        begin
+          result[:changes] = parse_ai_changes(
+            {changes: parsed["changes"]},
+            pr_data,
+            comments
+          )
+        rescue => e
+          Aidp.log_warn("change_request_processor", "Change parsing failed",
+            pr_number: pr_data[:number],
+            error: e.message)
+        end
+
+        Aidp.log_debug("change_request_processor", "Change request analysis result",
+          pr_number: pr_data[:number],
+          can_implement: result[:can_implement],
+          needs_clarification: result[:needs_clarification],
+          changes_count: result[:changes].length)
+
+        result
       rescue JSON::ParserError => e
         Aidp.log_error("change_request_processor", "Failed to parse AI response", error: e.message, content: content)
-        {can_implement: false, needs_clarification: false, reason: "Failed to parse AI analysis"}
+        {can_implement: false, needs_clarification: false, reason: "Failed to parse AI analysis", changes: []}
       rescue => e
         Aidp.log_error("change_request_processor", "AI analysis failed", error: e.message)
-        {can_implement: false, needs_clarification: false, reason: "AI analysis error: #{e.message}"}
+        {can_implement: false, needs_clarification: false, reason: "AI analysis error: #{e.message}", changes: []}
       end
 
       def change_request_system_prompt
@@ -313,129 +490,351 @@ module Aidp
       end
 
       def checkout_pr_branch(pr_data)
-        head_ref = pr_data[:head_ref]
         pr_number = pr_data[:number]
+        base_branch = pr_data[:base_ref]
 
-        worktree_path = resolve_worktree_for_pr(pr_data)
+        Aidp.log_debug(
+          "change_request_processor", "Starting PR branch checkout process",
+          pr_number: pr_number,
+          base_branch: base_branch,
+          project_dir: @project_dir
+        )
 
-        Dir.chdir(worktree_path) do
-          run_git(%w[fetch origin], allow_failure: true)
-          run_git(["checkout", head_ref])
-          run_git(%w[pull --ff-only], allow_failure: true)
-        end
+        # Get PR branch information
+        head_ref = @worktree_branch_manager.get_pr_branch(pr_number)
 
-        @project_dir = worktree_path
+        begin
+          # Advanced worktree strategy with more detailed lookup
+          existing_worktree = @worktree_branch_manager.find_worktree(
+            branch: head_ref,
+            pr_number: pr_number
+          )
 
-        Aidp.log_debug("change_request_processor", "Checked out PR branch", branch: head_ref, worktree: worktree_path)
-        display_message("🌿 Using worktree for PR ##{pr_number}: #{head_ref}", type: :info)
-      end
+          # Logging: Detailed worktree search strategy
+          log_worktree_strategy = {
+            pr_number: pr_number,
+            head_branch: head_ref,
+            existing_worktree: !!existing_worktree,
+            worktree_creation_strategy: @config.fetch(:worktree_strategy, "auto")
+          }
 
-      def resolve_worktree_for_pr(pr_data)
-        head_ref = pr_data[:head_ref]
-        pr_number = pr_data[:number]
+          # Enhanced logging for worktree preparation
+          Aidp.log_info(
+            "change_request_processor", "Preparing PR worktree",
+            **log_worktree_strategy
+          )
 
-        existing = Aidp::Worktree.find_by_branch(branch: head_ref, project_dir: @project_dir)
+          # Worktree creation or reuse strategy
+          case @config.fetch(:worktree_strategy, "auto")
+          when "always_create"
+            # Force new worktree creation, useful for complex/large PRs
+            Aidp.log_info("change_request_processor", "Forcing new worktree creation", pr: pr_number)
+            existing_worktree = nil
+          when "reuse_only"
+            # Use only existing worktrees, error if not found
+            unless existing_worktree
+              raise "No existing worktree found for PR ##{pr_number}"
+            end
+          else # "auto" or default
+            # Existing default behavior: find or create
+          end
 
-        if existing && existing[:active]
-          display_message("🔄 Using existing worktree for branch: #{head_ref}", type: :info)
-          Aidp.log_debug("change_request_processor", "worktree_reused", pr_number: pr_number, branch: head_ref, path: existing[:path])
-          return existing[:path]
-        end
+          # Use the enhanced find_or_create_pr_worktree method for PR-specific worktree handling
+          worktree_path = @worktree_branch_manager.find_or_create_pr_worktree(
+            pr_number: pr_number,
+            head_branch: head_ref,
+            base_branch: base_branch
+          )
 
-        issue_worktree = find_issue_worktree_for_pr(pr_data)
-        return issue_worktree if issue_worktree
+          # Detailed logging of worktree path and strategy
+          Aidp.log_debug(
+            "change_request_processor", "PR worktree determined",
+            worktree_path: worktree_path,
+            creation_strategy: existing_worktree ? "reused" : "created",
+            pr_number: pr_number
+          )
+
+          display_message("🔄 Using PR-specific worktree for branch: #{head_ref}", type: :info)
+
+          # Update project directory to use the worktree
+          @project_dir = worktree_path
+
+          # Ensure the branch is up-to-date with remote
+          Dir.chdir(@project_dir) do
+            run_git(["fetch", "origin", base_branch], allow_failure: true)
+            run_git(["fetch", "origin", head_ref], allow_failure: true)
+
+            # Checkout branch with more detailed tracking
+            checkout_result = run_git(["checkout", head_ref])
+
+            # Pull with fast-forward preference for cleaner history
+            pull_result = run_git(
+              ["pull", "--ff-only", "origin", head_ref],
+              allow_failure: true
+            )
+
+            # Enhanced branch state logging
+            Aidp.log_info(
+              "change_request_processor", "Branch synchronization complete",
+              pr_number: pr_number,
+              checkout_status: checkout_result.strip,
+              pull_status: pull_result.strip,
+              current_branch: run_git(["rev-parse", "--abbrev-ref", "HEAD"]).strip
+            )
+          end
+
+          # Additional validation and logging
+          worktree_details = {
+            path: worktree_path,
+            branch: head_ref,
+            base_branch: base_branch,
+            pr_number: pr_number,
+            timestamp: Time.now.utc.iso8601
+          }
+
+          Aidp.log_debug(
+            "change_request_processor", "Validated PR worktree",
+            **worktree_details
+          )
 
-        create_worktree_for_pr(pr_data)
+          worktree_path
+        rescue => e
+          Aidp.log_error(
+            "change_request_processor", "Critical worktree preparation failure",
+            pr_number: pr_number,
+            base_branch: base_branch,
+            head_branch: head_ref,
+            error: e.message,
+            error_class: e.class.name,
+            backtrace: e.backtrace.first(10)
+          )
+
+          # Provide detailed error handling and recovery
+          handle_worktree_error(pr_data, e)
+        end
       end
 
-      def find_issue_worktree_for_pr(pr_data)
-        pr_number = pr_data[:number]
-        linked_issue_numbers = extract_issue_numbers_from_pr(pr_data)
+      def handle_worktree_error(pr_data, error)
+        # Log the detailed error
+        Aidp.log_error(
+          "change_request_processor", "Worktree preparation critical error",
+          pr_number: pr_data[:number],
+          error_message: error.message,
+          error_class: error.class.name
+        )
+
+        # Post a comment to GitHub about the failure
+        comment_body = <<~COMMENT
+          #{COMMENT_HEADER}
 
-        build_match = @state_store.find_build_by_pr(pr_number)
-        linked_issue_numbers << build_match[:issue_number] if build_match
-        linked_issue_numbers = linked_issue_numbers.compact.uniq
+          ❌ Automated worktree preparation failed for this pull request.
 
-        linked_issue_numbers.each do |issue_number|
-          workstream = @state_store.workstream_for_issue(issue_number)
-          next unless workstream
+          **Error Details:**
+          ```
+          #{error.message}
+          ```
 
-          slug = workstream[:workstream]
-          branch = workstream[:branch]
+          **Possible Actions:**
+          1. Review the PR branch and its configuration
+          2. Check if the base repository is accessible
+          3. Try re-adding the `#{@change_request_label}` label
 
-          if slug
-            info = Aidp::Worktree.info(slug: slug, project_dir: @project_dir)
-            if info && info[:active]
-              Aidp.log_debug("change_request_processor", "issue_worktree_reused", pr_number: pr_number, issue_number: issue_number, branch: branch, path: info[:path])
-              display_message("🔄 Reusing worktree #{slug} for issue ##{issue_number} (PR ##{pr_number})", type: :info)
-              return info[:path]
-            end
-          end
+          This may require manual intervention or administrative access.
+        COMMENT
 
-          if branch
-            existing = Aidp::Worktree.find_by_branch(branch: branch, project_dir: @project_dir)
-            if existing && existing[:active]
-              Aidp.log_debug("change_request_processor", "issue_branch_worktree_reused", pr_number: pr_number, issue_number: issue_number, branch: branch, path: existing[:path])
-              display_message("🔄 Reusing branch worktree for issue ##{issue_number}: #{branch}", type: :info)
-              return existing[:path]
-            end
-          end
+        begin
+          @repository_client.post_comment(pr_data[:number], comment_body)
+
+          # Optionally remove or modify the label to indicate a problem
+          @repository_client.replace_labels(
+            pr_data[:number],
+            old_labels: [@change_request_label],
+            new_labels: ["aidp-worktree-error"]
+          )
+        rescue => comment_error
+          Aidp.log_warn(
+            "change_request_processor", "Failed to post error comment",
+            pr_number: pr_data[:number],
+            error: comment_error.message
+          )
         end
 
-        nil
+        # Re-raise the original error to halt processing
+        raise error
       end
 
-      def extract_issue_numbers_from_pr(pr_data)
-        body = pr_data[:body].to_s
-        issue_matches = body.scan(/(?:Fixes|Resolves|Closes)\s+#(\d+)/i).flatten
+      private
 
-        issue_matches.map { |num| num.to_i }.uniq
-      end
+      def apply_changes(changes)
+        # Validate we're in a worktree for enhanced change handling
+        in_worktree = @project_dir.include?(".worktrees")
+
+        Aidp.log_debug("change_request_processor", "Starting change application",
+          total_changes: changes.length,
+          in_worktree: in_worktree,
+          working_dir: @project_dir)
+
+        # Track overall change application results
+        results = {
+          total_changes: changes.length,
+          successful_changes: 0,
+          failed_changes: 0,
+          skipped_changes: 0,
+          errors: [],
+          worktree_context: in_worktree
+        }
 
-      def create_worktree_for_pr(pr_data)
-        head_ref = pr_data[:head_ref]
-        pr_number = pr_data[:number]
-        slug = "pr-#{pr_number}-change-requests"
+        # Enhanced change application with worktree context awareness
+        changes.each_with_index do |change, index|
+          file_path = change["file"].start_with?(@project_dir) ? change["file"] : File.join(@project_dir, change["file"])
 
-        display_message("🌿 Creating worktree for PR ##{pr_number}: #{head_ref}", type: :info)
+          Aidp.log_debug("change_request_processor", "Preparing to apply change",
+            index: index + 1,
+            total: changes.length,
+            action: change["action"],
+            file: change["file"],
+            content_length: change["content"]&.length || 0,
+            in_worktree: in_worktree)
 
-        Dir.chdir(@project_dir) do
-          run_git(%w[fetch origin], allow_failure: true)
-        end
+          begin
+            case change["action"]
+            when "create", "edit"
+              # Enhanced file change strategy with worktree awareness
+              unless change["content"]
+                results[:skipped_changes] += 1
+                Aidp.log_warn("change_request_processor", "Skipping change with empty content",
+                  file: change["file"], action: change["action"], in_worktree: in_worktree)
+                next
+              end
+
+              # Enhanced directory creation with worktree validation
+              target_directory = File.dirname(file_path)
+              unless File.directory?(target_directory)
+                FileUtils.mkdir_p(target_directory)
+                Aidp.log_debug("change_request_processor", "Created directory structure",
+                  directory: target_directory, in_worktree: in_worktree)
+              end
+
+              # Preserve file permissions if file already exists
+              old_permissions = File.exist?(file_path) ? File.stat(file_path).mode : 0o644
+
+              # Enhanced content writing with validation
+              File.write(file_path, change["content"])
+              File.chmod(old_permissions, file_path)
+
+              # Validate file was written correctly
+              unless File.exist?(file_path) && File.readable?(file_path)
+                raise "File creation validation failed"
+              end
+
+              display_message("  ✓ #{change["action"]} #{change["file"]}", type: :muted) if @verbose
+              Aidp.log_debug("change_request_processor", "Applied file change",
+                action: change["action"],
+                file: change["file"],
+                content_preview: change["content"]&.slice(0, 100),
+                file_size: File.size(file_path),
+                original_source_comments: change["source_comment_urls"],
+                in_worktree: in_worktree)
+
+              results[:successful_changes] += 1
+
+            when "delete"
+              # Enhanced delete strategy with worktree context
+              if File.exist?(file_path)
+                # Additional validation for worktree safety
+                if in_worktree
+                  canonical_path = File.expand_path(file_path)
+                  worktree_canonical = File.expand_path(@project_dir)
+                  unless canonical_path.start_with?(worktree_canonical)
+                    raise SecurityError, "Attempted to delete file outside worktree"
+                  end
+                end
+
+                File.delete(file_path)
+                display_message("  ✓ Deleted #{change["file"]}", type: :muted) if @verbose
+                Aidp.log_debug("change_request_processor", "Deleted file",
+                  file: change["file"],
+                  original_source_comments: change["source_comment_urls"],
+                  in_worktree: in_worktree)
+                results[:successful_changes] += 1
+              else
+                results[:skipped_changes] += 1
+                Aidp.log_warn("change_request_processor", "File to delete does not exist",
+                  file: change["file"], in_worktree: in_worktree)
+              end
+
+            else
+              results[:skipped_changes] += 1
+              error_msg = "Unknown change action: #{change["action"]}"
+              display_message("  ⚠️  #{error_msg} for #{change["file"]}", type: :warn)
+
+              results[:errors] << {
+                file: change["file"],
+                action: change["action"],
+                error: error_msg,
+                worktree_context: in_worktree
+              }
 
-        result = Aidp::Worktree.create(
-          slug: slug,
-          project_dir: @project_dir,
-          branch: head_ref,
-          base_branch: pr_data[:base_ref]
-        )
+              Aidp.log_warn("change_request_processor", "Unhandled change action",
+                action: change["action"],
+                file: change["file"],
+                in_worktree: in_worktree)
+            end
+          rescue SecurityError => e
+            results[:failed_changes] += 1
+            error_details = {
+              file: change["file"],
+              action: change["action"],
+              error: e.message,
+              error_type: "security_violation",
+              worktree_context: in_worktree
+            }
+
+            results[:errors] << error_details
+
+            Aidp.log_error("change_request_processor", "Security violation during change application",
+              **error_details)
+
+            display_message("  🔒 Security error applying change to #{change["file"]}: #{e.message}", type: :error)
+          rescue => e
+            results[:failed_changes] += 1
+            error_details = {
+              file: change["file"],
+              action: change["action"],
+              error: e.message,
+              backtrace: e.backtrace&.first(3),
+              worktree_context: in_worktree
+            }
 
-        worktree_path = result[:path]
-        Aidp.log_debug("change_request_processor", "worktree_created", pr_number: pr_number, branch: head_ref, path: worktree_path)
-        display_message("✅ Worktree created at #{worktree_path}", type: :success)
+            results[:errors] << error_details
 
-        worktree_path
-      end
+            Aidp.log_error("change_request_processor", "Change application failed",
+              **error_details)
 
-      def apply_changes(changes)
-        changes.each do |change|
-          file_path = File.join(@project_dir, change["file"])
-
-          case change["action"]
-          when "create", "edit"
-            FileUtils.mkdir_p(File.dirname(file_path))
-            File.write(file_path, change["content"])
-            display_message("  ✓ #{change["action"]} #{change["file"]}", type: :muted) if @verbose
-            Aidp.log_debug("change_request_processor", "Applied change", action: change["action"], file: change["file"])
-          when "delete"
-            File.delete(file_path) if File.exist?(file_path)
-            display_message("  ✓ Deleted #{change["file"]}", type: :muted) if @verbose
-            Aidp.log_debug("change_request_processor", "Deleted file", file: change["file"])
-          else
-            display_message("  ⚠️  Unknown action: #{change["action"]} for #{change["file"]}", type: :warn)
-            Aidp.log_warn("change_request_processor", "Unknown change action", action: change["action"], file: change["file"])
+            display_message("  ❌ Failed to apply change to #{change["file"]}: #{e.message}", type: :error)
           end
         end
+
+        # Enhanced logging of overall change application results
+        Aidp.log_info("change_request_processor", "Change application summary",
+          total_changes: results[:total_changes],
+          successful_changes: results[:successful_changes],
+          skipped_changes: results[:skipped_changes],
+          failed_changes: results[:failed_changes],
+          errors_count: results[:errors].length,
+          success_rate: (results[:successful_changes].to_f / results[:total_changes] * 100).round(2),
+          in_worktree: in_worktree,
+          working_directory: @project_dir)
+
+        # Additional worktree-specific logging
+        if in_worktree && results[:successful_changes] > 0
+          Aidp.log_info("change_request_processor", "Worktree changes applied successfully",
+            worktree_path: @project_dir,
+            files_modified: results[:successful_changes])
+        end
+
+        # Return enhanced results for potential additional handling
+        results
       end
 
       def run_tests_and_linters
@@ -466,30 +865,96 @@ module Aidp
 
       def commit_and_push(pr_data, analysis)
         Dir.chdir(@project_dir) do
-          # Check if there are changes
-          status_output = run_git(%w[status --porcelain])
-          if status_output.strip.empty?
-            display_message("ℹ️  No changes to commit after applying changes.", type: :muted)
+          # Validate we're in a worktree
+          unless @project_dir.include?(".worktrees")
+            Aidp.log_error(
+              "change_request_processor", "Invalid project directory for commit_and_push",
+              pr_number: pr_data[:number],
+              project_dir: @project_dir
+            )
             return false
           end
 
-          # Stage all changes
-          run_git(%w[add -A])
-
-          # Create commit
-          commit_message = build_commit_message(pr_data, analysis)
-          run_git(["commit", "-m", commit_message])
-
-          display_message("💾 Created commit: #{commit_message.lines.first.strip}", type: :info)
-          Aidp.log_debug("change_request_processor", "Created commit", pr: pr_data[:number])
-
-          # Push to origin
-          head_ref = pr_data[:head_ref]
-          run_git(["push", "origin", head_ref])
-
-          display_message("⬆️  Pushed changes to #{head_ref}", type: :success)
-          Aidp.log_info("change_request_processor", "Pushed changes", pr: pr_data[:number], branch: head_ref)
-          true
+          begin
+            # Check for changes
+            status_result = run_git(%w[status --porcelain])
+            return false if status_result.strip.empty?
+
+            modified_files = status_result.split("\n").map { |l| l.strip.split(" ", 2).last }
+
+            # Stage all changes
+            run_git(%w[add -A])
+            Aidp.log_debug("change_request_processor", "Staged changes",
+              pr_number: pr_data[:number],
+              staged_files: modified_files)
+
+            # Create commit
+            commit_message = build_commit_message(pr_data, analysis)
+            commit_result = run_git(["commit", "-m", commit_message])
+            first_line = commit_message.lines.first.strip
+            Aidp.log_info(
+              "change_request_processor", "Created commit",
+              pr: pr_data[:number],
+              commit_message: first_line,
+              files_changed: modified_files,
+              commit_result: commit_result.strip
+            )
+            display_message("💾 Created commit: #{first_line}", type: :info)
+
+            # Push changes
+            head_ref = pr_data[:head_ref]
+            begin
+              push_result = run_git(["push", "origin", head_ref], allow_failure: false)
+              Aidp.log_info("change_request_processor", "Pushed changes",
+                pr: pr_data[:number],
+                branch: head_ref,
+                push_result: push_result.strip)
+              display_message("⬆️  Pushed changes to #{head_ref}", type: :success)
+              return true
+            rescue => push_error
+              # Push failed, but commit was successful
+              Aidp.log_warn("change_request_processor", "Push failed, but commit was successful",
+                pr_number: pr_data[:number],
+                error: push_error.message)
+
+              # Post a detailed comment about the push failure
+              comment_body = <<~COMMENT
+                #{COMMENT_HEADER}
+
+                ⚠️ Automated changes were committed successfully, but pushing to the branch failed.
+
+                **Error Details:**
+                ```
+                #{push_error.message}
+                ```
+
+                **Suggested Actions:**
+                1. Check branch permissions
+                2. Verify remote repository configuration
+                3. Manually push the changes
+                4. Contact repository administrator
+              COMMENT
+
+              begin
+                @repository_client.post_comment(pr_data[:number], comment_body)
+              rescue => comment_error
+                Aidp.log_warn("change_request_processor", "Failed to post push error comment",
+                  pr_number: pr_data[:number],
+                  error: comment_error.message)
+              end
+
+              return false
+            end
+          rescue => error
+            # Catch any other unexpected errors during the entire process
+            Aidp.log_error(
+              "change_request_processor", "Unexpected error in commit_and_push",
+              pr_number: pr_data[:number],
+              error: error.message,
+              backtrace: error.backtrace.first(5)
+            )
+            return false
+          end
         end
       end
 
@@ -774,36 +1239,56 @@ module Aidp
       end
 
       def post_diff_too_large_comment(pr, diff_size)
+        # Configure handling based on repository/project config
+        handling_strategy = case @config[:large_pr_strategy]
+        when "create_worktree"
+          "Creating a dedicated git worktree"
+        when "manual"
+          "Requiring manual intervention"
+        else
+          "Skipping processing"
+        end
+
         comment = <<~COMMENT
           #{COMMENT_HEADER}
 
-          ⚠️  PR diff is too large for default change requests.
+          ⚠️  PR diff is too large for standard automated change requests.
 
           **Current size:** #{diff_size} lines
           **Maximum allowed:** #{@config[:max_diff_size]} lines
+          **Handling strategy:** #{handling_strategy}
 
-          For large PRs, you have several options:
-          1. Enable worktree-based large PR handling:
-             Set `allow_large_pr_worktree_bypass: true` in your `aidp.yml`
-          2. Break the PR into smaller chunks
-          3. Implement changes manually
-          4. Increase `max_diff_size` in your configuration
-
-          The worktree bypass allows processing large PRs by working directly in the branch
-          instead of using diff-based changes.
+          Options:
+          1. Break the PR into smaller chunks
+          2. Implement changes manually
+          3. Increase `max_diff_size` in your `aidp.yml` configuration
+          4. Configure `large_pr_strategy` to customize processing
         COMMENT
 
+        Aidp.log_debug(
+          "change_request_processor", "Large PR detected",
+          pr_number: pr[:number],
+          diff_size: diff_size,
+          max_diff_size: @config[:max_diff_size],
+          strategy: handling_strategy
+        )
+
         begin
           @repository_client.post_comment(pr[:number], comment)
           @repository_client.remove_labels(pr[:number], @change_request_label)
-        rescue
-          nil
+        rescue => e
+          Aidp.log_warn(
+            "change_request_processor", "Failed to post large PR comment",
+            pr_number: pr[:number],
+            error: e.message
+          )
         end
       end
 
       def run_git(args, allow_failure: false)
-        stdout, stderr, status = Open3.capture3("git", *Array(args))
-        raise "git #{args.join(" ")} failed: #{stderr.strip}" unless status.success? || allow_failure
+        args_array = Array(args)
+        stdout, stderr, status = Open3.capture3("git", *args_array)
+        raise "git #{args_array.join(" ")} failed: #{stderr.strip}" unless status.success? || allow_failure
         stdout
       end
 
@@ -839,6 +1324,53 @@ module Aidp
         end
       end
 
+      def parse_ai_changes(ai_response, pr_data, comments)
+        changes = ai_response[:changes] || []
+
+        # Log raw changes for debugging
+        Aidp.log_debug(
+          "change_request_processor", "Extracted changes",
+          pr_number: pr_data[:number],
+          changes_count: changes.length
+        )
+
+        # Validate and sanitize changes
+        validated_changes = changes.map do |change|
+          # Sanitize file path
+          file_path = change["file"].to_s.gsub(%r{^/|\.\.}, "")
+          file_path = File.join(@project_dir, file_path) unless file_path.start_with?(@project_dir)
+
+          # Validate change structure
+          {
+            "file" => file_path,
+            "action" => %w[create edit delete].include?(change["action"]) ? change["action"] : "edit",
+            "content" => change["content"].to_s,
+            "description" => change["description"].to_s.slice(0, 500), # Limit description length
+            "line_start" => change["line_start"]&.to_i,
+            "line_end" => change["line_end"]&.to_i
+          }
+        end.select do |change|
+          # Filter out invalid or empty changes
+          change["file"].present? &&
+            (change["action"] == "delete" || change["content"].present?)
+        end
+
+        # Add source reference for traceability
+        validated_changes.each do |change|
+          change["source_comment_urls"] = comments
+            .select { |c| c[:body].include?(change["description"]) }
+            .map { |c| c[:url] }
+        end
+
+        Aidp.log_debug(
+          "change_request_processor", "Validated changes",
+          pr_number: pr_data[:number],
+          validated_changes_count: validated_changes.length
+        )
+
+        validated_changes
+      end
+
       def symbolize_keys(hash)
         return {} unless hash