aidp 0.33.0 → 0.34.0

data/lib/aidp/execute/prompt_evaluator.rb

@@ -0,0 +1,359 @@
+# frozen_string_literal: true
+
+require_relative "../harness/ai_decision_engine"
+
+module Aidp
+  module Execute
+    # Evaluates prompt effectiveness using ZFC after multiple iterations
+    #
+    # FIX for issue #391: When the work loop reaches 10+ iterations without completion,
+    # this evaluator assesses prompt quality and suggests improvements.
+    #
+    # Uses Zero Framework Cognition (ZFC) to analyze:
+    # - Whether the prompt clearly defines completion criteria
+    # - If task breakdown instructions are adequate
+    # - Whether the agent has sufficient context
+    # - If there are blockers preventing progress
+    #
+    # @example
+    #   evaluator = PromptEvaluator.new(config)
+    #   result = evaluator.evaluate(
+    #     prompt_content: prompt_manager.read,
+    #     iteration_count: 12,
+    #     task_summary: persistent_tasklist.summary,
+    #     recent_failures: all_results
+    #   )
+    #   # => { effective: false, issues: [...], suggestions: [...] }
+    #
+    class PromptEvaluator
+      # Threshold for triggering evaluation
+      EVALUATION_ITERATION_THRESHOLD = 10
+
+      # Re-evaluate periodically after threshold
+      EVALUATION_INTERVAL = 5
+
+      # Expose for testability
+      attr_reader :ai_decision_engine
+
+      def initialize(config, ai_decision_engine: nil)
+        @config = config
+        @ai_decision_engine = ai_decision_engine || safely_build_ai_decision_engine
+      end
+
+      # Safely build AIDecisionEngine, returning nil if config doesn't support it
+      # This allows tests with mock configs to work without AI calls
+      def safely_build_ai_decision_engine
+        # Check if config supports the methods AIDecisionEngine needs
+        return nil unless @config.respond_to?(:default_provider)
+
+        build_default_ai_decision_engine
+      rescue => e
+        Aidp.log_debug("prompt_evaluator", "skipping_ai_decision_engine",
+          reason: e.message)
+        nil
+      end
+
+      # Check if evaluation should be triggered based on iteration count
+      # @param iteration_count [Integer] Current iteration number
+      # @return [Boolean]
+      def should_evaluate?(iteration_count)
+        return false unless iteration_count >= EVALUATION_ITERATION_THRESHOLD
+
+        # Evaluate at threshold and every EVALUATION_INTERVAL after
+        (iteration_count - EVALUATION_ITERATION_THRESHOLD) % EVALUATION_INTERVAL == 0
+      end
+
+      # Evaluate prompt effectiveness
+      # @param prompt_content [String] Current PROMPT.md content
+      # @param iteration_count [Integer] Current iteration number
+      # @param task_summary [Hash] Summary of task statuses
+      # @param recent_failures [Hash] Recent test/lint failures
+      # @param step_name [String] Name of current step
+      # @return [Hash] Evaluation result with :effective, :issues, :suggestions
+      def evaluate(prompt_content:, iteration_count:, task_summary:, recent_failures:, step_name: nil)
+        Aidp.log_debug("prompt_evaluator", "starting_evaluation",
+          iteration: iteration_count,
+          step: step_name,
+          prompt_size: prompt_content&.length || 0)
+
+        # When AI decision engine is unavailable (e.g., in tests with mock configs),
+        # return a neutral result that doesn't trigger feedback appending
+        unless @ai_decision_engine
+          Aidp.log_debug("prompt_evaluator", "skipping_evaluation_no_ai_engine")
+          return {
+            effective: true,  # Assume effective to avoid unnecessary feedback
+            issues: [],
+            suggestions: [],
+            likely_blockers: [],
+            recommended_actions: [],
+            confidence: 0.0,
+            skipped: true,
+            skip_reason: "AI decision engine not available"
+          }
+        end
+
+        prompt = build_evaluation_prompt(
+          prompt_content: prompt_content,
+          iteration_count: iteration_count,
+          task_summary: task_summary,
+          recent_failures: recent_failures
+        )
+
+        schema = {
+          type: "object",
+          properties: {
+            effective: {
+              type: "boolean",
+              description: "True if the prompt is likely to lead to completion within a few more iterations"
+            },
+            issues: {
+              type: "array",
+              items: {type: "string"},
+              description: "Specific problems identified with the current prompt"
+            },
+            suggestions: {
+              type: "array",
+              items: {type: "string"},
+              description: "Actionable suggestions to improve prompt effectiveness"
+            },
+            likely_blockers: {
+              type: "array",
+              items: {type: "string"},
+              description: "Potential blockers preventing progress"
+            },
+            recommended_actions: {
+              type: "array",
+              items: {
+                type: "object",
+                properties: {
+                  action: {type: "string"},
+                  priority: {type: "string", enum: ["high", "medium", "low"]},
+                  rationale: {type: "string"}
+                }
+              },
+              description: "Specific actions to take, prioritized"
+            },
+            confidence: {
+              type: "number",
+              minimum: 0.0,
+              maximum: 1.0,
+              description: "Confidence in this assessment"
+            }
+          },
+          required: ["effective", "issues", "suggestions", "confidence"]
+        }
+
+        begin
+          result = @ai_decision_engine.decide(
+            :prompt_evaluation,
+            context: {prompt: prompt},
+            schema: schema,
+            tier: :mini,
+            cache_ttl: nil  # Each evaluation is context-specific
+          )
+
+          Aidp.log_info("prompt_evaluator", "evaluation_complete",
+            iteration: iteration_count,
+            effective: result[:effective],
+            issue_count: result[:issues]&.size || 0,
+            confidence: result[:confidence])
+
+          result
+        rescue => e
+          Aidp.log_error("prompt_evaluator", "evaluation_failed",
+            error: e.message,
+            error_class: e.class.name)
+
+          build_fallback_result("Evaluation failed: #{e.message}")
+        end
+      end
+
+      # Generate improvement recommendations for the prompt template
+      # Used for AGD pattern - generating improved templates based on evaluation
+      # @param evaluation_result [Hash] Result from evaluate()
+      # @param original_template [String] The original template content
+      # @return [Hash] Template improvements
+      def generate_template_improvements(evaluation_result:, original_template:)
+        return nil unless @ai_decision_engine
+
+        Aidp.log_debug("prompt_evaluator", "generating_template_improvements",
+          issue_count: evaluation_result[:issues]&.size || 0)
+
+        prompt = build_improvement_prompt(evaluation_result, original_template)
+
+        schema = {
+          type: "object",
+          properties: {
+            improved_sections: {
+              type: "array",
+              items: {
+                type: "object",
+                properties: {
+                  section_name: {type: "string"},
+                  original: {type: "string"},
+                  improved: {type: "string"},
+                  rationale: {type: "string"}
+                }
+              }
+            },
+            additional_sections: {
+              type: "array",
+              items: {
+                type: "object",
+                properties: {
+                  section_name: {type: "string"},
+                  content: {type: "string"},
+                  rationale: {type: "string"}
+                }
+              }
+            },
+            completion_criteria_improvements: {
+              type: "array",
+              items: {type: "string"},
+              description: "Specific improvements to completion criteria definitions"
+            }
+          },
+          required: ["improved_sections", "completion_criteria_improvements"]
+        }
+
+        @ai_decision_engine.decide(
+          :template_improvement,
+          context: {prompt: prompt},
+          schema: schema,
+          tier: :standard,  # Use standard tier for more thoughtful improvements
+          cache_ttl: nil
+        )
+      rescue => e
+        Aidp.log_error("prompt_evaluator", "template_improvement_failed",
+          error: e.message)
+        nil
+      end
+
+      private
+
+      def build_evaluation_prompt(prompt_content:, iteration_count:, task_summary:, recent_failures:)
+        <<~PROMPT
+          You are evaluating the effectiveness of a work loop prompt that has been running for #{iteration_count} iterations without completion.
+
+          ## Current Prompt Content
+          #{truncate_content(prompt_content, 8000)}
+
+          ## Task Summary
+          #{format_task_summary(task_summary)}
+
+          ## Recent Check Results
+          #{format_failures(recent_failures)}
+
+          ## Evaluation Criteria
+
+          Analyze why this prompt may not be leading to completion:
+
+          1. **Clarity of Goals**: Are the implementation requirements clearly defined?
+          2. **Task Breakdown**: Does the prompt guide proper task decomposition?
+          3. **Completion Criteria**: Are the completion criteria specific and achievable?
+          4. **Context Sufficiency**: Does the agent have enough context to proceed?
+          5. **Blockers**: Are there technical blockers or missing information?
+          6. **Scope**: Is the scope realistic for an AI agent to complete?
+
+          ## Your Assessment
+
+          Provide:
+          - Whether this prompt is likely effective (true/false)
+          - Specific issues with the current prompt
+          - Actionable suggestions for improvement
+          - Likely blockers preventing progress
+          - Prioritized recommended actions
+          - Your confidence in this assessment (0.0-1.0)
+
+          Be specific and actionable. Focus on what can be changed to achieve completion.
+        PROMPT
+      end
+
+      def build_improvement_prompt(evaluation_result, original_template)
+        <<~PROMPT
+          Based on the following prompt evaluation, suggest improvements to the template.
+
+          ## Evaluation Results
+          - Effective: #{evaluation_result[:effective]}
+          - Issues: #{(evaluation_result[:issues] || []).join(", ")}
+          - Suggestions: #{(evaluation_result[:suggestions] || []).join(", ")}
+
+          ## Original Template
+          #{truncate_content(original_template, 4000)}
+
+          ## Your Task
+
+          Suggest specific improvements to make the template more effective:
+          1. Identify sections that need improvement
+          2. Propose new sections if needed
+          3. Focus especially on completion criteria clarity
+          4. Ensure task breakdown instructions are explicit
+          5. Add guidance for common failure modes
+
+          Be specific - provide actual text that could replace or supplement the template.
+        PROMPT
+      end
+
+      def format_task_summary(task_summary)
+        return "_No task summary available_" if task_summary.nil? || task_summary.empty?
+
+        if task_summary.is_a?(Hash)
+          parts = []
+          parts << "Total: #{task_summary[:total] || 0}"
+          parts << "Done: #{task_summary[:done] || 0}"
+          parts << "In Progress: #{task_summary[:in_progress] || 0}"
+          parts << "Pending: #{task_summary[:pending] || 0}"
+          parts << "Abandoned: #{task_summary[:abandoned] || 0}"
+          parts.join(" | ")
+        else
+          task_summary.to_s
+        end
+      end
+
+      def format_failures(recent_failures)
+        return "_No recent failures_" if recent_failures.nil? || recent_failures.empty?
+
+        parts = []
+        recent_failures.each do |check_type, result|
+          next unless result.is_a?(Hash)
+
+          status = result[:success] ? "✅ passed" : "❌ failed"
+          parts << "- #{check_type}: #{status}"
+
+          if !result[:success] && result[:failures]
+            failures = result[:failures].take(3)
+            failures.each { |f| parts << "  - #{truncate_content(f.to_s, 200)}" }
+          end
+        end
+
+        parts.empty? ? "_No failures to report_" : parts.join("\n")
+      end
+
+      def truncate_content(content, max_length)
+        return "_No content_" if content.nil? || content.empty?
+        return content if content.length <= max_length
+
+        "#{content[0, max_length]}\n\n[... truncated, showing first #{max_length} characters ...]"
+      end
+
+      def build_fallback_result(reason)
+        {
+          effective: nil,
+          issues: ["Unable to evaluate: #{reason}"],
+          suggestions: ["Check AI configuration and try again"],
+          likely_blockers: [],
+          recommended_actions: [],
+          confidence: 0.0
+        }
+      end
+
+      def build_default_ai_decision_engine
+        Aidp::Harness::AIDecisionEngine.new(@config)
+      rescue => e
+        Aidp.log_warn("prompt_evaluator", "failed_to_create_ai_decision_engine",
+          error: e.message)
+        nil
+      end
+    end
+  end
+end

data/lib/aidp/execute/repl_macros.rb

@@ -9,7 +9,9 @@ module Aidp
     # - /split - Divide work into smaller contracts
     # - /halt-on <pattern> - Pause on specific test failures
     class ReplMacros
-      attr_reader :pinned_files, :focus_patterns, :halt_patterns, :split_mode, :current_workstream, :current_skill
+      attr_reader :pinned_files, :focus_patterns, :halt_patterns, :split_mode, :current_workstream
+      # Expose current_skill for testability
+      attr_accessor :current_skill
 
       def initialize(project_dir: Dir.pwd)
         @pinned_files = Set.new
@@ -299,6 +301,12 @@ module Aidp
             usage: "/tasks <list|show|done|abandon|stats> [args]",
             example: "/tasks list pending",
             handler: method(:cmd_tasks)
+          },
+          "/rate" => {
+            description: "Rate the current output (good/neutral/bad)",
+            usage: "/rate <good|neutral|bad> [comment]",
+            example: "/rate good 'Clean code generated'",
+            handler: method(:cmd_rate)
           }
         }
       end
@@ -2071,6 +2079,97 @@ module Aidp
         {success: false, message: "Error: #{e.message}", action: :none}
       end
 
+      # Command: /rate <good|neutral|bad> [comment]
+      # Rate the current output with optional comment
+      def cmd_rate(args)
+        rating = args.shift
+        comment = args.join(" ")
+        comment = nil if comment.empty?
+
+        unless rating
+          return {
+            success: false,
+            message: "Usage: /rate <good|neutral|bad> [comment]\n\nExamples:\n  /rate good\n  /rate bad 'Generated code had bugs'\n  /rate neutral 'Acceptable but could be better'",
+            action: :none
+          }
+        end
+
+        # Validate rating
+        unless %w[good neutral bad].include?(rating.downcase)
+          return {
+            success: false,
+            message: "Invalid rating '#{rating}'. Must be: good, neutral, or bad",
+            action: :none
+          }
+        end
+
+        Aidp.log_debug("repl_macros", "rate_command", rating: rating, has_comment: !comment.nil?)
+
+        begin
+          require_relative "../evaluations"
+
+          # Capture context
+          context_capture = Aidp::Evaluations::ContextCapture.new(project_dir: @project_dir)
+          context = context_capture.capture(
+            step_name: @current_step_name,
+            iteration: @current_iteration
+          )
+
+          # Create evaluation record
+          record = Aidp::Evaluations::EvaluationRecord.new(
+            rating: rating.downcase,
+            comment: comment,
+            target_type: "work_loop",
+            context: context
+          )
+
+          # Store evaluation
+          storage = Aidp::Evaluations::EvaluationStorage.new(project_dir: @project_dir)
+          result = storage.store(record)
+
+          if result[:success]
+            rating_display = case rating.downcase
+            when "good" then "good (+)"
+            when "neutral" then "neutral (~)"
+            when "bad" then "bad (-)"
+            else rating
+            end
+
+            msg_lines = ["Evaluation recorded: #{record.id}"]
+            msg_lines << "  Rating: #{rating_display}"
+            msg_lines << "  Comment: #{comment}" if comment
+            msg_lines << ""
+            msg_lines << "View all evaluations: aidp eval list"
+
+            {
+              success: true,
+              message: msg_lines.join("\n"),
+              action: :evaluation_recorded,
+              data: {id: record.id, rating: rating.downcase, comment: comment}
+            }
+          else
+            {
+              success: false,
+              message: "Failed to store evaluation: #{result[:error]}",
+              action: :none
+            }
+          end
+        rescue ArgumentError => e
+          {
+            success: false,
+            message: "Error: #{e.message}",
+            action: :none
+          }
+        rescue => e
+          Aidp.log_error("repl_macros", "rate_command_failed", error: e.message)
+          {
+            success: false,
+            message: "Failed to record evaluation: #{e.message}",
+            action: :none
+          }
+        end
+      end
+
       private
 
       # List tasks with optional status filter