aidp 0.32.0 → 0.34.0

data/lib/aidp/evaluations/evaluation_storage.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require_relative "evaluation_record"
+require_relative "../config_paths"
+require_relative "../rescue_logging"
+
+module Aidp
+  module Evaluations
+    # Storage manager for evaluation records
+    #
+    # Stores evaluations in `.aidp/evaluations/` with append-only semantics:
+    # - Individual evaluations stored as JSON files: `eval_YYYYMMDD_HHMMSS_xxxx.json`
+    # - Indexed summary file for efficient lookups: `index.json`
+    #
+    # @example Storing an evaluation
+    #   storage = EvaluationStorage.new(project_dir: Dir.pwd)
+    #   storage.store(record)
+    #
+    # @example Listing evaluations
+    #   storage.list(limit: 10)
+    #   storage.list(rating: "bad")
+    class EvaluationStorage
+      include Aidp::RescueLogging
+
+      def initialize(project_dir: Dir.pwd)
+        @project_dir = project_dir
+        @evaluations_dir = ConfigPaths.evaluations_dir(project_dir)
+        @index_file = ConfigPaths.evaluations_index_file(project_dir)
+
+        Aidp.log_debug("evaluation_storage", "initialize",
+          project_dir: project_dir, evaluations_dir: @evaluations_dir)
+      end
+
+      # Store a new evaluation record
+      #
+      # @param record [EvaluationRecord] The evaluation to store
+      # @return [Hash] Result with :success and :id keys
+      def store(record)
+        ensure_directory
+        file_path = File.join(@evaluations_dir, "#{record.id}.json")
+
+        Aidp.log_debug("evaluation_storage", "store",
+          id: record.id, rating: record.rating, file_path: file_path)
+
+        File.write(file_path, JSON.pretty_generate(record.to_h))
+        update_index(record)
+
+        {success: true, id: record.id, file_path: file_path}
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "store",
+          fallback: {success: false},
+          id: record.id)
+        {success: false, error: error.message, id: record.id}
+      end
+
+      # Load a specific evaluation by ID
+      #
+      # @param id [String] The evaluation ID
+      # @return [EvaluationRecord, nil] The record or nil if not found
+      def load(id)
+        file_path = File.join(@evaluations_dir, "#{id}.json")
+        return nil unless File.exist?(file_path)
+
+        Aidp.log_debug("evaluation_storage", "load", id: id)
+
+        data = JSON.parse(File.read(file_path))
+        EvaluationRecord.from_h(data)
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "load",
+          fallback: nil,
+          id: id)
+        nil
+      end
+
+      # List evaluations with optional filtering
+      #
+      # @param limit [Integer] Maximum number of records to return
+      # @param rating [String, nil] Filter by rating (good/neutral/bad)
+      # @param target_type [String, nil] Filter by target type
+      # @return [Array<EvaluationRecord>] Matching records, newest first
+      def list(limit: 50, rating: nil, target_type: nil)
+        Aidp.log_debug("evaluation_storage", "list",
+          limit: limit, rating: rating, target_type: target_type)
+
+        index = load_index
+        entries = index[:entries] || []
+
+        # Apply filters
+        entries = entries.select { |e| e[:rating] == rating } if rating
+        entries = entries.select { |e| e[:target_type] == target_type } if target_type
+
+        # Sort by created_at descending, take limit
+        entries = entries.sort_by { |e| e[:created_at] || "" }.reverse.take(limit)
+
+        # Load full records
+        entries.filter_map { |entry| load(entry[:id]) }
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "list",
+          fallback: [],
+          limit: limit)
+        []
+      end
+
+      # Get statistics about evaluations
+      #
+      # @return [Hash] Statistics including counts by rating
+      def stats
+        Aidp.log_debug("evaluation_storage", "stats")
+
+        index = load_index
+        entries = index[:entries] || []
+
+        total = entries.size
+        by_rating = entries.group_by { |e| e[:rating] }
+        by_target = entries.group_by { |e| e[:target_type] }
+
+        {
+          total: total,
+          by_rating: {
+            good: (by_rating["good"] || []).size,
+            neutral: (by_rating["neutral"] || []).size,
+            bad: (by_rating["bad"] || []).size
+          },
+          by_target_type: by_target.transform_values(&:size),
+          first_evaluation: entries.min_by { |e| e[:created_at] || "" }&.dig(:created_at),
+          last_evaluation: entries.max_by { |e| e[:created_at] || "" }&.dig(:created_at)
+        }
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "stats",
+          fallback: {total: 0, by_rating: {good: 0, neutral: 0, bad: 0}})
+        {total: 0, by_rating: {good: 0, neutral: 0, bad: 0}, by_target_type: {}}
+      end
+
+      # Delete an evaluation by ID
+      #
+      # @param id [String] The evaluation ID
+      # @return [Hash] Result with :success key
+      def delete(id)
+        file_path = File.join(@evaluations_dir, "#{id}.json")
+        return {success: true, message: "Evaluation not found"} unless File.exist?(file_path)
+
+        Aidp.log_debug("evaluation_storage", "delete", id: id)
+
+        File.delete(file_path)
+        remove_from_index(id)
+
+        {success: true, id: id}
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "delete",
+          fallback: {success: false},
+          id: id)
+        {success: false, error: error.message}
+      end
+
+      # Clear all evaluations
+      #
+      # @return [Hash] Result with :success and :count keys
+      def clear
+        Aidp.log_debug("evaluation_storage", "clear")
+
+        return {success: true, count: 0} unless Dir.exist?(@evaluations_dir)
+
+        count = Dir.glob(File.join(@evaluations_dir, "eval_*.json")).size
+        FileUtils.rm_rf(@evaluations_dir)
+
+        {success: true, count: count}
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "clear",
+          fallback: {success: false})
+        {success: false, error: error.message}
+      end
+
+      # Check if evaluations directory exists and has evaluations
+      def any?
+        Dir.exist?(@evaluations_dir) && Dir.glob(File.join(@evaluations_dir, "eval_*.json")).any?
+      end
+
+      private
+
+      def ensure_directory
+        ConfigPaths.ensure_evaluations_dir(@project_dir)
+      end
+
+      def load_index
+        return {entries: []} unless File.exist?(@index_file)
+
+        data = JSON.parse(File.read(@index_file))
+        symbolize_index(data)
+      rescue
+        {entries: []}
+      end
+
+      def update_index(record)
+        index = load_index
+        index[:entries] ||= []
+
+        # Add new entry to index (stores minimal data for quick lookups)
+        index[:entries] << {
+          id: record.id,
+          rating: record.rating,
+          target_type: record.target_type,
+          target_id: record.target_id,
+          created_at: record.created_at
+        }
+
+        index[:updated_at] = Time.now.iso8601
+
+        File.write(@index_file, JSON.pretty_generate(index))
+      end
+
+      def remove_from_index(id)
+        index = load_index
+        index[:entries]&.reject! { |e| e[:id] == id }
+        index[:updated_at] = Time.now.iso8601
+
+        File.write(@index_file, JSON.pretty_generate(index))
+      end
+
+      def symbolize_index(data)
+        return data unless data.is_a?(Hash)
+        result = {}
+        data.each do |key, value|
+          sym_key = key.is_a?(String) ? key.to_sym : key
+          result[sym_key] = if value.is_a?(Array)
+            value.map { |v| v.is_a?(Hash) ? symbolize_index(v) : v }
+          elsif value.is_a?(Hash)
+            symbolize_index(value)
+          else
+            value
+          end
+        end
+        result
+      end
+    end
+  end
+end

data/lib/aidp/evaluations.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "evaluations/evaluation_record"
+require_relative "evaluations/evaluation_storage"
+require_relative "evaluations/context_capture"
+
+module Aidp
+  # Evaluation and feedback system for AIDP outputs
+  #
+  # Enables users to rate generated outputs (prompts, work units, work loops)
+  # as good, neutral, or bad while capturing rich execution context.
+  #
+  # @example Creating and storing an evaluation
+  #   record = Aidp::Evaluations::EvaluationRecord.new(
+  #     rating: "good",
+  #     comment: "Clean code generated",
+  #     target_type: "work_unit"
+  #   )
+  #   storage = Aidp::Evaluations::EvaluationStorage.new
+  #   storage.store(record)
+  module Evaluations
+  end
+end

data/lib/aidp/execute/async_work_loop_runner.rb

@@ -21,6 +21,9 @@ module Aidp
 
       attr_reader :state, :instruction_queue, :work_thread
 
+      # Expose sync_runner for testability
+      attr_accessor :sync_runner
+
       def initialize(project_dir, provider_manager, config, options = {})
         @project_dir = project_dir
         @provider_manager = provider_manager
@@ -175,7 +178,7 @@ module Aidp
       def save_cancellation_checkpoint
         return unless @sync_runner
 
-        checkpoint = @sync_runner.instance_variable_get(:@checkpoint)
+        checkpoint = @sync_runner.checkpoint
         return unless checkpoint
 
         checkpoint.record_checkpoint(

data/lib/aidp/execute/interactive_repl.rb

@@ -23,6 +23,10 @@ module Aidp
     class InteractiveRepl
       include Aidp::RescueLogging
 
+      # Expose running state and repl_macros for testability
+      attr_accessor :running
+      attr_reader :repl_macros, :async_runner, :completion_setup_needed, :output_display_thread
+
       def initialize(project_dir, provider_manager, config, options = {})
         @project_dir = project_dir
         @provider_manager = provider_manager
@@ -30,8 +34,8 @@ module Aidp
         @options = options
         @prompt = options[:prompt] || TTY::Prompt.new
         @async_runner_class = options[:async_runner_class] || AsyncWorkLoopRunner
-        @async_runner = nil
-        @repl_macros = ReplMacros.new
+        @async_runner = options[:async_runner]
+        @repl_macros = options[:repl_macros] || ReplMacros.new
         @output_display_thread = nil
         @running = false
         @completion_setup_needed = true

data/lib/aidp/execute/prompt_evaluator.rb

@@ -0,0 +1,359 @@
+# frozen_string_literal: true
+
+require_relative "../harness/ai_decision_engine"
+
+module Aidp
+  module Execute
+    # Evaluates prompt effectiveness using ZFC after multiple iterations
+    #
+    # FIX for issue #391: When the work loop reaches 10+ iterations without completion,
+    # this evaluator assesses prompt quality and suggests improvements.
+    #
+    # Uses Zero Framework Cognition (ZFC) to analyze:
+    # - Whether the prompt clearly defines completion criteria
+    # - If task breakdown instructions are adequate
+    # - Whether the agent has sufficient context
+    # - If there are blockers preventing progress
+    #
+    # @example
+    #   evaluator = PromptEvaluator.new(config)
+    #   result = evaluator.evaluate(
+    #     prompt_content: prompt_manager.read,
+    #     iteration_count: 12,
+    #     task_summary: persistent_tasklist.summary,
+    #     recent_failures: all_results
+    #   )
+    #   # => { effective: false, issues: [...], suggestions: [...] }
+    #
+    class PromptEvaluator
+      # Threshold for triggering evaluation
+      EVALUATION_ITERATION_THRESHOLD = 10
+
+      # Re-evaluate periodically after threshold
+      EVALUATION_INTERVAL = 5
+
+      # Expose for testability
+      attr_reader :ai_decision_engine
+
+      def initialize(config, ai_decision_engine: nil)
+        @config = config
+        @ai_decision_engine = ai_decision_engine || safely_build_ai_decision_engine
+      end
+
+      # Safely build AIDecisionEngine, returning nil if config doesn't support it
+      # This allows tests with mock configs to work without AI calls
+      def safely_build_ai_decision_engine
+        # Check if config supports the methods AIDecisionEngine needs
+        return nil unless @config.respond_to?(:default_provider)
+
+        build_default_ai_decision_engine
+      rescue => e
+        Aidp.log_debug("prompt_evaluator", "skipping_ai_decision_engine",
+          reason: e.message)
+        nil
+      end
+
+      # Check if evaluation should be triggered based on iteration count
+      # @param iteration_count [Integer] Current iteration number
+      # @return [Boolean]
+      def should_evaluate?(iteration_count)
+        return false unless iteration_count >= EVALUATION_ITERATION_THRESHOLD
+
+        # Evaluate at threshold and every EVALUATION_INTERVAL after
+        (iteration_count - EVALUATION_ITERATION_THRESHOLD) % EVALUATION_INTERVAL == 0
+      end
+
+      # Evaluate prompt effectiveness
+      # @param prompt_content [String] Current PROMPT.md content
+      # @param iteration_count [Integer] Current iteration number
+      # @param task_summary [Hash] Summary of task statuses
+      # @param recent_failures [Hash] Recent test/lint failures
+      # @param step_name [String] Name of current step
+      # @return [Hash] Evaluation result with :effective, :issues, :suggestions
+      def evaluate(prompt_content:, iteration_count:, task_summary:, recent_failures:, step_name: nil)
+        Aidp.log_debug("prompt_evaluator", "starting_evaluation",
+          iteration: iteration_count,
+          step: step_name,
+          prompt_size: prompt_content&.length || 0)
+
+        # When AI decision engine is unavailable (e.g., in tests with mock configs),
+        # return a neutral result that doesn't trigger feedback appending
+        unless @ai_decision_engine
+          Aidp.log_debug("prompt_evaluator", "skipping_evaluation_no_ai_engine")
+          return {
+            effective: true,  # Assume effective to avoid unnecessary feedback
+            issues: [],
+            suggestions: [],
+            likely_blockers: [],
+            recommended_actions: [],
+            confidence: 0.0,
+            skipped: true,
+            skip_reason: "AI decision engine not available"
+          }
+        end
+
+        prompt = build_evaluation_prompt(
+          prompt_content: prompt_content,
+          iteration_count: iteration_count,
+          task_summary: task_summary,
+          recent_failures: recent_failures
+        )
+
+        schema = {
+          type: "object",
+          properties: {
+            effective: {
+              type: "boolean",
+              description: "True if the prompt is likely to lead to completion within a few more iterations"
+            },
+            issues: {
+              type: "array",
+              items: {type: "string"},
+              description: "Specific problems identified with the current prompt"
+            },
+            suggestions: {
+              type: "array",
+              items: {type: "string"},
+              description: "Actionable suggestions to improve prompt effectiveness"
+            },
+            likely_blockers: {
+              type: "array",
+              items: {type: "string"},
+              description: "Potential blockers preventing progress"
+            },
+            recommended_actions: {
+              type: "array",
+              items: {
+                type: "object",
+                properties: {
+                  action: {type: "string"},
+                  priority: {type: "string", enum: ["high", "medium", "low"]},
+                  rationale: {type: "string"}
+                }
+              },
+              description: "Specific actions to take, prioritized"
+            },
+            confidence: {
+              type: "number",
+              minimum: 0.0,
+              maximum: 1.0,
+              description: "Confidence in this assessment"
+            }
+          },
+          required: ["effective", "issues", "suggestions", "confidence"]
+        }
+
+        begin
+          result = @ai_decision_engine.decide(
+            :prompt_evaluation,
+            context: {prompt: prompt},
+            schema: schema,
+            tier: :mini,
+            cache_ttl: nil  # Each evaluation is context-specific
+          )
+
+          Aidp.log_info("prompt_evaluator", "evaluation_complete",
+            iteration: iteration_count,
+            effective: result[:effective],
+            issue_count: result[:issues]&.size || 0,
+            confidence: result[:confidence])
+
+          result
+        rescue => e
+          Aidp.log_error("prompt_evaluator", "evaluation_failed",
+            error: e.message,
+            error_class: e.class.name)
+
+          build_fallback_result("Evaluation failed: #{e.message}")
+        end
+      end
+
+      # Generate improvement recommendations for the prompt template
+      # Used for AGD pattern - generating improved templates based on evaluation
+      # @param evaluation_result [Hash] Result from evaluate()
+      # @param original_template [String] The original template content
+      # @return [Hash] Template improvements
+      def generate_template_improvements(evaluation_result:, original_template:)
+        return nil unless @ai_decision_engine
+
+        Aidp.log_debug("prompt_evaluator", "generating_template_improvements",
+          issue_count: evaluation_result[:issues]&.size || 0)
+
+        prompt = build_improvement_prompt(evaluation_result, original_template)
+
+        schema = {
+          type: "object",
+          properties: {
+            improved_sections: {
+              type: "array",
+              items: {
+                type: "object",
+                properties: {
+                  section_name: {type: "string"},
+                  original: {type: "string"},
+                  improved: {type: "string"},
+                  rationale: {type: "string"}
+                }
+              }
+            },
+            additional_sections: {
+              type: "array",
+              items: {
+                type: "object",
+                properties: {
+                  section_name: {type: "string"},
+                  content: {type: "string"},
+                  rationale: {type: "string"}
+                }
+              }
+            },
+            completion_criteria_improvements: {
+              type: "array",
+              items: {type: "string"},
+              description: "Specific improvements to completion criteria definitions"
+            }
+          },
+          required: ["improved_sections", "completion_criteria_improvements"]
+        }
+
+        @ai_decision_engine.decide(
+          :template_improvement,
+          context: {prompt: prompt},
+          schema: schema,
+          tier: :standard,  # Use standard tier for more thoughtful improvements
+          cache_ttl: nil
+        )
+      rescue => e
+        Aidp.log_error("prompt_evaluator", "template_improvement_failed",
+          error: e.message)
+        nil
+      end
+
+      private
+
+      def build_evaluation_prompt(prompt_content:, iteration_count:, task_summary:, recent_failures:)
+        <<~PROMPT
+          You are evaluating the effectiveness of a work loop prompt that has been running for #{iteration_count} iterations without completion.
+
+          ## Current Prompt Content
+          #{truncate_content(prompt_content, 8000)}
+
+          ## Task Summary
+          #{format_task_summary(task_summary)}
+
+          ## Recent Check Results
+          #{format_failures(recent_failures)}
+
+          ## Evaluation Criteria
+
+          Analyze why this prompt may not be leading to completion:
+
+          1. **Clarity of Goals**: Are the implementation requirements clearly defined?
+          2. **Task Breakdown**: Does the prompt guide proper task decomposition?
+          3. **Completion Criteria**: Are the completion criteria specific and achievable?
+          4. **Context Sufficiency**: Does the agent have enough context to proceed?
+          5. **Blockers**: Are there technical blockers or missing information?
+          6. **Scope**: Is the scope realistic for an AI agent to complete?
+
+          ## Your Assessment
+
+          Provide:
+          - Whether this prompt is likely effective (true/false)
+          - Specific issues with the current prompt
+          - Actionable suggestions for improvement
+          - Likely blockers preventing progress
+          - Prioritized recommended actions
+          - Your confidence in this assessment (0.0-1.0)
+
+          Be specific and actionable. Focus on what can be changed to achieve completion.
+        PROMPT
+      end
+
+      def build_improvement_prompt(evaluation_result, original_template)
+        <<~PROMPT
+          Based on the following prompt evaluation, suggest improvements to the template.
+
+          ## Evaluation Results
+          - Effective: #{evaluation_result[:effective]}
+          - Issues: #{(evaluation_result[:issues] || []).join(", ")}
+          - Suggestions: #{(evaluation_result[:suggestions] || []).join(", ")}
+
+          ## Original Template
+          #{truncate_content(original_template, 4000)}
+
+          ## Your Task
+
+          Suggest specific improvements to make the template more effective:
+          1. Identify sections that need improvement
+          2. Propose new sections if needed
+          3. Focus especially on completion criteria clarity
+          4. Ensure task breakdown instructions are explicit
+          5. Add guidance for common failure modes
+
+          Be specific - provide actual text that could replace or supplement the template.
+        PROMPT
+      end
+
+      def format_task_summary(task_summary)
+        return "_No task summary available_" if task_summary.nil? || task_summary.empty?
+
+        if task_summary.is_a?(Hash)
+          parts = []
+          parts << "Total: #{task_summary[:total] || 0}"
+          parts << "Done: #{task_summary[:done] || 0}"
+          parts << "In Progress: #{task_summary[:in_progress] || 0}"
+          parts << "Pending: #{task_summary[:pending] || 0}"
+          parts << "Abandoned: #{task_summary[:abandoned] || 0}"
+          parts.join(" | ")
+        else
+          task_summary.to_s
+        end
+      end
+
+      def format_failures(recent_failures)
+        return "_No recent failures_" if recent_failures.nil? || recent_failures.empty?
+
+        parts = []
+        recent_failures.each do |check_type, result|
+          next unless result.is_a?(Hash)
+
+          status = result[:success] ? "✅ passed" : "❌ failed"
+          parts << "- #{check_type}: #{status}"
+
+          if !result[:success] && result[:failures]
+            failures = result[:failures].take(3)
+            failures.each { |f| parts << "  - #{truncate_content(f.to_s, 200)}" }
+          end
+        end
+
+        parts.empty? ? "_No failures to report_" : parts.join("\n")
+      end
+
+      def truncate_content(content, max_length)
+        return "_No content_" if content.nil? || content.empty?
+        return content if content.length <= max_length
+
+        "#{content[0, max_length]}\n\n[... truncated, showing first #{max_length} characters ...]"
+      end
+
+      def build_fallback_result(reason)
+        {
+          effective: nil,
+          issues: ["Unable to evaluate: #{reason}"],
+          suggestions: ["Check AI configuration and try again"],
+          likely_blockers: [],
+          recommended_actions: [],
+          confidence: 0.0
+        }
+      end
+
+      def build_default_ai_decision_engine
+        Aidp::Harness::AIDecisionEngine.new(@config)
+      rescue => e
+        Aidp.log_warn("prompt_evaluator", "failed_to_create_ai_decision_engine",
+          error: e.message)
+        nil
+      end
+    end
+  end
+end