aidp 0.33.0 → 0.34.0

data/lib/aidp/evaluations/context_capture.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require_relative "../config_paths"
+
+module Aidp
+  module Evaluations
+    # Captures rich context for evaluation records
+    #
+    # Gathers information about:
+    # - Prompt metadata (template, persona, skills, provider, model, tokens, settings)
+    # - Work-loop data (unit count, checkpoints, retries, file modifications)
+    # - Environment details (devcontainer status, Ruby version, branch info)
+    #
+    # @example Capturing context
+    #   context = ContextCapture.new(project_dir: Dir.pwd)
+    #   data = context.capture(step_name: "01_INIT", iteration: 3)
+    class ContextCapture
+      def initialize(project_dir: Dir.pwd, config: nil)
+        @project_dir = project_dir
+        @config = config
+
+        Aidp.log_debug("context_capture", "initialize", project_dir: project_dir)
+      end
+
+      # Capture full context for an evaluation
+      #
+      # @param step_name [String, nil] Current work loop step
+      # @param iteration [Integer, nil] Current iteration number
+      # @param provider [String, nil] AI provider being used
+      # @param model [String, nil] AI model being used
+      # @param additional [Hash] Additional context to include
+      # @return [Hash] Captured context
+      def capture(step_name: nil, iteration: nil, provider: nil, model: nil, additional: {})
+        Aidp.log_debug("context_capture", "capture",
+          step_name: step_name, iteration: iteration, provider: provider)
+
+        {
+          prompt: capture_prompt_context(step_name),
+          work_loop: capture_work_loop_context(step_name, iteration),
+          environment: capture_environment_context,
+          provider: {
+            name: provider,
+            model: model
+          },
+          timestamp: Time.now.iso8601
+        }.merge(additional)
+      end
+
+      # Capture minimal context (for quick evaluations)
+      #
+      # @return [Hash] Minimal context with timestamp and environment basics
+      def capture_minimal
+        Aidp.log_debug("context_capture", "capture_minimal")
+
+        {
+          environment: {
+            ruby_version: RUBY_VERSION,
+            branch: current_git_branch,
+            aidp_version: aidp_version
+          },
+          timestamp: Time.now.iso8601
+        }
+      end
+
+      # Capture watch mode context for evaluating watch outputs
+      #
+      # @param repo [String] Repository in owner/repo format
+      # @param number [Integer] Issue or PR number
+      # @param processor_type [String] Type of processor (plan, review, build, ci_fix, change_request)
+      # @return [Hash] Watch mode context
+      def capture_watch(repo:, number:, processor_type:)
+        Aidp.log_debug("context_capture", "capture_watch",
+          repo: repo, number: number, processor_type: processor_type)
+
+        {
+          watch: {
+            repo: repo,
+            number: number,
+            processor_type: processor_type,
+            state: load_watch_state(repo, number, processor_type)
+          },
+          environment: capture_environment_context,
+          timestamp: Time.now.iso8601
+        }
+      end
+
+      private
+
+      def load_watch_state(repo, number, processor_type)
+        # Try to load state from the watch state store
+        state_file = find_watch_state_file(repo)
+        return nil unless state_file && File.exist?(state_file)
+
+        require "yaml"
+        state = YAML.safe_load_file(state_file, permitted_classes: [Time, Date, Symbol])
+        return nil unless state
+
+        # Extract relevant state based on processor type
+        case processor_type
+        when "plan"
+          state.dig("issues", number.to_s, "plan") ||
+            state.dig(:issues, number, :plan)
+        when "review"
+          state.dig("pull_requests", number.to_s, "review") ||
+            state.dig(:pull_requests, number, :review)
+        when "build"
+          state.dig("issues", number.to_s, "build") ||
+            state.dig(:issues, number, :build)
+        when "ci_fix"
+          state.dig("pull_requests", number.to_s, "ci_fix") ||
+            state.dig(:pull_requests, number, :ci_fix)
+        when "change_request"
+          state.dig("pull_requests", number.to_s, "change_request") ||
+            state.dig(:pull_requests, number, :change_request)
+        end
+      rescue => e
+        Aidp.log_error("context_capture", "load_watch_state failed", error: e.message)
+        nil
+      end
+
+      def find_watch_state_file(repo)
+        watch_dir = File.join(@project_dir, ".aidp", "watch")
+        return nil unless Dir.exist?(watch_dir)
+
+        # Sanitize repo name the same way StateStore does
+        sanitized = repo.tr("/", "_").gsub(/[^a-zA-Z0-9_-]/, "")
+        state_file = File.join(watch_dir, "#{sanitized}.yml")
+
+        File.exist?(state_file) ? state_file : nil
+      end
+
+      def capture_prompt_context(step_name)
+        prompt_file = File.join(@project_dir, ".aidp", "PROMPT.md")
+        return {} unless File.exist?(prompt_file)
+
+        content = File.read(prompt_file)
+        {
+          step_name: step_name,
+          prompt_length: content.length,
+          has_prompt: true
+        }
+      rescue => e
+        Aidp.log_error("context_capture", "capture_prompt_context failed", error: e.message)
+        {}
+      end
+
+      def capture_work_loop_context(step_name, iteration)
+        checkpoint_file = ConfigPaths.checkpoint_file(@project_dir)
+        checkpoint_data = if File.exist?(checkpoint_file)
+          require "yaml"
+          YAML.safe_load_file(checkpoint_file, permitted_classes: [Time, Date, Symbol])
+        end
+
+        {
+          step_name: step_name,
+          iteration: iteration,
+          checkpoint: checkpoint_data ? {
+            status: checkpoint_data["status"] || checkpoint_data[:status],
+            metrics: checkpoint_data["metrics"] || checkpoint_data[:metrics]
+          } : nil
+        }
+      rescue => e
+        Aidp.log_error("context_capture", "capture_work_loop_context failed", error: e.message)
+        {step_name: step_name, iteration: iteration}
+      end
+
+      def capture_environment_context
+        {
+          ruby_version: RUBY_VERSION,
+          platform: RUBY_PLATFORM,
+          branch: current_git_branch,
+          commit: current_git_commit,
+          devcontainer: in_devcontainer?,
+          aidp_version: aidp_version
+        }
+      end
+
+      def current_git_branch
+        Dir.chdir(@project_dir) do
+          `git rev-parse --abbrev-ref HEAD 2>/dev/null`.strip
+        end
+      rescue
+        nil
+      end
+
+      def current_git_commit
+        Dir.chdir(@project_dir) do
+          `git rev-parse --short HEAD 2>/dev/null`.strip
+        end
+      rescue
+        nil
+      end
+
+      def in_devcontainer?
+        File.exist?("/.dockerenv") || ENV["REMOTE_CONTAINERS"] == "true"
+      end
+
+      def aidp_version
+        Aidp::VERSION if defined?(Aidp::VERSION)
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/aidp/evaluations/evaluation_record.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Aidp
+  module Evaluations
+    # Represents a single evaluation record
+    #
+    # An evaluation captures user feedback (good/neutral/bad) for AIDP outputs
+    # such as prompts, work units, or full work loops, along with rich context.
+    #
+    # @example Creating an evaluation
+    #   record = EvaluationRecord.new(
+    #     rating: "good",
+    #     comment: "Generated code was clean and well-structured",
+    #     target_type: "work_unit",
+    #     target_id: "01_INIT"
+    #   )
+    class EvaluationRecord
+      VALID_RATINGS = %w[good neutral bad].freeze
+      VALID_TARGET_TYPES = %w[prompt work_unit work_loop step plan review build ci_fix change_request].freeze
+
+      attr_reader :id, :rating, :comment, :target_type, :target_id,
+        :context, :created_at
+
+      def initialize(rating:, comment: nil, target_type: nil, target_id: nil, context: {}, id: nil, created_at: nil)
+        @id = id || generate_id
+        @rating = validate_rating(rating)
+        @comment = comment
+        @target_type = validate_target_type(target_type)
+        @target_id = target_id
+        @context = context || {}
+        @created_at = created_at || Time.now.iso8601
+
+        Aidp.log_debug("evaluation_record", "create",
+          id: @id, rating: @rating, target_type: @target_type, target_id: @target_id)
+      end
+
+      # Convert to hash for storage
+      def to_h
+        {
+          id: @id,
+          rating: @rating,
+          comment: @comment,
+          target_type: @target_type,
+          target_id: @target_id,
+          context: @context,
+          created_at: @created_at
+        }
+      end
+
+      # Create record from stored hash
+      def self.from_h(hash)
+        hash = symbolize_keys(hash)
+        new(
+          id: hash[:id],
+          rating: hash[:rating],
+          comment: hash[:comment],
+          target_type: hash[:target_type],
+          target_id: hash[:target_id],
+          context: hash[:context] || {},
+          created_at: hash[:created_at]
+        )
+      end
+
+      # Check if rating is positive
+      def good?
+        @rating == "good"
+      end
+
+      # Check if rating is negative
+      def bad?
+        @rating == "bad"
+      end
+
+      # Check if rating is neutral
+      def neutral?
+        @rating == "neutral"
+      end
+
+      private
+
+      def generate_id
+        "eval_#{Time.now.strftime("%Y%m%d_%H%M%S")}_#{SecureRandom.hex(4)}"
+      end
+
+      def validate_rating(rating)
+        rating_str = rating.to_s.downcase
+        unless VALID_RATINGS.include?(rating_str)
+          raise ArgumentError, "Invalid rating '#{rating}'. Must be one of: #{VALID_RATINGS.join(", ")}"
+        end
+        rating_str
+      end
+
+      def validate_target_type(target_type)
+        return nil if target_type.nil?
+        type_str = target_type.to_s.downcase
+        unless VALID_TARGET_TYPES.include?(type_str)
+          raise ArgumentError, "Invalid target_type '#{target_type}'. Must be one of: #{VALID_TARGET_TYPES.join(", ")}"
+        end
+        type_str
+      end
+
+      class << self
+        private
+
+        def symbolize_keys(hash)
+          return hash unless hash.is_a?(Hash)
+          hash.transform_keys { |k| k.is_a?(String) ? k.to_sym : k }
+        end
+      end
+    end
+  end
+end

data/lib/aidp/evaluations/evaluation_storage.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require_relative "evaluation_record"
+require_relative "../config_paths"
+require_relative "../rescue_logging"
+
+module Aidp
+  module Evaluations
+    # Storage manager for evaluation records
+    #
+    # Stores evaluations in `.aidp/evaluations/` with append-only semantics:
+    # - Individual evaluations stored as JSON files: `eval_YYYYMMDD_HHMMSS_xxxx.json`
+    # - Indexed summary file for efficient lookups: `index.json`
+    #
+    # @example Storing an evaluation
+    #   storage = EvaluationStorage.new(project_dir: Dir.pwd)
+    #   storage.store(record)
+    #
+    # @example Listing evaluations
+    #   storage.list(limit: 10)
+    #   storage.list(rating: "bad")
+    class EvaluationStorage
+      include Aidp::RescueLogging
+
+      def initialize(project_dir: Dir.pwd)
+        @project_dir = project_dir
+        @evaluations_dir = ConfigPaths.evaluations_dir(project_dir)
+        @index_file = ConfigPaths.evaluations_index_file(project_dir)
+
+        Aidp.log_debug("evaluation_storage", "initialize",
+          project_dir: project_dir, evaluations_dir: @evaluations_dir)
+      end
+
+      # Store a new evaluation record
+      #
+      # @param record [EvaluationRecord] The evaluation to store
+      # @return [Hash] Result with :success and :id keys
+      def store(record)
+        ensure_directory
+        file_path = File.join(@evaluations_dir, "#{record.id}.json")
+
+        Aidp.log_debug("evaluation_storage", "store",
+          id: record.id, rating: record.rating, file_path: file_path)
+
+        File.write(file_path, JSON.pretty_generate(record.to_h))
+        update_index(record)
+
+        {success: true, id: record.id, file_path: file_path}
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "store",
+          fallback: {success: false},
+          id: record.id)
+        {success: false, error: error.message, id: record.id}
+      end
+
+      # Load a specific evaluation by ID
+      #
+      # @param id [String] The evaluation ID
+      # @return [EvaluationRecord, nil] The record or nil if not found
+      def load(id)
+        file_path = File.join(@evaluations_dir, "#{id}.json")
+        return nil unless File.exist?(file_path)
+
+        Aidp.log_debug("evaluation_storage", "load", id: id)
+
+        data = JSON.parse(File.read(file_path))
+        EvaluationRecord.from_h(data)
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "load",
+          fallback: nil,
+          id: id)
+        nil
+      end
+
+      # List evaluations with optional filtering
+      #
+      # @param limit [Integer] Maximum number of records to return
+      # @param rating [String, nil] Filter by rating (good/neutral/bad)
+      # @param target_type [String, nil] Filter by target type
+      # @return [Array<EvaluationRecord>] Matching records, newest first
+      def list(limit: 50, rating: nil, target_type: nil)
+        Aidp.log_debug("evaluation_storage", "list",
+          limit: limit, rating: rating, target_type: target_type)
+
+        index = load_index
+        entries = index[:entries] || []
+
+        # Apply filters
+        entries = entries.select { |e| e[:rating] == rating } if rating
+        entries = entries.select { |e| e[:target_type] == target_type } if target_type
+
+        # Sort by created_at descending, take limit
+        entries = entries.sort_by { |e| e[:created_at] || "" }.reverse.take(limit)
+
+        # Load full records
+        entries.filter_map { |entry| load(entry[:id]) }
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "list",
+          fallback: [],
+          limit: limit)
+        []
+      end
+
+      # Get statistics about evaluations
+      #
+      # @return [Hash] Statistics including counts by rating
+      def stats
+        Aidp.log_debug("evaluation_storage", "stats")
+
+        index = load_index
+        entries = index[:entries] || []
+
+        total = entries.size
+        by_rating = entries.group_by { |e| e[:rating] }
+        by_target = entries.group_by { |e| e[:target_type] }
+
+        {
+          total: total,
+          by_rating: {
+            good: (by_rating["good"] || []).size,
+            neutral: (by_rating["neutral"] || []).size,
+            bad: (by_rating["bad"] || []).size
+          },
+          by_target_type: by_target.transform_values(&:size),
+          first_evaluation: entries.min_by { |e| e[:created_at] || "" }&.dig(:created_at),
+          last_evaluation: entries.max_by { |e| e[:created_at] || "" }&.dig(:created_at)
+        }
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "stats",
+          fallback: {total: 0, by_rating: {good: 0, neutral: 0, bad: 0}})
+        {total: 0, by_rating: {good: 0, neutral: 0, bad: 0}, by_target_type: {}}
+      end
+
+      # Delete an evaluation by ID
+      #
+      # @param id [String] The evaluation ID
+      # @return [Hash] Result with :success key
+      def delete(id)
+        file_path = File.join(@evaluations_dir, "#{id}.json")
+        return {success: true, message: "Evaluation not found"} unless File.exist?(file_path)
+
+        Aidp.log_debug("evaluation_storage", "delete", id: id)
+
+        File.delete(file_path)
+        remove_from_index(id)
+
+        {success: true, id: id}
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "delete",
+          fallback: {success: false},
+          id: id)
+        {success: false, error: error.message}
+      end
+
+      # Clear all evaluations
+      #
+      # @return [Hash] Result with :success and :count keys
+      def clear
+        Aidp.log_debug("evaluation_storage", "clear")
+
+        return {success: true, count: 0} unless Dir.exist?(@evaluations_dir)
+
+        count = Dir.glob(File.join(@evaluations_dir, "eval_*.json")).size
+        FileUtils.rm_rf(@evaluations_dir)
+
+        {success: true, count: count}
+      rescue => error
+        log_rescue(error,
+          component: "evaluation_storage",
+          action: "clear",
+          fallback: {success: false})
+        {success: false, error: error.message}
+      end
+
+      # Check if evaluations directory exists and has evaluations
+      def any?
+        Dir.exist?(@evaluations_dir) && Dir.glob(File.join(@evaluations_dir, "eval_*.json")).any?
+      end
+
+      private
+
+      def ensure_directory
+        ConfigPaths.ensure_evaluations_dir(@project_dir)
+      end
+
+      def load_index
+        return {entries: []} unless File.exist?(@index_file)
+
+        data = JSON.parse(File.read(@index_file))
+        symbolize_index(data)
+      rescue
+        {entries: []}
+      end
+
+      def update_index(record)
+        index = load_index
+        index[:entries] ||= []
+
+        # Add new entry to index (stores minimal data for quick lookups)
+        index[:entries] << {
+          id: record.id,
+          rating: record.rating,
+          target_type: record.target_type,
+          target_id: record.target_id,
+          created_at: record.created_at
+        }
+
+        index[:updated_at] = Time.now.iso8601
+
+        File.write(@index_file, JSON.pretty_generate(index))
+      end
+
+      def remove_from_index(id)
+        index = load_index
+        index[:entries]&.reject! { |e| e[:id] == id }
+        index[:updated_at] = Time.now.iso8601
+
+        File.write(@index_file, JSON.pretty_generate(index))
+      end
+
+      def symbolize_index(data)
+        return data unless data.is_a?(Hash)
+        result = {}
+        data.each do |key, value|
+          sym_key = key.is_a?(String) ? key.to_sym : key
+          result[sym_key] = if value.is_a?(Array)
+            value.map { |v| v.is_a?(Hash) ? symbolize_index(v) : v }
+          elsif value.is_a?(Hash)
+            symbolize_index(value)
+          else
+            value
+          end
+        end
+        result
+      end
+    end
+  end
+end

data/lib/aidp/evaluations.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "evaluations/evaluation_record"
+require_relative "evaluations/evaluation_storage"
+require_relative "evaluations/context_capture"
+
+module Aidp
+  # Evaluation and feedback system for AIDP outputs
+  #
+  # Enables users to rate generated outputs (prompts, work units, work loops)
+  # as good, neutral, or bad while capturing rich execution context.
+  #
+  # @example Creating and storing an evaluation
+  #   record = Aidp::Evaluations::EvaluationRecord.new(
+  #     rating: "good",
+  #     comment: "Clean code generated",
+  #     target_type: "work_unit"
+  #   )
+  #   storage = Aidp::Evaluations::EvaluationStorage.new
+  #   storage.store(record)
+  module Evaluations
+  end
+end

data/lib/aidp/execute/async_work_loop_runner.rb

@@ -21,6 +21,9 @@ module Aidp
 
       attr_reader :state, :instruction_queue, :work_thread
 
+      # Expose sync_runner for testability
+      attr_accessor :sync_runner
+
       def initialize(project_dir, provider_manager, config, options = {})
         @project_dir = project_dir
         @provider_manager = provider_manager
@@ -175,7 +178,7 @@ module Aidp
       def save_cancellation_checkpoint
         return unless @sync_runner
 
-        checkpoint = @sync_runner.instance_variable_get(:@checkpoint)
+        checkpoint = @sync_runner.checkpoint
         return unless checkpoint
 
         checkpoint.record_checkpoint(

data/lib/aidp/execute/interactive_repl.rb

@@ -23,6 +23,10 @@ module Aidp
     class InteractiveRepl
       include Aidp::RescueLogging
 
+      # Expose running state and repl_macros for testability
+      attr_accessor :running
+      attr_reader :repl_macros, :async_runner, :completion_setup_needed, :output_display_thread
+
       def initialize(project_dir, provider_manager, config, options = {})
         @project_dir = project_dir
         @provider_manager = provider_manager
@@ -30,8 +34,8 @@ module Aidp
         @options = options
         @prompt = options[:prompt] || TTY::Prompt.new
         @async_runner_class = options[:async_runner_class] || AsyncWorkLoopRunner
-        @async_runner = nil
-        @repl_macros = ReplMacros.new
+        @async_runner = options[:async_runner]
+        @repl_macros = options[:repl_macros] || ReplMacros.new
         @output_display_thread = nil
         @running = false
         @completion_setup_needed = true