aidp 0.32.0 → 0.34.0

data/lib/aidp/harness/test_runner.rb

@@ -3,16 +3,19 @@
 require "open3"
 require_relative "../tooling_detector"
 require_relative "output_filter"
+require_relative "output_filter_config"
 
 module Aidp
   module Harness
     # Executes test and linter commands configured in aidp.yml
     # Returns results with exit status and output
+    # Applies intelligent output filtering to reduce token consumption
     class TestRunner
       def initialize(project_dir, config)
         @project_dir = project_dir
         @config = config
         @iteration_count = 0
+        @filter_stats = {total_input_bytes: 0, total_output_bytes: 0}
       end
 
       # Run all configured tests
@@ -47,6 +50,29 @@ module Aidp
         run_command_category(:documentation, "Documentation")
       end
 
+      # Preview the commands that will run for each category so callers can log intent
+      # Returns a hash of category => array of normalized command entries
+      def planned_commands
+        {
+          tests: resolved_commands(:test),
+          lints: resolved_commands(:lint),
+          formatters: resolved_commands(:formatter),
+          builds: resolved_commands(:build),
+          docs: resolved_commands(:documentation)
+        }
+      end
+
+      # Get current iteration count
+      attr_reader :iteration_count
+
+      # Get filtering statistics
+      attr_reader :filter_stats
+
+      # Reset iteration counter (useful for testing)
+      def reset_iteration_count
+        @iteration_count = 0
+      end
+
       private
 
       # Run commands for a specific category (test, lint, formatter, build, documentation)
@@ -59,6 +85,12 @@ module Aidp
         # Determine output mode based on category
         mode = determine_output_mode(category)
 
+        Aidp.log_debug("test_runner", "running_category",
+          category: category,
+          command_count: commands.length,
+          iteration: @iteration_count,
+          output_mode: mode)
+
         # Execute all commands
         results = commands.map do |cmd_config|
           # Handle both string commands (legacy) and hash format (new)
@@ -71,6 +103,23 @@ module Aidp
           end
         end
 
+        aggregate_results(results, display_name, mode: mode)
+      rescue NameError
+        # Logging not available
+        commands = resolved_commands(category)
+        return {success: true, output: "", failures: [], required_failures: []} if commands.empty?
+
+        mode = determine_output_mode(category)
+        results = commands.map do |cmd_config|
+          if cmd_config.is_a?(String)
+            result = execute_command(cmd_config, category.to_s)
+            result.merge(required: true)
+          else
+            result = execute_command(cmd_config[:command], category.to_s)
+            result.merge(required: cmd_config[:required])
+          end
+        end
+
         aggregate_results(results, display_name, mode: mode)
       end
 
@@ -107,6 +156,11 @@ module Aidp
             (optional_failures.any? ? "\n" + format_failures(optional_failures, "#{category} - Optional", mode: mode) : "")
         end
 
+        # Add filtering summary if filtering was applied
+        if mode != :full && all_failures.any?
+          output += "\n[Output filtered: mode=#{mode}]"
+        end
+
         {
           success: success,
           output: output,
@@ -124,9 +178,12 @@ module Aidp
           output << "Exit Code: #{failure[:exit_code]}"
           output << "--- Output ---"
 
+          # Detect framework for filtering
+          framework = detect_framework_from_command(failure[:command])
+
           # Apply filtering based on mode and framework
-          filtered_stdout = filter_output(failure[:stdout], mode, detect_framework_from_command(failure[:command]))
-          filtered_stderr = filter_output(failure[:stderr], mode, :unknown)
+          filtered_stdout = filter_output(failure[:stdout], mode, framework, :test)
+          filtered_stderr = filter_output(failure[:stderr], mode, :unknown, :lint)
 
           output << filtered_stdout unless filtered_stdout.strip.empty?
           output << filtered_stderr unless filtered_stderr.strip.empty?
@@ -136,67 +193,139 @@ module Aidp
         output.join("\n")
       end
 
-      def filter_output(raw_output, mode, framework)
+      def filter_output(raw_output, mode, framework, category)
         return raw_output if mode == :full || raw_output.nil? || raw_output.empty?
 
-        filter_config = {
+        # Build filter config from configuration or defaults
+        filter_config = build_filter_config(mode, category)
+
+        # Load AI-generated filter definition if available (AGD pattern)
+        filter_definition = load_filter_definition_for(category)
+
+        # Track input size for stats
+        @filter_stats[:total_input_bytes] += raw_output.bytesize
+
+        filter = OutputFilter.new(filter_config, filter_definition: filter_definition)
+        filtered = filter.filter(raw_output, framework: framework)
+
+        # Track output size for stats
+        @filter_stats[:total_output_bytes] += filtered.bytesize
+
+        Aidp.log_debug("test_runner", "output_filtered",
+          framework: framework,
           mode: mode,
-          include_context: true,
-          context_lines: 3,
-          max_lines: 500
-        }
+          input_size: raw_output.bytesize,
+          output_size: filtered.bytesize,
+          has_custom_definition: !filter_definition.nil?)
 
-        filter = OutputFilter.new(filter_config)
-        filter.filter(raw_output, framework: framework)
+        filtered
       rescue NameError
         # Logging infrastructure not available
         raw_output
       rescue => e
-        Aidp.log_warn("test_runner", "filter_failed",
-          error: e.message,
-          framework: framework)
+        begin
+          Aidp.log_warn("test_runner", "filter_failed",
+            error: e.message,
+            framework: framework)
+        rescue NameError
+          # Logging not available
+        end
         raw_output  # Fallback to unfiltered on error
       end
 
-      def detect_framework_from_command(command)
-        case command
-        when /rspec/
-          :rspec
-        when /minitest/
-          :minitest
-        when /jest/
-          :jest
-        when /pytest/
-          :pytest
-        else
-          :unknown
+      # Load AI-generated filter definition for a category
+      # @param category [Symbol] :test or :lint
+      # @return [FilterDefinition, nil]
+      def load_filter_definition_for(category)
+        return nil unless @config.respond_to?(:filter_definition_for)
+
+        # Map category to definition key (matches setup wizard keys)
+        definition_key = case category
+        when :test then "unit_test"
+        when :lint then "lint"
+        else category.to_s
         end
+
+        @config.filter_definition_for(definition_key)
+      end
+
+      def build_filter_config(mode, category)
+        config_hash = {
+          mode: mode,
+          include_context: output_filtering_include_context,
+          context_lines: output_filtering_context_lines,
+          max_lines: max_output_lines_for_category(category)
+        }
+
+        OutputFilterConfig.from_hash(config_hash)
+      end
+
+      def detect_framework_from_command(command)
+        # Use ToolingDetector's enhanced framework detection
+        Aidp::ToolingDetector.framework_from_command(command)
       end
 
       def determine_output_mode(category)
         # Check config for category-specific mode
         case category
         when :test
-          if @config.respond_to?(:test_output_mode)
-            @config.test_output_mode
-          elsif @iteration_count > 1
-            :failures_only
-          else
-            :full
-          end
+          configured_mode = test_output_mode_from_config
+          return configured_mode if configured_mode
+
+          # Default: full on first iteration, failures_only after
+          (@iteration_count > 1) ? :failures_only : :full
         when :lint
-          if @config.respond_to?(:lint_output_mode)
-            @config.lint_output_mode
-          elsif @iteration_count > 1
-            :failures_only
-          else
-            :full
-          end
+          configured_mode = lint_output_mode_from_config
+          return configured_mode if configured_mode
+
+          # Default: full on first iteration, failures_only after
+          (@iteration_count > 1) ? :failures_only : :full
         else
           :full
         end
       end
 
+      def test_output_mode_from_config
+        return nil unless @config.respond_to?(:test_output_mode)
+        mode = @config.test_output_mode
+        return nil if mode.nil?
+        mode.to_sym
+      rescue
+        nil
+      end
+
+      def lint_output_mode_from_config
+        return nil unless @config.respond_to?(:lint_output_mode)
+        mode = @config.lint_output_mode
+        return nil if mode.nil?
+        mode.to_sym
+      rescue
+        nil
+      end
+
+      def max_output_lines_for_category(category)
+        case category
+        when :test
+          return @config.test_max_output_lines if @config.respond_to?(:test_max_output_lines) && @config.test_max_output_lines
+          OutputFilterConfig::DEFAULT_MAX_LINES
+        when :lint
+          return @config.lint_max_output_lines if @config.respond_to?(:lint_max_output_lines) && @config.lint_max_output_lines
+          300  # Default smaller for lint output
+        else
+          OutputFilterConfig::DEFAULT_MAX_LINES
+        end
+      end
+
+      def output_filtering_include_context
+        return @config.output_filtering_include_context if @config.respond_to?(:output_filtering_include_context)
+        OutputFilterConfig::DEFAULT_INCLUDE_CONTEXT
+      end
+
+      def output_filtering_context_lines
+        return @config.output_filtering_context_lines if @config.respond_to?(:output_filtering_context_lines)
+        OutputFilterConfig::DEFAULT_CONTEXT_LINES
+      end
+
       # Resolve commands for a specific category
       # Returns normalized command configs (array of {command:, required:} hashes)
       def resolved_commands(category)
@@ -239,8 +368,17 @@ module Aidp
       def detected_tooling
         @detected_tooling ||= Aidp::ToolingDetector.detect(@project_dir)
       rescue => e
-        Aidp.log_warn("test_runner", "tooling_detection_failed", error: e.message)
-        Aidp::ToolingDetector::Result.new(test_commands: [], lint_commands: [])
+        begin
+          Aidp.log_warn("test_runner", "tooling_detection_failed", error: e.message)
+        rescue NameError
+          # Logging not available
+        end
+        Aidp::ToolingDetector::Result.new(
+          test_commands: [],
+          lint_commands: [],
+          formatter_commands: [],
+          frameworks: {}
+        )
       end
 
       def log_fallback(type, commands)

data/lib/aidp/harness/thinking_depth_manager.rb

@@ -15,7 +15,7 @@ module Aidp
 
       def initialize(configuration, registry: nil, root_dir: nil)
         @configuration = configuration
-        @registry = registry || CapabilityRegistry.new(root_dir: root_dir || configuration.instance_variable_get(:@project_dir))
+        @registry = registry || CapabilityRegistry.new(root_dir: root_dir || configuration.project_dir)
         @current_tier = nil
         @session_max_tier = nil
         @tier_history = []
@@ -152,6 +152,15 @@ module Aidp
       def select_model_for_tier(tier = nil, provider: nil)
         tier ||= current_tier
         validate_tier!(tier)
+        provider_has_no_tiers = provider && configuration.configured_tiers(provider).empty?
+        provider_has_catalog_models = provider && !@registry.models_for_provider(provider).empty?
+
+        if provider_has_no_tiers && !provider_has_catalog_models
+          Aidp.log_info("thinking_depth_manager", "No configured tiers for provider, deferring to provider auto model selection",
+            requested_tier: tier,
+            provider: provider)
+          return [provider, nil, {auto_model: true, reason: "provider_has_no_tiers"}]
+        end
 
         # First, try to get models from user's configuration for this tier and provider
         if provider
@@ -234,43 +243,47 @@ module Aidp
               model: model_name)
             return [provider, model_name, model_data]
           end
-
-          # If provider doesn't support tier and switching allowed, try others
-          unless configuration.allow_provider_switch_for_tier?
-            Aidp.log_warn("thinking_depth_manager", "Provider lacks tier in catalog, switching disabled",
-              tier: tier,
-              provider: provider)
-            return nil
-          end
-        end
-
-        # Try all providers in catalog
-        if provider && !configuration.allow_provider_switch_for_tier?
-          return nil
+          # Per issue #323: Don't return nil here - let fallback logic handle missing tiers
         end
 
-        providers_to_try = provider ? [@registry.provider_names - [provider]].flatten : @registry.provider_names
+        # Try all providers in catalog if provider switching is allowed
+        if configuration.allow_provider_switch_for_tier?
+          providers_to_try = provider ? (@registry.provider_names - [provider]) : @registry.provider_names
 
-        providers_to_try.each do |prov_name|
-          model_name, model_data = @registry.best_model_for_tier(tier, prov_name)
-          if model_name
-            Aidp.log_info("thinking_depth_manager", "Selected model from catalog (alternate provider)",
-              tier: tier,
-              original_provider: provider,
-              selected_provider: prov_name,
-              model: model_name)
-            return [prov_name, model_name, model_data]
+          providers_to_try.each do |prov_name|
+            model_name, model_data = @registry.best_model_for_tier(tier, prov_name)
+            if model_name
+              Aidp.log_info("thinking_depth_manager", "Selected model from catalog (alternate provider)",
+                tier: tier,
+                original_provider: provider,
+                selected_provider: prov_name,
+                model: model_name)
+              return [prov_name, model_name, model_data]
+            end
           end
         end
 
         # No model found for requested tier - try fallback to other tiers
-        Aidp.log_warn("thinking_depth_manager", "No model found for requested tier, trying fallback",
+        # Per issue #323: fallback events log at debug level
+        Aidp.log_debug("thinking_depth_manager", "tier_not_found_trying_fallback",
           tier: tier,
           provider: provider)
 
         result = try_fallback_tiers(tier, provider)
 
+        # If no model found after fallback, defer to provider auto model selection
+        # This allows providers to select their own model when no explicit tier config exists
+        # Per issue #323: log at debug level, don't constrain model selection
+        if result.nil? && provider
+          Aidp.log_debug("thinking_depth_manager", "no_model_for_tier_deferring_to_provider",
+            requested_tier: tier,
+            provider: provider,
+            reason: provider_has_no_tiers ? "provider_has_no_tiers" : "tier_not_configured")
+          return [provider, nil, {auto_model: true, reason: provider_has_no_tiers ? "provider_has_no_tiers" : "tier_not_configured"}]
+        end
+
         unless result
+          # This path should only be reached when no provider is specified
           # Enhanced error message with discovery hints
           display_enhanced_tier_error(tier, provider)
 
@@ -436,7 +449,8 @@ module Aidp
 
             if configured_models.any?
               model_name = configured_models.first
-              Aidp.log_warn("thinking_depth_manager", "Falling back to different tier (from config)",
+              # Per issue #323: fallback events log at debug level
+              Aidp.log_debug("thinking_depth_manager", "tier_fallback_from_config",
                 requested_tier: requested_tier,
                 fallback_tier: fallback_tier,
                 provider: provider,
@@ -450,7 +464,8 @@ module Aidp
           if provider
             model_name, model_data = @registry.best_model_for_tier(fallback_tier, provider)
             if model_name
-              Aidp.log_warn("thinking_depth_manager", "Falling back to different tier (from catalog)",
+              # Per issue #323: fallback events log at debug level
+              Aidp.log_debug("thinking_depth_manager", "tier_fallback_from_catalog",
                 requested_tier: requested_tier,
                 fallback_tier: fallback_tier,
                 provider: provider,
@@ -466,7 +481,8 @@ module Aidp
 
               model_name, model_data = @registry.best_model_for_tier(fallback_tier, prov_name)
               if model_name
-                Aidp.log_warn("thinking_depth_manager", "Falling back to different tier and provider (from catalog)",
+                # Per issue #323: fallback events log at debug level
+                Aidp.log_debug("thinking_depth_manager", "tier_fallback_provider_switch",
                   requested_tier: requested_tier,
                   fallback_tier: fallback_tier,
                   requested_provider: provider,

data/lib/aidp/harness/ui/enhanced_tui.rb

@@ -20,6 +20,10 @@ module Aidp
 
         class DisplayError < TUIError; end
 
+        # Expose state for testability
+        attr_accessor :headless, :current_mode, :workflow_active, :current_step
+        attr_reader :jobs
+
         def initialize(prompt: TTY::Prompt.new, tty: $stdin)
           @cursor = TTY::Cursor
           @screen = TTY::Screen

data/lib/aidp/harness/ui/enhanced_workflow_selector.rb

@@ -11,6 +11,10 @@ module Aidp
       class EnhancedWorkflowSelector
         class WorkflowError < StandardError; end
 
+        # Expose state for testability
+        attr_reader :tui, :project_dir, :user_input
+        attr_accessor :workflow_selector
+
         def initialize(tui = nil, project_dir: Dir.pwd)
           @tui = tui || EnhancedTUI.new
           @user_input = {}

data/lib/aidp/harness/ui/error_handler.rb

@@ -18,6 +18,9 @@ module Aidp
 
         class InteractionError < UIError; end
 
+        # Expose for testability
+        attr_reader :logger
+
         def initialize(ui_components = {})
           @logger = ui_components[:logger] || default_logger
           @formatter = ui_components[:formatter] || ErrorFormatter.new

data/lib/aidp/harness/ui/job_monitor.rb

@@ -17,6 +17,10 @@ module Aidp
 
         class MonitorError < JobMonitorError; end
 
+        # Expose for testability
+        attr_accessor :monitor_thread
+        attr_reader :update_callbacks
+
         JOB_STATUSES = {
           pending: "Pending",
           running: "Running",

data/lib/aidp/harness/ui/navigation/submenu.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "main_menu"
-
 module Aidp
   module Harness
     module UI
@@ -24,6 +22,8 @@ module Aidp
 
           attr_reader :title, :parent_menu
           attr_accessor :drill_down_enabled, :max_depth
+          # Expose current_level for testability
+          attr_accessor :current_level
 
           def add_submenu_item(item)
             validate_submenu_item(item)

data/lib/aidp/harness/ui/navigation/workflow_selector.rb

@@ -17,6 +17,9 @@ module Aidp
 
           class SelectionError < WorkflowError; end
 
+          # Expose for testability
+          attr_reader :prompt
+
           WORKFLOW_MODES = {
             simple: {
               name: "Simple Mode",
@@ -132,6 +135,9 @@ module Aidp
 
         # Formats workflow selection display
         class WorkflowFormatter
+          # Expose for testability
+          attr_reader :pastel
+
           def initialize
             @pastel = Pastel.new
           end

data/lib/aidp/harness/ui/spinner_helper.rb

@@ -11,6 +11,9 @@ module Aidp
       class SpinnerHelper
         class SpinnerError < StandardError; end
 
+        # Expose for testability
+        attr_accessor :active_spinners
+
         def initialize
           @pastel = Pastel.new
           @active_spinners = []

data/lib/aidp/harness/ui/workflow_controller.rb

@@ -16,6 +16,9 @@ module Aidp
 
         class ControlError < WorkflowError; end
 
+        # Expose for testability
+        attr_accessor :pause_time, :control_thread
+
         WORKFLOW_STATES = {
           running: "Running",
           paused: "Paused",

data/lib/aidp/harness/user_interface.rb

@@ -9,6 +9,9 @@ module Aidp
     class UserInterface
       include Aidp::MessageDisplay
 
+      # Expose for testability
+      attr_reader :auto_confirm_defaults, :show_help_automatically, :verbose_mode, :file_selection_enabled
+
       def initialize(prompt: TTY::Prompt.new)
         @input_history = []
         @file_selection_enabled = false