aidp 0.25.0 → 0.27.0

data/lib/aidp/harness/model_discovery_service.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+require_relative "model_cache"
+require_relative "model_registry"
+
+module Aidp
+  module Harness
+    # Service for discovering available models from providers
+    #
+    # Orchestrates model discovery across multiple providers:
+    # 1. Checks cache first (with TTL)
+    # 2. Falls back to dynamic discovery via provider.discover_models
+    # 3. Merges with static registry for comprehensive results
+    # 4. Caches results for future use
+    #
+    # Usage:
+    #   service = ModelDiscoveryService.new
+    #   models = service.discover_models("anthropic")
+    #   all_models = service.discover_all_models
+    class ModelDiscoveryService
+      attr_reader :cache, :registry
+
+      def initialize(cache: nil, registry: nil)
+        @cache = cache || ModelCache.new
+        @registry = registry || ModelRegistry.new
+        @provider_classes = discover_provider_classes
+        Aidp.log_debug("model_discovery_service", "initialized",
+          providers: @provider_classes.keys)
+      end
+
+      # Discover models for a specific provider
+      #
+      # @param provider [String] Provider name (e.g., "anthropic", "cursor")
+      # @param use_cache [Boolean] Whether to use cached results (default: true)
+      # @return [Array<Hash>] Discovered models
+      def discover_models(provider, use_cache: true)
+        Aidp.log_info("model_discovery_service", "discovering models",
+          provider: provider, use_cache: use_cache)
+
+        # Check cache first
+        if use_cache
+          cached = @cache.get_cached_models(provider)
+          if cached
+            Aidp.log_debug("model_discovery_service", "using cached models",
+              provider: provider, count: cached.size)
+            return cached
+          end
+        end
+
+        # Perform discovery
+        models = perform_discovery(provider)
+
+        # Cache the results
+        @cache.cache_models(provider, models) if models.any?
+
+        models
+      rescue => e
+        Aidp.log_error("model_discovery_service", "discovery failed",
+          provider: provider, error: e.message, backtrace: e.backtrace.first(3))
+        []
+      end
+
+      # Discover models from all available providers
+      #
+      # @param use_cache [Boolean] Whether to use cached results
+      # @return [Hash] Hash of provider => models array
+      def discover_all_models(use_cache: true)
+        results = {}
+
+        @provider_classes.each_key do |provider|
+          models = discover_models(provider, use_cache: use_cache)
+          results[provider] = models if models.any?
+        end
+
+        Aidp.log_info("model_discovery_service", "discovered all models",
+          providers: results.keys, total_models: results.values.flatten.size)
+        results
+      end
+
+      # Discover models concurrently from multiple providers
+      #
+      # @param providers [Array<String>] List of provider names
+      # @param use_cache [Boolean] Whether to use cached results
+      # @return [Hash] Hash of provider => models array
+      def discover_concurrent(providers, use_cache: true)
+        require "concurrent"
+
+        results = {}
+        mutex = Mutex.new
+
+        # Create a thread pool
+        pool = Concurrent::FixedThreadPool.new(providers.size)
+
+        # Submit discovery tasks
+        futures = providers.map do |provider|
+          Concurrent::Future.execute(executor: pool) do
+            models = discover_models(provider, use_cache: use_cache)
+            mutex.synchronize { results[provider] = models }
+          end
+        end
+
+        # Wait for all to complete
+        futures.each(&:wait)
+
+        pool.shutdown
+        pool.wait_for_termination(30)
+
+        Aidp.log_info("model_discovery_service", "concurrent discovery complete",
+          providers: results.keys, total_models: results.values.flatten.size)
+        results
+      rescue LoadError => e
+        # Fallback to sequential if concurrent gem not available
+        Aidp.log_warn("model_discovery_service", "concurrent gem not available, using sequential",
+          error: e.message)
+        providers.each_with_object({}) do |provider, hash|
+          hash[provider] = discover_models(provider, use_cache: use_cache)
+        end
+      rescue => e
+        Aidp.log_error("model_discovery_service", "concurrent discovery failed",
+          error: e.message)
+        {}
+      end
+
+      # Get all available models (discovery + static registry)
+      #
+      # Combines dynamically discovered models with static registry
+      #
+      # @param use_cache [Boolean] Whether to use cached results
+      # @return [Hash] Hash with :discovered and :registry keys
+      def all_available_models(use_cache: true)
+        discovered = discover_all_models(use_cache: use_cache)
+        registry_families = @registry.all_families
+
+        {
+          discovered: discovered,
+          registry: registry_families.map { |family| @registry.get_model_info(family) }.compact
+        }
+      end
+
+      # Find which providers support a given model family
+      #
+      # @param family_name [String] Model family name
+      # @return [Array<String>] List of provider names
+      def providers_supporting(family_name)
+        providers = []
+
+        @provider_classes.each do |provider_name, class_name|
+          provider_class = constantize_provider(class_name)
+          next unless provider_class
+
+          if provider_class.respond_to?(:supports_model_family?)
+            providers << provider_name if provider_class.supports_model_family?(family_name)
+          end
+        end
+
+        providers
+      end
+
+      # Refresh cache for all providers
+      def refresh_all_caches
+        @cache.invalidate_all
+        discover_all_models(use_cache: false)
+      end
+
+      # Refresh cache for specific provider
+      #
+      # @param provider [String] Provider name
+      def refresh_cache(provider)
+        @cache.invalidate(provider)
+        discover_models(provider, use_cache: false)
+      end
+
+      private
+
+      def perform_discovery(provider)
+        provider_class = get_provider_class(provider)
+        unless provider_class
+          Aidp.log_warn("model_discovery_service", "unknown provider",
+            provider: provider)
+          return []
+        end
+
+        unless provider_class.respond_to?(:available?) && provider_class.available?
+          Aidp.log_debug("model_discovery_service", "provider not available",
+            provider: provider)
+          return []
+        end
+
+        unless provider_class.respond_to?(:discover_models)
+          Aidp.log_warn("model_discovery_service", "provider missing discover_models",
+            provider: provider)
+          return []
+        end
+
+        models = provider_class.discover_models
+        Aidp.log_info("model_discovery_service", "discovered models",
+          provider: provider, count: models.size)
+        models
+      end
+
+      def get_provider_class(provider)
+        class_name = @provider_classes[provider]
+        return nil unless class_name
+
+        constantize_provider(class_name)
+      end
+
+      def constantize_provider(class_name)
+        # Safely constantize the provider class
+        parts = class_name.split("::")
+        parts.reduce(Object) { |mod, name| mod.const_get(name) }
+      rescue NameError => e
+        Aidp.log_debug("model_discovery_service", "provider class not found",
+          class: class_name, error: e.message)
+        nil
+      end
+
+      # Dynamically discover all provider classes from the providers directory
+      #
+      # @return [Hash] Hash of provider_name => class_name
+      def discover_provider_classes
+        providers_dir = File.join(__dir__, "../providers")
+        provider_files = Dir.glob("*.rb", base: providers_dir)
+
+        # Exclude base classes and utility files
+        excluded_files = ["base.rb", "adapter.rb", "error_taxonomy.rb", "capability_registry.rb"]
+        provider_files -= excluded_files
+
+        providers = {}
+
+        provider_files.each do |file|
+          provider_name = File.basename(file, ".rb")
+          # Convert to class name (e.g., "anthropic" -> "Anthropic", "github_copilot" -> "GithubCopilot")
+          class_name = provider_name.split("_").map(&:capitalize).join
+          full_class_name = "Aidp::Providers::#{class_name}"
+
+          # Try to load and verify the provider class exists
+          begin
+            require_relative "../providers/#{provider_name}"
+            provider_class = constantize_provider(full_class_name)
+            if provider_class&.respond_to?(:discover_models)
+              providers[provider_name] = full_class_name
+            end
+          rescue => e
+            # Skip providers that can't be loaded or don't implement discover_models
+            if ENV["DEBUG"]
+              Aidp.log_debug("model_discovery_service", "skipping provider",
+                provider: provider_name, reason: e.message)
+            end
+          end
+        end
+
+        Aidp.log_debug("model_discovery_service", "discovered provider classes",
+          count: providers.size, providers: providers.keys)
+        providers
+      end
+    end
+  end
+end

data/lib/aidp/harness/model_registry.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "pathname"
+
+module Aidp
+  module Harness
+    # ModelRegistry manages the static registry of known model families and their tier classifications
+    #
+    # The registry uses model families (e.g., "claude-3-5-sonnet") rather than specific versioned
+    # model IDs (e.g., "claude-3-5-sonnet-20241022"). This design provides:
+    #   - No version tracking burden - registry tracks families, not every dated version
+    #   - Future-proofing - new model versions automatically inherit family tier
+    #   - Provider autonomy - each provider handles version-specific naming
+    #
+    # Usage:
+    #   registry = ModelRegistry.new
+    #   registry.get_model_info("claude-3-5-sonnet")
+    #   # => { name: "Claude 3.5 Sonnet", tier: "standard", ... }
+    #
+    #   registry.models_for_tier("standard")
+    #   # => ["claude-3-5-sonnet", "gpt-4-turbo", ...]
+    #
+    #   registry.match_to_family("claude-3-5-sonnet-20241022")
+    #   # => "claude-3-5-sonnet"
+    class ModelRegistry
+      class RegistryError < StandardError; end
+      class InvalidRegistrySchema < RegistryError; end
+      class ModelNotFound < RegistryError; end
+
+      VALID_TIERS = %w[mini standard advanced].freeze
+      VALID_CAPABILITIES = %w[chat code vision tool_use streaming json_mode].freeze
+      VALID_SPEEDS = %w[very_fast fast medium slow].freeze
+
+      attr_reader :registry_data
+
+      def initialize(registry_path: nil)
+        @registry_path = registry_path || default_registry_path
+        @registry_data = load_static_registry
+        validate_registry_schema
+        Aidp.log_debug("model_registry", "initialized", models: @registry_data["model_families"].keys.size)
+      end
+
+      # Get complete model information for a family
+      #
+      # @param family_name [String] The model family name (e.g., "claude-3-5-sonnet")
+      # @return [Hash, nil] Model metadata hash or nil if not found
+      def get_model_info(family_name)
+        info = @registry_data["model_families"][family_name]
+        return nil unless info
+
+        info.merge("family" => family_name)
+      end
+
+      # Get all model families for a specific tier
+      #
+      # @param tier [String, Symbol] The tier name (mini, standard, advanced)
+      # @return [Array<String>] List of model family names for the tier
+      def models_for_tier(tier)
+        tier_str = tier.to_s
+        unless VALID_TIERS.include?(tier_str)
+          Aidp.log_warn("model_registry", "invalid tier requested", tier: tier_str, valid: VALID_TIERS)
+          return []
+        end
+
+        families = @registry_data["model_families"].select { |_family, info|
+          info["tier"] == tier_str
+        }.keys
+
+        Aidp.log_debug("model_registry", "found models for tier", tier: tier_str, count: families.size)
+        families
+      end
+
+      # Classify a model family's tier
+      #
+      # @param family_name [String] The model family name
+      # @return [String, nil] The tier name or nil if not found
+      def classify_model_tier(family_name)
+        info = get_model_info(family_name)
+        tier = info&.fetch("tier", nil)
+        Aidp.log_debug("model_registry", "classified tier", family: family_name, tier: tier)
+        tier
+      end
+
+      # Match a versioned model name to its family using pattern matching
+      #
+      # This method attempts to normalize versioned model names (e.g., "claude-3-5-sonnet-20241022")
+      # to their family name (e.g., "claude-3-5-sonnet") by testing against version_pattern regexes.
+      #
+      # @param versioned_name [String] The versioned model name
+      # @return [String, nil] The family name if matched, nil otherwise
+      def match_to_family(versioned_name)
+        @registry_data["model_families"].each do |family, info|
+          pattern = info["version_pattern"]
+          next unless pattern
+
+          begin
+            regex = Regexp.new("^#{pattern}$")
+            if regex.match?(versioned_name)
+              Aidp.log_debug("model_registry", "matched to family", versioned: versioned_name, family: family)
+              return family
+            end
+          rescue RegexpError => e
+            Aidp.log_error("model_registry", "invalid pattern", family: family, pattern: pattern, error: e.message)
+          end
+        end
+
+        # If no pattern matches, check if the versioned_name is itself a family
+        if @registry_data["model_families"].key?(versioned_name)
+          Aidp.log_debug("model_registry", "exact family match", name: versioned_name)
+          return versioned_name
+        end
+
+        Aidp.log_debug("model_registry", "no family match", versioned: versioned_name)
+        nil
+      end
+
+      # Get all registered model families
+      #
+      # @return [Array<String>] List of all model family names
+      def all_families
+        @registry_data["model_families"].keys
+      end
+
+      # Check if a model family exists in the registry
+      #
+      # @param family_name [String] The model family name
+      # @return [Boolean] True if the family exists
+      def family_exists?(family_name)
+        @registry_data["model_families"].key?(family_name)
+      end
+
+      # Get all tiers that have at least one model
+      #
+      # @return [Array<String>] List of tier names
+      def available_tiers
+        @registry_data["model_families"].values.map { |info| info["tier"] }.uniq.sort
+      end
+
+      private
+
+      def default_registry_path
+        Pathname.new(__dir__).parent.join("data", "model_registry.yml")
+      end
+
+      def load_static_registry
+        unless File.exist?(@registry_path)
+          raise RegistryError, "Model registry file not found at #{@registry_path}"
+        end
+
+        data = YAML.load_file(@registry_path)
+        unless data.is_a?(Hash) && data.key?("model_families")
+          raise InvalidRegistrySchema, "Registry must contain 'model_families' key"
+        end
+
+        Aidp.log_info("model_registry", "loaded registry", path: @registry_path, families: data["model_families"].size)
+        data
+      rescue Psych::SyntaxError => e
+        raise InvalidRegistrySchema, "Invalid YAML in registry file: #{e.message}"
+      end
+
+      def validate_registry_schema
+        @registry_data["model_families"].each do |family, info|
+          validate_model_entry(family, info)
+        end
+      end
+
+      def validate_model_entry(family, info)
+        # Required fields
+        unless info["tier"]
+          raise InvalidRegistrySchema, "Model family '#{family}' missing required 'tier' field"
+        end
+        unless VALID_TIERS.include?(info["tier"])
+          raise InvalidRegistrySchema, "Model family '#{family}' has invalid tier '#{info["tier"]}'. Valid: #{VALID_TIERS.join(", ")}"
+        end
+
+        # Optional but validated if present
+        if info["capabilities"]
+          unless info["capabilities"].is_a?(Array)
+            raise InvalidRegistrySchema, "Model family '#{family}' capabilities must be an array"
+          end
+          invalid_caps = info["capabilities"] - VALID_CAPABILITIES
+          unless invalid_caps.empty?
+            Aidp.log_warn("model_registry", "unknown capabilities", family: family, unknown: invalid_caps)
+          end
+        end
+
+        if info["speed"] && !VALID_SPEEDS.include?(info["speed"])
+          Aidp.log_warn("model_registry", "invalid speed", family: family, speed: info["speed"], valid: VALID_SPEEDS)
+        end
+
+        # Validate numeric fields if present
+        %w[context_window max_output cost_per_1m_input cost_per_1m_output].each do |field|
+          if info[field] && !info[field].is_a?(Numeric)
+            raise InvalidRegistrySchema, "Model family '#{family}' field '#{field}' must be numeric"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aidp/harness/output_filter.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Aidp
+  module Harness
+    # Filters test and linter output to reduce token consumption
+    # Uses framework-specific strategies to extract relevant information
+    class OutputFilter
+      # Output modes
+      MODES = {
+        full: :full,                   # No filtering (default for first run)
+        failures_only: :failures_only, # Only failure information
+        minimal: :minimal              # Minimal failure info + summary
+      }.freeze
+
+      # @param config [Hash] Configuration options
+      # @option config [Symbol] :mode Output mode (:full, :failures_only, :minimal)
+      # @option config [Boolean] :include_context Include surrounding lines
+      # @option config [Integer] :context_lines Number of context lines
+      # @option config [Integer] :max_lines Maximum output lines
+      def initialize(config = {})
+        @mode = config[:mode] || :full
+        @include_context = config.fetch(:include_context, true)
+        @context_lines = config.fetch(:context_lines, 3)
+        @max_lines = config.fetch(:max_lines, 500)
+
+        validate_mode!
+
+        Aidp.log_debug("output_filter", "initialized",
+          mode: @mode,
+          include_context: @include_context,
+          max_lines: @max_lines)
+      rescue NameError
+        # Logging infrastructure not available in some tests
+      end
+
+      # Filter output based on framework and mode
+      # @param output [String] Raw output
+      # @param framework [Symbol] Framework identifier
+      # @return [String] Filtered output
+      def filter(output, framework: :unknown)
+        return output if @mode == :full
+        return "" if output.nil? || output.empty?
+
+        Aidp.log_debug("output_filter", "filtering_start",
+          framework: framework,
+          input_lines: output.lines.count)
+
+        strategy = strategy_for_framework(framework)
+        filtered = strategy.filter(output, self)
+
+        truncated = truncate_if_needed(filtered)
+
+        Aidp.log_debug("output_filter", "filtering_complete",
+          output_lines: truncated.lines.count,
+          reduction: reduction_stats(output, truncated))
+
+        truncated
+      rescue NameError
+        # Logging infrastructure not available
+        return output if @mode == :full
+        return "" if output.nil? || output.empty?
+
+        strategy = strategy_for_framework(framework)
+        filtered = strategy.filter(output, self)
+        truncate_if_needed(filtered)
+      rescue => e
+        # External failure - graceful degradation
+        begin
+          Aidp.log_error("output_filter", "filtering_failed",
+            framework: framework,
+            mode: @mode,
+            error: e.message,
+            error_class: e.class.name)
+        rescue NameError
+          # Logging not available
+        end
+
+        # Return original output as fallback
+        output
+      end
+
+      # Accessors for strategy use
+      attr_reader :mode, :include_context, :context_lines, :max_lines
+
+      private
+
+      def validate_mode!
+        unless MODES.key?(@mode)
+          raise ArgumentError, "Invalid mode: #{@mode}. Must be one of #{MODES.keys}"
+        end
+      end
+
+      def strategy_for_framework(framework)
+        case framework
+        when :rspec
+          require_relative "rspec_filter_strategy"
+          RSpecFilterStrategy.new
+        when :minitest
+          require_relative "generic_filter_strategy"
+          GenericFilterStrategy.new
+        when :jest
+          require_relative "generic_filter_strategy"
+          GenericFilterStrategy.new
+        when :pytest
+          require_relative "generic_filter_strategy"
+          GenericFilterStrategy.new
+        else
+          require_relative "generic_filter_strategy"
+          GenericFilterStrategy.new
+        end
+      end
+
+      def truncate_if_needed(output)
+        lines = output.lines
+        return output if lines.count <= @max_lines
+
+        truncated = lines.first(@max_lines).join
+        # Only add newline if truncated doesn't already end with one
+        separator = truncated.end_with?("\n") ? "" : "\n"
+        truncated + separator + "[Output truncated - #{lines.count - @max_lines} more lines omitted]"
+      end
+
+      def reduction_stats(input, output)
+        input_size = input.bytesize
+        output_size = output.bytesize
+        reduction = ((input_size - output_size).to_f / input_size * 100).round(1)
+
+        {
+          input_bytes: input_size,
+          output_bytes: output_size,
+          reduction_percent: reduction
+        }
+      end
+    end
+  end
+end

data/lib/aidp/harness/provider_manager.rb

@@ -1394,14 +1394,27 @@ module Aidp
 
       # Execute a prompt with a specific provider
       def execute_with_provider(provider_type, prompt, options = {})
+        # Extract model from options if provided
+        model_name = options.delete(:model)
+
         # Create provider factory instance
         provider_factory = ProviderFactory.new
 
+        # Add model to provider options if specified
+        provider_options = options.dup
+        provider_options[:model] = model_name if model_name
+
         # Create provider instance
-        provider = provider_factory.create_provider(provider_type, options)
+        provider = provider_factory.create_provider(provider_type, provider_options)
 
-        # Set current provider
+        # Set current provider and model
         @current_provider = provider_type
+        @current_model = model_name if model_name
+
+        Aidp.logger.debug("provider_manager", "Executing with provider",
+          provider: provider_type,
+          model: model_name,
+          prompt_length: prompt.length)
 
         # Execute the prompt with the provider
         result = provider.send_message(prompt: prompt, session: nil)
@@ -1410,15 +1423,17 @@ module Aidp
         {
           status: "completed",
           provider: provider_type,
+          model: model_name,
           output: result,
           metadata: {
             provider_type: provider_type,
+            model: model_name,
             prompt_length: prompt.length,
             timestamp: Time.now.strftime("%Y-%m-%dT%H:%M:%S.%3N%z")
           }
         }
       rescue => e
-        log_rescue(e, component: "provider_manager", action: "execute_with_provider", fallback: "error_result", provider: provider_type, prompt_length: prompt.length)
+        log_rescue(e, component: "provider_manager", action: "execute_with_provider", fallback: "error_result", provider: provider_type, model: model_name, prompt_length: prompt.length)
         # Return error result
         {
           status: "error",