aidp 0.26.0 → 0.27.0

data/lib/aidp/providers/anthropic.rb

@@ -9,10 +9,143 @@ module Aidp
     class Anthropic < Base
       include Aidp::DebugMixin
 
+      # Model name pattern for Anthropic Claude models
+      MODEL_PATTERN = /^claude-[\d.-]+-(?:opus|sonnet|haiku)(?:-\d{8})?$/i
+
       def self.available?
         !!Aidp::Util.which("claude")
       end
 
+      # Normalize a provider-specific model name to its model family
+      #
+      # Anthropic uses date-versioned models (e.g., "claude-3-5-sonnet-20241022").
+      # This method strips the date suffix to get the family name.
+      #
+      # @param provider_model_name [String] The versioned model name
+      # @return [String] The model family name (e.g., "claude-3-5-sonnet")
+      def self.model_family(provider_model_name)
+        # Strip date suffix: "claude-3-5-sonnet-20241022" → "claude-3-5-sonnet"
+        provider_model_name.sub(/-\d{8}$/, "")
+      end
+
+      # Convert a model family name to the provider's preferred model name
+      #
+      # Returns the family name as-is. Users can configure specific versions in aidp.yml.
+      #
+      # @param family_name [String] The model family name
+      # @return [String] The model name (same as family for flexibility)
+      def self.provider_model_name(family_name)
+        family_name
+      end
+
+      # Check if this provider supports a given model family
+      #
+      # @param family_name [String] The model family name
+      # @return [Boolean] True if it matches Claude model pattern
+      def self.supports_model_family?(family_name)
+        MODEL_PATTERN.match?(family_name)
+      end
+
+      # Discover available models from Claude CLI
+      #
+      # @return [Array<Hash>] Array of discovered models
+      def self.discover_models
+        return [] unless available?
+
+        begin
+          require "open3"
+          output, _, status = Open3.capture3("claude", "models", "list", {timeout: 10})
+          return [] unless status.success?
+
+          parse_models_list(output)
+        rescue => e
+          Aidp.log_debug("anthropic_provider", "discovery failed", error: e.message)
+          []
+        end
+      end
+
+      class << self
+        private
+
+        def parse_models_list(output)
+          return [] if output.nil? || output.empty?
+
+          models = []
+          lines = output.lines.map(&:strip)
+
+          # Skip header and separator lines
+          lines.reject! { |line| line.empty? || line.match?(/^[-=]+$/) || line.match?(/^(Model|Name)/i) }
+
+          lines.each do |line|
+            model_info = parse_model_line(line)
+            models << model_info if model_info
+          end
+
+          Aidp.log_info("anthropic_provider", "discovered models", count: models.size)
+          models
+        end
+
+        def parse_model_line(line)
+          # Format 1: Simple list of model names
+          if line.match?(/^claude-\d/)
+            model_name = line.split.first
+            return build_model_info(model_name)
+          end
+
+          # Format 2: Table format with columns
+          parts = line.split(/\s{2,}/)
+          if parts.size >= 1 && parts[0].match?(/^claude/)
+            model_name = parts[0]
+            model_name = "#{model_name}-#{parts[1]}" if parts.size > 1 && parts[1].match?(/^\d{8}$/)
+            return build_model_info(model_name)
+          end
+
+          # Format 3: JSON-like or key-value pairs
+          if line.match?(/name:\s*(.+)/)
+            model_name = $1.strip
+            return build_model_info(model_name)
+          end
+
+          nil
+        end
+
+        def build_model_info(model_name)
+          family = model_family(model_name)
+          tier = classify_tier(model_name)
+
+          {
+            name: model_name,
+            family: family,
+            tier: tier,
+            capabilities: extract_capabilities(model_name),
+            context_window: infer_context_window(family),
+            provider: "anthropic"
+          }
+        end
+
+        def classify_tier(model_name)
+          name_lower = model_name.downcase
+          return "advanced" if name_lower.include?("opus")
+          return "mini" if name_lower.include?("haiku")
+          return "standard" if name_lower.include?("sonnet")
+          "standard"
+        end
+
+        def extract_capabilities(model_name)
+          capabilities = ["chat", "code"]
+          name_lower = model_name.downcase
+          capabilities << "vision" unless name_lower.include?("haiku")
+          capabilities
+        end
+
+        def infer_context_window(family)
+          family.match?(/claude-3/) ? 200_000 : nil
+        end
+      end
+
+      # Public instance methods (called from workflows and harness)
+      public
+
       def name
         "anthropic"
       end
@@ -53,8 +186,7 @@ module Aidp
           supports_json_mode: true,
           supports_tool_use: true,
           supports_vision: false,
-          supports_file_upload: true,
-          streaming: true
+          supports_file_upload: true
         }
       end
 
@@ -115,19 +247,8 @@ module Aidp
         debug_provider("claude", "Starting execution", {timeout: timeout_seconds})
         debug_log("📝 Sending prompt to claude...", level: :info)
 
-        # Check if streaming mode is enabled
-        streaming_enabled = ENV["AIDP_STREAMING"] == "1" || ENV["DEBUG"] == "1"
-
-        # Build command arguments with proper streaming support
-        args = ["--print"]
-        if streaming_enabled
-          # Claude CLI requires --verbose when using --print with --output-format=stream-json
-          args += ["--verbose", "--output-format=stream-json", "--include-partial-messages"]
-          display_message("📺 True streaming enabled - real-time chunks from Claude API", type: :info)
-        else
-          # Use text format for non-streaming (default behavior)
-          args += ["--output-format=text"]
-        end
+        # Build command arguments
+        args = ["--print", "--output-format=text"]
 
         # Check if we should skip permissions (devcontainer support)
         if should_skip_permissions?
@@ -136,26 +257,20 @@ module Aidp
         end
 
         begin
-          # Use debug_execute_command with streaming support
-          result = debug_execute_command("claude", args: args, input: prompt, timeout: timeout_seconds, streaming: streaming_enabled)
+          result = debug_execute_command("claude", args: args, input: prompt, timeout: timeout_seconds)
 
           # Log the results
           debug_command("claude", args: args, input: prompt, output: result.out, error: result.err, exit_code: result.exit_status)
 
           if result.exit_status == 0
-            # Handle different output formats
-            if streaming_enabled && args.include?("--output-format=stream-json")
-              # Parse stream-json output and extract final content
-              parse_stream_json_output(result.out)
-            else
-              # Return text output as-is
-              result.out
-            end
+            result.out
           else
             # Detect auth issues in stdout/stderr (Claude sometimes prints JSON with auth error to stdout)
             combined = [result.out, result.err].compact.join("\n")
             if combined.downcase.include?("oauth token has expired") || combined.downcase.include?("authentication_error")
-              error_message = "Authentication error from Claude CLI: token expired or invalid. Run 'claude /login' or refresh credentials."
+              error_message = "Authentication error from Claude CLI: token expired or invalid.\n" \
+                              "Run 'claude /login' or refresh credentials.\n" \
+                              "Note: Model discovery requires valid authentication."
               debug_error(StandardError.new(error_message), {exit_code: result.exit_status, stdout: result.out, stderr: result.err})
               # Raise a recognizable error for classifier
               raise error_message
@@ -172,60 +287,6 @@ module Aidp
 
       private
 
-      def calculate_timeout
-        # Priority order for timeout calculation:
-        # 1. Quick mode (for testing)
-        # 2. Environment variable override
-        # 3. Adaptive timeout based on step type
-        # 4. Default timeout
-
-        if ENV["AIDP_QUICK_MODE"]
-          display_message("⚡ Quick mode enabled - #{TIMEOUT_QUICK_MODE / 60} minute timeout", type: :highlight)
-          return TIMEOUT_QUICK_MODE
-        end
-
-        if ENV["AIDP_ANTHROPIC_TIMEOUT"]
-          return ENV["AIDP_ANTHROPIC_TIMEOUT"].to_i
-        end
-
-        if adaptive_timeout
-          display_message("🧠 Using adaptive timeout: #{adaptive_timeout} seconds", type: :info)
-          return adaptive_timeout
-        end
-
-        # Default timeout
-        display_message("📋 Using default timeout: #{TIMEOUT_DEFAULT / 60} minutes", type: :info)
-        TIMEOUT_DEFAULT
-      end
-
-      def adaptive_timeout
-        @adaptive_timeout ||= begin
-          # Timeout recommendations based on step type patterns
-          step_name = ENV["AIDP_CURRENT_STEP"] || ""
-
-          case step_name
-          when /REPOSITORY_ANALYSIS/
-            TIMEOUT_REPOSITORY_ANALYSIS
-          when /ARCHITECTURE_ANALYSIS/
-            TIMEOUT_ARCHITECTURE_ANALYSIS
-          when /TEST_ANALYSIS/
-            TIMEOUT_TEST_ANALYSIS
-          when /FUNCTIONALITY_ANALYSIS/
-            TIMEOUT_FUNCTIONALITY_ANALYSIS
-          when /DOCUMENTATION_ANALYSIS/
-            TIMEOUT_DOCUMENTATION_ANALYSIS
-          when /STATIC_ANALYSIS/
-            TIMEOUT_STATIC_ANALYSIS
-          when /REFACTORING_RECOMMENDATIONS/
-            TIMEOUT_REFACTORING_RECOMMENDATIONS
-          when /IMPLEMENTATION/
-            TIMEOUT_IMPLEMENTATION
-          else
-            nil # Use default
-          end
-        end
-      end
-
       # Check if we should skip permissions based on devcontainer configuration
       # Overrides base class to add logging and Claude-specific config check
       def should_skip_permissions?
@@ -243,54 +304,6 @@ module Aidp
         false
       end
 
-      # Parse stream-json output from Claude CLI
-      def parse_stream_json_output(output)
-        return output if output.nil? || output.empty?
-
-        # Stream-json output contains multiple JSON objects, one per line
-        # We want to extract the final content from the last complete message
-        lines = output.strip.split("\n")
-        content_parts = []
-
-        lines.each do |line|
-          next if line.strip.empty?
-
-          begin
-            json_obj = JSON.parse(line)
-
-            # Look for content in various possible structures
-            if json_obj["type"] == "content_block_delta" && json_obj["delta"] && json_obj["delta"]["text"]
-              content_parts << json_obj["delta"]["text"]
-            elsif json_obj["content"]&.is_a?(Array)
-              json_obj["content"].each do |content_item|
-                content_parts << content_item["text"] if content_item["text"]
-              end
-            elsif json_obj["message"] && json_obj["message"]["content"]
-              if json_obj["message"]["content"].is_a?(Array)
-                json_obj["message"]["content"].each do |content_item|
-                  content_parts << content_item["text"] if content_item["text"]
-                end
-              elsif json_obj["message"]["content"].is_a?(String)
-                content_parts << json_obj["message"]["content"]
-              end
-            end
-          rescue JSON::ParserError => e
-            debug_log("⚠️ Failed to parse JSON line: #{e.message}", level: :warn, data: {line: line})
-            # If JSON parsing fails, treat as plain text
-            content_parts << line
-          end
-        end
-
-        result = content_parts.join
-
-        # Fallback: if no content found in JSON, return original output
-        result.empty? ? output : result
-      rescue => e
-        debug_log("⚠️ Failed to parse stream-json output: #{e.message}", level: :warn)
-        # Return original output if parsing fails
-        output
-      end
-
       # Parse Claude MCP server list output
       def parse_claude_mcp_output(output)
         servers = []

data/lib/aidp/providers/base.rb

@@ -324,6 +324,42 @@ module Aidp
         end
       end
 
+      # Helper method for registry-based model discovery
+      #
+      # Providers that use the model registry can call this method to discover models
+      # based on a model family pattern.
+      #
+      # @param model_pattern [Regexp] Pattern to match model families
+      # @param provider_name [String] Name of the provider
+      # @return [Array<Hash>] Array of discovered models
+      def self.discover_models_from_registry(model_pattern, provider_name)
+        require_relative "../harness/model_registry"
+        registry = Aidp::Harness::ModelRegistry.new
+
+        # Get all models from registry that match the pattern
+        models = registry.all_families.filter_map do |family|
+          next unless model_pattern.match?(family)
+
+          info = registry.get_model_info(family)
+          next unless info
+
+          {
+            name: family,
+            family: family,
+            tier: info["tier"],
+            capabilities: info["capabilities"] || [],
+            context_window: info["context_window"],
+            provider: provider_name
+          }
+        end
+
+        Aidp.log_info("#{provider_name}_provider", "using registry models", count: models.size)
+        models
+      rescue => e
+        Aidp.log_debug("#{provider_name}_provider", "discovery failed", error: e.message)
+        []
+      end
+
       protected
 
       # Log message to job if in background mode
@@ -371,8 +407,6 @@ module Aidp
         (success_rate * 50) + ((1 - rate_limit_ratio) * 30) + (response_time_score * 0.2)
       end
 
-      protected
-
       # Update spinner status with elapsed time
       # This is a shared method used by all providers to display progress
       def update_spinner_status(spinner, elapsed, provider_name)
@@ -419,6 +453,68 @@ module Aidp
         false # Base implementation returns false, subclasses should override
       end
 
+      # Calculate timeout for provider operations
+      #
+      # Priority order:
+      # 1. Quick mode (for testing)
+      # 2. Provider-specific environment variable override
+      # 3. Adaptive timeout based on step type
+      # 4. Default timeout
+      #
+      # Override provider_env_var to customize the environment variable name
+      def calculate_timeout
+        if ENV["AIDP_QUICK_MODE"]
+          display_message("⚡ Quick mode enabled - #{TIMEOUT_QUICK_MODE / 60} minute timeout", type: :highlight)
+          return TIMEOUT_QUICK_MODE
+        end
+
+        provider_env_var = "AIDP_#{name.upcase}_TIMEOUT"
+        return ENV[provider_env_var].to_i if ENV[provider_env_var]
+
+        step_timeout = adaptive_timeout
+        if step_timeout
+          display_message("🧠 Using adaptive timeout: #{step_timeout} seconds", type: :info)
+          return step_timeout
+        end
+
+        # Default timeout
+        display_message("📋 Using default timeout: #{TIMEOUT_DEFAULT / 60} minutes", type: :info)
+        TIMEOUT_DEFAULT
+      end
+
+      # Get adaptive timeout based on step type
+      #
+      # This method returns different timeout values based on the type of operation
+      # being performed, as indicated by the AIDP_CURRENT_STEP environment variable.
+      # Returns nil for unknown steps to allow calculate_timeout to use the default.
+      def adaptive_timeout
+        @adaptive_timeout ||= begin
+          # Timeout recommendations based on step type patterns
+          step_name = ENV["AIDP_CURRENT_STEP"] || ""
+
+          case step_name
+          when /REPOSITORY_ANALYSIS/
+            TIMEOUT_REPOSITORY_ANALYSIS
+          when /ARCHITECTURE_ANALYSIS/
+            TIMEOUT_ARCHITECTURE_ANALYSIS
+          when /TEST_ANALYSIS/
+            TIMEOUT_TEST_ANALYSIS
+          when /FUNCTIONALITY_ANALYSIS/
+            TIMEOUT_FUNCTIONALITY_ANALYSIS
+          when /DOCUMENTATION_ANALYSIS/
+            TIMEOUT_DOCUMENTATION_ANALYSIS
+          when /STATIC_ANALYSIS/
+            TIMEOUT_STATIC_ANALYSIS
+          when /REFACTORING_RECOMMENDATIONS/
+            TIMEOUT_REFACTORING_RECOMMENDATIONS
+          when /IMPLEMENTATION/
+            TIMEOUT_IMPLEMENTATION
+          else
+            nil # Return nil for unknown steps
+          end
+        end
+      end
+
       private
     end
   end

data/lib/aidp/providers/capability_registry.rb

@@ -16,7 +16,6 @@ module Aidp
         :supports_tool_use,    # Boolean: supports tool/function calling
         :supports_vision,      # Boolean: supports image/vision inputs
         :supports_file_upload, # Boolean: supports file uploads
-        :streaming,            # Boolean: supports streaming responses
         :supports_mcp,         # Boolean: supports Model Context Protocol
         :max_tokens,           # Maximum tokens per response
         :supports_dangerous_mode # Boolean: supports elevated permissions mode

data/lib/aidp/providers/codex.rb

@@ -10,10 +10,35 @@ module Aidp
     class Codex < Base
       include Aidp::DebugMixin
 
+      # Model name pattern for OpenAI models (since Codex uses OpenAI)
+      MODEL_PATTERN = /^gpt-[\d.o-]+(?:-turbo)?(?:-mini)?$/i
+      LONG_PROMPT_THRESHOLD = 8000
+      LONG_PROMPT_TIMEOUT = 900 # 15 minutes for large prompts
+
       def self.available?
         !!Aidp::Util.which("codex")
       end
 
+      # Check if this provider supports a given model family
+      #
+      # @param family_name [String] The model family name
+      # @return [Boolean] True if it matches OpenAI model pattern
+      def self.supports_model_family?(family_name)
+        MODEL_PATTERN.match?(family_name)
+      end
+
+      # Discover available models from registry
+      #
+      # Note: Codex CLI doesn't have a standard model listing command
+      # Returns registry-based models that match OpenAI patterns
+      #
+      # @return [Array<Hash>] Array of discovered models
+      def self.discover_models
+        return [] unless available?
+
+        discover_models_from_registry(MODEL_PATTERN, "codex")
+      end
+
       def name
         "codex"
       end
@@ -37,18 +62,13 @@ module Aidp
       def send_message(prompt:, session: nil)
         raise "codex CLI not available" unless self.class.available?
 
-        # Smart timeout calculation
+        # Smart timeout calculation (store prompt length for adaptive logic)
+        @current_codex_prompt_length = prompt.length
         timeout_seconds = calculate_timeout
 
         debug_provider("codex", "Starting execution", {timeout: timeout_seconds})
         debug_log("📝 Sending prompt to codex (length: #{prompt.length})", level: :info)
 
-        # Check if streaming mode is enabled
-        streaming_enabled = ENV["AIDP_STREAMING"] == "1" || ENV["DEBUG"] == "1"
-        if streaming_enabled
-          display_message("📺 Streaming mode enabled - output will appear in real-time", type: :info)
-        end
-
         # Set up activity monitoring
         setup_activity_monitoring("codex", method(:activity_callback))
         record_activity("Starting codex execution")
@@ -94,7 +114,7 @@ module Aidp
           end
 
           # Use debug_execute_command for better debugging
-          result = debug_execute_command("codex", args: args, timeout: timeout_seconds, streaming: streaming_enabled)
+          result = debug_execute_command("codex", args: args, timeout: timeout_seconds)
 
           # Log the results
           debug_command("codex", args: args, input: prompt, output: result.out, error: result.err, exit_code: result.exit_status)
@@ -116,6 +136,7 @@ module Aidp
           raise
         ensure
           cleanup_activity_display(activity_display_thread, spinner)
+          @current_codex_prompt_length = nil
         end
       end
 
@@ -161,22 +182,17 @@ module Aidp
 
       # Internal helper for send_with_options - executes with custom arguments
       def send_with_custom_args(prompt:, args:)
+        @current_codex_prompt_length = prompt.length
         timeout_seconds = calculate_timeout
 
         debug_provider("codex", "Starting execution", {timeout: timeout_seconds, args: args})
         debug_log("📝 Sending prompt to codex with custom args", level: :info)
 
-        # Check if streaming mode is enabled
-        streaming_enabled = ENV["AIDP_STREAMING"] == "1" || ENV["DEBUG"] == "1"
-        if streaming_enabled
-          display_message("📺 Display streaming enabled - output buffering reduced (codex CLI does not support true streaming)", type: :info)
-        end
-
         setup_activity_monitoring("codex", method(:activity_callback))
         record_activity("Starting codex execution with custom args")
 
         begin
-          result = debug_execute_command("codex", args: args, timeout: timeout_seconds, streaming: streaming_enabled)
+          result = debug_execute_command("codex", args: args, timeout: timeout_seconds)
           debug_command("codex", args: args, output: result.out, error: result.err, exit_code: result.exit_status)
 
           if result.exit_status == 0
@@ -191,58 +207,8 @@ module Aidp
           mark_failed("codex execution failed: #{e.message}")
           debug_error(e, {provider: "codex", prompt_length: prompt.length})
           raise
-        end
-      end
-
-      def calculate_timeout
-        # Priority order for timeout calculation:
-        # 1. Quick mode (for testing)
-        # 2. Environment variable override
-        # 3. Adaptive timeout based on step type
-        # 4. Default timeout
-
-        if ENV["AIDP_QUICK_MODE"]
-          display_message("⚡ Quick mode enabled - #{TIMEOUT_QUICK_MODE / 60} minute timeout", type: :highlight)
-          return TIMEOUT_QUICK_MODE
-        end
-
-        if ENV["AIDP_CODEX_TIMEOUT"]
-          return ENV["AIDP_CODEX_TIMEOUT"].to_i
-        end
-
-        if adaptive_timeout
-          display_message("🧠 Using adaptive timeout: #{adaptive_timeout} seconds", type: :info)
-          return adaptive_timeout
-        end
-
-        # Default timeout
-        display_message("📋 Using default timeout: #{TIMEOUT_DEFAULT / 60} minutes", type: :info)
-        TIMEOUT_DEFAULT
-      end
-
-      def adaptive_timeout
-        @adaptive_timeout ||= begin
-          # Timeout recommendations based on step type patterns
-          step_name = ENV["AIDP_CURRENT_STEP"] || ""
-
-          case step_name
-          when /REPOSITORY_ANALYSIS/
-            TIMEOUT_REPOSITORY_ANALYSIS
-          when /ARCHITECTURE_ANALYSIS/
-            TIMEOUT_ARCHITECTURE_ANALYSIS
-          when /TEST_ANALYSIS/
-            TIMEOUT_TEST_ANALYSIS
-          when /FUNCTIONALITY_ANALYSIS/
-            TIMEOUT_FUNCTIONALITY_ANALYSIS
-          when /DOCUMENTATION_ANALYSIS/
-            TIMEOUT_DOCUMENTATION_ANALYSIS
-          when /STATIC_ANALYSIS/
-            TIMEOUT_STATIC_ANALYSIS
-          when /REFACTORING_RECOMMENDATIONS/
-            TIMEOUT_REFACTORING_RECOMMENDATIONS
-          else
-            nil # Use default
-          end
+        ensure
+          @current_codex_prompt_length = nil
         end
       end
 
@@ -257,6 +223,22 @@ module Aidp
           display_message("\n❌ Codex CLI failed: #{message}", type: :error)
         end
       end
+
+      def calculate_timeout
+        env_override = ENV["AIDP_CODEX_TIMEOUT"]
+        return env_override.to_i if env_override&.match?(/^\d+$/)
+
+        base_timeout = super
+
+        prompt_length = @current_codex_prompt_length
+        return base_timeout unless prompt_length && prompt_length >= LONG_PROMPT_THRESHOLD
+
+        extended_timeout = [base_timeout, LONG_PROMPT_TIMEOUT].max
+        if extended_timeout > base_timeout
+          display_message("⏱️  Codex prompt length #{prompt_length} detected - extending timeout to #{extended_timeout} seconds", type: :info)
+        end
+        extended_timeout
+      end
     end
   end
 end