ruby_llm 1.0.1 → 1.1.0rc1

data/lib/ruby_llm/models.rb

@@ -26,10 +26,21 @@ module RubyLLM
         File.expand_path('models.json', __dir__)
       end
 
-      # Class method to refresh model data
-      def refresh!
-        models = RubyLLM.providers.flat_map(&:list_models).sort_by(&:id)
-        @instance = new(models)
+      def refresh! # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        configured = Provider.configured_providers
+
+        # Log provider status
+        skipped = Provider.providers.values - configured
+        RubyLLM.logger.info "Refreshing models from #{configured.map(&:slug).join(', ')}" if configured.any?
+        RubyLLM.logger.info "Skipping #{skipped.map(&:slug).join(', ')} - providers not configured" if skipped.any?
+
+        # Store current models except from configured providers
+        current = instance.load_models
+        preserved = current.reject { |m| configured.map(&:slug).include?(m.provider) }
+
+        all = (preserved + configured.flat_map(&:list_models)).sort_by(&:id)
+        @instance = new(all)
+        @instance
       end
 
       def method_missing(method, ...)
@@ -52,10 +63,10 @@ module RubyLLM
 
     # Load models from the JSON file
     def load_models
-      data = JSON.parse(File.read(self.class.models_file))
-      data.map { |model| ModelInfo.new(model.transform_keys(&:to_sym)) }
-    rescue Errno::ENOENT
-      [] # Return empty array if file doesn't exist yet
+      data = File.exist?(self.class.models_file) ? File.read(self.class.models_file) : '[]'
+      JSON.parse(data).map { |model| ModelInfo.new(model.transform_keys(&:to_sym)) }
+    rescue JSON::ParserError
+      []
     end
 
     def save_models
@@ -73,9 +84,12 @@ module RubyLLM
     end
 
     # Find a specific model by ID
-    def find(model_id)
-      all.find { |m| m.id == model_id } or
-        raise ModelNotFoundError, "Unknown model: #{model_id}"
+    def find(model_id, provider = nil)
+      if provider
+        find_with_provider(model_id, provider)
+      else
+        find_without_provider(model_id)
+      end
     end
 
     # Filter to only chat models
@@ -112,5 +126,20 @@ module RubyLLM
     def refresh!
       self.class.refresh!
     end
+
+    private
+
+    def find_with_provider(model_id, provider)
+      resolved_id = Aliases.resolve(model_id, provider)
+      all.find { |m| m.id == model_id && m.provider == provider.to_s } ||
+        all.find { |m| m.id == resolved_id && m.provider == provider.to_s } ||
+        raise(ModelNotFoundError, "Unknown model: #{model_id} for provider: #{provider}")
+    end
+
+    def find_without_provider(model_id)
+      all.find { |m| m.id == model_id } ||
+        all.find { |m| m.id == Aliases.resolve(model_id) } ||
+        raise(ModelNotFoundError, "Unknown model: #{model_id}")
+    end
   end
 end

data/lib/ruby_llm/provider.rb

@@ -7,7 +7,9 @@ module RubyLLM
   module Provider
     # Common functionality for all LLM providers. Implements the core provider
     # interface so specific providers only need to implement a few key methods.
-    module Methods # rubocop:disable Metrics/ModuleLength
+    module Methods
+      extend Streaming
+
       def complete(messages, tools:, temperature:, model:, &block) # rubocop:disable Metrics/MethodLength
         normalized_temperature = if capabilities.respond_to?(:normalize_temperature)
                                    capabilities.normalize_temperature(temperature, model)
@@ -49,24 +51,35 @@ module RubyLLM
         parse_image_response(response)
       end
 
+      def configured?
+        missing_configs.empty?
+      end
+
       private
 
-      def sync_response(payload)
-        response = post completion_url, payload
-        parse_completion_response response
+      def missing_configs
+        configuration_requirements.select do |key|
+          value = RubyLLM.config.send(key)
+          value.nil? || value.empty?
+        end
       end
 
-      def stream_response(payload, &block)
-        accumulator = StreamAccumulator.new
+      def ensure_configured!
+        return if configured?
 
-        post stream_url, payload do |req|
-          req.options.on_data = handle_stream do |chunk|
-            accumulator.add chunk
-            block.call chunk
+        config_block = <<~RUBY
+          RubyLLM.configure do |config|
+            #{missing_configs.map { |key| "config.#{key} = ENV['#{key.to_s.upcase}']" }.join("\n  ")}
           end
-        end
+        RUBY
+
+        raise ConfigurationError,
+              "#{slug} provider is not configured. Add this to your initialization:\n\n#{config_block}"
+      end
 
-        accumulator.to_message
+      def sync_response(payload)
+        response = post completion_url, payload
+        parse_completion_response response
       end
 
       def post(url, payload)
@@ -77,6 +90,8 @@ module RubyLLM
       end
 
       def connection # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        ensure_configured!
+
         @connection ||= Faraday.new(api_base) do |f| # rubocop:disable Metrics/BlockLength
           f.options.timeout = RubyLLM.config.request_timeout
 
@@ -115,33 +130,6 @@ module RubyLLM
           f.use :llm_errors, provider: self
         end
       end
-
-      def to_json_stream(&block) # rubocop:disable Metrics/MethodLength
-        buffer = String.new
-        parser = EventStreamParser::Parser.new
-
-        proc do |chunk, _bytes, env|
-          if env && env.status != 200
-            # Accumulate error chunks
-            buffer << chunk
-            begin
-              error_data = JSON.parse(buffer)
-              error_response = env.merge(body: error_data)
-              ErrorMiddleware.parse_error(provider: self, response: error_response)
-            rescue JSON::ParserError
-              # Keep accumulating if we don't have complete JSON yet
-              RubyLLM.logger.debug "Accumulating error chunk: #{chunk}"
-            end
-          else
-            parser.feed(chunk) do |_type, data|
-              unless data == '[DONE]'
-                parsed_data = JSON.parse(data)
-                block.call(parsed_data)
-              end
-            end
-          end
-        end
-      end
     end
 
     def try_parse_json(maybe_json)
@@ -181,6 +169,7 @@ module RubyLLM
     class << self
       def extended(base)
         base.extend(Methods)
+        base.extend(Streaming)
       end
 
       def register(name, provider_module)
@@ -195,6 +184,10 @@ module RubyLLM
       def providers
         @providers ||= {}
       end
+
+      def configured_providers
+        providers.select { |_name, provider| provider.configured? }.values
+      end
     end
   end
 end

data/lib/ruby_llm/providers/anthropic/capabilities.rb

@@ -20,8 +20,8 @@ module RubyLLM
         # @return [Integer] the maximum output tokens
         def determine_max_tokens(model_id)
           case model_id
-          when /claude-3-(7-sonnet|5)/ then 8_192 # Can be increased to 64K with extended thinking
-          else 4_096 # Claude 3 Opus and Haiku
+          when /claude-3-7-sonnet/, /claude-3-5/ then 8_192
+          else 4_096
           end
         end
 
@@ -92,13 +92,12 @@ module RubyLLM
 
         # Pricing information for Anthropic models (per million tokens)
         PRICES = {
-          claude37_sonnet: { input: 3.0, output: 15.0 },   # $3.00/$15.00 per million tokens
-          claude35_sonnet: { input: 3.0, output: 15.0 },   # $3.00/$15.00 per million tokens
-          claude35_haiku: { input: 0.80, output: 4.0 },    # $0.80/$4.00 per million tokens
-          claude3_opus: { input: 15.0, output: 75.0 },     # $15.00/$75.00 per million tokens
-          claude3_sonnet: { input: 3.0, output: 15.0 },    # $3.00/$15.00 per million tokens
-          claude3_haiku: { input: 0.25, output: 1.25 },    # $0.25/$1.25 per million tokens
-          claude2: { input: 3.0, output: 15.0 }            # Default pricing for Claude 2.x models
+          claude37_sonnet: { input: 3.0, output: 15.0 },
+          claude35_sonnet: { input: 3.0, output: 15.0 },
+          claude35_haiku: { input: 0.80, output: 4.0 },
+          claude3_opus: { input: 15.0, output: 75.0 },
+          claude3_haiku: { input: 0.25, output: 1.25 },
+          claude2: { input: 3.0, output: 15.0 }
         }.freeze
 
         # Default input price if model not found in PRICES

data/lib/ruby_llm/providers/anthropic/chat.rb

@@ -12,15 +12,42 @@ module RubyLLM
         end
 
         def render_payload(messages, tools:, temperature:, model:, stream: false)
+          system_messages, chat_messages = separate_messages(messages)
+          system_content = build_system_content(system_messages)
+
+          build_base_payload(chat_messages, temperature, model, stream).tap do |payload|
+            add_optional_fields(payload, system_content:, tools:)
+          end
+        end
+
+        def separate_messages(messages)
+          messages.partition { |msg| msg.role == :system }
+        end
+
+        def build_system_content(system_messages)
+          if system_messages.length > 1
+            RubyLLM.logger.warn(
+              "Anthropic's Claude implementation only supports a single system message. " \
+              'Multiple system messages will be combined into one.'
+            )
+          end
+
+          system_messages.map { |msg| format_message(msg)[:content] }.join("\n\n")
+        end
+
+        def build_base_payload(chat_messages, temperature, model, stream)
           {
             model: model,
-            messages: messages.map { |msg| format_message(msg) },
+            messages: chat_messages.map { |msg| format_message(msg) },
             temperature: temperature,
             stream: stream,
             max_tokens: RubyLLM.models.find(model).max_tokens
-          }.tap do |payload|
-            payload[:tools] = tools.values.map { |t| function_for(t) } if tools.any?
-          end
+          }
+        end
+
+        def add_optional_fields(payload, system_content:, tools:)
+          payload[:tools] = tools.values.map { |t| function_for(t) } if tools.any?
+          payload[:system] = system_content unless system_content.empty?
         end
 
         def parse_completion_response(response)

data/lib/ruby_llm/providers/anthropic/streaming.rb

@@ -11,12 +11,6 @@ module RubyLLM
           completion_url
         end
 
-        def handle_stream(&block)
-          to_json_stream do |data|
-            block.call(build_chunk(data))
-          end
-        end
-
         def build_chunk(data)
           Chunk.new(
             role: :assistant,
@@ -31,6 +25,18 @@ module RubyLLM
         def json_delta?(data)
           data['type'] == 'content_block_delta' && data.dig('delta', 'type') == 'input_json_delta'
         end
+
+        def parse_streaming_error(data)
+          error_data = JSON.parse(data)
+          return unless error_data['type'] == 'error'
+
+          case error_data.dig('error', 'type')
+          when 'overloaded_error'
+            [529, error_data['error']['message']]
+          else
+            [500, error_data['error']['message']]
+          end
+        end
       end
     end
   end

data/lib/ruby_llm/providers/anthropic.rb

@@ -33,6 +33,10 @@ module RubyLLM
       def slug
         'anthropic'
       end
+
+      def configuration_requirements
+        %i[anthropic_api_key]
+      end
     end
   end
 end

data/lib/ruby_llm/providers/bedrock/capabilities.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    module Bedrock
+      # Determines capabilities and pricing for AWS Bedrock models
+      module Capabilities
+        module_function
+
+        # Returns the context window size for the given model ID
+        # @param model_id [String] the model identifier
+        # @return [Integer] the context window size in tokens
+        def context_window_for(model_id)
+          case model_id
+          when /anthropic\.claude-2/ then 100_000
+          else 200_000
+          end
+        end
+
+        # Returns the maximum output tokens for the given model ID
+        # @param model_id [String] the model identifier
+        # @return [Integer] the maximum output tokens
+        def max_tokens_for(_model_id)
+          4_096
+        end
+
+        # Returns the input price per million tokens for the given model ID
+        # @param model_id [String] the model identifier
+        # @return [Float] the price per million tokens for input
+        def input_price_for(model_id)
+          PRICES.dig(model_family(model_id), :input) || default_input_price
+        end
+
+        # Returns the output price per million tokens for the given model ID
+        # @param model_id [String] the model identifier
+        # @return [Float] the price per million tokens for output
+        def output_price_for(model_id)
+          PRICES.dig(model_family(model_id), :output) || default_output_price
+        end
+
+        # Determines if the model supports chat capabilities
+        # @param model_id [String] the model identifier
+        # @return [Boolean] true if the model supports chat
+        def supports_chat?(model_id)
+          model_id.match?(/anthropic\.claude/)
+        end
+
+        # Determines if the model supports streaming capabilities
+        # @param model_id [String] the model identifier
+        # @return [Boolean] true if the model supports streaming
+        def supports_streaming?(model_id)
+          model_id.match?(/anthropic\.claude/)
+        end
+
+        # Determines if the model supports image input/output
+        # @param model_id [String] the model identifier
+        # @return [Boolean] true if the model supports images
+        def supports_images?(model_id)
+          model_id.match?(/anthropic\.claude/)
+        end
+
+        # Determines if the model supports vision capabilities
+        # @param model_id [String] the model identifier
+        # @return [Boolean] true if the model supports vision
+        def supports_vision?(model_id)
+          model_id.match?(/anthropic\.claude/)
+        end
+
+        # Determines if the model supports function calling
+        # @param model_id [String] the model identifier
+        # @return [Boolean] true if the model supports functions
+        def supports_functions?(model_id)
+          model_id.match?(/anthropic\.claude/)
+        end
+
+        # Determines if the model supports audio input/output
+        # @param model_id [String] the model identifier
+        # @return [Boolean] true if the model supports audio
+        def supports_audio?(_model_id)
+          false
+        end
+
+        # Determines if the model supports JSON mode
+        # @param model_id [String] the model identifier
+        # @return [Boolean] true if the model supports JSON mode
+        def supports_json_mode?(model_id)
+          model_id.match?(/anthropic\.claude/)
+        end
+
+        # Formats the model ID into a human-readable display name
+        # @param model_id [String] the model identifier
+        # @return [String] the formatted display name
+        def format_display_name(model_id)
+          model_id.then { |id| humanize(id) }
+        end
+
+        # Determines the type of model
+        # @param model_id [String] the model identifier
+        # @return [String] the model type (chat, embedding, image, audio)
+        def model_type(_model_id)
+          'chat'
+        end
+
+        # Determines if the model supports structured output
+        # @param model_id [String] the model identifier
+        # @return [Boolean] true if the model supports structured output
+        def supports_structured_output?(model_id)
+          model_id.match?(/anthropic\.claude/)
+        end
+
+        # Model family patterns for capability lookup
+        MODEL_FAMILIES = {
+          /anthropic\.claude-3-opus/ => :claude3_opus,
+          /anthropic\.claude-3-sonnet/ => :claude3_sonnet,
+          /anthropic\.claude-3-5-sonnet/ => :claude3_sonnet,
+          /anthropic\.claude-3-7-sonnet/ => :claude3_sonnet,
+          /anthropic\.claude-3-haiku/ => :claude3_haiku,
+          /anthropic\.claude-3-5-haiku/ => :claude3_5_haiku,
+          /anthropic\.claude-v2/ => :claude2,
+          /anthropic\.claude-instant/ => :claude_instant
+        }.freeze
+
+        # Determines the model family for pricing and capability lookup
+        # @param model_id [String] the model identifier
+        # @return [Symbol] the model family identifier
+        def model_family(model_id)
+          MODEL_FAMILIES.find { |pattern, _family| model_id.match?(pattern) }&.last || :other
+        end
+
+        # Pricing information for Bedrock models (per million tokens)
+        PRICES = {
+          claude3_opus: { input: 15.0, output: 75.0 },
+          claude3_sonnet: { input: 3.0, output: 15.0 },
+          claude3_haiku: { input: 0.25, output: 1.25 },
+          claude3_5_haiku: { input: 0.8, output: 4.0 },
+          claude2: { input: 8.0, output: 24.0 },
+          claude_instant: { input: 0.8, output: 2.4 }
+        }.freeze
+
+        # Default input price when model-specific pricing is not available
+        # @return [Float] the default price per million tokens
+        def default_input_price
+          0.1
+        end
+
+        # Default output price when model-specific pricing is not available
+        # @return [Float] the default price per million tokens
+        def default_output_price
+          0.2
+        end
+
+        private
+
+        # Converts a model ID to a human-readable format
+        # @param id [String] the model identifier
+        # @return [String] the humanized model name
+        def humanize(id)
+          id.tr('-', ' ')
+            .split('.')
+            .last
+            .split
+            .map(&:capitalize)
+            .join(' ')
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/bedrock/chat.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    module Bedrock
+      # Chat methods for the AWS Bedrock API implementation
+      module Chat
+        private
+
+        def completion_url
+          "model/#{@model_id}/invoke"
+        end
+
+        def render_payload(messages, tools:, temperature:, model:, stream: false) # rubocop:disable Lint/UnusedMethodArgument
+          # Hold model_id in instance variable for use in completion_url and stream_url
+          @model_id = model
+
+          system_messages, chat_messages = separate_messages(messages)
+          system_content = build_system_content(system_messages)
+
+          build_base_payload(chat_messages, temperature, model).tap do |payload|
+            add_optional_fields(payload, system_content:, tools:)
+          end
+        end
+
+        def separate_messages(messages)
+          messages.partition { |msg| msg.role == :system }
+        end
+
+        def build_system_content(system_messages)
+          if system_messages.length > 1
+            RubyLLM.logger.warn(
+              "Amazon Bedrock's Claude implementation only supports a single system message. " \
+              'Multiple system messages will be combined into one.'
+            )
+          end
+
+          system_messages.map { |msg| format_message(msg)[:content] }.join("\n\n")
+        end
+
+        def build_base_payload(chat_messages, temperature, model)
+          {
+            anthropic_version: 'bedrock-2023-05-31',
+            messages: chat_messages.map { |msg| format_message(msg) },
+            temperature: temperature,
+            max_tokens: RubyLLM.models.find(model).max_tokens
+          }
+        end
+
+        def add_optional_fields(payload, system_content:, tools:)
+          payload[:tools] = tools.values.map { |t| function_for(t) } if tools.any?
+          payload[:system] = system_content unless system_content.empty?
+        end
+
+        def format_message(msg)
+          if msg.tool_call?
+            format_tool_call(msg)
+          elsif msg.tool_result?
+            format_tool_result(msg)
+          else
+            format_basic_message(msg)
+          end
+        end
+
+        def format_basic_message(msg)
+          {
+            role: convert_role(msg.role),
+            content: Anthropic::Media.format_content(msg.content)
+          }
+        end
+
+        def convert_role(role)
+          case role
+          when :tool, :user then 'user'
+          when :system then 'system'
+          else 'assistant'
+          end
+        end
+
+        def parse_completion_response(response)
+          data = response.body
+          content_blocks = data['content'] || []
+
+          text_content = extract_text_content(content_blocks)
+          tool_use = find_tool_use(content_blocks)
+
+          build_message(data, text_content, tool_use)
+        end
+
+        def extract_text_content(blocks)
+          text_blocks = blocks.select { |c| c['type'] == 'text' }
+          text_blocks.map { |c| c['text'] }.join
+        end
+
+        def build_message(data, content, tool_use)
+          Message.new(
+            role: :assistant,
+            content: content,
+            tool_calls: parse_tool_calls(tool_use),
+            input_tokens: data.dig('usage', 'input_tokens'),
+            output_tokens: data.dig('usage', 'output_tokens'),
+            model_id: data['model']
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/bedrock/models.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    module Bedrock
+      # Models methods for the AWS Bedrock API implementation
+      module Models
+        def list_models
+          @connection = nil # reset connection since base url is different
+          @api_base = "https://bedrock.#{RubyLLM.config.bedrock_region}.amazonaws.com"
+          full_models_url = "#{@api_base}/#{models_url}"
+          signature = sign_request(full_models_url, method: :get)
+          response = connection.get(models_url) do |req|
+            req.headers.merge! signature.headers
+          end
+          @connection = nil # reset connection since base url is different
+
+          parse_list_models_response(response, slug, capabilities)
+        end
+
+        module_function
+
+        def models_url
+          'foundation-models'
+        end
+
+        def parse_list_models_response(response, slug, capabilities)
+          data = response.body['modelSummaries'] || []
+          data.filter { |model| model['modelId'].include?('claude') }
+              .map { |model| create_model_info(model, slug, capabilities) }
+        end
+
+        def create_model_info(model, slug, capabilities)
+          model_id = model['modelId']
+          ModelInfo.new(
+            **base_model_attributes(model_id, model, slug),
+            **capability_attributes(model_id, capabilities),
+            **pricing_attributes(model_id, capabilities),
+            metadata: build_metadata(model)
+          )
+        end
+
+        def base_model_attributes(model_id, model, slug)
+          {
+            id: model_id,
+            created_at: nil,
+            display_name: model['modelName'] || capabilities.format_display_name(model_id),
+            provider: slug
+          }
+        end
+
+        def capability_attributes(model_id, capabilities)
+          {
+            context_window: capabilities.context_window_for(model_id),
+            max_tokens: capabilities.max_tokens_for(model_id),
+            type: capabilities.model_type(model_id),
+            family: capabilities.model_family(model_id).to_s,
+            supports_vision: capabilities.supports_vision?(model_id),
+            supports_functions: capabilities.supports_functions?(model_id),
+            supports_json_mode: capabilities.supports_json_mode?(model_id)
+          }
+        end
+
+        def pricing_attributes(model_id, capabilities)
+          {
+            input_price_per_million: capabilities.input_price_for(model_id),
+            output_price_per_million: capabilities.output_price_for(model_id)
+          }
+        end
+
+        def build_metadata(model)
+          {
+            provider_name: model['providerName'],
+            customizations_supported: model['customizationsSupported'] || [],
+            inference_configurations: model['inferenceTypesSupported'] || [],
+            response_streaming_supported: model['responseStreamingSupported'] || false,
+            input_modalities: model['inputModalities'] || [],
+            output_modalities: model['outputModalities'] || []
+          }
+        end
+      end
+    end
+  end
+end