ruby_llm 1.16.0 → 2.0.0.rc2

data/lib/ruby_llm/protocols/interactions.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    # The Gemini Interactions API for conversations and hosted tools.
+    class Interactions < Protocol
+      include Interactions::Chat
+      include Interactions::Content
+      include Interactions::Tools
+      include Interactions::Streaming
+      include Interactions::Transcription
+
+      public :render
+
+      SERVER_TOOL_ALIASES = {
+        mcp: { tool: { type: 'mcp_server' } },
+        web_search: { tool: { type: 'google_search' } },
+        web_fetch: { tool: { type: 'url_context' } },
+        code_execution: { tool: { type: 'code_execution' } },
+        file_search: { tool: { type: 'file_search' } },
+        google_maps: { tool: { type: 'google_maps' } }
+      }.freeze
+
+      def server_tool_aliases
+        SERVER_TOOL_ALIASES
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/invoke_model/cohere_embeddings.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class InvokeModel
+      # Cohere embedding models over Bedrock InvokeModel.
+      class CohereEmbeddings < InvokeModel
+        # rubocop:disable-next Lint/UnusedMethodArgument
+        def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
+          ensure_no_embedding_media!(with)
+          track_usage(:embedding) do
+            payload = render_embedding_payload(text, model:, dimensions:, task_type:, provider_options:)
+            response = signed_post(embedding_url(model:), payload)
+
+            parse_embedding_response(response, model:, text:)
+          end
+        end
+
+        private
+
+        def render_embedding_payload(text, model:, dimensions:, provider_options:, task_type: nil)
+          payload = {
+            input_type: task_type || 'search_document'
+          }
+          texts = [text].flatten.compact.map(&:to_s).reject(&:empty?)
+          payload[:texts] = texts unless texts.empty?
+
+          if dimensions
+            raise Error, "#{model} does not support custom dimensions" unless cohere_v4?(model)
+
+            payload[:output_dimension] = dimensions
+          end
+
+          deep_merge_provider_options(payload, provider_options)
+        end
+
+        def parse_embedding_response(response, model:, text:)
+          vectors = response.body['embeddings']
+          vectors = vectors['float'] || vectors.values.first if vectors.is_a?(Hash)
+          vectors = vectors.first unless text.is_a?(Array)
+
+          Embedding.new(vectors:, model:)
+        end
+
+        def cohere_v4?(model)
+          model.to_s.include?('cohere.embed-v4')
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/invoke_model/embedding_batches.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class InvokeModel
+      # Titan embedding requests over Bedrock's S3 batch job API.
+      module EmbeddingBatches
+        include Protocols::Bedrock::Batches
+
+        private
+
+        def bedrock_invocation_type
+          'InvokeModel'
+        end
+
+        def validate_bedrock_batch_requests!(requests)
+          requests.each do |request|
+            values = request.fetch(:text)
+            values = [values] unless values.is_a?(Array)
+            unless values.any? && values.all? { |value| value.is_a?(String) && !value.empty? }
+              raise ArgumentError, 'Bedrock embedding batches require nonempty text strings'
+            end
+            unless request.fetch(:payload).key?(:inputText)
+              raise ArgumentError, 'Bedrock embedding batches require Titan embedding requests'
+            end
+          end
+        end
+
+        def bedrock_batch_jsonl(requests)
+          requests.flat_map do |request|
+            array = request.fetch(:text).is_a?(Array)
+            texts = array ? request.fetch(:text) : [request.fetch(:text)]
+            texts.each_with_index.map do |text, index|
+              id = "rllm-#{request.fetch(:custom_id)}-#{index}-#{texts.size}-#{array ? 'a' : 's'}"
+              JSON.generate(recordId: id, modelInput: batch_payload(request).merge(inputText: text))
+            end
+          end.join("\n")
+        end
+
+        def bedrock_job_name(input_uri, requests:)
+          "ruby-llm-embed-#{requests.size}-#{Digest::SHA256.hexdigest(input_uri)[0, 16]}"
+        end
+
+        def parse_batch_response(data)
+          super.merge(request_count: data['jobName'].to_s[/\Aruby-llm-embed-(\d+)-[0-9a-f]+\z/, 1]&.to_i)
+        end
+
+        def parse_bedrock_outputs(outputs, model:)
+          records = outputs.flat_map { |body| body.to_s.each_line.map { |line| JSON.parse(line) } }
+          groups = records.group_by { |record| embedding_record_id(record).first }
+          groups.filter_map { |index, group| parse_embedding_batch_group(index, group, model:) }
+        end
+
+        def embedding_record_id(record)
+          id = record['recordId'] || record['record_id']
+          match = /\Arllm-(\d+)-(\d+)-(\d+)-([as])\z/.match(id.to_s)
+          raise Error, "Unknown Bedrock embedding record ID: #{id.inspect}" unless match
+
+          [Integer(match[1]), Integer(match[2]), Integer(match[3]), match[4] == 'a']
+        end
+
+        def parse_embedding_batch_group(index, records, model:)
+          metadata = records.map { |record| embedding_record_id(record) }
+          count = metadata.first[2]
+          array = metadata.first[3]
+          error = embedding_group_error(records, metadata)
+          return [index, nil, batch_failure(index, error)] if error
+          return if records.size < count
+
+          ordered = records.sort_by { |record| embedding_record_id(record)[1] }
+          unless ordered.map { |record| embedding_record_id(record)[1] } == (0...count).to_a
+            return [index, nil, batch_failure(index, 'Invalid embedding record positions')]
+          end
+
+          parse_embedding_batch_result(index, ordered, model:, array:)
+        end
+
+        def embedding_group_error(records, metadata)
+          return 'Invalid or duplicate embedding record positions' unless consistent_embedding_metadata?(metadata)
+
+          error = records.find { |record| !record['modelOutput'] }
+          return unless error
+
+          batch_error_value(error['error']) || error['errorMessage'] || 'Bedrock returned no model output'
+        end
+
+        def consistent_embedding_metadata?(metadata)
+          metadata.map { |item| item[2..] }.uniq.one? &&
+            metadata.map { |item| item[1] }.uniq.size == metadata.size &&
+            (metadata.first[3] || metadata.first[2] == 1)
+        end
+
+        def parse_embedding_batch_result(index, records, model:, array:)
+          bodies = records.map { |record| record.fetch('modelOutput') }
+          vectors = embedding_batch_vectors(bodies)
+          return [index, nil, batch_failure(index, 'Bedrock returned no embedding')] unless vectors
+
+          counts = bodies.map { |body| body['inputTextTokenCount'] }
+          tokens = counts.sum if counts.all?
+          result = Embedding.new(vectors: array ? vectors : vectors.first, model:, input_tokens: tokens)
+          [index, result]
+        end
+
+        def embedding_batch_vectors(bodies)
+          vectors = bodies.map { |body| extract_embedding(body) }
+          vectors if vectors.all? { |vector| vector.is_a?(Array) && !vector.empty? && vector.all?(Numeric) }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/invoke_model/nova_embeddings.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class InvokeModel
+      # Amazon Nova multimodal embedding models over Bedrock InvokeModel.
+      class NovaEmbeddings < InvokeModel
+        DEFAULT_EMBEDDING_PURPOSE = 'GENERIC_INDEX'
+
+        # rubocop:disable-next Lint/UnusedMethodArgument
+        def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
+          ensure_no_embedding_media!(with)
+          track_usage(:embedding) do
+            responses = [text].flatten.map do |value|
+              payload = render_embedding_payload(value, dimensions:, task_type:, provider_options:)
+              signed_post(embedding_url(model:), payload).tap { |response| record_embedding_attempt(response) }
+            end
+
+            parse_single_embedding_responses(responses, model:, text:)
+          end
+        end
+
+        private
+
+        def render_embedding_payload(text, dimensions:, task_type:, provider_options:)
+          params = {
+            embeddingPurpose: task_type || DEFAULT_EMBEDDING_PURPOSE,
+            text: {
+              truncationMode: 'END',
+              value: text.to_s
+            }
+          }
+          params[:embeddingDimension] = dimensions if dimensions
+
+          deep_merge_provider_options(
+            {
+              taskType: 'SINGLE_EMBEDDING',
+              singleEmbeddingParams: params
+            },
+            provider_options
+          )
+        end
+
+        def extract_embedding(body)
+          body.dig('embeddings', 0, 'embedding')
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/invoke_model/stability_images.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class InvokeModel
+      # Stability image generation and inpainting over Bedrock InvokeModel.
+      class StabilityImages < InvokeModel
+        GENERATION_MODELS = %w[
+          stability.sd3-5-large-v1:0 stability.stable-image-core-v1:1 stability.stable-image-ultra-v1:1
+        ].freeze
+        INPAINT_MODEL = 'stability.stable-image-inpaint-v1:0'
+        MODELS = (GENERATION_MODELS + [INPAINT_MODEL]).freeze
+        ASPECT_RATIOS = %w[16:9 1:1 21:9 2:3 3:2 4:5 5:4 9:16 9:21].freeze
+        IMAGE_TYPES = %w[image/jpeg image/png image/webp].freeze
+
+        def paint(prompt, model:, size:, count: nil, with: nil, mask: nil, provider_options: {})
+          track_usage(:image) do
+            payload = render_image_payload(prompt, model:, size:, count:, with:, mask:, provider_options:)
+            response = signed_post("/model/#{model}/invoke", payload)
+            images = parse_image_responses(response, model:)
+            images.each { |image| image.config = @config }
+            images.one? ? images.first : images
+          end
+        end
+
+        def render_image_payload(prompt, model:, size:, count: nil, with: nil, mask: nil, provider_options: {})
+          RubyLLM.logger.debug { 'Stability image models return one image per request' } if count && count > 1
+          attachments = Attachment.wrap(with, config: @config)
+          payload = { prompt: }.merge(render_image_inputs(attachments, model:, mask:, size:))
+          payload.merge(provider_options.transform_keys(&:to_sym))
+        end
+
+        def parse_image_responses(response, model:)
+          images = Array(response.body['images']).compact.reject(&:empty?)
+          if images.empty?
+            reasons = Array(response.body['finish_reasons']).compact.join(', ')
+            message = reasons.empty? ? 'Bedrock returned no images' : "Bedrock image generation failed: #{reasons}"
+            raise Error.new(message, response:)
+          end
+
+          images.map do |data|
+            mime_type = RubyLLM::Files::MimeType.for(StringIO.new(Base64.decode64(data)))
+            raise Error.new('Bedrock returned invalid image data', response:) unless IMAGE_TYPES.include?(mime_type)
+
+            Image.new(data:, model:, mime_type:)
+          end
+        end
+
+        private
+
+        def render_image_inputs(attachments, model:, mask:, size:)
+          return render_inpaint_inputs(attachments, mask:, size:) if model.end_with?(INPAINT_MODEL)
+          raise UnsupportedAttachmentError, 'image mask' if mask
+          return { aspect_ratio: image_aspect_ratio(size) }.compact if attachments.empty?
+          raise UnsupportedAttachmentError, 'image reference' unless model.end_with?('stability.sd3-5-large-v1:0')
+
+          { image: single_image(attachments, size:), mode: 'image-to-image', strength: 0.5 }
+        end
+
+        def render_inpaint_inputs(attachments, mask:, size:)
+          payload = { image: single_image(attachments, size:) }
+          payload[:mask] = single_image(Attachment.wrap(mask, config: @config), size: nil) if mask
+          payload
+        end
+
+        def single_image(attachments, size:)
+          raise ArgumentError, 'with: must contain exactly one image' unless attachments.one?
+          raise ArgumentError, 'size: cannot change dimensions when editing a Stability image' if size
+
+          encoded_image(attachments.first)
+        end
+
+        def encoded_image(attachment)
+          raise UnsupportedAttachmentError, attachment.mime_type unless IMAGE_TYPES.include?(attachment.mime_type)
+
+          attachment.encoded
+        end
+
+        def image_aspect_ratio(size)
+          return unless size
+
+          match = size.to_s.match(/\A(\d+)\s*[x×:]\s*(\d+)\z/i)
+          raise ArgumentError, "Invalid image size: #{size.inspect}" unless match
+
+          width, height = match.captures.map(&:to_i)
+          raise ArgumentError, "Invalid image size: #{size.inspect}" unless width.positive? && height.positive?
+
+          ratio = matching_aspect_ratio(width, height)
+          raise ArgumentError, "Unsupported Stability image aspect ratio: #{size}" unless ratio
+
+          ratio
+        end
+
+        def matching_aspect_ratio(width, height)
+          ASPECT_RATIOS.find do |candidate|
+            ratio_width, ratio_height = candidate.split(':').map(&:to_i)
+            width * ratio_height == height * ratio_width
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/invoke_model/titan_multimodal_embeddings.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class InvokeModel
+      # Amazon Titan multimodal embedding models over Bedrock InvokeModel.
+      class TitanMultimodalEmbeddings < InvokeModel
+        # rubocop:disable-next Lint/UnusedMethodArgument
+        def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
+          ensure_no_embedding_media!(with)
+          track_usage(:embedding) do
+            responses = [text].flatten.map do |value|
+              payload = render_embedding_payload(value, dimensions:, provider_options:)
+              signed_post(embedding_url(model:), payload).tap { |response| record_embedding_attempt(response) }
+            end
+
+            parse_single_embedding_responses(responses, model:, text:)
+          end
+        end
+
+        private
+
+        def render_embedding_payload(text, dimensions:, provider_options:, **)
+          payload = {}
+          payload[:inputText] = text.to_s unless text.nil? || text.to_s.empty?
+          payload[:embeddingConfig] = { outputEmbeddingLength: dimensions } if dimensions
+
+          deep_merge_provider_options(payload, provider_options)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/invoke_model/titan_text_embeddings.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class InvokeModel
+      # Amazon Titan text embedding models over Bedrock InvokeModel.
+      class TitanTextEmbeddings < InvokeModel
+        # rubocop:disable-next Lint/UnusedMethodArgument
+        def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
+          ensure_no_embedding_media!(with)
+          track_usage(:embedding) do
+            responses = [text].flatten.map do |value|
+              payload = render_embedding_payload(value, model:, dimensions:, provider_options:)
+              signed_post(embedding_url(model:), payload).tap { |response| record_embedding_attempt(response) }
+            end
+
+            parse_single_embedding_responses(responses, model:, text:)
+          end
+        end
+
+        private
+
+        # The G1 and V1 models take inputText alone; Bedrock rejects the V2
+        # tuning keys as extraneous.
+        def render_embedding_payload(text, model:, dimensions:, provider_options:, **)
+          payload = { inputText: text.to_s }
+
+          if titan_v2?(model)
+            payload[:dimensions] = dimensions if dimensions
+            payload[:normalize] = true
+          elsif dimensions
+            raise Error, "#{model} does not support custom dimensions"
+          end
+
+          deep_merge_provider_options(payload, provider_options)
+        end
+
+        def titan_v2?(model)
+          model.to_s.include?('titan-embed-text-v2')
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/invoke_model.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    # The AWS InvokeModel API, where the envelope is shared but each model
+    # family owns its request and response body. Subclasses under
+    # Protocols::InvokeModel implement one family each. Requests are signed
+    # by the provider, which knows the credentials and the signing service.
+    class InvokeModel < Protocol
+      private
+
+      def signed_post(url, payload, additional_headers = {})
+        body = JSON.generate(payload)
+
+        @connection.post(url, payload, usage: @usage_tracker) do |req|
+          req.headers.merge!(@provider.sign_headers('POST', url, body))
+          req.headers.merge!(additional_headers) unless additional_headers.empty?
+        end
+      end
+
+      def embedding_url(model:)
+        "/model/#{model}/invoke"
+      end
+
+      def ensure_no_embedding_media!(with)
+        attachments = Attachment.wrap(with)
+        raise UnsupportedAttachmentError, attachments.first.mime_type if attachments.any?
+      end
+
+      def parse_single_embedding_responses(responses, model:, text:)
+        vectors = responses.map { |response| extract_embedding(response.body) }
+        input_tokens = Tokens.aggregate(embedding_attempt_tokens(responses)).input
+        vectors = vectors.first unless text.is_a?(Array)
+
+        Embedding.new(vectors:, model:, input_tokens:)
+      end
+
+      def record_embedding_attempt(response)
+        @usage_tracker&.succeed_attempts(tokens: embedding_attempt_tokens([response]))
+      end
+
+      def embedding_attempt_tokens(responses)
+        responses.map { |response| Tokens.new(input: response.body['inputTextTokenCount']) }
+      end
+
+      def deep_merge_provider_options(payload, provider_options)
+        return payload if provider_options.empty?
+
+        Support::Utils.deep_merge(payload, provider_options)
+      end
+
+      def extract_embedding(body)
+        body['embedding'] || body.dig('embeddingsByType', 'float') || body['embeddingsByType']&.values&.first
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/mistral/content.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    module Mistral
+      module Content # :nodoc:
+        module_function
+
+        def parse_conversation_content(output)
+          result = { text: +'', thinking: +'', attachments: [], citations: [] }
+          output.select { |entry| entry['type'] == 'message.output' }.each do |entry|
+            parse_conversation_parts(entry['content'], result)
+          end
+          result[:thinking] = nil if result[:thinking].empty?
+          result
+        end
+
+        def parse_conversation_parts(content, result)
+          return result[:text] << content if content.is_a?(String)
+
+          Array(content).compact.each do |part|
+            case part['type']
+            when 'text' then result[:text] << part['text'].to_s
+            when 'thinking' then result[:thinking] << Array(part['thinking']).filter_map { |item| item['text'] }.join
+            when 'tool_reference' then result[:citations] << parse_conversation_citation(part, result[:text].length)
+            when 'tool_file' then result[:attachments] << parse_conversation_file(part)
+            when 'image_url'
+              url = part['image_url'].is_a?(Hash) ? part['image_url']['url'] : part['image_url']
+              result[:attachments] << Attachment.new(url, config: @config)
+            end
+          end
+        end
+
+        def parse_conversation_citation(part, offset)
+          Citation.new(url: part['url'], title: part['title'], cited_text: part['description'],
+                       start_index: offset, end_index: offset)
+        end
+
+        def parse_conversation_file(part)
+          format = part['file_type'].to_s
+          mime_type = format.include?('/') ? format : RubyLLM::Files::MimeType.for(name: "file.#{format}")
+          file = UploadedFile.new(id: part.fetch('file_id'), provider: @provider.slug,
+                                  filename: part['file_name'], mime_type: mime_type, downloadable: true, metadata: part)
+          Attachment.new(file, config: @config)
+        end
+      end
+    end
+  end
+end