ruby_llm 2.1.1 → 2.2.0

data/lib/ruby_llm/protocols/converse.rb

@@ -16,15 +16,31 @@ module RubyLLM
       SERVER_TOOL_ALIASES = {
         web_search: lambda { |options|
           { tool: { systemTool: { name: 'nova_grounding' }.merge(Support::Utils.deep_symbolize_keys(options)) } }
-        }
+        },
+        code_execution: { tool: { systemTool: { name: 'nova_code_interpreter' } } },
+        code_interpreter: { tool: { systemTool: { name: 'nova_code_interpreter' } } }
       }.freeze
 
       def server_tool_aliases
         SERVER_TOOL_ALIASES
       end
 
+      def render(...)
+        keep_binding_beta(super)
+      end
+
       private
 
+      # Provider options replace an array, and Bedrock refuses the binding
+      # without its beta.
+      def keep_binding_beta(payload)
+        fields = payload[:additionalModelRequestFields]
+        return payload unless fields&.dig(:thinking, :block_binding)
+
+        fields[:anthropic_beta] = Array(fields[:anthropic_beta]) | [Anthropic::BlockBinding::BETA]
+        payload
+      end
+
       # Converse carries tools under toolConfig.tools rather than a
       # top-level tools array.
       def merge_server_tool_entries(payload, entries)

data/lib/ruby_llm/protocols/deepgram/speech.rb

@@ -51,8 +51,7 @@ module RubyLLM
                           .merge(provider_options).compact
         end
 
-        # rubocop:disable-next Lint/UnusedMethodArgument
-        def render_speech_payload(input, model:, voice: nil, format: nil, provider_options: {})
+        def render_speech_payload(input, **)
           { text: input }
         end
 

data/lib/ruby_llm/protocols/deepgram/streaming_transcription.rb

@@ -37,9 +37,7 @@ module RubyLLM
 
         def streaming_transcription_url(model:, language:, speaker_names:, provider_options:, prompt:)
           params = { model:, language:, smart_format: true, interim_results: true }
-          params[:diarize_model] = DIARIZE_MODEL if speaker_names
-          params[:keyterm] = Array(prompt) if prompt
-          params.merge!(provider_options)
+          params.merge!(transcription_hint_params(speaker_names:, prompt:), provider_options)
           params = params.compact.flat_map { |key, value| Array(value).map { |item| [key, item] } }
           uri = URI.join("#{@provider.api_base.sub(%r{/+\z}, '')}/", "v1/listen?#{URI.encode_www_form(params)}")
           uri.scheme = uri.scheme == 'https' ? 'wss' : 'ws'

data/lib/ruby_llm/protocols/deepgram/transcription.rb

@@ -7,9 +7,9 @@ module RubyLLM
     class Deepgram
       # Transcription dialect for the Deepgram speech-to-text API. Every
       # option is a query parameter on v1/listen, and the body is either the
-      # audio bytes or a JSON pointer to a remote url. Giving speaker names
-      # turns on diarization, which labels each word and utterance with a
-      # numeric speaker index.
+      # audio bytes or a JSON pointer to a remote url. Requesting speaker
+      # labels turns on diarization, which labels each word and utterance
+      # with a numeric speaker index.
       module Transcription
         def render_transcription_options(timestamps:, **)
           return {} if timestamps.nil? || timestamps == :word
@@ -17,6 +17,10 @@ module RubyLLM
           raise ArgumentError, 'Deepgram transcription timestamps must be word'
         end
 
+        def rejected_options(operation)
+          operation == :transcribe ? %i[format temperature speaker_references named_speakers] : super
+        end
+
         # Turned on by default so the transcript reads like the transcripts
         # every other provider returns. Smart formatting punctuates and
         # formats numbers, dates, and currency; utterances split the
@@ -28,36 +32,31 @@ module RubyLLM
         # it deprecates does not.
         DIARIZE_MODEL = 'latest'
 
+        # rubocop:disable-next Lint/UnusedMethodArgument
         def transcribe(audio_file, model:, language:, format: nil, speaker_names: nil,
                        speaker_references: nil, provider_options: {}, prompt: nil, temperature: nil, &block)
           if block
-            if format || speaker_references || temperature
-              raise ArgumentError,
-                    'Deepgram streaming transcription does not accept format, speaker_references or temperature'
-            end
-
             return stream_live_transcription(audio_file, model:, language:, speaker_names:, provider_options:, prompt:,
                                              &block)
           end
 
           track_usage(:transcription) do
-            attachment = Attachment.new(audio_file, config: @config)
-            url = transcription_url(model:, language:, speaker_names:, provider_options:)
+            attachment = audio_file.is_a?(Attachment) ? audio_file : Attachment.new(audio_file, config: @config)
+            url = transcription_url(model:, language:, speaker_names:, provider_options:, prompt:)
             payload = render_transcription_payload(attachment)
             response = post_transcription(url, payload, attachment)
-            parse_transcription_response(response, model:)
+            parse_transcription_response(response, model:, language:)
           end
         end
 
-        def transcription_url(model:, language: nil, speaker_names: nil, provider_options: {})
+        def transcription_url(model:, language: nil, speaker_names: nil, provider_options: {}, prompt: nil)
           "v1/listen?#{URI.encode_www_form(transcription_params(model:, language:, speaker_names:,
-                                                                provider_options:))}"
+                                                                provider_options:, prompt:))}"
         end
 
-        def transcription_params(model:, language: nil, speaker_names: nil, provider_options: {})
+        def transcription_params(model:, language: nil, speaker_names: nil, provider_options: {}, prompt: nil)
           params = { model: model, language: language }.merge(DEFAULT_PARAMS)
-          params[:diarize_model] = DIARIZE_MODEL if speaker_names
-          params.merge(provider_options).compact
+          params.merge(transcription_hint_params(speaker_names:, prompt:)).merge(provider_options).compact
         end
 
         # Deepgram fetches remote audio itself, so a url attachment is sent
@@ -68,16 +67,19 @@ module RubyLLM
           attachment.content
         end
 
-        def parse_transcription_response(response, model:)
+        def parse_transcription_response(response, model:, language: nil)
           data = response.body || {}
           channel = data.dig('results', 'channels', 0) || {}
           alternative = channel.dig('alternatives', 0) || {}
 
+          duration = data.dig('metadata', 'duration')
+
           RubyLLM::Transcription.new(
             text: alternative['transcript'],
             model: model,
-            language: channel['detected_language'],
-            duration: data.dig('metadata', 'duration'),
+            language: channel['detected_language'] || language,
+            duration:,
+            audio_seconds: duration,
             segments: data.dig('results', 'utterances'),
             words: alternative['words']
           )
@@ -85,6 +87,13 @@ module RubyLLM
 
         private
 
+        def transcription_hint_params(speaker_names:, prompt:)
+          params = {}
+          params[:diarize_model] = DIARIZE_MODEL if speaker_names
+          params[:keyterm] = Array(prompt) if prompt
+          params
+        end
+
         def post_transcription(url, payload, attachment)
           @connection.post(url, payload, usage: @usage_tracker) do |request|
             request.headers['Content-Type'] = attachment.mime_type || 'application/octet-stream' unless attachment.url?

data/lib/ruby_llm/protocols/deepgram.rb

@@ -3,8 +3,9 @@
 module RubyLLM
   module Protocols
     # The Deepgram API: speech to text on v1/listen, text to speech on
-    # v1/speak, and the model catalog behind them. Deepgram has no chat,
-    # embedding, or image endpoints, so those seams are left unimplemented.
+    # v1/speak, and the model catalog behind them. Deepgram has no embedding
+    # or image endpoints, so those seams are left unimplemented. Chat is a
+    # Chat Completions dialect, declared by the provider.
     #
     # Deepgram carries its request options in the query string rather than
     # the body, so +provider_options:+ joins the query on both audio

data/lib/ruby_llm/protocols/deepseek/files.rb

@@ -11,7 +11,7 @@ module RubyLLM
 
         private
 
-        def render_upload_payload(attachment, purpose: nil, **)
+        def render_upload_fields(attachment, purpose: nil, **)
           super(attachment, purpose: purpose || 'user_data', **)
         end
 

data/lib/ruby_llm/protocols/elevenlabs/assets.rb

@@ -10,7 +10,7 @@ module RubyLLM
           url = file.metadata['content_url']
           raise Error, 'ElevenLabs asset is still processing; retrieve it again before downloading' unless url
 
-          Transport::Connection.basic(@config).get(url).body
+          Transport::Connection.download(url, @config)
         end
 
         private
@@ -19,10 +19,12 @@ module RubyLLM
           'v1/assets'
         end
 
-        def render_upload_payload(attachment, purpose: nil, expires_in: nil, name: nil)
-          raise ArgumentError, 'ElevenLabs assets do not accept purpose or expires_in' if purpose || expires_in
+        def upload_file_field
+          :asset
+        end
 
-          { asset: file_part(attachment), name: name || attachment.filename }
+        def render_upload_fields(attachment, **)
+          { name: attachment.filename }
         end
 
         def parse_file_response(data)

data/lib/ruby_llm/protocols/elevenlabs/flows/images.rb

@@ -13,6 +13,10 @@ module RubyLLM
             @connection.post images_url, payload, usage: @usage_tracker, idempotent: false
           end
 
+          def rejected_options(operation)
+            operation == :paint ? %i[count] : super
+          end
+
           def render_image_payload(prompt, model:, size:, with: nil, mask: nil, provider_options: {}, **)
             images = Attachment.wrap(with, config: @config)
             payload = { model_id: model, prompt: }

data/lib/ruby_llm/protocols/elevenlabs/speech.rb

@@ -34,12 +34,12 @@ module RubyLLM
         end
 
         def speech_url(voice: nil, format: nil, streaming: false)
-          path = "v1/text-to-speech/#{voice || DEFAULT_VOICE}"
+          path = "v1/text-to-speech/#{URI.encode_uri_component(voice || DEFAULT_VOICE).gsub('.', '%2E')}"
           path += '/stream' if streaming
-          "#{path}?output_format=#{output_format_for(format)}"
+          "#{path}?output_format=#{URI.encode_uri_component(output_format_for(format))}"
         end
 
-        def render_speech_payload(input, model:, voice: nil, format: nil, provider_options: {}) # rubocop:disable Lint/UnusedMethodArgument
+        def render_speech_payload(input, model:, provider_options: {}, **)
           Support::Utils.deep_merge({ text: input, model_id: model }, provider_options)
         end
 

data/lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb

@@ -28,7 +28,8 @@ module RubyLLM
             write = ->(connection) { send_transcription_audio(connection, audio, commits) }
             socket.each_message(write:) do |message|
               event = JSON.parse(message)
-              transcript = process_transcription_event(event, prefix: transcripts.any?, &block)
+              prefix = transcripts.any? { |item| !item.fetch('text').empty? }
+              transcript = process_transcription_event(event, prefix:, &block)
               next unless transcript
 
               transcripts << transcript
@@ -45,7 +46,7 @@ module RubyLLM
 
         def build_streaming_transcription(transcripts, audio, model:)
           RubyLLM::Transcription.new(
-            text: transcripts.map { |item| item.fetch('text') }.join(' '), model:,
+            text: transcripts.map { |item| item.fetch('text') }.reject(&:empty?).join(' '), model:,
             duration: audio.duration, language: transcripts.last['language_code'],
             words: transcripts.flat_map { |item| item['words'] || [] }
           )
@@ -110,6 +111,8 @@ module RubyLLM
             yield TranscriptionChunk.new(type: TranscriptionChunk::PARTIAL, text: event['text'], raw: event)
           when 'committed_transcript'
             text = event['text']
+            return if text.to_s.empty?
+
             yield TranscriptionChunk.new(type: TranscriptionChunk::DELTA, delta: prefix ? " #{text}" : text, raw: event)
           when 'committed_transcript_with_timestamps'
             return event

data/lib/ruby_llm/protocols/elevenlabs/transcription.rb

@@ -20,10 +20,12 @@ module RubyLLM
           streaming ? { include_timestamps: true } : { timestamps_granularity: value }
         end
 
-        # rubocop:disable-next Lint/UnusedMethodArgument
+        def rejected_options(operation)
+          operation == :transcribe ? %i[prompt speaker_references] : super
+        end
+
         def render_transcription_payload(file_part, model:, language:, format: nil, speaker_names: nil,
-                                         speaker_references: nil, provider_options: {}, prompt: nil,
-                                         temperature: nil)
+                                         provider_options: {}, temperature: nil, **)
           payload = {
             model_id: model,
             file: file_part,
@@ -52,6 +54,7 @@ module RubyLLM
             model: model,
             language: data['language_code'],
             duration: data['audio_duration_secs'],
+            audio_seconds: data['audio_duration_secs'],
             words: data['words']
           )
         end

data/lib/ruby_llm/protocols/files.rb

@@ -6,16 +6,20 @@ module RubyLLM
   module Protocols
     # Provider-managed file storage APIs.
     class Files < Protocol
+      def rejected_options(operation)
+        operation == :upload ? %i[purpose expires_in uri] : super
+      end
+
       # rubocop:disable-next Lint/UnusedMethodArgument
       def upload(file, filename: nil, purpose: nil, expires_in: nil, uri: nil, content_type: nil, provider_options: {})
+        @content_type = content_type
         attachment = file_attachment(file, filename:)
-        options = { purpose:, expires_in: }.compact.merge(provider_options.transform_keys(&:to_sym))
-        response = @connection.post(files_url, render_upload_payload(attachment, **options),
-                                    idempotent: false) do |request|
+        payload = render_upload_payload(attachment, purpose:, expires_in:, provider_options:)
+        response = @connection.post(files_url, payload, idempotent: false) do |request|
           request.headers.delete('Content-Type')
           upload_headers(request)
         end
-        parse_file_response(response.body)
+        parse_upload_response(response.body, attachment)
       end
 
       def find(file_id)
@@ -24,7 +28,7 @@ module RubyLLM
       end
 
       def download(file_id)
-        response = @connection.get(download_file_url(file_id)) do |request|
+        response = @connection.get(download_file_url(file_id), raw: true) do |request|
           request.headers['Accept'] = 'application/octet-stream'
           file_headers(request)
         end
@@ -49,14 +53,25 @@ module RubyLLM
         "#{file_info_url(file_id)}/content"
       end
 
-      # rubocop:disable-next Lint/UnusedMethodArgument
-      def render_upload_payload(attachment, purpose: nil, expires_in: nil, visibility: nil,
-                                display_name: nil, uri: nil, content_type: nil)
-        { file: file_part(attachment) }
+      # The file part goes last: OpenAI can misread a field sent after a large
+      # file part, rejecting a valid purpose as "user_data\r", and xAI requires
+      # its fields to arrive before the file.
+      def render_upload_payload(attachment, purpose: nil, expires_in: nil, provider_options: {})
+        fields = render_upload_fields(attachment, purpose:, expires_in:)
+        fields.merge(provider_options.transform_keys(&:to_sym)).except(upload_file_field).compact
+              .merge(upload_file_field => file_part(attachment))
+      end
+
+      def render_upload_fields(_attachment, **)
+        {}
+      end
+
+      def upload_file_field
+        :file
       end
 
-      def multipart_payload(attachment, **fields)
-        { file: file_part(attachment) }.merge(fields.compact)
+      def parse_upload_response(data, _attachment)
+        parse_file_response(data)
       end
 
       def upload_headers(_request); end
@@ -76,7 +91,7 @@ module RubyLLM
       end
 
       def file_content_type(attachment)
-        attachment.extension == 'jsonl' ? 'application/jsonl' : attachment.mime_type
+        @content_type || (attachment.extension == 'jsonl' ? 'application/jsonl' : attachment.mime_type)
       end
 
       def file_part_source(attachment)

data/lib/ruby_llm/protocols/gemini/caches.rb

@@ -18,15 +18,15 @@ module RubyLLM
         def render_cache_payload(content, model:, ttl: nil, instructions: nil, attachments: [])
           payload = {
             model: cache_model_name(model),
-            contents: [{ role: 'user', parts: Media.format_content(content, attachments) }]
+            contents: [{ role: 'user', parts: Media.render_content(content, attachments) }]
           }
           payload[:systemInstruction] = { parts: [{ text: instructions }] } if instructions
-          payload[:ttl] = format_cache_ttl(ttl) if ttl
+          payload[:ttl] = render_cache_ttl(ttl) if ttl
           payload
         end
 
         def render_cache_update_payload(ttl:)
-          { ttl: format_cache_ttl(ttl) }
+          { ttl: render_cache_ttl(ttl) }
         end
 
         def parse_cache_response(data)
@@ -50,7 +50,7 @@ module RubyLLM
           "models/#{model_id}"
         end
 
-        def format_cache_ttl(ttl)
+        def render_cache_ttl(ttl)
           ttl.is_a?(String) ? ttl : "#{ttl.to_i}s"
         end
       end