ruby_llm 1.16.0 → 2.0.0.rc2

data/lib/ruby_llm/protocols/converse.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    # The AWS Bedrock Converse API. Requests are SigV4-signed by the provider.
+    class Converse < Protocol
+      REGION_PREFIXES = %w[global us-gov us eu apac ap sa ca me af il au jp].freeze
+
+      include Converse::Chat
+      include Converse::Media
+      include Converse::Streaming
+
+      # Nova 2 models execute built-in tools server-side when the request
+      # names them as system tools.
+      SERVER_TOOL_ALIASES = {
+        web_search: lambda { |options|
+          { tool: { systemTool: { name: 'nova_grounding' }.merge(Support::Utils.deep_symbolize_keys(options)) } }
+        }
+      }.freeze
+
+      def server_tool_aliases
+        SERVER_TOOL_ALIASES
+      end
+
+      private
+
+      # Converse carries tools under toolConfig.tools rather than a
+      # top-level tools array.
+      def merge_server_tool_entries(payload, entries)
+        tool_config = payload[:toolConfig] ||= {}
+        tool_config[:tools] = Array(tool_config[:tools]) + entries
+        payload
+      end
+
+      def sync_response(payload, additional_headers = {})
+        response = signed_post(completion_url, payload, additional_headers)
+        parse_completion_response(response)
+      end
+
+      def post_count_tokens(payload)
+        signed_post(count_tokens_url, payload)
+      end
+
+      def signed_post(url, payload, additional_headers = {})
+        body = JSON.generate(payload)
+
+        @connection.post(url, payload, usage: @usage_tracker) do |req|
+          req.headers.merge!(@provider.sign_headers('POST', url, body))
+          req.headers.merge!(additional_headers) unless additional_headers.empty?
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/deepgram/models.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Deepgram
+      # Model catalog for Deepgram. GET v1/models splits the catalog into
+      # stt and tts groups, and names each entry twice: +name+ is the short
+      # name, such as the voice 'zeus', while +canonical_name+ is the id the
+      # listen and speak endpoints accept.
+      module Models
+        module_function
+
+        def models_url
+          'v1/models'
+        end
+
+        def parse_list_models_response(response, slug)
+          body = response.body || {}
+
+          %w[stt tts].flat_map do |group|
+            Array(body[group]).group_by { |data| data['canonical_name'] || data['name'] }
+                              .map { |_, rows| build_model(merge_language_rows(rows), slug, group) }
+          end
+        end
+
+        # Deepgram lists a model once per language it was trained on, so
+        # nova-3-general arrives 146 times with a different language and
+        # version each time. One model answers to that id, so the rows
+        # collapse into it, carrying every language between them. Fields
+        # the rows disagree on describe a single language rather than the
+        # model, and are left out.
+        def merge_language_rows(rows)
+          merged = rows.first.merge('languages' => rows.flat_map { |row| Array(row['languages']) }.uniq.sort)
+          %w[version architecture].each do |key|
+            merged.delete(key) unless rows.map { |row| row[key] }.uniq.one?
+          end
+          merged
+        end
+
+        def build_model(data, slug, group)
+          model_id = data['canonical_name'] || data['name']
+          transcription = group == 'stt' && !model_id.start_with?('flux-')
+          modalities = if group == 'stt'
+                         { input: ['audio'], output: ['text'] }
+                       else
+                         { input: ['text'], output: ['audio'] }
+                       end
+
+          Model.new(
+            id: model_id,
+            name: model_id,
+            provider: slug,
+            family: data['architecture'],
+            modalities: modalities,
+            capabilities: transcription ? ['transcription'] : [],
+            pricing: {},
+            metadata: build_metadata(data)
+          )
+        end
+
+        def build_metadata(data)
+          metadata = {
+            architecture: data['architecture'],
+            languages: data['languages'],
+            version: data['version']
+          }
+          metadata[:voice] = data['name'] if data['name'] && data['name'] != data['canonical_name']
+          metadata.merge!(data['metadata'].transform_keys(&:to_sym)) if data['metadata'].is_a?(Hash)
+          metadata.compact
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/deepgram/speech.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module RubyLLM
+  module Protocols
+    class Deepgram
+      # Speech dialect for the Deepgram text-to-speech API. Deepgram names
+      # the voice inside the model id, so aura-2-thalia-en is the Thalia
+      # voice of Aura 2: asking for a +voice:+ swaps that segment of the
+      # model id. The container is an encoding query value, the body carries
+      # only the text, and the response is raw audio bytes.
+      module Speech
+        DEFAULT_FORMAT = 'mp3'
+
+        # RubyLLM format names in Deepgram's encoding vocabulary. Deepgram
+        # wraps linear16 in a WAV header unless the container is turned off,
+        # and the compressed encodings carry their own.
+        OUTPUT_FORMATS = {
+          'aac' => { encoding: 'aac' },
+          'alaw' => { encoding: 'alaw', container: 'wav' },
+          'flac' => { encoding: 'flac' },
+          'mp3' => { encoding: 'mp3' },
+          'mulaw' => { encoding: 'mulaw', container: 'wav' },
+          'opus' => { encoding: 'opus' },
+          'pcm' => { encoding: 'linear16', container: 'none' },
+          'wav' => { encoding: 'linear16', container: 'wav' }
+        }.freeze
+
+        def speak(input, model:, voice:, format:, provider_options: {}, &block)
+          track_usage(:speech) do
+            spoken_model = speech_model_for(model, voice)
+            payload = render_speech_payload(input, model: spoken_model, voice:, format:)
+            url = speech_url(model: spoken_model, format:, provider_options:)
+            if block
+              next stream_speech_response(url, payload, model: spoken_model, voice: voice_for(spoken_model),
+                                                        format:, &block)
+            end
+
+            response = @connection.post url, payload, usage: @usage_tracker
+            parse_speech_response(response, model: spoken_model, voice: voice_for(spoken_model), format:)
+          end
+        end
+
+        def speech_url(model:, format: nil, provider_options: {})
+          "v1/speak?#{URI.encode_www_form(speech_params(model:, format:, provider_options:))}"
+        end
+
+        def speech_params(model:, format: nil, provider_options: {})
+          { model: model }.merge(OUTPUT_FORMATS.fetch(format_for(format), encoding: format_for(format)))
+                          .merge(provider_options).compact
+        end
+
+        # rubocop:disable-next Lint/UnusedMethodArgument
+        def render_speech_payload(input, model:, voice: nil, format: nil, provider_options: {})
+          { text: input }
+        end
+
+        def parse_speech_response(response, model:, voice:, format:)
+          RubyLLM::Speech.new(
+            data: response.body,
+            model: model,
+            voice: voice,
+            format: format_for(format)
+          )
+        end
+
+        # Puts +voice+ into the voice segment of +model+, so Aura 2 with
+        # voice 'zeus' is aura-2-zeus-en. A voice that already names a whole
+        # model is used as it stands.
+        def speech_model_for(model, voice)
+          return model unless voice
+
+          segments = model.to_s.split('-')
+          return voice if voice.include?('-') || segments.size < 3
+
+          (segments[0..-3] + [voice, segments[-1]]).join('-')
+        end
+
+        # The voice segment of a Deepgram model id, so aura-2-thalia-en
+        # speaks as 'thalia'.
+        def voice_for(model)
+          segments = model.to_s.split('-')
+          segments[-2] if segments.size >= 3
+        end
+
+        def format_for(format)
+          (format || DEFAULT_FORMAT).to_s
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/deepgram/streaming_transcription.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Deepgram
+      module StreamingTranscription # :nodoc: all
+        def stream_live_transcription(audio_file, model:, language:, speaker_names:, provider_options:, prompt:, &block)
+          attachment = Attachment.wrap(audio_file, config: @config)
+          raise ArgumentError, 'Transcription requires exactly one audio file' unless attachment.one?
+
+          track_usage(:transcription) do
+            url = streaming_transcription_url(model:, language:, speaker_names:, provider_options:, prompt:)
+            segments = []
+            metadata = nil
+            @usage_tracker.start
+            Transport::WebsocketConnection.open(url, headers: @provider.headers, config: @config) do |socket|
+              write = ->(connection) { send_transcription_audio(connection, attachment.first.content) }
+              socket.each_message(write:) do |message|
+                event = JSON.parse(message)
+                case event['type']
+                when 'Results'
+                  process_transcription_result(event, segments, &block)
+                when 'Metadata'
+                  metadata = event
+                when 'Error'
+                  raise Error, event['description'] || event['message'] || 'Deepgram transcription failed'
+                end
+              end
+            end
+            raise Error, 'Deepgram transcription ended before its completion metadata' unless metadata
+
+            result = build_live_transcription(segments, metadata, model:, language:)
+            block.call(TranscriptionChunk.new(type: TranscriptionChunk::DONE, text: result.text, raw: metadata))
+            result
+          end
+        end
+
+        def streaming_transcription_url(model:, language:, speaker_names:, provider_options:, prompt:)
+          params = { model:, language:, smart_format: true, interim_results: true }
+          params[:diarize_model] = DIARIZE_MODEL if speaker_names
+          params[:keyterm] = Array(prompt) if prompt
+          params.merge!(provider_options)
+          params = params.compact.flat_map { |key, value| Array(value).map { |item| [key, item] } }
+          uri = URI.join("#{@provider.api_base.sub(%r{/+\z}, '')}/", "v1/listen?#{URI.encode_www_form(params)}")
+          uri.scheme = uri.scheme == 'https' ? 'wss' : 'ws'
+          uri.to_s
+        end
+
+        def send_transcription_audio(socket, audio)
+          offset = 0
+          while offset < audio.bytesize
+            socket.send_binary(audio.byteslice(offset, 16_384))
+            offset += 16_384
+          end
+          socket.send_text(JSON.generate(type: 'CloseStream'))
+        end
+
+        def process_transcription_result(event, segments)
+          alternative = event.dig('channel', 'alternatives', 0) || {}
+          text = alternative['transcript'].to_s
+          return if text.empty?
+
+          unless event['is_final']
+            yield TranscriptionChunk.new(type: TranscriptionChunk::PARTIAL, text:, raw: event)
+            return
+          end
+
+          delta = segments.empty? ? text : " #{text}"
+          segment = {
+            'text' => text,
+            'start' => event['start'],
+            'end' => event['start'].to_f + event['duration'].to_f,
+            'channel' => event.dig('channel_index', 0),
+            'words' => alternative['words']
+          }.compact
+          segments << segment
+          yield TranscriptionChunk.new(type: TranscriptionChunk::SEGMENT, delta:, segment:, raw: event)
+        end
+
+        def build_live_transcription(segments, metadata, model:, language:)
+          RubyLLM::Transcription.new(
+            text: segments.map { |segment| segment.fetch('text') }.join(' '), model:, language:,
+            duration: metadata['duration'], segments:, words: segments.flat_map { |segment| segment['words'] || [] }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/deepgram/transcription.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module RubyLLM
+  module Protocols
+    class Deepgram
+      # Transcription dialect for the Deepgram speech-to-text API. Every
+      # option is a query parameter on v1/listen, and the body is either the
+      # audio bytes or a JSON pointer to a remote url. Giving speaker names
+      # turns on diarization, which labels each word and utterance with a
+      # numeric speaker index.
+      module Transcription
+        def render_transcription_options(timestamps:, **)
+          return {} if timestamps.nil? || timestamps == :word
+
+          raise ArgumentError, 'Deepgram transcription timestamps must be word'
+        end
+
+        # Turned on by default so the transcript reads like the transcripts
+        # every other provider returns. Smart formatting punctuates and
+        # formats numbers, dates, and currency; utterances split the
+        # transcript into timed segments.
+        DEFAULT_PARAMS = { smart_format: true, utterances: true }.freeze
+
+        # The batch diarizer to run. 'latest' tracks Deepgram's current
+        # generally available diarizer, which the boolean diarize parameter
+        # it deprecates does not.
+        DIARIZE_MODEL = 'latest'
+
+        def transcribe(audio_file, model:, language:, format: nil, speaker_names: nil,
+                       speaker_references: nil, provider_options: {}, prompt: nil, temperature: nil, &block)
+          if block
+            if format || speaker_references || temperature
+              raise ArgumentError,
+                    'Deepgram streaming transcription does not accept format, speaker_references or temperature'
+            end
+
+            return stream_live_transcription(audio_file, model:, language:, speaker_names:, provider_options:, prompt:,
+                                             &block)
+          end
+
+          track_usage(:transcription) do
+            attachment = Attachment.new(audio_file, config: @config)
+            url = transcription_url(model:, language:, speaker_names:, provider_options:)
+            payload = render_transcription_payload(attachment)
+            response = post_transcription(url, payload, attachment)
+            parse_transcription_response(response, model:)
+          end
+        end
+
+        def transcription_url(model:, language: nil, speaker_names: nil, provider_options: {})
+          "v1/listen?#{URI.encode_www_form(transcription_params(model:, language:, speaker_names:,
+                                                                provider_options:))}"
+        end
+
+        def transcription_params(model:, language: nil, speaker_names: nil, provider_options: {})
+          params = { model: model, language: language }.merge(DEFAULT_PARAMS)
+          params[:diarize_model] = DIARIZE_MODEL if speaker_names
+          params.merge(provider_options).compact
+        end
+
+        # Deepgram fetches remote audio itself, so a url attachment is sent
+        # as a JSON pointer and everything else is uploaded as raw bytes.
+        def render_transcription_payload(attachment)
+          return { url: attachment.source.to_s } if attachment.url?
+
+          attachment.content
+        end
+
+        def parse_transcription_response(response, model:)
+          data = response.body || {}
+          channel = data.dig('results', 'channels', 0) || {}
+          alternative = channel.dig('alternatives', 0) || {}
+
+          RubyLLM::Transcription.new(
+            text: alternative['transcript'],
+            model: model,
+            language: channel['detected_language'],
+            duration: data.dig('metadata', 'duration'),
+            segments: data.dig('results', 'utterances'),
+            words: alternative['words']
+          )
+        end
+
+        private
+
+        def post_transcription(url, payload, attachment)
+          @connection.post(url, payload, usage: @usage_tracker) do |request|
+            request.headers['Content-Type'] = attachment.mime_type || 'application/octet-stream' unless attachment.url?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/deepgram.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    # The Deepgram API: speech to text on v1/listen, text to speech on
+    # v1/speak, and the model catalog behind them. Deepgram has no chat,
+    # embedding, or image endpoints, so those seams are left unimplemented.
+    #
+    # Deepgram carries its request options in the query string rather than
+    # the body, so +provider_options:+ joins the query on both audio
+    # operations.
+    class Deepgram < Protocol
+      include Deepgram::Models
+      include Deepgram::Speech
+      include Deepgram::Transcription
+      include Deepgram::StreamingTranscription
+    end
+  end
+end

data/lib/ruby_llm/protocols/deepseek/files.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    module DeepSeek
+      # DeepSeek stores image files for reuse by vision models.
+      class Files < Protocols::OpenAI::Files
+        IMAGE_TYPES = %w[image/jpeg image/png image/gif image/webp].freeze
+        MAX_FILE_SIZE = 64 * 1024 * 1024
+
+        def download(_file_id)
+          raise Error, 'DeepSeek does not support downloading uploaded files'
+        end
+
+        private
+
+        def render_upload_payload(attachment, purpose: nil, **options)
+          raise UnsupportedAttachmentError, attachment.mime_type unless IMAGE_TYPES.include?(attachment.mime_type)
+          raise ArgumentError, 'DeepSeek image uploads cannot exceed 64 MiB' if file_size(attachment) > MAX_FILE_SIZE
+          raise ArgumentError, 'DeepSeek file uploads require purpose: user_data' if purpose && purpose != 'user_data'
+
+          super(attachment, purpose: 'user_data', **options)
+        end
+
+        def uploaded_file(data, **attributes)
+          super(data, **attributes, downloadable: false)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/assets.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      # Workspace media assets used by the Image & Video API.
+      class Assets < Protocols::Files
+        def download(file_id)
+          file = find(file_id)
+          url = file.metadata['content_url']
+          raise Error, 'ElevenLabs asset is still processing; retrieve it again before downloading' unless url
+
+          Transport::Connection.basic(@config).get(url).body
+        end
+
+        private
+
+        def files_url
+          'v1/assets'
+        end
+
+        def render_upload_payload(attachment, purpose: nil, expires_in: nil, name: nil)
+          raise ArgumentError, 'ElevenLabs assets do not accept purpose or expires_in' if purpose || expires_in
+
+          { asset: file_part(attachment), name: name || attachment.filename }
+        end
+
+        def parse_file_response(data)
+          uploaded_file(data, id: data.fetch('asset_id'), filename: data['name'], mime_type: data['mime_type'],
+                              created_at: timestamp(data['created_at_unix']), downloadable: true,
+                              status: data['content_url'] ? 'ready' : 'processing')
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/flows/images.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      class Flows
+        module Images # :nodoc: all
+          MASK_MODELS = %w[gpt-image-1 gpt-image-1.5 gpt-image-2].freeze
+
+          def images_url(**)
+            'v1/flows/image'
+          end
+
+          def post_image(payload, **)
+            @connection.post images_url, payload, usage: @usage_tracker, idempotent: false
+          end
+
+          def render_image_payload(prompt, model:, size:, with: nil, mask: nil, provider_options: {}, **)
+            images = Attachment.wrap(with, config: @config)
+            payload = { model_id: model, prompt: }
+            payload[:images] = images.map { |image| render_media_reference(image) } if images.any?
+            payload[:aspect_ratio] = image_aspect_ratio(size) if size && size != 'auto'
+            if mask
+              raise ArgumentError, 'ElevenLabs image masks require a GPT Image model' unless MASK_MODELS.include?(model)
+
+              payload[:mask] = render_media_reference(Attachment.wrap(mask, config: @config).first)
+            end
+            Support::Utils.deep_merge(payload, provider_options)
+          end
+
+          def parse_image_response(response, model:)
+            id = response.body.fetch('id')
+            body = wait_for_image(id)
+            Image.new(url: body.fetch('content_url'), mime_type: body.fetch('content_mime_type'), model:)
+          end
+
+          def wait_for_image(id)
+            deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @config.request_timeout
+            loop do
+              body = @connection.get("#{images_url}/#{id}").body
+              state = parse_generation_status(body)
+              return body if state[:status] == :completed
+              raise Error, "ElevenLabs image generation failed: #{state[:error]}" if state[:status] == :failed
+              if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+                raise Error, "ElevenLabs image generation timed out: #{id}"
+              end
+
+              sleep 1
+            end
+          end
+
+          def validate_paint_inputs!(with:, mask:)
+            images = Attachment.wrap(with, config: @config)
+            raise ArgumentError, 'An image mask requires a source image' if mask && images.empty?
+
+            images += Attachment.wrap(mask, config: @config) if mask
+            images.each do |image|
+              raise UnsupportedAttachmentError, image.mime_type unless image.image?
+            end
+          end
+
+          def image_aspect_ratio(size)
+            match = size.to_s.match(/\A([1-9]\d*)x([1-9]\d*)\z/)
+            raise ArgumentError, 'size must be widthxheight or auto' unless match
+
+            width, height = match.captures.map(&:to_i)
+            divisor = width.gcd(height)
+            "#{width / divisor}:#{height / divisor}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/flows/media.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      class Flows
+        module Media # :nodoc: all
+          def render_media_reference(attachment)
+            unless attachment.image? || attachment.audio? || attachment.video?
+              raise UnsupportedAttachmentError, attachment.mime_type
+            end
+
+            if attachment.provider_file?
+              unless attachment.source.provider.to_s == @provider.slug
+                raise ArgumentError, 'ElevenLabs media references require an asset from the same provider'
+              end
+
+              { type: 'asset', asset_id: attachment.provider_file_id }
+            else
+              { type: 'inline_base64', content_base64: attachment.encoded, mime_type: attachment.mime_type }
+            end
+          end
+
+          def parse_generation_status(body)
+            case body['status']
+            when 'pending', 'generating'
+              { status: :pending, raw: body }
+            when 'completed'
+              raise Error, 'ElevenLabs completed a generation without an output URL' unless body['content_url']
+
+              { status: :completed, raw: body }
+            when 'failed'
+              { status: :failed, raw: body, error: body['error_message'] || body['failure_reason'] }
+            else
+              raise Error, "ElevenLabs returned an unknown generation status: #{body['status'].inspect}"
+            end
+          end
+        end
+      end
+    end
+  end
+end