ruby_llm 1.16.0 → 2.0.0.rc1

data/lib/ruby_llm/protocols/elevenlabs/models.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      # Model catalog for ElevenLabs. GET v1/models returns a bare array of
+      # the speech synthesis models and leaves out the Scribe transcription
+      # models, so those are appended from the published catalog.
+      module Models
+        module_function
+
+        TRANSCRIPTION_MODELS = {
+          'scribe_v2' => 'Scribe v2',
+          'scribe_v2_realtime' => 'Scribe v2 Realtime'
+        }.freeze
+
+        def models_url
+          'v1/models'
+        end
+
+        def parse_list_models_response(response, slug)
+          listed = Array(response.body).map do |data|
+            build_model(data['model_id'], slug, name: data['name'], description: data['description'], data: data)
+          end
+          listed_ids = listed.map(&:id)
+
+          listed + TRANSCRIPTION_MODELS.except(*listed_ids).map do |model_id, name|
+            build_model(model_id, slug, name: name, transcription: true)
+          end
+        end
+
+        def build_model(model_id, slug, name: nil, description: nil, data: {}, transcription: false)
+          transcription ||= TRANSCRIPTION_MODELS.key?(model_id)
+
+          Model.new(
+            id: model_id,
+            name: name || model_id,
+            provider: slug,
+            family: transcription ? 'scribe' : 'eleven',
+            modalities: modalities_for(data, transcription),
+            capabilities: capabilities_from(data, transcription),
+            pricing: {},
+            metadata: { description: description }.compact
+          )
+        end
+
+        def modalities_for(data, transcription)
+          return { input: ['audio'], output: ['text'] } if transcription
+          return { input: ['audio'], output: ['audio'] } if data['can_do_voice_conversion']
+
+          { input: ['text'], output: ['audio'] }
+        end
+
+        def capabilities_from(data, transcription)
+          capabilities = []
+          capabilities << 'transcription' if transcription
+          capabilities << 'speech_generation' if data['can_do_text_to_speech']
+          capabilities << 'fine_tuning' if data['can_be_finetuned']
+          capabilities
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/speech.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      # Speech dialect for the ElevenLabs text-to-speech API. The voice id
+      # is a path segment and the container is an output_format query
+      # value, so the endpoint cannot be named without the voice and format
+      # the caller asked for. The response is raw audio bytes.
+      module Speech
+        DEFAULT_VOICE = 'JBFqnCBsd6RMkjVDRZzb'
+        DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128'
+
+        OUTPUT_FORMATS = {
+          'alaw' => 'alaw_8000',
+          'mp3' => 'mp3_44100_128',
+          'opus' => 'opus_48000_128',
+          'pcm' => 'pcm_44100',
+          'ulaw' => 'ulaw_8000',
+          'wav' => 'wav_44100'
+        }.freeze
+
+        def speak(input, model:, voice:, format:, provider_options: {}, &block)
+          track_usage(:speech) do
+            payload = render_speech_payload(input, model:, voice:, format:, provider_options:)
+            if block
+              next stream_speech_response(speech_url(voice:, format:, streaming: true), payload,
+                                          model:, voice:, format:, &block)
+            end
+
+            response = @connection.post speech_url(voice:, format:), payload, usage: @usage_tracker
+            parse_speech_response(response, model:, voice:, format:)
+          end
+        end
+
+        def speech_url(voice: nil, format: nil, streaming: false)
+          path = "v1/text-to-speech/#{voice || DEFAULT_VOICE}"
+          path += '/stream' if streaming
+          "#{path}?output_format=#{output_format_for(format)}"
+        end
+
+        def render_speech_payload(input, model:, voice: nil, format: nil, provider_options: {}) # rubocop:disable Lint/UnusedMethodArgument
+          Support::Utils.deep_merge({ text: input, model_id: model }, provider_options)
+        end
+
+        def parse_speech_response(response, model:, voice:, format:)
+          RubyLLM::Speech.new(
+            data: response.body,
+            model: model,
+            voice: voice || DEFAULT_VOICE,
+            format: container_for(format)
+          )
+        end
+
+        def output_format_for(format)
+          return DEFAULT_OUTPUT_FORMAT unless format
+
+          OUTPUT_FORMATS.fetch(format.to_s, format.to_s)
+        end
+
+        def container_for(format)
+          output_format_for(format).split('_').first
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      module StreamingTranscription # :nodoc: all
+        SAMPLE_RATES = [8000, 16_000, 22_050, 24_000, 44_100, 48_000].freeze
+
+        def stream_transcription(payload, model:, &block)
+          validate_streaming_transcription(payload)
+          audio = RubyLLM::Transcription::WavAudio.new(payload.fetch(:file).io.read)
+          raise ArgumentError, 'Streaming transcription requires non-empty audio' if audio.data.empty?
+
+          audio_format = streaming_audio_format(audio)
+          url = streaming_transcription_url(payload, audio_format:)
+          @usage_tracker.start
+          expected_commits = (audio.data.bytesize.to_f / transcription_segment_bytes(audio)).ceil
+          transcripts = collect_transcription(url, audio, expected_commits, &block)
+          result = build_streaming_transcription(transcripts, audio, model:)
+          block.call(TranscriptionChunk.new(type: TranscriptionChunk::DONE, text: result.text, raw: transcripts.last))
+          result
+        ensure
+          payload[:file]&.io&.close
+        end
+
+        def collect_transcription(url, audio, expected_commits, &block)
+          transcripts = []
+          commits = Queue.new
+          Transport::WebsocketConnection.open(url, headers: @provider.headers, config: @config) do |socket|
+            write = ->(connection) { send_transcription_audio(connection, audio, commits) }
+            socket.each_message(write:) do |message|
+              event = JSON.parse(message)
+              transcript = process_transcription_event(event, prefix: transcripts.any?, &block)
+              next unless transcript
+
+              transcripts << transcript
+              commits << true
+              socket.close if transcripts.size == expected_commits
+            end
+          end
+          unless transcripts.size == expected_commits && transcripts.any?
+            raise Error, 'ElevenLabs transcription ended before its committed transcript'
+          end
+
+          transcripts
+        end
+
+        def build_streaming_transcription(transcripts, audio, model:)
+          RubyLLM::Transcription.new(
+            text: transcripts.map { |item| item.fetch('text') }.join(' '), model:,
+            duration: audio.duration, language: transcripts.last['language_code'],
+            words: transcripts.flat_map { |item| item['words'] || [] }
+          )
+        end
+
+        def validate_streaming_transcription(payload)
+          unsupported = payload.keys & %i[diarize num_speakers temperature timestamps_granularity]
+          unless unsupported.empty?
+            raise ArgumentError, "ElevenLabs streaming transcription does not accept #{unsupported.join(', ')}"
+          end
+
+          if payload.fetch(:commit_strategy, 'manual') != 'manual' || payload[:include_timestamps] == false ||
+             payload[:filter_background_audio]
+            raise ArgumentError, 'ElevenLabs file streaming requires manual commits and word timestamps'
+          end
+        end
+
+        def streaming_audio_format(audio)
+          return "pcm_#{audio.sample_rate}" if pcm_audio?(audio)
+          return 'ulaw_8000' if [audio.channels, audio.encoding, audio.sample_rate,
+                                 audio.bits_per_sample] == [1, 7, 8000, 8]
+
+          raise ArgumentError, 'ElevenLabs streaming requires mono 16-bit PCM WAV or 8 kHz mu-law WAV audio'
+        end
+
+        def pcm_audio?(audio)
+          [audio.channels, audio.encoding, audio.bits_per_sample] == [1, 1, 16] &&
+            SAMPLE_RATES.include?(audio.sample_rate)
+        end
+
+        def streaming_transcription_url(payload, audio_format:)
+          language_detection = payload.fetch(:include_language_detection, true)
+          params = payload.except(:file).merge(audio_format:, commit_strategy: 'manual', include_timestamps: true,
+                                               include_language_detection: language_detection)
+          uri = URI.join("#{@provider.api_base.sub(%r{/+\z}, '')}/",
+                         "v1/speech-to-text/realtime?#{URI.encode_www_form(params)}")
+          uri.scheme = uri.scheme == 'https' ? 'wss' : 'ws'
+          uri.to_s
+        end
+
+        def transcription_segment_bytes(audio)
+          audio.sample_rate * audio.channels * audio.bits_per_sample / 8 * 20
+        end
+
+        def send_transcription_audio(socket, audio, commits)
+          offset = 0
+          segment_bytes = transcription_segment_bytes(audio)
+          while offset < audio.data.bytesize
+            length = [16_384, segment_bytes - (offset % segment_bytes)].min
+            data = audio.data.byteslice(offset, length)
+            offset += data.bytesize
+            commit = (offset % segment_bytes).zero? || offset == audio.data.bytesize
+            socket.send_text(JSON.generate(message_type: 'input_audio_chunk',
+                                           'audio_base_64' => Base64.strict_encode64(data),
+                                           sample_rate: audio.sample_rate, commit:))
+            commits.pop if commit
+          end
+        end
+
+        def process_transcription_event(event, prefix: false)
+          case event['message_type']
+          when 'partial_transcript'
+            yield TranscriptionChunk.new(type: TranscriptionChunk::PARTIAL, text: event['text'], raw: event)
+          when 'committed_transcript'
+            text = event['text']
+            yield TranscriptionChunk.new(type: TranscriptionChunk::DELTA, delta: prefix ? " #{text}" : text, raw: event)
+          when 'committed_transcript_with_timestamps'
+            return event
+          else
+            raise Error, event['error'] if event['error']
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/transcription.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      # Transcription dialect for the ElevenLabs speech-to-text API. The
+      # request is multipart with the model id in the form, and giving
+      # speaker names turns on diarization and caps the speaker count at
+      # the number of names.
+      module Transcription
+        def render_transcription_options(timestamps:, format:, streaming:)
+          return {} if timestamps.nil?
+
+          value = timestamps.to_s
+          allowed = streaming ? ['word'] : %w[none word character]
+          unless allowed.include?(value) && (format.nil? || format == value)
+            raise ArgumentError, "ElevenLabs timestamps must be #{allowed.join(', ')} and match format when provided"
+          end
+
+          streaming ? { include_timestamps: true } : { timestamps_granularity: value }
+        end
+
+        # rubocop:disable-next Lint/UnusedMethodArgument
+        def render_transcription_payload(file_part, model:, language:, format: nil, speaker_names: nil,
+                                         speaker_references: nil, provider_options: {}, prompt: nil,
+                                         temperature: nil)
+          payload = {
+            model_id: model,
+            file: file_part,
+            language_code: language,
+            temperature: temperature,
+            timestamps_granularity: format
+          }.compact
+
+          if speaker_names
+            payload[:diarize] = true
+            payload[:num_speakers] = speaker_names.size
+          end
+
+          payload.merge(provider_options)
+        end
+
+        def transcription_url
+          'v1/speech-to-text'
+        end
+
+        def parse_transcription_response(response, model:)
+          data = response.body
+
+          RubyLLM::Transcription.new(
+            text: data['text'],
+            model: model,
+            language: data['language_code'],
+            duration: data['audio_duration_secs'],
+            words: data['words']
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    # The ElevenLabs audio API: text to speech, speech to text, and the model
+    # catalog behind them. Image and video generation and workspace assets use
+    # separate protocols.
+    class ElevenLabs < Protocol
+      include ElevenLabs::Models
+      include ElevenLabs::Speech
+      include ElevenLabs::Transcription
+      include ElevenLabs::StreamingTranscription
+    end
+  end
+end

data/lib/ruby_llm/protocols/files.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+module RubyLLM
+  module Protocols
+    # Provider-managed file storage APIs.
+    class Files < Protocol
+      # rubocop:disable-next Lint/UnusedMethodArgument
+      def upload(file, filename: nil, purpose: nil, expires_in: nil, uri: nil, content_type: nil, provider_options: {})
+        attachment = file_attachment(file, filename:)
+        options = { purpose:, expires_in: }.compact.merge(provider_options.transform_keys(&:to_sym))
+        response = @connection.post(files_url, render_upload_payload(attachment, **options),
+                                    idempotent: false) do |request|
+          request.headers.delete('Content-Type')
+          upload_headers(request)
+        end
+        parse_file_response(response.body)
+      end
+
+      def find(file_id)
+        response = @connection.get(file_info_url(file_id)) { |request| file_headers(request) }
+        parse_file_response(response.body)
+      end
+
+      def download(file_id)
+        response = @connection.get(download_file_url(file_id)) do |request|
+          request.headers['Accept'] = 'application/octet-stream'
+          file_headers(request)
+        end
+        response.body
+      end
+
+      def list_uris(_uri)
+        raise Error, "#{@provider.slug} doesn't support file listing"
+      end
+
+      private
+
+      def files_url
+        'files'
+      end
+
+      def file_info_url(file_id)
+        "#{files_url}/#{file_id}"
+      end
+
+      def download_file_url(file_id)
+        "#{file_info_url(file_id)}/content"
+      end
+
+      # rubocop:disable-next Lint/UnusedMethodArgument
+      def render_upload_payload(attachment, purpose: nil, expires_in: nil, visibility: nil,
+                                display_name: nil, uri: nil, content_type: nil)
+        { file: file_part(attachment) }
+      end
+
+      def multipart_payload(attachment, **fields)
+        { file: file_part(attachment) }.merge(fields.compact)
+      end
+
+      def upload_headers(_request); end
+
+      def file_headers(_request); end
+
+      def file_attachment(file, filename: nil)
+        return file if file.is_a?(Attachment) && filename.nil?
+
+        file = file.source if file.is_a?(Attachment)
+        Attachment.new(file, filename:, config:)
+      end
+
+      def file_part(attachment, content_type: nil)
+        Faraday::Multipart::FilePart.new(file_part_source(attachment), content_type || file_content_type(attachment),
+                                         attachment.filename)
+      end
+
+      def file_content_type(attachment)
+        attachment.extension == 'jsonl' ? 'application/jsonl' : attachment.mime_type
+      end
+
+      def file_part_source(attachment)
+        if attachment.path?
+          attachment.source.to_s
+        elsif attachment.io_like?
+          attachment.source.tap { |io| io.rewind if io.respond_to?(:rewind) }
+        else
+          StringIO.new(attachment.content)
+        end
+      end
+
+      def timestamp(value)
+        return if value.nil?
+        return Time.at(value) if value.is_a?(Numeric)
+        return Time.at(value.to_i) if value.to_s.match?(/\A\d+\z/)
+
+        Time.iso8601(value.to_s)
+      end
+
+      def uploaded_file(data, **attributes)
+        UploadedFile.new(**attributes, provider: @provider.slug, metadata: data)
+      end
+
+      def with_file_body(attachment, &)
+        if attachment.path?
+          File.open(attachment.source, 'rb', &)
+        else
+          body = attachment.io_like? ? attachment.source : StringIO.new(attachment.content)
+          body.rewind if body.respond_to?(:rewind)
+          yield body
+        end
+      end
+
+      def file_size(attachment)
+        attachment.path? ? File.size(attachment.source) : attachment.content.bytesize
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/gemini/batches.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Gemini
+      # Gemini Batch API with inlined generateContent requests.
+      module Batches
+        include RubyLLM::Batch::Helpers
+        include Gemini::EmbeddingBatches
+
+        # The wire enum is BATCH_STATE_*; the SDKs print JOB_STATE_*. Match the
+        # suffix so either spelling works.
+        TERMINAL = %w[SUCCEEDED FAILED CANCELLED EXPIRED].freeze
+        private_constant :TERMINAL
+
+        def create_batch(requests)
+          model = single_batch_model!(requests, 'gemini')
+          action = embedding_batch?(requests) ? 'asyncBatchEmbedContent' : 'batchGenerateContent'
+          response = @connection.post("models/#{model}:#{action}", {
+                                        batch: {
+                                          displayName: "ruby_llm_#{SecureRandom.hex(8)}",
+                                          inputConfig: {
+                                            requests: {
+                                              requests: requests.flat_map do |request|
+                                                if embedding_batch_payload?(request.fetch(:payload))
+                                                  embedding_batch_requests(request, model)
+                                                else
+                                                  [gemini_batch_request(request, model)]
+                                                end
+                                              end
+                                            }
+                                          }
+                                        }
+                                      }, idempotent: false)
+
+          parse_batch_response(response.body)
+        end
+
+        def find_batch(id)
+          parse_batch_response @connection.get(batch_name(id)).body
+        end
+
+        def cancel_batch(id)
+          @connection.post("#{batch_name(id)}:cancel", {})
+          find_batch(id)
+        end
+
+        # Inline answers are correlated by the key we sent (the submission index);
+        # Gemini also returns them in order, so we fall back to position.
+        def batch_results(id)
+          body = @connection.get(batch_name(id)).body
+          inlined = inline_batch_responses(body)
+          return parse_embedding_batch_results(inlined) if embedding_batch_response?(body)
+
+          inlined.each_with_index.map { |response, index| parse_inline_response(response, index) }
+        end
+
+        private
+
+        def inline_batch_responses(body)
+          body.dig('response', 'inlinedEmbedContentResponses', 'inlinedResponses') ||
+            body.dig('output', 'inlinedEmbedContentResponses', 'inlinedResponses') ||
+            body.dig('response', 'inlinedResponses', 'inlinedResponses') ||
+            body.dig('output', 'inlinedResponses', 'inlinedResponses') ||
+            body.dig('metadata', 'output', 'inlinedResponses', 'inlinedResponses') || []
+        end
+
+        def gemini_batch_request(request, model)
+          {
+            request: batch_schema_payload(batch_payload(request)).merge(model: "models/#{model}"),
+            metadata: {
+              custom_id: request[:custom_id]
+            }
+          }
+        end
+
+        # batchGenerateContent ignores responseJsonSchema and returns JSON of
+        # its own shape, while the legacy responseSchema field is honored, so
+        # batches carry the schema in Gemini's Schema dialect.
+        def batch_schema_payload(payload)
+          config = payload[:generationConfig]
+          return payload unless config.is_a?(Hash) && config.key?(:responseJsonSchema)
+
+          schema = response_schema(config[:responseJsonSchema])
+          payload.merge(generationConfig: config.except(:responseJsonSchema).merge(responseSchema: schema))
+        end
+
+        JSON_SCHEMA_ONLY_KEYS = %w[$schema $id additionalProperties strict].freeze
+        private_constant :JSON_SCHEMA_ONLY_KEYS
+
+        def response_schema(node)
+          case node
+          when Hash then response_schema_hash(node)
+          when Array then node.map { |value| response_schema(value) }
+          else node
+          end
+        end
+
+        def response_schema_hash(node)
+          schema = node.each_with_object({}) do |(key, value), converted|
+            next if JSON_SCHEMA_ONLY_KEYS.include?(key.to_s)
+
+            converted[key] = if key.to_s == 'properties' && value.is_a?(Hash)
+                               value.transform_values { |property| response_schema(property) }
+                             else
+                               response_schema(value)
+                             end
+          end
+          nullable_type(schema)
+        end
+
+        def nullable_type(schema)
+          key = schema.key?(:type) ? :type : 'type'
+          types = Array(schema[key])
+          return schema unless types.length > 1 && types.include?('null')
+
+          schema.merge(key => (types - ['null']).first, nullable: true)
+        end
+
+        def batch_name(id)
+          id.to_s.start_with?('batches/') ? id : "batches/#{id}"
+        end
+
+        # A batch starts as an Operation wrapping the batch in `metadata`; polling
+        # returns the batch directly. Read either shape.
+        def parse_batch_response(data)
+          batch = data['metadata'] || data
+          state = batch['state']
+          request_counts = batch['batchStats']
+
+          {
+            id: data['name'] || batch['name'],
+            raw_status: state,
+            completed: TERMINAL.any? { |terminal| state&.end_with?(terminal) },
+            request_counts:,
+            request_count: (request_counts&.fetch('requestCount', nil)&.to_i unless embedding_batch_response?(data))
+          }
+        end
+
+        def parse_batch_status(raw_status, completed:)
+          return :pending unless completed
+          return :succeeded if raw_status&.end_with?('SUCCEEDED')
+          return :cancelled if raw_status&.end_with?('CANCELLED')
+
+          :failed
+        end
+
+        def parse_inline_response(inline, index)
+          key = inline.dig('metadata', 'custom_id') || inline.dig('metadata', 'key')
+          index = batch_result_index(key) if key
+
+          if inline['response']
+            body = inline['response']
+            [index, parse_completion_body(body, raw: body)]
+          else
+            [index, nil, batch_failure(key || index, inline.dig('error', 'message'))]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/gemini/caches.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Gemini
+      # Explicit content caching via the Gemini cachedContents API.
+      module Caches
+        module_function
+
+        def caches_url
+          'cachedContents'
+        end
+
+        def cache_url(name)
+          cache_name(name)
+        end
+
+        def render_cache_payload(content, model:, ttl: nil, instructions: nil, attachments: [])
+          payload = {
+            model: cache_model_name(model),
+            contents: [{ role: 'user', parts: Media.format_content(content, attachments) }]
+          }
+          payload[:systemInstruction] = { parts: [{ text: instructions }] } if instructions
+          payload[:ttl] = format_cache_ttl(ttl) if ttl
+          payload
+        end
+
+        def render_cache_update_payload(ttl:)
+          { ttl: format_cache_ttl(ttl) }
+        end
+
+        def parse_cache_response(data)
+          CachedContent.new(
+            name: data['name'],
+            model: data['model']&.split('/')&.last,
+            provider_instance: @provider,
+            created_at: (Time.iso8601(data['createTime']) if data['createTime']),
+            expires_at: (Time.iso8601(data['expireTime']) if data['expireTime']),
+            tokens: data.dig('usageMetadata', 'totalTokenCount'),
+            metadata: data
+          )
+        end
+
+        def cache_name(name)
+          name = name.name if name.is_a?(CachedContent)
+          name.to_s.include?('/') ? name.to_s : "cachedContents/#{name}"
+        end
+
+        def cache_model_name(model_id)
+          "models/#{model_id}"
+        end
+
+        def format_cache_ttl(ttl)
+          ttl.is_a?(String) ? ttl : "#{ttl.to_i}s"
+        end
+      end
+    end
+  end
+end