ruby_llm 1.16.0 → 2.0.0.rc2

data/lib/ruby_llm/protocols/elevenlabs/flows/videos.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      class Flows
+        module Videos # :nodoc: all
+          REFERENCE_VIDEO_MODELS = %w[
+            bytedance-seedance-v2 bytedance-seedance-v2-fast bytedance-seedance-v2-mini bytedance-seedance-v2.5
+          ].freeze
+
+          def video_url
+            'v1/flows/video'
+          end
+
+          def render_video_payload(prompt, model:, with: [], provider_options: {})
+            payload = if model == 'creatify-aurora'
+                        render_character_animation(prompt, with:)
+                      else
+                        render_video_inputs(prompt, model:, with:)
+                      end
+            Support::Utils.deep_merge({ model_id: model }.merge(payload), provider_options)
+          end
+
+          def render_video_inputs(prompt, model:, with:)
+            raise ArgumentError, 'This ElevenLabs video model requires a prompt' if prompt.to_s.empty?
+
+            payload = { prompt: }
+            if with.all?(&:image?)
+              raise ArgumentError, 'Video generation accepts at most two frame images' if with.size > 2
+
+              payload[:start_frame] = render_media_reference(with[0]) if with[0]
+              payload[:end_frame] = render_media_reference(with[1]) if with[1]
+            else
+              render_video_references(payload, with, model:)
+            end
+            payload
+          end
+
+          def render_character_animation(prompt, with:)
+            unless prompt.nil? || prompt.empty?
+              raise ArgumentError, 'Creatify Aurora uses image and audio input; omit the prompt'
+            end
+
+            image = with.select(&:image?)
+            audio = with.select(&:audio?)
+            unless with.size == 2 && image.one? && audio.one?
+              raise ArgumentError, 'Creatify Aurora requires exactly one image and one audio attachment'
+            end
+
+            { image: render_media_reference(image.first), audio: render_media_reference(audio.first) }
+          end
+
+          def render_video_references(payload, attachments, model:)
+            unless REFERENCE_VIDEO_MODELS.include?(model)
+              raise ArgumentError, 'This ElevenLabs video model only accepts image attachments'
+            end
+
+            { images: :image?, audios: :audio?, videos: :video? }.each do |field, predicate|
+              references = attachments.select { |attachment| attachment.public_send(predicate) }
+              payload[field] = references.map { |attachment| render_media_reference(attachment) } if references.any?
+            end
+          end
+
+          def parse_video_job(response, model:)
+            VideoJob.new(id: response.body.fetch('id'), protocol: self, model:, raw: response.body)
+          end
+
+          def video_job_url(job)
+            "#{video_url}/#{job.id}"
+          end
+
+          def parse_video_job_status(response, **)
+            parse_generation_status(response.body)
+          end
+
+          def download_video(job)
+            Video.new(url: job.raw.fetch('content_url'), mime_type: job.raw.fetch('content_mime_type'),
+                      model: job.model, raw: job.raw)
+          end
+
+          def validate_animate_inputs!(with:)
+            with.each do |attachment|
+              unless attachment.image? || attachment.audio? || attachment.video?
+                raise UnsupportedAttachmentError, attachment.mime_type
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/flows.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      # The asynchronous ElevenLabs Image & Video API.
+      class Flows < Protocol
+        include Flows::Media
+        include Flows::Images
+        include Flows::Videos
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/models.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      # Model catalog for ElevenLabs. GET v1/models returns a bare array of
+      # the speech synthesis models and leaves out the Scribe transcription
+      # models, so those are appended from the published catalog.
+      module Models
+        module_function
+
+        TRANSCRIPTION_MODELS = {
+          'scribe_v2' => 'Scribe v2',
+          'scribe_v2_realtime' => 'Scribe v2 Realtime'
+        }.freeze
+
+        def models_url
+          'v1/models'
+        end
+
+        def parse_list_models_response(response, slug)
+          listed = Array(response.body).map do |data|
+            build_model(data['model_id'], slug, name: data['name'], description: data['description'], data: data)
+          end
+          listed_ids = listed.map(&:id)
+
+          listed + TRANSCRIPTION_MODELS.except(*listed_ids).map do |model_id, name|
+            build_model(model_id, slug, name: name, transcription: true)
+          end
+        end
+
+        def build_model(model_id, slug, name: nil, description: nil, data: {}, transcription: false)
+          transcription ||= TRANSCRIPTION_MODELS.key?(model_id)
+
+          Model.new(
+            id: model_id,
+            name: name || model_id,
+            provider: slug,
+            family: transcription ? 'scribe' : 'eleven',
+            modalities: modalities_for(data, transcription),
+            capabilities: capabilities_from(data, transcription),
+            pricing: {},
+            metadata: { description: description }.compact
+          )
+        end
+
+        def modalities_for(data, transcription)
+          return { input: ['audio'], output: ['text'] } if transcription
+          return { input: ['audio'], output: ['audio'] } if data['can_do_voice_conversion']
+
+          { input: ['text'], output: ['audio'] }
+        end
+
+        def capabilities_from(data, transcription)
+          capabilities = []
+          capabilities << 'transcription' if transcription
+          capabilities << 'speech_generation' if data['can_do_text_to_speech']
+          capabilities << 'fine_tuning' if data['can_be_finetuned']
+          capabilities
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/speech.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      # Speech dialect for the ElevenLabs text-to-speech API. The voice id
+      # is a path segment and the container is an output_format query
+      # value, so the endpoint cannot be named without the voice and format
+      # the caller asked for. The response is raw audio bytes.
+      module Speech
+        DEFAULT_VOICE = 'JBFqnCBsd6RMkjVDRZzb'
+        DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128'
+
+        OUTPUT_FORMATS = {
+          'alaw' => 'alaw_8000',
+          'mp3' => 'mp3_44100_128',
+          'opus' => 'opus_48000_128',
+          'pcm' => 'pcm_44100',
+          'ulaw' => 'ulaw_8000',
+          'wav' => 'wav_44100'
+        }.freeze
+
+        def speak(input, model:, voice:, format:, provider_options: {}, &block)
+          track_usage(:speech) do
+            payload = render_speech_payload(input, model:, voice:, format:, provider_options:)
+            if block
+              next stream_speech_response(speech_url(voice:, format:, streaming: true), payload,
+                                          model:, voice:, format:, &block)
+            end
+
+            response = @connection.post speech_url(voice:, format:), payload, usage: @usage_tracker
+            parse_speech_response(response, model:, voice:, format:)
+          end
+        end
+
+        def speech_url(voice: nil, format: nil, streaming: false)
+          path = "v1/text-to-speech/#{voice || DEFAULT_VOICE}"
+          path += '/stream' if streaming
+          "#{path}?output_format=#{output_format_for(format)}"
+        end
+
+        def render_speech_payload(input, model:, voice: nil, format: nil, provider_options: {}) # rubocop:disable Lint/UnusedMethodArgument
+          Support::Utils.deep_merge({ text: input, model_id: model }, provider_options)
+        end
+
+        def parse_speech_response(response, model:, voice:, format:)
+          RubyLLM::Speech.new(
+            data: response.body,
+            model: model,
+            voice: voice || DEFAULT_VOICE,
+            format: container_for(format)
+          )
+        end
+
+        def output_format_for(format)
+          return DEFAULT_OUTPUT_FORMAT unless format
+
+          OUTPUT_FORMATS.fetch(format.to_s, format.to_s)
+        end
+
+        def container_for(format)
+          output_format_for(format).split('_').first
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      module StreamingTranscription # :nodoc: all
+        SAMPLE_RATES = [8000, 16_000, 22_050, 24_000, 44_100, 48_000].freeze
+
+        def stream_transcription(payload, model:, &block)
+          validate_streaming_transcription(payload)
+          audio = RubyLLM::Transcription::WavAudio.new(payload.fetch(:file).io.read)
+          raise ArgumentError, 'Streaming transcription requires non-empty audio' if audio.data.empty?
+
+          audio_format = streaming_audio_format(audio)
+          url = streaming_transcription_url(payload, audio_format:)
+          @usage_tracker.start
+          expected_commits = (audio.data.bytesize.to_f / transcription_segment_bytes(audio)).ceil
+          transcripts = collect_transcription(url, audio, expected_commits, &block)
+          result = build_streaming_transcription(transcripts, audio, model:)
+          block.call(TranscriptionChunk.new(type: TranscriptionChunk::DONE, text: result.text, raw: transcripts.last))
+          result
+        ensure
+          payload[:file]&.io&.close
+        end
+
+        def collect_transcription(url, audio, expected_commits, &block)
+          transcripts = []
+          commits = Queue.new
+          Transport::WebsocketConnection.open(url, headers: @provider.headers, config: @config) do |socket|
+            write = ->(connection) { send_transcription_audio(connection, audio, commits) }
+            socket.each_message(write:) do |message|
+              event = JSON.parse(message)
+              transcript = process_transcription_event(event, prefix: transcripts.any?, &block)
+              next unless transcript
+
+              transcripts << transcript
+              commits << true
+              socket.close if transcripts.size == expected_commits
+            end
+          end
+          unless transcripts.size == expected_commits && transcripts.any?
+            raise Error, 'ElevenLabs transcription ended before its committed transcript'
+          end
+
+          transcripts
+        end
+
+        def build_streaming_transcription(transcripts, audio, model:)
+          RubyLLM::Transcription.new(
+            text: transcripts.map { |item| item.fetch('text') }.join(' '), model:,
+            duration: audio.duration, language: transcripts.last['language_code'],
+            words: transcripts.flat_map { |item| item['words'] || [] }
+          )
+        end
+
+        def validate_streaming_transcription(payload)
+          unsupported = payload.keys & %i[diarize num_speakers temperature timestamps_granularity]
+          unless unsupported.empty?
+            raise ArgumentError, "ElevenLabs streaming transcription does not accept #{unsupported.join(', ')}"
+          end
+
+          if payload.fetch(:commit_strategy, 'manual') != 'manual' || payload[:include_timestamps] == false ||
+             payload[:filter_background_audio]
+            raise ArgumentError, 'ElevenLabs file streaming requires manual commits and word timestamps'
+          end
+        end
+
+        def streaming_audio_format(audio)
+          return "pcm_#{audio.sample_rate}" if pcm_audio?(audio)
+          return 'ulaw_8000' if [audio.channels, audio.encoding, audio.sample_rate,
+                                 audio.bits_per_sample] == [1, 7, 8000, 8]
+
+          raise ArgumentError, 'ElevenLabs streaming requires mono 16-bit PCM WAV or 8 kHz mu-law WAV audio'
+        end
+
+        def pcm_audio?(audio)
+          [audio.channels, audio.encoding, audio.bits_per_sample] == [1, 1, 16] &&
+            SAMPLE_RATES.include?(audio.sample_rate)
+        end
+
+        def streaming_transcription_url(payload, audio_format:)
+          language_detection = payload.fetch(:include_language_detection, true)
+          params = payload.except(:file).merge(audio_format:, commit_strategy: 'manual', include_timestamps: true,
+                                               include_language_detection: language_detection)
+          uri = URI.join("#{@provider.api_base.sub(%r{/+\z}, '')}/",
+                         "v1/speech-to-text/realtime?#{URI.encode_www_form(params)}")
+          uri.scheme = uri.scheme == 'https' ? 'wss' : 'ws'
+          uri.to_s
+        end
+
+        def transcription_segment_bytes(audio)
+          audio.sample_rate * audio.channels * audio.bits_per_sample / 8 * 20
+        end
+
+        def send_transcription_audio(socket, audio, commits)
+          offset = 0
+          segment_bytes = transcription_segment_bytes(audio)
+          while offset < audio.data.bytesize
+            length = [16_384, segment_bytes - (offset % segment_bytes)].min
+            data = audio.data.byteslice(offset, length)
+            offset += data.bytesize
+            commit = (offset % segment_bytes).zero? || offset == audio.data.bytesize
+            socket.send_text(JSON.generate(message_type: 'input_audio_chunk',
+                                           'audio_base_64' => Base64.strict_encode64(data),
+                                           sample_rate: audio.sample_rate, commit:))
+            commits.pop if commit
+          end
+        end
+
+        def process_transcription_event(event, prefix: false)
+          case event['message_type']
+          when 'partial_transcript'
+            yield TranscriptionChunk.new(type: TranscriptionChunk::PARTIAL, text: event['text'], raw: event)
+          when 'committed_transcript'
+            text = event['text']
+            yield TranscriptionChunk.new(type: TranscriptionChunk::DELTA, delta: prefix ? " #{text}" : text, raw: event)
+          when 'committed_transcript_with_timestamps'
+            return event
+          else
+            raise Error, event['error'] if event['error']
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs/transcription.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ElevenLabs
+      # Transcription dialect for the ElevenLabs speech-to-text API. The
+      # request is multipart with the model id in the form, and giving
+      # speaker names turns on diarization and caps the speaker count at
+      # the number of names.
+      module Transcription
+        def render_transcription_options(timestamps:, format:, streaming:)
+          return {} if timestamps.nil?
+
+          value = timestamps.to_s
+          allowed = streaming ? ['word'] : %w[none word character]
+          unless allowed.include?(value) && (format.nil? || format == value)
+            raise ArgumentError, "ElevenLabs timestamps must be #{allowed.join(', ')} and match format when provided"
+          end
+
+          streaming ? { include_timestamps: true } : { timestamps_granularity: value }
+        end
+
+        # rubocop:disable-next Lint/UnusedMethodArgument
+        def render_transcription_payload(file_part, model:, language:, format: nil, speaker_names: nil,
+                                         speaker_references: nil, provider_options: {}, prompt: nil,
+                                         temperature: nil)
+          payload = {
+            model_id: model,
+            file: file_part,
+            language_code: language,
+            temperature: temperature,
+            timestamps_granularity: format
+          }.compact
+
+          if speaker_names
+            payload[:diarize] = true
+            payload[:num_speakers] = speaker_names.size
+          end
+
+          payload.merge(provider_options)
+        end
+
+        def transcription_url
+          'v1/speech-to-text'
+        end
+
+        def parse_transcription_response(response, model:)
+          data = response.body
+
+          RubyLLM::Transcription.new(
+            text: data['text'],
+            model: model,
+            language: data['language_code'],
+            duration: data['audio_duration_secs'],
+            words: data['words']
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/elevenlabs.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    # The ElevenLabs audio API: text to speech, speech to text, and the model
+    # catalog behind them. Image and video generation and workspace assets use
+    # separate protocols.
+    class ElevenLabs < Protocol
+      include ElevenLabs::Models
+      include ElevenLabs::Speech
+      include ElevenLabs::Transcription
+      include ElevenLabs::StreamingTranscription
+    end
+  end
+end

data/lib/ruby_llm/protocols/files.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+module RubyLLM
+  module Protocols
+    # Provider-managed file storage APIs.
+    class Files < Protocol
+      # rubocop:disable-next Lint/UnusedMethodArgument
+      def upload(file, filename: nil, purpose: nil, expires_in: nil, uri: nil, content_type: nil, provider_options: {})
+        attachment = file_attachment(file, filename:)
+        options = { purpose:, expires_in: }.compact.merge(provider_options.transform_keys(&:to_sym))
+        response = @connection.post(files_url, render_upload_payload(attachment, **options),
+                                    idempotent: false) do |request|
+          request.headers.delete('Content-Type')
+          upload_headers(request)
+        end
+        parse_file_response(response.body)
+      end
+
+      def find(file_id)
+        response = @connection.get(file_info_url(file_id)) { |request| file_headers(request) }
+        parse_file_response(response.body)
+      end
+
+      def download(file_id)
+        response = @connection.get(download_file_url(file_id)) do |request|
+          request.headers['Accept'] = 'application/octet-stream'
+          file_headers(request)
+        end
+        response.body
+      end
+
+      def list_uris(_uri)
+        raise Error, "#{@provider.slug} doesn't support file listing"
+      end
+
+      private
+
+      def files_url
+        'files'
+      end
+
+      def file_info_url(file_id)
+        "#{files_url}/#{file_id}"
+      end
+
+      def download_file_url(file_id)
+        "#{file_info_url(file_id)}/content"
+      end
+
+      # rubocop:disable-next Lint/UnusedMethodArgument
+      def render_upload_payload(attachment, purpose: nil, expires_in: nil, visibility: nil,
+                                display_name: nil, uri: nil, content_type: nil)
+        { file: file_part(attachment) }
+      end
+
+      def multipart_payload(attachment, **fields)
+        { file: file_part(attachment) }.merge(fields.compact)
+      end
+
+      def upload_headers(_request); end
+
+      def file_headers(_request); end
+
+      def file_attachment(file, filename: nil)
+        return file if file.is_a?(Attachment) && filename.nil?
+
+        file = file.source if file.is_a?(Attachment)
+        Attachment.new(file, filename:, config:)
+      end
+
+      def file_part(attachment, content_type: nil)
+        Faraday::Multipart::FilePart.new(file_part_source(attachment), content_type || file_content_type(attachment),
+                                         attachment.filename)
+      end
+
+      def file_content_type(attachment)
+        attachment.extension == 'jsonl' ? 'application/jsonl' : attachment.mime_type
+      end
+
+      def file_part_source(attachment)
+        if attachment.path?
+          attachment.source.to_s
+        elsif attachment.io_like?
+          attachment.source.tap { |io| io.rewind if io.respond_to?(:rewind) }
+        else
+          StringIO.new(attachment.content)
+        end
+      end
+
+      def timestamp(value)
+        return if value.nil?
+        return Time.at(value) if value.is_a?(Numeric)
+        return Time.at(value.to_i) if value.to_s.match?(/\A\d+\z/)
+
+        Time.iso8601(value.to_s)
+      end
+
+      def uploaded_file(data, **attributes)
+        UploadedFile.new(**attributes, provider: @provider.slug, metadata: data)
+      end
+
+      def with_file_body(attachment, &)
+        if attachment.path?
+          File.open(attachment.source, 'rb', &)
+        else
+          body = attachment.io_like? ? attachment.source : StringIO.new(attachment.content)
+          body.rewind if body.respond_to?(:rewind)
+          yield body
+        end
+      end
+
+      def file_size(attachment)
+        attachment.path? ? File.size(attachment.source) : attachment.content.bytesize
+      end
+    end
+  end
+end