ruby_llm 2.1.0 → 2.2.0

data/lib/ruby_llm/protocols/invoke_model/titan_multimodal_embeddings.rb

@@ -7,10 +7,10 @@ module RubyLLM
       class TitanMultimodalEmbeddings < InvokeModel
         # rubocop:disable-next Lint/UnusedMethodArgument
         def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
-          ensure_no_embedding_media!(with)
+          image = embedding_image(embedding_attachments(text, with))
           track_usage(:embedding) do
             responses = [text].flatten.map do |value|
-              payload = render_embedding_payload(value, dimensions:, provider_options:)
+              payload = render_embedding_payload(value, dimensions:, image:, provider_options:)
               signed_post(embedding_url(model:), payload).tap { |response| record_embedding_attempt(response) }
             end
 
@@ -20,9 +20,17 @@ module RubyLLM
 
         private
 
-        def render_embedding_payload(text, dimensions:, provider_options:, **)
+        def embedding_image(attachments)
+          attachment = single_embedding_attachment(attachments, 'Titan Multimodal Embeddings')
+          raise UnsupportedAttachmentError, attachment.mime_type if attachment && !attachment.image?
+
+          attachment
+        end
+
+        def render_embedding_payload(text, dimensions:, provider_options:, image: nil, **)
           payload = {}
           payload[:inputText] = text.to_s unless text.nil? || text.to_s.empty?
+          payload[:inputImage] = image.encoded if image
           payload[:embeddingConfig] = { outputEmbeddingLength: dimensions } if dimensions
 
           deep_merge_provider_options(payload, provider_options)

data/lib/ruby_llm/protocols/invoke_model.rb

@@ -7,6 +7,10 @@ module RubyLLM
     # Protocols::InvokeModel implement one family each. Requests are signed
     # by the provider, which knows the credentials and the signing service.
     class InvokeModel < Protocol
+      def rejected_options(operation)
+        operation == :embed ? %i[task_type title] : super
+      end
+
       private
 
       def signed_post(url, payload, additional_headers = {})
@@ -24,8 +28,23 @@ module RubyLLM
         raise UnsupportedAttachmentError, attachments.first.mime_type if attachments.any?
       end
 
+      def embedding_attachments(text, with)
+        attachments = Attachment.wrap(with, config: @config)
+        if attachments.any? && text.is_a?(Array)
+          raise ArgumentError, 'embed one text at a time when embedding attachments'
+        end
+
+        attachments
+      end
+
+      def single_embedding_attachment(attachments, label)
+        raise ArgumentError, "#{label} embeds one attachment at a time" if attachments.size > 1
+
+        attachments.first
+      end
+
       def parse_single_embedding_responses(responses, model:, text:)
-        vectors = responses.map { |response| extract_embedding(response.body) }
+        vectors = responses.map { |response| parse_embedding(response.body) }
         input_tokens = Tokens.aggregate(embedding_attempt_tokens(responses)).input
         vectors = vectors.first unless text.is_a?(Array)
 
@@ -46,7 +65,7 @@ module RubyLLM
         Support::Utils.deep_merge(payload, provider_options)
       end
 
-      def extract_embedding(body)
+      def parse_embedding(body)
         body['embedding'] || body.dig('embeddingsByType', 'float') || body['embeddingsByType']&.values&.first
       end
     end

data/lib/ruby_llm/protocols/meta/transcription/stream.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    module Meta
+      module Transcription
+        # Assembles a transcript from Meta's voice events. Without turns the
+        # transcript arrives whole in the final transcript event. With turns
+        # each speechComplete carries a turn's final text, and the transcript,
+        # speaker, and timing events between speechStart and it belong to the
+        # most recently opened turn. Turns can overlap and finish out of order,
+        # so each is held until every turn opened before it has finished.
+        class Stream # :nodoc:
+          def initialize(model:, &block)
+            @model = model
+            @block = block
+            @turns = {}
+            @segments = []
+            @latest_ms = 0
+          end
+
+          def process(event)
+            @latest_ms = [@latest_ms, event['audioProcessedMs'].to_i].max
+            @last_event = event
+            case event['type']
+            when 'transcript' then process_transcript(event)
+            when 'speechStart' then open_turn(event)
+            when 'speaker' then @turns[@current_turn]&.store(:speaker, event['label'])
+            when 'speechEnd' then @turns[event['turnId']]&.store(:end_ms, event['audioProcessedMs'])
+            when 'speechComplete' then complete_turn(event)
+            when 'error' then raise Error, event['message'] || 'Meta transcription failed'
+            end
+          end
+
+          def finish
+            raise Error, 'Meta transcription ended before its final transcript' unless complete?
+
+            duration = @latest_ms / 1000.0
+            result = RubyLLM::Transcription.new(
+              text: @turns.empty? ? @final_text : @segments.map { |segment| segment['text'] }.join(' '),
+              model: @model, duration:, audio_seconds: duration, segments: @segments.empty? ? nil : @segments
+            )
+            @block.call(TranscriptionChunk.new(type: TranscriptionChunk::DONE, text: result.text, raw: @last_event))
+            result
+          end
+
+          private
+
+          def process_transcript(event)
+            text = event['transcript'].to_s.strip
+            return finish_transcript(text, event) if event['final']
+            return if text.empty?
+
+            @block.call(TranscriptionChunk.new(type: TranscriptionChunk::PARTIAL, text:, raw: event))
+          end
+
+          def finish_transcript(text, event)
+            @final_text = text
+            return if text.empty? || @turns.any?
+
+            @block.call(TranscriptionChunk.new(type: TranscriptionChunk::DELTA, delta: text, raw: event))
+          end
+
+          def open_turn(event)
+            @current_turn = event['turnId']
+            @turns[@current_turn] = { start_ms: event['audioProcessedMs'] }
+          end
+
+          def complete_turn(event)
+            turn = @turns[event['turnId']] ||= {}
+            text = event['transcript'].to_s.strip
+            segment = text.empty? ? nil : turn_segment(event['turnId'], text, turn, event)
+            turn.merge!(complete: true, segment:, event:)
+            emit_completed_turns
+          end
+
+          def emit_completed_turns
+            @turns.each_value do |turn|
+              break unless turn[:complete]
+
+              emit_turn(turn) unless turn[:emitted]
+            end
+          end
+
+          def emit_turn(turn)
+            turn[:emitted] = true
+            segment = turn[:segment]
+            return unless segment
+
+            delta = @segments.empty? ? segment['text'] : " #{segment['text']}"
+            @segments << segment
+            @block.call(TranscriptionChunk.new(type: TranscriptionChunk::SEGMENT, delta:, segment:, raw: turn[:event]))
+          end
+
+          def turn_segment(turn_id, text, turn, event)
+            end_ms = turn[:end_ms] || event['audioProcessedMs']
+            { 'text' => text, 'turnId' => turn_id, 'start' => turn[:start_ms] && (turn[:start_ms] / 1000.0),
+              'end' => end_ms && (end_ms / 1000.0), 'speaker' => turn[:speaker] }.compact
+          end
+
+          def complete?
+            @turns.values.all? { |turn| turn[:complete] } && (!@final_text.nil? || @segments.any?)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/meta/transcription.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    module Meta
+      # Muse Voice Transcribe on /v1/asr/transcribe. The settings travel as a
+      # JSON multipart part beside the WAV audio. The mode decides what comes
+      # back: one transcript by default, timed turns with ENDPOINTING, and
+      # turns labelled with a speaker with DIARIZATION. Accepting an event
+      # stream returns the same events as Meta's realtime endpoint.
+      # https://dev.meta.ai/docs/speech-to-text
+      module Transcription
+        STREAM_HEADERS = { 'Accept' => 'text/event-stream' }.freeze
+
+        def rejected_options(operation)
+          operation == :transcribe ? %i[format temperature speaker_references named_speakers] : super
+        end
+
+        def render_transcription_options(timestamps:, streaming:, **)
+          options = streaming ? { emitAudioProgress: false } : {}
+          return options if timestamps.nil?
+          unless Array(timestamps).map(&:to_s) == ['segment']
+            raise ArgumentError, 'Meta transcription timestamps must be segment'
+          end
+
+          options.merge(mode: 'ENDPOINTING')
+        end
+
+        def transcription_url
+          'asr/transcribe'
+        end
+
+        def render_transcription_payload(file_part, model:, language:, speaker_names: nil, provider_options: {},
+                                         prompt: nil, **)
+          settings = { model:, audioEncoding: 'WAV', languageBias: language && Array(language),
+                       keywords: prompt && Array(prompt) }.compact
+          settings = settings.merge(provider_options.transform_keys(&:to_sym))
+          settings[:mode] = 'DIARIZATION' if speaker_names
+          { request: Faraday::Multipart::ParamPart.new(JSON.generate(settings), 'application/json'),
+            audio: render_transcription_audio(file_part) }
+        end
+
+        def parse_transcription_response(response, model:)
+          data = response.body
+          duration = data['audioDurationMs'] && (data['audioDurationMs'] / 1000.0)
+          turns = data['turns'].to_a
+
+          RubyLLM::Transcription.new(
+            text: data['transcript'], model:, duration:, audio_seconds: duration,
+            segments: turns.empty? ? nil : turns.map { |turn| parse_transcription_turn(turn) }
+          )
+        end
+
+        def stream_transcription(payload, model:, &)
+          stream = Stream.new(model:, &)
+          stream_events(transcription_url, payload, STREAM_HEADERS) { |event| stream.process(event) }
+          stream.finish
+        end
+
+        private
+
+        # Meta answers "Malformed multipart body" to a part with its own
+        # Content-Length, which a file part always carries.
+        def render_transcription_audio(file_part)
+          audio = file_part.io.read
+          file_part.io.close
+          Faraday::Multipart::ParamPart.new(audio, file_part.content_type)
+        end
+
+        def parse_transcription_turn(turn)
+          turn.merge('text' => turn['transcript'], 'start' => turn['startMs'] / 1000.0, 'end' => turn['endMs'] / 1000.0)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/minimax/files.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class MiniMax
+      # MiniMax's Files API, which requires a purpose on every upload.
+      class Files < Protocols::Files
+        INPUT_PURPOSES = %w[voice_clone prompt_audio t2a_async_input video_understanding video_generation_input].freeze
+
+        def rejected_options(operation)
+          operation == :upload ? %i[uri expires_in] : super
+        end
+
+        private
+
+        def files_url
+          media_url('v1/files/upload')
+        end
+
+        def file_info_url(file_id)
+          media_url("v1/files/retrieve?file_id=#{URI.encode_uri_component(file_id.to_s)}")
+        end
+
+        def download_file_url(file_id)
+          media_url("v1/files/retrieve_content?file_id=#{URI.encode_uri_component(file_id.to_s)}")
+        end
+
+        def media_url(path)
+          "#{@provider.media_base}/#{path}"
+        end
+
+        def render_upload_fields(_attachment, purpose: nil, **)
+          { purpose: }
+        end
+
+        def parse_upload_response(data, attachment)
+          parse_file_response(data, mime_type: file_content_type(attachment))
+        end
+
+        def parse_file_response(data, mime_type: nil)
+          file = data['file']
+          raise Error, 'MiniMax did not return a file' unless file
+
+          uploaded_file(
+            data,
+            id: file['file_id'].to_s,
+            filename: file['filename'],
+            byte_size: file['bytes'],
+            created_at: timestamp(file['created_at']),
+            purpose: file['purpose'],
+            downloadable: !INPUT_PURPOSES.include?(file['purpose']),
+            mime_type:
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/minimax/images.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class MiniMax
+      # Image generation on v1/image_generation. Source images become
+      # subject references, which keep a person's likeness in a new scene;
+      # there is no mask.
+      module Images
+        def images_url(**)
+          media_url('v1/image_generation')
+        end
+
+        def render_image_payload(prompt, model:, size: nil, count: nil, with: nil, provider_options: {}, **)
+          payload = { model: model, prompt: prompt, n: count, response_format: 'base64' }.compact
+          payload.merge!(image_dimensions(size))
+          references = subject_references(with)
+          payload[:subject_reference] = references unless references.empty?
+          payload.merge(provider_options)
+        end
+
+        def parse_image_responses(response, model:)
+          data = response.body['data'] || {}
+          sources = Array(data['image_base64']).map { |image| { data: image } } +
+                    Array(data['image_urls']).map { |url| { url: } }
+          raise Error.new('MiniMax did not return an image', response:) if sources.empty?
+
+          sources.map { |source| Image.new(**source, mime_type: 'image/jpeg', model:) }
+        end
+
+        private
+
+        def validate_paint_inputs!(mask:, **)
+          raise UnsupportedAttachmentError, 'image mask' if mask
+        end
+
+        def image_dimensions(size)
+          return {} unless size
+
+          dimensions = size.to_s.match(/\A(\d+)x(\d+)\z/)
+          return { aspect_ratio: size.to_s } unless dimensions
+
+          { width: dimensions[1].to_i, height: dimensions[2].to_i }
+        end
+
+        def subject_references(sources)
+          Attachment.wrap(sources, config: @config).map do |attachment|
+            { type: 'character', image_file: attachment.url_or_data_uri }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/minimax/speech.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class MiniMax
+      # Synchronous text to speech on v1/t2a_v2, which returns the audio
+      # hex-encoded inside a JSON body.
+      module Speech
+        DEFAULT_VOICE = 'English_expressive_narrator'
+        DEFAULT_FORMAT = 'mp3'
+
+        def speech_url(**)
+          media_url('v1/t2a_v2')
+        end
+
+        def render_speech_payload(input, model:, voice: nil, format: nil, provider_options: {})
+          payload = {
+            model: model,
+            text: input,
+            stream: false,
+            output_format: 'hex',
+            voice_setting: { voice_id: voice || DEFAULT_VOICE },
+            audio_setting: { format: (format || DEFAULT_FORMAT).to_s }
+          }
+          Support::Utils.deep_merge(payload, provider_options).tap do |rendered|
+            @effective_voice = Support::Utils.deep_symbolize_keys(rendered).dig(:voice_setting, :voice_id)
+          end
+        end
+
+        def parse_speech_response(response, model:, voice:, format:)
+          audio = response.body.dig('data', 'audio')
+          raise Error.new('MiniMax did not return audio', response:) unless audio
+
+          RubyLLM::Speech.new(
+            data: decode_speech_audio(audio),
+            model: model,
+            voice: @effective_voice || voice || DEFAULT_VOICE,
+            format: response.body.dig('extra_info', 'audio_format') || (format || DEFAULT_FORMAT).to_s,
+            characters: response.body.dig('extra_info', 'usage_characters')
+          )
+        end
+
+        private
+
+        def decode_speech_audio(audio)
+          return [audio].pack('H*') unless audio.start_with?('http')
+
+          Transport::Connection.download(audio, @config)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/minimax/transcription.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class MiniMax
+      # Speech to text on v1/speech_to_text. The language hint travels as a
+      # request header, and verbose_json is the format that carries timed,
+      # speaker-labelled segments.
+      module Transcription
+        TIMESTAMP_LEVELS = { segment: nil, word: 'word' }.freeze
+
+        def rejected_options(operation)
+          operation == :transcribe ? %i[temperature prompt speaker_references named_speakers] : super
+        end
+
+        def render_transcription_options(timestamps:, format: nil, **)
+          return {} if timestamps.nil?
+
+          level = timestamps.respond_to?(:to_sym) ? timestamps.to_sym : timestamps
+          unless TIMESTAMP_LEVELS.key?(level)
+            raise ArgumentError, 'MiniMax transcription timestamps must be word or segment'
+          end
+
+          { response_format: format || 'verbose_json', timestamp_level: TIMESTAMP_LEVELS[level] }.compact
+        end
+
+        def transcription_url
+          media_url('v1/speech_to_text')
+        end
+
+        def transcribe(audio_file, model:, language:, format: nil, speaker_names: nil,
+                       provider_options: {}, **, &block)
+          headers = language ? { 'language' => language.to_s } : {}
+          track_usage(:transcription) do
+            payload = render_transcription_payload(build_audio_file_part(audio_file), model:, format:, speaker_names:,
+                                                                                      provider_options:)
+            next stream_transcription(payload, model:, language:, headers:, &block) if block
+
+            response = @connection.post(transcription_url, payload, usage: @usage_tracker) do |request|
+              request.headers.merge!(headers)
+            end
+            parse_transcription_response(response, model:, language:, word_level: payload[:timestamp_level] == 'word')
+          end
+        end
+
+        def render_transcription_payload(file_part, model:, format: nil, speaker_names: nil, provider_options: {}, **)
+          payload = { model: model, response_format: format || ('verbose_json' if speaker_names) }.compact
+          payload.merge(provider_options.transform_keys(&:to_sym)).except(:file).merge(file: file_part)
+        end
+
+        def parse_transcription_response(response, model:, language: nil, word_level: false)
+          data = response.body
+          return RubyLLM::Transcription.new(text: data, model:, language:) if data.is_a?(String)
+
+          RubyLLM::Transcription.new(
+            text: data['text'], model:, language:, duration: data['duration'], audio_seconds: data['duration'],
+            segments: (data['segments'] unless word_level), words: (data['segments'] if word_level)
+          )
+        end
+
+        def stream_transcription(payload, model:, language: nil, headers: {}, &block)
+          text = +''
+          duration = nil
+          stream_events(transcription_url, payload.merge(stream: 'true'), headers) do |event|
+            delta = event['delta'].to_s
+            text << delta
+            duration = event['duration'] || duration
+            block.call(transcription_chunk(:delta, delta:, raw: event)) unless delta.empty?
+            block.call(transcription_chunk(:done, text: text.dup, raw: event)) if event['finish']
+          end
+          RubyLLM::Transcription.new(text:, model:, language:, duration:, audio_seconds: duration)
+        end
+
+        private
+
+        def transcription_chunk(type, **attributes)
+          RubyLLM::TranscriptionChunk.new(type:, **attributes)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/minimax/videos.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class MiniMax
+      # Asynchronous video generation on v2/video_generation. The endpoint
+      # requires a resolution and duration, and a ratio for text-only
+      # requests, so those are defaulted to values every MiniMax-H3 variant
+      # takes. A single image is the first frame; other attachments are
+      # references.
+      module Videos
+        DEFAULT_SETTINGS = { resolution: '768P', duration: 5 }.freeze
+        DEFAULT_RATIO = '16:9'
+        UNTYPED_MIME_TYPES = [nil, 'application/octet-stream'].freeze
+        REFERENCE_KINDS = { image: :image_url, video: :video_url, audio: :audio_url }.freeze
+
+        def video_url
+          media_url('v2/video_generation')
+        end
+
+        def video_job_url(job)
+          media_url("v2/query/video_generation/#{job.id}")
+        end
+
+        def render_video_payload(prompt, model:, with: [], provider_options: {})
+          payload = { model: model, content: [{ type: 'text', text: prompt }] + video_references(with) }
+          payload.merge!(DEFAULT_SETTINGS)
+          payload[:ratio] = DEFAULT_RATIO if with.empty?
+          Support::Utils.deep_merge(payload, provider_options)
+        end
+
+        def parse_video_job(response, model:, **request)
+          id = response.body['task_id']
+          raise Error.new('MiniMax did not return a video task id', response:) unless id
+
+          VideoJob.new(id: id.to_s, protocol: self, model: model, raw: response.body, **request)
+        end
+
+        def parse_video_job_status(response, **)
+          task = response.body['task']
+          raise Error.new('MiniMax did not return a video task', response:) unless task
+
+          state = case task['status']
+                  when 'succeeded' then { status: :completed }
+                  when 'failed' then { status: :failed, error: task.dig('error', 'message') || 'failed' }
+                  else { status: :pending }
+                  end
+          state.merge(raw: response.body)
+        end
+
+        def download_video(job)
+          task = job.raw['task'] || {}
+          Video.new(url: task.dig('content', 'url'), mime_type: 'video/mp4', duration: task['duration'],
+                    model: task['model'] || job.model, raw: job.raw)
+        end
+
+        private
+
+        def validate_animate_inputs!(with:)
+          with.each do |attachment|
+            next if reference_kind(attachment) || UNTYPED_MIME_TYPES.include?(attachment.mime_type)
+
+            raise UnsupportedAttachmentError, attachment.mime_type
+          end
+        end
+
+        def video_references(attachments)
+          first_frame = attachments.one? && reference_kind_or_image(attachments.first) == :image
+          attachments.map do |attachment|
+            kind = reference_kind_or_image(attachment)
+            part = REFERENCE_KINDS.fetch(kind)
+            { type: part.to_s, part => { url: video_reference_url(attachment) },
+              role: first_frame ? 'first_frame' : "reference_#{kind}" }
+          end
+        end
+
+        def reference_kind(attachment)
+          %i[image video audio].find { |kind| attachment.public_send(:"#{kind}?") }
+        end
+
+        def reference_kind_or_image(attachment)
+          reference_kind(attachment) || :image
+        end
+
+        def video_reference_url(attachment)
+          return "mm_file://#{attachment.provider_file_id}" if attachment.provider_file?
+
+          attachment.url_or_data_uri
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/minimax.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    # MiniMax's own media API at the root of its host, beside the
+    # Anthropic-compatible chat endpoint: images, videos, speech, and
+    # transcription.
+    class MiniMax < Protocol
+      include MiniMax::Images
+      include MiniMax::Videos
+      include MiniMax::Speech
+      include MiniMax::Transcription
+
+      private
+
+      def media_url(path)
+        "#{@provider.media_base}/#{path}"
+      end
+    end
+  end
+end