ruby_llm 1.16.0 → 2.0.0.rc1

data/lib/ruby_llm/protocols/gpustack/videos.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    module GPUStack
+      # vLLM-Omni video jobs through GPUStack's model proxy.
+      module Videos
+        def video_url
+          "#{@provider.backend_api_base}/v1/videos"
+        end
+
+        def render_video_payload(prompt, model:, with: [], provider_options: {})
+          raise ArgumentError, 'vLLM-Omni video generation requires a prompt' if prompt.nil?
+
+          options = video_options(provider_options)
+          payload = { model: model, prompt: prompt }.merge(video_references(with)).merge(options).compact
+          payload.transform_values do |value|
+            value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : value
+          end
+        end
+
+        def post_video(url, payload)
+          @connection.post(url, payload, idempotent: false) do |request|
+            request.headers['Content-Type'] = 'multipart/form-data'
+          end
+        end
+
+        def parse_video_job(response, model:)
+          body = response.body
+          id = body['id']
+          raise Error.new('GPUStack did not return a video job id', response:) unless id
+
+          VideoJob.new(id: id, protocol: self, model: body['model'] || model, **video_job_state(body))
+        end
+
+        def video_job_url(job)
+          "#{video_url}/#{job.id}"
+        end
+
+        def parse_video_job_status(response, **)
+          video_job_state(response.body)
+        end
+
+        def download_video(job)
+          response = @connection.get("#{video_job_url(job)}/content")
+          Video.new(data: response.body, mime_type: response.headers['content-type'] || job.raw['media_type'],
+                    model: job.model, raw: job.raw)
+        end
+
+        private
+
+        def video_options(options)
+          options = Support::Utils.deep_symbolize_keys(options)
+          if options[:num_outputs_per_prompt] && options[:num_outputs_per_prompt] != 1
+            raise ArgumentError, 'animate returns one video; num_outputs_per_prompt must be 1'
+          end
+
+          options
+        end
+
+        def video_references(attachments)
+          attachments.group_by(&:type).to_h do |type, group|
+            values = group.map { |attachment| { "#{type}_url" => video_reference(attachment) } }
+            [:"#{type}_reference", values.one? ? values.first : values]
+          end
+        end
+
+        def video_job_state(body)
+          status = case body['status']
+                   when 'queued', 'in_progress' then :pending
+                   when 'completed' then :completed
+                   when 'failed' then :failed
+                   else raise Error, "Unknown GPUStack video status: #{body['status'].inspect}"
+                   end
+          { status: status, raw: body, error: body.dig('error', 'message') }
+        end
+
+        def validate_animate_inputs!(with:)
+          @provider.backend_api_base
+          with.each do |attachment|
+            if attachment.provider_file?
+              raise ArgumentError, 'vLLM-Omni video references require media bytes or URLs, not uploaded file ids'
+            end
+            next if attachment.image? || attachment.video? || attachment.audio?
+
+            raise UnsupportedAttachmentError, attachment.mime_type
+          end
+        end
+
+        def video_reference(attachment)
+          attachment.url? ? attachment.source.to_s : attachment.for_llm
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/interactions/chat.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Interactions
+      module Chat # :nodoc:
+        FINISH_REASONS = { 'completed' => :stop, 'requires_action' => :tool_calls, 'incomplete' => :max_tokens }.freeze
+
+        module_function
+
+        def finish_reasons = FINISH_REASONS
+
+        def completion_url
+          'interactions'
+        end
+
+        def render_payload(messages, tools:, temperature:, model:, stream: false, max_output_tokens: nil,
+                           schema: nil, thinking: nil, tool_prefs: nil, **)
+          config = { temperature: temperature, max_output_tokens: max_output_tokens }.compact
+          config.merge!(render_interaction_thinking(thinking)) if thinking
+          choice = tool_prefs&.dig(:choice)
+          config[:tool_choice] = render_interaction_choice(choice) if choice
+          payload = {
+            model: model.id, input: format_interaction_input(messages), stream: stream, store: false,
+            system_instruction: messages.select { |message| message.role == :system }.map(&:content).join("\n\n"),
+            generation_config: config, tools: render_interaction_tools(tools)
+          }
+          payload[:response_format] = { type: 'text', mime_type: 'application/json', schema: schema[:schema] } if schema
+          payload
+        end
+
+        def render_interaction_thinking(thinking)
+          if thinking.disabled? || thinking.budget&.zero?
+            raise ArgumentError, 'Gemini Interactions does not expose a thinking-off control'
+          end
+          raise ArgumentError, 'Gemini Interactions accepts thinking effort, not a token budget' if thinking.budget
+          if thinking.effort && !%i[minimal low medium high].include?(thinking.effort)
+            raise ArgumentError, 'Gemini Interactions thinking effort must be minimal, low, medium, or high'
+          end
+
+          { thinking_level: thinking.effort&.to_s,
+            thinking_summaries: render_interaction_summaries(thinking.display) }.compact
+        end
+
+        def render_interaction_summaries(display)
+          unless [nil, :summarized, :omitted].include?(display)
+            raise ArgumentError, 'Gemini Interactions thinking display must be summarized or omitted'
+          end
+
+          display == :omitted ? 'none' : 'auto'
+        end
+
+        def format_interaction_input(messages)
+          calls = messages.flat_map { |message| message.tool_calls.to_h.values }.to_h { |call| [call.id, call] }
+          messages.reject { |message| message.role == :system }.flat_map do |message|
+            render_interaction_turn(message, calls)
+          end
+        end
+
+        def render_interaction_turn(message, calls)
+          if interaction_state?(message.raw_content)
+            render_interaction_history(message.raw_content.dig('response', 'steps') || [])
+          elsif message.tool_result?
+            [render_interaction_result(message, calls)]
+          else
+            render_interaction_message(message)
+          end
+        end
+
+        def render_interaction_result(message, calls)
+          { type: 'function_result', call_id: message.tool_call_id, name: calls[message.tool_call_id]&.name,
+            result: render_interaction_content(message.content, message.attachments) }.compact
+        end
+
+        def render_interaction_history(steps)
+          steps.map do |step|
+            step['type'].to_s.start_with?('mcp_server_') ? step.except('signature') : step
+          end
+        end
+
+        def render_interaction_message(message)
+          steps = []
+          if message.content || message.attachments.any?
+            steps << { type: message.role == :assistant ? 'model_output' : 'user_input',
+                       content: render_interaction_content(message.content, message.attachments) }
+          end
+          message.tool_calls&.each_value do |call|
+            steps << { type: 'function_call', id: call.id, name: call.name, arguments: call.arguments }
+          end
+          steps
+        end
+
+        def interaction_state?(content)
+          content.is_a?(Hash) && content.dig('response', 'object') == 'interaction'
+        end
+
+        def parse_completion_body(data, raw:, model: data['model'] || @model&.id, cost: nil)
+          unless %w[completed requires_action incomplete].include?(data['status'])
+            message = Array(data['errors']).filter_map { |error| error['message'] }.join('; ')
+            raise Error.new(message.empty? ? "Gemini interaction ended with status #{data['status']}" : message,
+                            response: raw)
+          end
+
+          steps = data.fetch('steps', [])
+          content = parse_interaction_content(steps)
+          calls = parse_interaction_calls(steps)
+          if data['status'] == 'requires_action' && calls.empty?
+            raise Error.new('Gemini interaction requires an unsupported action', response: raw)
+          end
+
+          Message.new(role: :assistant, content: content[:text], attachments: content[:attachments],
+                      citations: content[:citations], thinking: parse_interaction_thinking(steps),
+                      tool_calls: calls, server_tool_calls: parse_interaction_server_calls(steps),
+                      raw_content: { 'response' => data },
+                      model: model, raw: raw, cost: cost,
+                      finish_reason: interaction_finish_reason(data['status'], calls),
+                      **parse_interaction_usage(data['usage'] || {}))
+        end
+
+        def interaction_finish_reason(status, calls)
+          return :tool_calls unless calls.empty?
+
+          finish_reasons.fetch(status)
+        end
+
+        def parse_interaction_thinking(steps)
+          thoughts = steps.select { |step| step['type'] == 'thought' }
+          text = thoughts.flat_map { |step| Array(step['summary']) }.filter_map { |part| part['text'] }.join
+          Thinking.build(text: text.empty? ? nil : text, signature: thoughts.last&.dig('signature'))
+        end
+
+        def parse_interaction_usage(usage)
+          prompt = usage['total_input_tokens']
+          cached = usage['total_cached_tokens'].to_i
+          thoughts = usage['total_thought_tokens'].to_i
+          {
+            input_tokens: prompt && [prompt + usage['total_tool_use_tokens'].to_i - cached, 0].max,
+            output_tokens: usage['total_output_tokens'] && (usage['total_output_tokens'] + thoughts),
+            cache_read_tokens: usage['total_cached_tokens'], thinking_tokens: usage['total_thought_tokens']
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/interactions/content.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+module RubyLLM
+  module Protocols
+    class Interactions
+      module Content # :nodoc:
+        CITATION_TYPES = %w[url_citation file_citation place_citation].freeze
+
+        module_function
+
+        def render_interaction_content(text, attachments)
+          parts = text.nil? ? [] : [{ type: 'text', text: text.to_s }]
+          parts + attachments.map { |attachment| render_interaction_attachment(attachment) }
+        end
+
+        def render_interaction_attachment(attachment)
+          return { type: 'text', text: attachment.content } if attachment.text?
+
+          type = interaction_attachment_type(attachment)
+          raise ArgumentError, "Gemini Interactions does not support #{attachment.mime_type} input" unless type
+
+          part = { type: type, mime_type: attachment.mime_type }
+          if attachment.provider_file?
+            part.merge(uri: attachment.provider_file_uri)
+          elsif attachment.url?
+            part.merge(uri: attachment.source.to_s)
+          else
+            part.merge(data: attachment.encoded)
+          end
+        end
+
+        def interaction_attachment_type(attachment)
+          return 'image' if attachment.image?
+          return 'audio' if attachment.audio?
+          return 'video' if attachment.video?
+
+          'document' if attachment.pdf?
+        end
+
+        def parse_interaction_content(steps)
+          result = { text: +'', attachments: [], citations: [] }
+          steps.select { |step| step['type'] == 'model_output' }.each do |step|
+            Array(step['content']).each { |part| parse_interaction_part(part, result) }
+          end
+          result
+        end
+
+        def parse_interaction_part(part, result)
+          if part['type'] == 'text'
+            text = part['text'].to_s
+            result[:citations].concat(parse_interaction_citations(part, result[:text].length))
+            result[:text] << text
+          elsif part['data'] || part['uri']
+            source = part['uri'] || StringIO.new(Base64.decode64(part['data']))
+            result[:attachments] << Attachment.new(source, config: @config)
+          end
+        end
+
+        def parse_interaction_citations(part, offset)
+          Array(part['annotations']).filter_map do |annotation|
+            next unless CITATION_TYPES.include?(annotation['type'])
+
+            Citation.new(**interaction_citation_source(annotation),
+                         **interaction_citation_span(part['text'].to_s, annotation, offset))
+          end
+        end
+
+        def interaction_citation_source(annotation)
+          { url: annotation['url'] || annotation['document_uri'],
+            title: annotation['title'] || annotation['file_name'] || annotation['name'],
+            source_id: annotation['media_id'] || annotation['place_id'], cited_text: annotation['source'],
+            start_page: annotation['page_number'], end_page: annotation['page_number'] }
+        end
+
+        def interaction_citation_span(text, annotation, offset)
+          start_index = interaction_citation_index(text, annotation['start_index'])
+          end_index = interaction_citation_index(text, annotation['end_index'])
+          { start_index: start_index && (offset + start_index), end_index: end_index && (offset + end_index),
+            text: start_index && end_index && text[start_index...end_index] }
+        end
+
+        def interaction_citation_index(text, bytes)
+          text.byteslice(0, bytes)&.length unless bytes.nil?
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/interactions/streaming.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Interactions
+      module Streaming # :nodoc:
+        module_function
+
+        def stream_response(payload, additional_headers = {})
+          @interaction_steps = {}
+          @interaction_response = {}
+          @interaction_done = false
+          response = stream_events(completion_url, payload, additional_headers) { |data| yield build_chunk(data) }
+          raise Error.new('Gemini interaction stream ended before completion', response:) unless @interaction_done
+
+          parse_completion_body(streamed_interaction, raw: response)
+        end
+
+        def build_chunk(data)
+          @interaction_steps ||= {}
+          @interaction_response ||= {}
+          case data['event_type']
+          when 'interaction.created'
+            @interaction_response.merge!(data.fetch('interaction'))
+          when 'interaction.completed'
+            @interaction_done = true
+            @interaction_response.merge!(data.fetch('interaction'))
+            return final_interaction_chunk
+          when 'error'
+            raise Error, data.dig('error', 'message') || 'Gemini interaction failed'
+          when 'step.start'
+            @interaction_steps[data.fetch('index')] = Support::Utils.deep_dup(data.fetch('step'))
+          when 'step.delta'
+            step = @interaction_steps.fetch(data.fetch('index'))
+            return append_interaction_delta(step, data.fetch('delta'))
+          end
+          Chunk.new(role: :assistant, content: nil)
+        end
+
+        def append_interaction_delta(step, delta)
+          type = delta['type']
+          case type
+          when 'text'
+            append_interaction_text(step, delta['text'])
+            return Chunk.new(role: :assistant, content: delta['text'])
+          when 'text_annotation'
+            append_interaction_text(step, '')
+            (step['content'].last['annotations'] ||= []) << delta['annotation']
+          when 'thought_summary'
+            (step['summary'] ||= []) << delta['content']
+            return Chunk.new(role: :assistant, content: nil,
+                             thinking: Thinking.build(text: delta.dig('content', 'text')))
+          when 'thought_signature'
+            step['signature'] = delta['signature']
+          when 'arguments_delta'
+            step['arguments'] = +'' unless step['arguments'].is_a?(String)
+            step['arguments'] << delta['arguments'].to_s
+          when 'image', 'audio', 'video', 'document'
+            (step['content'] ||= []) << Support::Utils.deep_dup(delta)
+          else
+            step.merge!(delta.except('type'))
+          end
+          Chunk.new(role: :assistant, content: nil)
+        end
+
+        def append_interaction_text(step, text)
+          content = step['content'] ||= []
+          content << { 'type' => 'text', 'text' => +'' } unless content.last&.dig('type') == 'text'
+          content.last['text'] << text.to_s
+        end
+
+        def streamed_interaction
+          steps = @interaction_steps.sort.map do |_index, step|
+            next step unless step['type'] == 'function_call'
+
+            step.merge('arguments' => parse_interaction_arguments(step['arguments']))
+          end
+          @interaction_response.merge('steps' => steps)
+        end
+
+        def final_interaction_chunk
+          message = parse_completion_body(streamed_interaction, raw: nil)
+          Chunk.new(role: :assistant, content: nil, model: message.model, tokens: message.tokens,
+                    citations: message.citations, tool_calls: message.tool_calls,
+                    server_tool_calls: message.server_tool_calls, raw_content: message.raw_content,
+                    attachments: message.attachments, finish_reason: message.finish_reason)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/interactions/tools.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Interactions
+      module Tools # :nodoc:
+        module_function
+
+        def render_interaction_tools(tools)
+          tools.values.map do |tool|
+            parameters = tool.parameters_schema ||
+                         Tool::SchemaDefinition.from_parameters(tool.declared_parameters)&.json_schema
+            Support::Utils.deep_merge({ type: 'function', name: tool.name, description: tool.description,
+                                        parameters: parameters }.compact, tool.provider_options)
+          end
+        end
+
+        def render_interaction_choice(choice)
+          return 'any' if choice == :required
+          return choice.to_s if %i[auto none].include?(choice)
+
+          { allowed_tools: { mode: 'any', tools: [choice.to_s] } }
+        end
+
+        def parse_interaction_calls(steps)
+          steps.select { |step| step['type'] == 'function_call' }.to_h do |step|
+            [step.fetch('id'), ToolCall.new(id: step.fetch('id'), name: step.fetch('name'),
+                                            arguments: parse_interaction_arguments(step['arguments']),
+                                            thought_signature: step['signature'])]
+          end
+        end
+
+        def parse_interaction_arguments(arguments)
+          arguments.is_a?(String) ? JSON.parse(arguments) : arguments || {}
+        rescue JSON::ParserError => e
+          raise ToolCallParseError.new(finish_reason: :tool_calls), cause: e
+        end
+
+        def parse_interaction_server_calls(steps)
+          steps.filter_map do |step|
+            type = step['type'].to_s
+            next unless type.end_with?('_call', '_result') && !type.start_with?('function_')
+
+            ServerToolCall.new(type: type, id: step['id'] || step['call_id'], name: step['name'],
+                               input: step['arguments'], result: step['result'], raw: step)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/interactions/transcription.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Interactions
+      module Transcription # :nodoc: all
+        def render_transcription_options(timestamps:, **)
+          return {} if timestamps.nil?
+          raise ArgumentError, 'Gemini transcription timestamps must be word' unless timestamps == :word
+
+          { generation_config: { transcription_config: { mode: { type: 'verbatim',
+                                                                 timestamp_granularities: ['word'] } } } }
+        end
+
+        include Gemini::FileTranscription
+
+        def transcription_url(_model)
+          'interactions'
+        end
+
+        def render_transcription_payload(attachment, model:, language:, speaker_names:, provider_options:, prompt:)
+          config = { language_codes: language && Array(language), custom_vocabulary: prompt && Array(prompt) }.compact
+          config[:mode] = { type: 'verbatim', diarization_mode: 'speaker' } if speaker_names
+          payload = { model:, store: false,
+                      input: [{ type: 'audio', mime_type: attachment.mime_type, data: attachment.encoded }],
+                      generation_config: { transcription_config: config } }
+          payload = Support::Utils.deep_merge(payload, provider_options)
+          validate_transcription_config(payload.dig(:generation_config, :transcription_config))
+          payload
+        end
+
+        def validate_transcription_config(config)
+          mode = config[:mode]
+          return unless config[:custom_vocabulary] && mode.is_a?(Hash)
+          return unless mode[:diarization_mode] || mode[:timestamp_granularities]
+
+          raise ArgumentError, 'Gemini custom vocabulary cannot be combined with diarization or word timestamps'
+        end
+
+        def parse_transcription_response(response, model:)
+          data = response.body
+          message = parse_completion_body(data, raw: response)
+          annotations = Array(data['steps']).flat_map { |step| Array(step['content']) }
+                                            .flat_map { |part| Array(part['annotations']) }
+          words = annotations.filter_map { |item| parse_transcription_word(item) if item['type'] == 'word_info' }
+          RubyLLM::Transcription.new(text: message.content, model:, words: words.empty? ? nil : words,
+                                     **parse_interaction_usage(data['usage'] || {}))
+        end
+
+        def parse_transcription_word(word)
+          { 'word' => word['text'], 'speaker' => word['speaker'],
+            'start' => word['start_offset'] && Float(word['start_offset'].delete_suffix('s')),
+            'end' => word['end_offset'] && Float(word['end_offset'].delete_suffix('s')) }.compact
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/interactions.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    # The Gemini Interactions API for conversations and hosted tools.
+    class Interactions < Protocol
+      include Interactions::Chat
+      include Interactions::Content
+      include Interactions::Tools
+      include Interactions::Streaming
+      include Interactions::Transcription
+
+      public :render
+
+      SERVER_TOOL_ALIASES = {
+        mcp: { tool: { type: 'mcp_server' } },
+        web_search: { tool: { type: 'google_search' } },
+        web_fetch: { tool: { type: 'url_context' } },
+        code_execution: { tool: { type: 'code_execution' } },
+        file_search: { tool: { type: 'file_search' } },
+        google_maps: { tool: { type: 'google_maps' } }
+      }.freeze
+
+      def server_tool_aliases
+        SERVER_TOOL_ALIASES
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/invoke_model/cohere_embeddings.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class InvokeModel
+      # Cohere embedding models over Bedrock InvokeModel.
+      class CohereEmbeddings < InvokeModel
+        # rubocop:disable-next Lint/UnusedMethodArgument
+        def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
+          ensure_no_embedding_media!(with)
+          track_usage(:embedding) do
+            payload = render_embedding_payload(text, model:, dimensions:, task_type:, provider_options:)
+            response = signed_post(embedding_url(model:), payload)
+
+            parse_embedding_response(response, model:, text:)
+          end
+        end
+
+        private
+
+        def render_embedding_payload(text, model:, dimensions:, provider_options:, task_type: nil)
+          payload = {
+            input_type: task_type || 'search_document'
+          }
+          texts = [text].flatten.compact.map(&:to_s).reject(&:empty?)
+          payload[:texts] = texts unless texts.empty?
+
+          if dimensions
+            raise Error, "#{model} does not support custom dimensions" unless cohere_v4?(model)
+
+            payload[:output_dimension] = dimensions
+          end
+
+          deep_merge_provider_options(payload, provider_options)
+        end
+
+        def parse_embedding_response(response, model:, text:)
+          vectors = response.body['embeddings']
+          vectors = vectors['float'] || vectors.values.first if vectors.is_a?(Hash)
+          vectors = vectors.first unless text.is_a?(Array)
+
+          Embedding.new(vectors:, model:)
+        end
+
+        def cohere_v4?(model)
+          model.to_s.include?('cohere.embed-v4')
+        end
+      end
+    end
+  end
+end