ruby_llm 2.1.0 → 2.2.0

data/lib/ruby_llm/protocols/gemini/chat.rb

@@ -6,7 +6,7 @@ module RubyLLM
       # Chat methods for the Gemini API implementation
       module Chat
         FINISH_REASONS = {
-          'STOP' => :stop, 'MAX_TOKENS' => :max_tokens,
+          'STOP' => :stop, 'MAX_TOKENS' => :max_tokens, 'CONTINUATION' => nil,
           'SAFETY' => :content_filter, 'RECITATION' => :content_filter, 'BLOCKLIST' => :content_filter,
           'PROHIBITED_CONTENT' => :content_filter, 'SPII' => :content_filter, 'IMAGE_SAFETY' => :content_filter,
           'IMAGE_RECITATION' => :content_filter, 'IMAGE_PROHIBITED_CONTENT' => :content_filter,
@@ -23,7 +23,7 @@ module RubyLLM
         def normalize_finish_reason(reason)
           return nil if reason.nil?
 
-          finish_reasons.fetch(reason.to_s) { reason.to_s.to_sym }
+          finish_reasons.fetch(reason.to_s) { reason.to_s.downcase.to_sym }
         end
 
         def completion_url
@@ -35,11 +35,13 @@ module RubyLLM
                            thinking: nil, citations: false, caching: nil, tool_prefs: nil)
           warn_unsupported_citations(model) if citations && !model.supports?(:citations)
           tool_prefs ||= {}
+          unfinished = messages.last if messages.last&.continuation
           payload = {
-            contents: format_messages(messages.reject { |msg| msg.role == :system }),
+            contents: render_messages(messages.reject { |msg| msg.role == :system || msg.equal?(unfinished) }),
             generationConfig: {}
           }
-          system_instruction = format_system_instruction(messages)
+          payload[:continuationToken] = unfinished.continuation if unfinished
+          system_instruction = render_system_instruction(messages)
           payload[:systemInstruction] = system_instruction if system_instruction
 
           payload[:generationConfig][:temperature] = temperature unless temperature.nil?
@@ -49,7 +51,7 @@ module RubyLLM
           payload[:generationConfig][:thinkingConfig] = build_thinking_config(model, thinking) if thinking&.enabled?
 
           if tools.any?
-            payload[:tools] = format_tools(tools)
+            payload[:tools] = render_tools(tools)
             # Gemini doesn't support controlling parallel tool calls
             payload[:toolConfig] = build_tool_config(tool_prefs[:choice]) unless tool_prefs[:choice].nil?
           end
@@ -108,9 +110,10 @@ module RubyLLM
         end
 
         def build_thinking_config(_model, thinking)
-          return { includeThoughts: false, thinkingBudget: 0 } if thinking.enabled == false
+          off = thinking.enabled == false || (thinking.budget.is_a?(Integer) && thinking.budget.zero?)
+          return { includeThoughts: false, thinkingBudget: 0 } if off
 
-          config = { includeThoughts: true }
+          config = { includeThoughts: include_thoughts?(thinking.display) }
 
           config[:thinkingLevel] = thinking.effort.to_s if thinking.effort
           config[:thinkingBudget] = thinking.budget if thinking.budget.is_a?(Integer)
@@ -119,7 +122,15 @@ module RubyLLM
           config
         end
 
-        def supports_provider_file_references?
+        def include_thoughts?(display)
+          unless [nil, :summarized, :omitted].include?(display)
+            raise ArgumentError, 'Gemini thinking display must be summarized or omitted'
+          end
+
+          display != :omitted
+        end
+
+        def provider_file_references?
           true
         end
 
@@ -133,20 +144,20 @@ module RubyLLM
 
         private
 
-        def format_system_instruction(messages)
+        def render_system_instruction(messages)
           parts = messages.select { |msg| msg.role == :system }.flat_map do |msg|
             text = msg.content.to_s
-            Media.format_content(text.empty? ? nil : text, msg.attachments)
+            Media.render_content(text.empty? ? nil : text, msg.attachments)
           end
 
           { parts: parts } if parts.any?
         end
 
-        def format_messages(messages)
+        def render_messages(messages)
           MessageFormatter.new(self, messages).format
         end
 
-        def format_role(role)
+        def render_role(role)
           case role
           when :assistant then 'model'
           when :system, :tool then 'user'
@@ -154,20 +165,21 @@ module RubyLLM
           end
         end
 
-        def format_parts(msg)
+        def render_parts(msg)
           if msg.role == :assistant && msg.raw_content.is_a?(Array)
             msg.raw_content
           elsif msg.tool_call?
-            format_tool_call(msg)
+            render_tool_call(msg)
           elsif msg.tool_result?
-            format_tool_result(msg)
+            render_tool_result(msg)
           else
-            format_message_parts(msg)
+            render_message_parts(msg)
           end
         end
 
-        def format_message_parts(msg)
-          parts = Media.format_content(msg.content, msg.attachments)
+        def render_message_parts(msg)
+          content = msg.content unless msg.role == :assistant && msg.content.to_s.empty?
+          parts = Media.render_content(content, msg.attachments)
           thinking = msg.thinking if msg.role == :assistant
           return parts unless thinking
 
@@ -179,27 +191,29 @@ module RubyLLM
         # Gemini signs the last part of an answer. An answer without parts gets
         # the empty text part a stream ends with: Gemini refuses a part without data.
         def sign_last_part(parts, signature)
-          parts = [Media.format_text('')] if parts.empty?
+          parts = [Media.render_text('')] if parts.empty?
           [*parts[...-1], parts.last.merge(thoughtSignature: signature)]
         end
 
         def parse_completion_body(data, raw:)
           parts = data.dig('candidates', 0, 'content', 'parts') || []
-          tool_calls = extract_tool_calls(data)
+          tool_calls = parse_tool_calls(data)
           content, attachments = parse_content(data)
+          continuation = parse_continuation(data)
 
           Message.new(
             role: :assistant,
             content: content,
             attachments: attachments,
-            citations: extract_citations(data, content),
+            citations: parse_citations(data, parts),
             thinking: Thinking.build(
-              text: extract_thought_parts(parts),
-              signature: extract_thought_signature(parts)
+              text: parse_thought_parts(parts),
+              signature: parse_thought_signature(parts)
             ),
             tool_calls: tool_calls,
-            server_tool_calls: extract_server_tool_calls(data, parts),
-            raw_content: parts.any? { |part| server_tool_part?(part) } ? without_search_suggestions(parts) : nil,
+            server_tool_calls: parse_server_tool_calls(data, parts),
+            raw_content: (without_search_suggestions(parts) if continuation || parts.any? { |p| server_tool_part?(p) }),
+            continuation: continuation,
             input_tokens: input_tokens(data),
             output_tokens: calculate_output_tokens(data),
             cache_read_tokens: data.dig('usageMetadata', 'cachedContentTokenCount'),
@@ -213,6 +227,14 @@ module RubyLLM
           )
         end
 
+        # Long decode hands back a token whenever it pauses, including at
+        # stream checkpoints and on MAX_TOKENS; only CONTINUATION leaves the
+        # response unfinished.
+        def parse_continuation(data)
+          candidate = data.dig('candidates', 0) || {}
+          candidate['continuationToken'] if candidate['finishReason'] == 'CONTINUATION'
+        end
+
         # Gemini 3 bills each distinct, non-empty query a grounding ran.
         def parse_server_tool_use(data)
           searches = Array(data['candidates']).sum do |candidate|
@@ -221,11 +243,15 @@ module RubyLLM
           { 'web_search_requests' => searches }
         end
 
+        # Gemini reports what a server-side tool loaded for the model, such as
+        # the frames of an agentic video, apart from the prompt, and bills it
+        # as input.
         def input_tokens(data)
-          prompt_tokens = data.dig('usageMetadata', 'promptTokenCount')
+          usage = data['usageMetadata'] || {}
+          prompt_tokens = usage['promptTokenCount']
           return unless prompt_tokens
 
-          [prompt_tokens.to_i - data.dig('usageMetadata', 'cachedContentTokenCount').to_i, 0].max
+          [prompt_tokens.to_i + usage['toolUsePromptTokenCount'].to_i - usage['cachedContentTokenCount'].to_i, 0].max
         end
 
         def parse_content(data)
@@ -260,7 +286,7 @@ module RubyLLM
           end
         end
 
-        def extract_server_tool_calls(data, parts)
+        def parse_server_tool_calls(data, parts)
           calls = parts.select { |part| server_tool_part?(part) }.map do |part|
             ServerToolCall.new(
               type: part.key?('executableCode') ? 'executable_code' : 'code_execution_result',
@@ -280,6 +306,7 @@ module RubyLLM
           queries = candidate.dig('groundingMetadata', 'webSearchQueries')
           if queries&.any?
             calls << ServerToolCall.new(type: 'google_search', input: { 'queries' => queries },
+                                        result: web_grounding_chunks(candidate),
                                         raw: { 'webSearchQueries' => queries },
                                         search_suggestions: candidate.dig('groundingMetadata', 'searchEntryPoint',
                                                                           'renderedContent'))
@@ -290,8 +317,15 @@ module RubyLLM
           calls
         end
 
+        # Grounding on a store of documents reports retrievedContext chunks,
+        # which are not web pages.
+        def web_grounding_chunks(candidate)
+          chunks = Array(candidate.dig('groundingMetadata', 'groundingChunks')).select { |chunk| chunk['web'] }
+          chunks unless chunks.empty?
+        end
+
         # Normalizes grounding metadata (Google Search grounding) into citations.
-        def extract_citations(data, content)
+        def parse_citations(data, parts)
           metadata = data.dig('candidates', 0, 'groundingMetadata')
           return [] unless metadata
 
@@ -299,13 +333,12 @@ module RubyLLM
           supports = metadata['groundingSupports'] || []
           return chunk_citations(chunks) if supports.empty?
 
-          supports.flat_map { |support| support_citations(support, chunks, content) }
+          supports.flat_map { |support| support_citations(support, chunks, parts) }
         end
 
-        def support_citations(support, chunks, content)
+        def support_citations(support, chunks, parts)
           segment = support['segment'] || {}
-          end_index = segment['endIndex']
-          start_index = segment['startIndex'] || (0 if end_index)
+          start_index, end_index = citation_indices(parts, segment)
 
           Array(support['groundingChunkIndices']).filter_map do |index|
             source = chunk_source(chunks[index])
@@ -315,13 +348,39 @@ module RubyLLM
               url: source['uri'],
               title: source['title'],
               text: segment['text'],
-              start_index: byte_to_char_index(content, start_index),
-              end_index: byte_to_char_index(content, end_index),
+              start_index: start_index,
+              end_index: end_index,
               source_index: index
             )
           end
         end
 
+        def citation_indices(parts, segment)
+          indices = [segment.fetch('startIndex', 0), segment['endIndex']].map do |byte_index|
+            citation_offset(parts, segment, byte_index)
+          end
+          indices.all? && indices.first <= indices.last ? indices : [nil, nil]
+        end
+
+        # Without a partIndex, offsets count across the visible text, which
+        # keeps thinking parts ahead of the answer from absorbing the span.
+        def citation_offset(parts, segment, byte_index)
+          return byte_to_char_index(visible_text(parts), byte_index) unless segment.key?('partIndex')
+
+          part_index = segment['partIndex']
+          return unless part_index.is_a?(Integer) && part_index.between?(0, parts.length - 1)
+
+          part = parts[part_index]
+          return if part['thought']
+
+          index = byte_to_char_index(part['text'], byte_index)
+          visible_text(parts.take(part_index)).length + index if index
+        end
+
+        def visible_text(parts)
+          parts.reject { |part| part['thought'] }.sum('') { |part| part['text'].to_s }
+        end
+
         def chunk_citations(chunks)
           chunks.each_with_index.filter_map do |chunk, index|
             source = chunk_source(chunk)
@@ -337,21 +396,27 @@ module RubyLLM
           chunk['web'] || chunk['retrievedContext']
         end
 
-        # Grounding segment indices are byte offsets into the UTF-8 response text.
+        # Grounding segment indices are byte offsets within the selected UTF-8 part.
         def byte_to_char_index(content, byte_index)
-          return nil unless content.is_a?(String) && byte_index
+          return unless content.is_a?(String) && byte_index.is_a?(Integer) &&
+                        byte_index.between?(0, content.bytesize)
 
-          content.byteslice(0, byte_index)&.length
+          prefix = content.byteslice(0, byte_index)
+          prefix.length if prefix.valid_encoding?
         end
 
-        def extract_thought_parts(parts)
+        def parse_thought_parts(parts)
           thought_parts = parts.select { |p| p['thought'] }
           thoughts = thought_parts.filter_map { |p| p['text'] }.join
           thoughts.empty? ? nil : thoughts
         end
 
-        def extract_thought_signature(parts)
+        # Agentic video navigation steps carry signatures of their own, which
+        # Gemini refuses on the answer they precede.
+        def parse_thought_signature(parts)
           parts.each do |part|
+            next if part.key?('toolCall') || part.key?('toolResponse')
+
             signature = part['thoughtSignature'] ||
                         part['thought_signature'] ||
                         part.dig('functionCall', 'thoughtSignature') ||
@@ -363,16 +428,15 @@ module RubyLLM
         end
 
         def calculate_output_tokens(data)
-          candidates = data.dig('usageMetadata', 'candidatesTokenCount') || 0
-          thoughts = data.dig('usageMetadata', 'thoughtsTokenCount') || 0
-          candidates + thoughts
+          counts = (data['usageMetadata'] || {}).values_at('candidatesTokenCount', 'thoughtsTokenCount').compact
+          counts.sum unless counts.empty?
         end
 
         def build_json_schema(schema)
           normalized = RubyLLM::Support::Utils.deep_dup(schema[:schema])
           normalized.delete(:strict)
           normalized.delete('strict')
-          RubyLLM::Support::Utils.deep_stringify_keys(normalized)
+          Schemas.render_schema(RubyLLM::Support::Utils.deep_stringify_keys(normalized))
         end
 
         def structured_output_config(schema)
@@ -402,7 +466,10 @@ module RubyLLM
                 @index = next_index
               else
                 remember_tool_calls if current_message.tool_call?
-                formatted << build_standard_message(current_message)
+                turn = build_standard_message(current_message)
+                # Vertex AI refuses a turn without parts, which is what an empty
+                # answer is once another provider's signature is left out.
+                formatted << turn if turn[:parts].any? || current_message.role != :assistant
                 @index += 1
               end
             end
@@ -430,7 +497,7 @@ module RubyLLM
             end
 
             parts = in_call_order(results).flat_map do |tool_message|
-              format_tool_result(tool_message, @tool_call_names.delete(tool_message.tool_call_id))
+              render_tool_result(tool_message, @tool_call_names.delete(tool_message.tool_call_id))
             end
 
             [parts, index]
@@ -451,17 +518,17 @@ module RubyLLM
           end
 
           def remember_tool_calls
-            current_message.tool_calls.each do |tool_call_id, tool_call|
+            current_message.tool_calls.to_h.each do |tool_call_id, tool_call|
               @tool_call_names[tool_call_id] = tool_call.name
             end
           end
 
           def build_standard_message(message)
-            parts = @provider.send(:format_parts, message)
+            parts = @provider.send(:render_parts, message)
             parts = @provider.send(:sign_step, parts) if message.tool_call? && current_turn?
 
             {
-              role: @provider.send(:format_role, message.role),
+              role: @provider.send(:render_role, message.role),
               parts: parts
             }
           end
@@ -472,8 +539,8 @@ module RubyLLM
             @turn_start.nil? || @index > @turn_start
           end
 
-          def format_tool_result(message, tool_name)
-            @provider.send(:format_tool_result, message, tool_name)
+          def render_tool_result(message, tool_name)
+            @provider.send(:render_tool_result, message, tool_name)
           end
         end
       end

data/lib/ruby_llm/protocols/gemini/embedding_batches.rb

@@ -44,11 +44,11 @@ module RubyLLM
         end
 
         def parse_embedding_batch_results(responses)
-          groups = responses.each_with_index.group_by do |inline, index|
-            inline.dig('metadata', 'custom_id') || index.to_s
-          end
-          groups.filter_map do |key, indexed|
-            parse_embedding_batch_group(key, indexed.map(&:first))
+          identified, unidentified = responses.partition { |inline| inline.dig('metadata', 'custom_id') }
+          RubyLLM.logger.warn 'Gemini returned batch results without metadata' if unidentified.any?
+
+          identified.group_by { |inline| inline.dig('metadata', 'custom_id') }.filter_map do |key, group|
+            parse_embedding_batch_group(key, group)
           end
         end
 
@@ -60,6 +60,10 @@ module RubyLLM
           metadata = responses.first.fetch('metadata')
           return if responses.size < metadata.fetch('embedding_count')
 
+          parse_embedding_batch_slot(index, key, responses, metadata)
+        end
+
+        def parse_embedding_batch_slot(index, key, responses, metadata)
           positions = responses.map { |inline| inline.dig('metadata', 'embedding_index') }
           unless positions.sort == (0...responses.size).to_a
             return [index, nil, batch_failure(key, 'Invalid or duplicate embedding record positions')]

data/lib/ruby_llm/protocols/gemini/embeddings.rb

@@ -26,7 +26,7 @@ module RubyLLM
           Support::Utils.deep_merge({ requests: requests }, provider_options)
         end
 
-        def supports_embedding_media?
+        def embedding_media?
           true
         end
 
@@ -42,7 +42,7 @@ module RubyLLM
           vectors = response.body['embeddings']&.map { |e| e['values'] }
           vectors = vectors.first if vectors&.length == 1 && !text.is_a?(Array)
 
-          Embedding.new(vectors:, model:)
+          Embedding.new(vectors:, model:, input_tokens: response.body.dig('usageMetadata', 'promptTokenCount'))
         end
 
         private
@@ -52,7 +52,7 @@ module RubyLLM
         end
 
         def media_embedding_payload(text, attachments, model:, dimensions:, task_type: nil, title: nil)
-          embedding_payload(Media.format_content(text, attachments), model:, dimensions:, task_type:, title:)
+          embedding_payload(Media.render_content(text, attachments), model:, dimensions:, task_type:, title:)
         end
 
         def embedding_payload(parts, model:, dimensions:, task_type:, title:)

data/lib/ruby_llm/protocols/gemini/file_transcription.rb

@@ -4,6 +4,10 @@ module RubyLLM
   module Protocols
     class Gemini
       module FileTranscription # :nodoc: all
+        def rejected_options(operation)
+          operation == :transcribe ? %i[named_speakers] : super
+        end
+
         def transcribe(audio_file, model:, language:, format: nil, speaker_names: nil,
                        speaker_references: nil, provider_options: {}, prompt: nil, temperature: nil)
           raise_transcription_streaming_unsupported if block_given?

data/lib/ruby_llm/protocols/gemini/files.rb

@@ -11,9 +11,10 @@ module RubyLLM
         # rubocop:disable-next Lint/UnusedMethodArgument
         def upload(file, filename: nil, purpose: nil, expires_in: nil, uri: nil, content_type: nil,
                    provider_options: {})
+          @content_type = content_type
           attachment = file_attachment(file, filename:)
-          display_name = provider_options[:display_name] || attachment.filename
-          upload_url = start_resumable_upload(attachment, display_name:)
+          resource = { display_name: attachment.filename }.merge(provider_options.transform_keys(&:to_sym))
+          upload_url = start_resumable_upload(attachment, resource:)
           response = upload_file_bytes(upload_url, attachment)
           await_active(parse_file_response(response.body.fetch('file')))
         end
@@ -51,7 +52,7 @@ module RubyLLM
           gemini_file_name(file_id)
         end
 
-        def start_resumable_upload(attachment, display_name:)
+        def start_resumable_upload(attachment, resource:)
           response = gemini_connection(request_json: true, response_json: true).post(gemini_upload_url) do |request|
             request.headers.merge!(@provider.headers)
             request.headers['X-Goog-Upload-Protocol'] = 'resumable'
@@ -59,7 +60,7 @@ module RubyLLM
             request.headers['X-Goog-Upload-Header-Content-Length'] = attachment.content.bytesize.to_s
             request.headers['X-Goog-Upload-Header-Content-Type'] = file_content_type(attachment)
             request.headers['Content-Type'] = 'application/json'
-            request.body = { file: { display_name: display_name } }
+            request.body = { file: resource }
           end
 
           response.headers['x-goog-upload-url'] || response.headers['X-Goog-Upload-URL'] ||

data/lib/ruby_llm/protocols/gemini/images.rb

@@ -9,13 +9,13 @@ module RubyLLM
         ASPECT_RATIO = /\A\d+:\d+\z/
         PIXEL_DIMENSIONS = /\A(\d+)\s*[x×]\s*(\d+)\z/i
 
-        def images_url(with: nil, mask: nil) # rubocop:disable Lint/UnusedMethodArgument
+        def images_url(**)
           id = model_id(@model)
 
           "models/#{id}:#{image_endpoint_action(id)}"
         end
 
-        def render_image_payload(prompt, model:, size:, count: nil, with: nil, mask: nil, provider_options: {}) # rubocop:disable Lint/UnusedMethodArgument
+        def render_image_payload(prompt, model:, size:, count: nil, with: nil, provider_options: {}, **)
           @model = model
           payload = if gemini_image_model?(model)
                       render_gemini_image_payload(prompt, with:, count:, size:)
@@ -52,7 +52,8 @@ module RubyLLM
         end
 
         def render_imagen_payload(prompt, count: nil, size: nil)
-          RubyLLM.logger.debug { "Ignoring size #{size}. Imagen sizing is not supported." } if size
+          image_config = build_image_config(size) || {}
+          raise ArgumentError, "Imagen takes an aspect ratio, got size #{size.inspect}" if image_config.key?(:imageSize)
 
           {
             instances: [
@@ -62,7 +63,7 @@ module RubyLLM
             ],
             parameters: {
               sampleCount: count || 1
-            }
+            }.merge(image_config)
           }
         end
 
@@ -76,7 +77,7 @@ module RubyLLM
             contents: [
               {
                 role: 'user',
-                parts: Media.format_content(prompt, image_attachments(with))
+                parts: Media.render_content(prompt, image_attachments(with))
               }
             ],
             generationConfig: generation_config
@@ -142,9 +143,9 @@ module RubyLLM
         end
 
         def gemini_image_parts(data)
-          Array(data['candidates']).filter_map do |candidate|
+          Array(data['candidates']).flat_map do |candidate|
             parts = candidate.dig('content', 'parts') || []
-            parts.filter_map { |part| part['inlineData'] }.find do |inline_data|
+            parts.reject { |part| part['thought'] }.filter_map { |part| part['inlineData'] }.select do |inline_data|
               image_mime_type?(inline_data['mimeType']) && inline_data['data']
             end
           end
@@ -157,7 +158,8 @@ module RubyLLM
         def gemini_image_usage(data)
           {
             'input_tokens' => input_tokens(data),
-            'output_tokens' => calculate_output_tokens(data)
+            'output_tokens' => calculate_output_tokens(data),
+            'cache_read_tokens' => data.dig('usageMetadata', 'cachedContentTokenCount')
           }.compact
         end
 
@@ -169,7 +171,7 @@ module RubyLLM
 
         def gemini_image_model?(model)
           id = model_id(model).downcase
-          return true if id.start_with?('nano-banana', 'nanobanana')
+          return true if id.start_with?('nano-banana', 'nanobanana', 'gemini-nano-banana')
 
           id.start_with?('gemini-') && id.include?('-image')
         end

data/lib/ruby_llm/protocols/gemini/live_transcription.rb

@@ -92,7 +92,7 @@ module RubyLLM
               socket.close if event.dig('serverContent', 'generationComplete')
             end
           end
-          unless events.last&.dig('serverContent', 'generationComplete')
+          unless events.any? { |event| event.dig('serverContent', 'generationComplete') }
             raise Error, 'Google Live transcription ended before generation completed'
           end