ruby_llm 1.16.0 → 2.0.0.rc2

data/lib/ruby_llm/protocols/gemini/models.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Gemini
+      # Models methods for the Gemini API integration
+      module Models
+        def list_models
+          models = []
+          page_token = nil
+
+          loop do
+            response = @connection.get(models_url) do |req|
+              req.params = { pageSize: 1000 }
+              req.params[:pageToken] = page_token if page_token
+            end
+
+            models.concat(parse_list_models_response(response, @provider.slug))
+            page_token = response.body['nextPageToken']
+            break unless page_token
+          end
+
+          models
+        end
+
+        private
+
+        def models_url
+          'models'
+        end
+
+        def parse_list_models_response(response, slug)
+          Array(response.body['models']).map do |model_data|
+            model_id = model_data['name'].gsub('models/', '')
+            methods = Array(model_data['supportedGenerationMethods'])
+
+            Model.new(
+              id: model_id,
+              name: model_data['displayName'] || model_id,
+              provider: slug,
+              created_at: nil,
+              context_window: model_data['inputTokenLimit'],
+              max_output_tokens: model_data['outputTokenLimit'],
+              modalities: modalities_from(methods),
+              capabilities: capabilities_from(methods),
+              metadata: {
+                version: model_data['version'],
+                description: model_data['description'],
+                supported_generation_methods: methods
+              }
+            )
+          end
+        end
+
+        def modalities_from(methods)
+          return unless methods.include?('embedContent')
+
+          { input: ['text'], output: ['embeddings'] }
+        end
+
+        def capabilities_from(methods)
+          capabilities = []
+          capabilities << 'batch' if methods.intersect?(%w[batchGenerateContent asyncBatchEmbedContent])
+          capabilities << 'caching' if methods.include?('createCachedContent')
+          capabilities.push('streaming', 'realtime') if methods.include?('bidiGenerateContent')
+          capabilities
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/gemini/speech.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module RubyLLM
+  module Protocols
+    class Gemini
+      # Speech generation methods for the Gemini API implementation
+      module Speech
+        def speech_url(model:)
+          "models/#{model}:generateContent"
+        end
+
+        def render_speech_payload(input, model:, voice:, format:, provider_options: {})
+          RubyLLM.logger.debug { "Ignoring speech format #{format}. Gemini returns PCM audio." } if format
+
+          payload = {
+            contents: [
+              {
+                role: 'user',
+                parts: [
+                  { text: input }
+                ]
+              }
+            ],
+            generationConfig: {
+              responseModalities: ['AUDIO'],
+              speechConfig: {
+                voiceConfig: {
+                  prebuiltVoiceConfig: {
+                    voiceName: voice || 'Kore'
+                  }
+                }
+              }
+            },
+            model: model
+          }
+
+          Support::Utils.deep_merge(payload, provider_options)
+        end
+
+        def parse_speech_response(response, model:, voice:, format:) # rubocop:disable Lint/UnusedMethodArgument
+          audio = response.body.dig('candidates', 0, 'content', 'parts', 0, 'inlineData', 'data')
+          raise Error, 'Unexpected response format from Gemini speech generation API' unless audio
+
+          RubyLLM::Speech.new(
+            data: Base64.decode64(audio),
+            model: model,
+            voice: voice || 'Kore',
+            format: 'pcm'
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/gemini/streaming.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module RubyLLM
+  module Protocols
+    class Gemini
+      # Streaming methods for the Gemini API implementation
+      module Streaming
+        def stream_url
+          "models/#{@model.id}:streamGenerateContent?alt=sse"
+        end
+
+        def build_chunk(data)
+          parts = data.dig('candidates', 0, 'content', 'parts') || []
+          track_stream_parts(data, parts)
+
+          Chunk.new(
+            role: :assistant,
+            model: data['modelVersion'],
+            content: extract_text_content(parts),
+            citations: extract_citations(data, nil),
+            thinking: Thinking.build(
+              text: extract_thought_parts(parts),
+              signature: extract_thought_signature(parts)
+            ),
+            input_tokens: input_tokens(data),
+            output_tokens: extract_output_tokens(data),
+            cache_read_tokens: data.dig('usageMetadata', 'cachedContentTokenCount'),
+            thinking_tokens: data.dig('usageMetadata', 'thoughtsTokenCount'),
+            finish_reason: normalize_finish_reason(
+              data.dig('candidates', 0, 'finishReason') || data.dig('promptFeedback', 'blockReason')
+            ),
+            tool_calls: extract_tool_calls(data),
+            **stream_end_fields(data)
+          )
+        end
+
+        private
+
+        # Accumulates streamed parts so code-execution turns can be replayed
+        # verbatim, mirroring the non-streaming path.
+        def track_stream_parts(data, parts)
+          @stream_parts = (@stream_parts || []).concat(parts)
+          @saw_server_part = true if parts.any? { |part| server_tool_part?(part) }
+          metadata_calls = metadata_server_tool_calls(data)
+          @stream_metadata_calls = metadata_calls if metadata_calls.any?
+        end
+
+        def stream_end_fields(data)
+          return {} unless data.dig('candidates', 0, 'finishReason')
+
+          part_calls = @stream_parts.to_a.select { |part| server_tool_part?(part) }.map do |part|
+            ServerToolCall.new(
+              type: part.key?('executableCode') ? 'executable_code' : 'code_execution_result',
+              input: part['executableCode'],
+              result: part['codeExecutionResult'],
+              raw: part
+            )
+          end
+          calls = part_calls + @stream_metadata_calls.to_a
+          return {} if calls.empty?
+
+          {
+            server_tool_calls: calls,
+            raw_content: @saw_server_part ? @stream_parts : nil
+          }
+        end
+
+        def extract_text_content(parts)
+          text_parts = parts.reject { |p| p['thought'] }
+          text = text_parts.filter_map { |p| p['text'] }.join
+          text.empty? ? nil : text
+        end
+
+        def extract_output_tokens(data)
+          candidates = data.dig('usageMetadata', 'candidatesTokenCount') || 0
+          thoughts = data.dig('usageMetadata', 'thoughtsTokenCount') || 0
+          total = candidates + thoughts
+          total.positive? ? total : nil
+        end
+
+        def parse_streaming_error(data)
+          error_data = JSON.parse(data)
+          error = error_data.is_a?(Hash) ? error_data['error'] || error_data : error_data
+          return [nil, error.to_s] unless error.is_a?(Hash)
+
+          [error['code'], error['message']]
+        rescue JSON::ParserError => e
+          RubyLLM.logger.debug { "Failed to parse streaming error: #{e.message}" }
+          [500, "Failed to parse error: #{data}"]
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/gemini/tools.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require 'rubygems/version'
+require 'securerandom'
+
+module RubyLLM
+  module Protocols
+    class Gemini
+      # Tools methods for the Gemini API implementation
+      module Tools
+        MULTIMODAL_FUNCTION_RESPONSE_GENERATION = Gem::Version.new('3')
+
+        def format_tools(tools)
+          return [] if tools.empty?
+
+          [{
+            functionDeclarations: tools.values.map { |tool| function_declaration_for(tool) }
+          }]
+        end
+
+        def format_tool_call(msg) # rubocop:disable Metrics/PerceivedComplexity
+          parts = []
+
+          parts.concat(Media.format_content(msg.content, msg.attachments)) if msg.content && !msg.content.empty?
+
+          fallback_signature = msg.thinking&.signature
+          used_fallback = false
+
+          msg.tool_calls.each_value do |tool_call|
+            part = {
+              functionCall: {
+                name: tool_call.name,
+                args: tool_call.arguments
+              }
+            }
+
+            signature = tool_call.thought_signature
+            if signature.nil? && fallback_signature && !used_fallback
+              signature = fallback_signature
+              used_fallback = true
+            end
+            part[:thoughtSignature] = signature if signature
+            parts << part
+          end
+
+          parts
+        end
+
+        def format_tool_result(msg, function_name = nil)
+          function_name ||= msg.tool_call_id
+          content = msg.content
+          content = nil if content && content.empty?
+          content = '(no output)' if content.nil? && msg.attachments.empty?
+
+          function_response = {
+            name: function_name,
+            response: {
+              name: function_name,
+              content: Media.format_content(content)
+            }
+          }
+
+          media_parts, sibling_parts = partition_tool_result_attachments(msg.attachments)
+          function_response[:parts] = media_parts if media_parts.any?
+
+          [{ functionResponse: function_response }, *sibling_parts]
+        end
+
+        def extract_tool_calls(data) # rubocop:disable Metrics/PerceivedComplexity
+          return nil unless data
+
+          candidate = data.is_a?(Hash) ? data.dig('candidates', 0) : nil
+          return nil unless candidate
+
+          parts = candidate.dig('content', 'parts')
+          return nil unless parts.is_a?(Array)
+
+          tool_calls = parts.each_with_object({}) do |part, result|
+            function_data = part['functionCall']
+            next unless function_data
+
+            id = SecureRandom.uuid
+            thought_signature = part['thoughtSignature'] || part['thought_signature']
+
+            result[id] = ToolCall.new(
+              id:,
+              name: function_data['name'],
+              arguments: function_data['args'] || {},
+              thought_signature: thought_signature
+            )
+          end
+
+          tool_calls.empty? ? nil : tool_calls
+        end
+
+        private
+
+        # functionResponse.parts only accepts inline bytes, and pre-Gemini 3 models reject it
+        def partition_tool_result_attachments(attachments)
+          return [[], []] if attachments.empty?
+
+          parts = attachments.map { |attachment| Media.format_content_attachment(attachment) }
+          return [[], parts] unless multimodal_function_responses_supported?(@model)
+
+          parts.partition { |part| part.key?(:inline_data) }
+        end
+
+        # Neither the model listing nor the registry distinguishes a Gemini
+        # generation, so the id is the only signal available. The -latest
+        # aliases always track the newest release.
+        def multimodal_function_responses_supported?(model)
+          id = model.respond_to?(:id) ? model.id.to_s : model.to_s
+          return false unless id.start_with?('gemini-')
+          return true if id.end_with?('-latest')
+
+          generation = id[/\Agemini-(\d+(?:\.\d+)?)(?:-|\z)/, 1]
+          generation ? Gem::Version.new(generation) >= MULTIMODAL_FUNCTION_RESPONSE_GENERATION : false
+        end
+
+        def function_declaration_for(tool)
+          parameters_schema = tool.parameters_schema ||
+                              RubyLLM::Tool::SchemaDefinition.from_parameters(tool.declared_parameters)&.json_schema
+
+          declaration = {
+            name: tool.name,
+            description: tool.description
+          }
+
+          declaration[:parametersJsonSchema] = parameters_schema if parameters_schema
+
+          return declaration if tool.provider_options.empty?
+
+          RubyLLM::Support::Utils.deep_merge(declaration, tool.provider_options)
+        end
+
+        def build_tool_config(tool_choice)
+          {
+            functionCallingConfig: {
+              mode: forced_tool_choice?(tool_choice) ? 'any' : tool_choice
+            }.tap do |config|
+              # Use allowedFunctionNames to simulate specific tool choice
+              config[:allowedFunctionNames] = [tool_choice] if specific_tool_choice?(tool_choice)
+            end
+          }
+        end
+
+        def forced_tool_choice?(tool_choice)
+          tool_choice == :required || specific_tool_choice?(tool_choice)
+        end
+
+        def specific_tool_choice?(tool_choice)
+          !%i[auto none required].include?(tool_choice)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/{providers → protocols}/gemini/transcription.rb

@@ -1,17 +1,24 @@
 # frozen_string_literal: true
 
 module RubyLLM
-  module Providers
+  module Protocols
     class Gemini
       # Audio transcription helpers for the Gemini API implementation
       module Transcription
         DEFAULT_PROMPT = 'Transcribe the provided audio and respond with only the transcript text.'
 
-        def transcribe(audio_file, model:, language:, **options)
-          attachment = Attachment.new(audio_file)
-          payload = render_transcription_payload(attachment, language:, **options)
-          response = @connection.post(transcription_url(model), payload)
-          parse_transcription_response(response, model:)
+        # rubocop:disable-next  Lint/UnusedMethodArgument
+        def transcribe(audio_file, model:, language:, format: nil, speaker_names: nil,
+                       speaker_references: nil, provider_options: {}, prompt: nil, temperature: nil)
+          raise_transcription_streaming_unsupported if block_given?
+
+          track_usage(:transcription) do
+            attachment = Attachment.new(audio_file, config: @config)
+            payload = render_transcription_payload(attachment, language:, format:, provider_options:, prompt:,
+                                                               temperature:)
+            response = @connection.post(transcription_url(model), payload, usage: @usage_tracker)
+            parse_transcription_response(response, model:)
+          end
         end
 
         private
@@ -20,8 +27,9 @@ module RubyLLM
           "models/#{model}:generateContent"
         end
 
-        def render_transcription_payload(attachment, language:, **options)
-          prompt = build_prompt(options[:prompt], language)
+        def render_transcription_payload(attachment, language:, format: nil, provider_options: {}, prompt: nil,
+                                         temperature: nil)
+          prompt = build_prompt(prompt, language)
           audio_part = format_audio_part(attachment)
 
           raise UnsupportedAttachmentError, attachment.mime_type unless attachment.audio?
@@ -35,24 +43,16 @@ module RubyLLM
                   audio_part
                 ]
               }
-            ]
+            ],
+            generationConfig: build_generation_config(format:, temperature:)
           }
 
-          generation_config = build_generation_config(options)
-          payload[:generationConfig] = generation_config unless generation_config.empty?
-          payload[:safetySettings] = options[:safety_settings] if options[:safety_settings]
-
-          payload
+          Support::Utils.deep_merge(payload, provider_options)
         end
 
-        def build_generation_config(options)
-          config = {}
-          response_mime_type = options.fetch(:response_mime_type, 'text/plain')
-
-          config[:responseMimeType] = response_mime_type if response_mime_type
-          config[:temperature] = options[:temperature] if options.key?(:temperature)
-          config[:maxOutputTokens] = options[:max_output_tokens] if options[:max_output_tokens]
-
+        def build_generation_config(format:, temperature:)
+          config = { responseMimeType: format || 'text/plain' }
+          config[:temperature] = temperature if temperature
           config
         end
 

data/lib/ruby_llm/protocols/gemini/videos.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class Gemini
+      # Veo video generation through the Gemini API's long-running predict
+      # operations. The finished video is a Files API URI that requires the
+      # API key to download.
+      module Videos
+        def video_url
+          "models/#{model_id(@model)}:predictLongRunning"
+        end
+
+        def render_video_payload(prompt, model:, with: [], provider_options: {}) # rubocop:disable Lint/UnusedMethodArgument
+          instance = { prompt: prompt }
+          instance[:image] = render_video_image(with.first) if with.first
+
+          Support::Utils.deep_merge({ instances: [instance] }, provider_options)
+        end
+
+        def parse_video_job(response, model:)
+          name = response.body['name']
+          raise Error.new('Gemini did not return a video generation operation', response:) unless name
+
+          VideoJob.new(id: name, protocol: self, model: model, raw: response.body)
+        end
+
+        def render_video_extension_payload(prompt, extend:, provider_options: {}, **)
+          video = render_video_extension(extend)
+          Support::Utils.deep_merge({ instances: [{ prompt: prompt, video: video }] }, provider_options)
+        end
+
+        def video_job_url(job)
+          job.id
+        end
+
+        def parse_video_job_status(response, job:) # rubocop:disable Lint/UnusedMethodArgument
+          body = response.body
+          if body['error']
+            { status: :failed, raw: body, error: body.dig('error', 'message') }
+          elsif body['done']
+            video = generated_video(body)
+            video ? { status: :completed, raw: body } : filtered_video_failure(body)
+          else
+            { status: :pending, raw: body }
+          end
+        end
+
+        # The file URI answers with a redirect to the download host, and
+        # both hops require the API key.
+        def download_video(job)
+          video = generated_video(job.raw)
+          response = @connection.get video['uri']
+          response = @connection.get response.headers['location'] if response.status / 100 == 3
+
+          Video.new(
+            data: response.body,
+            mime_type: video['mimeType'] || 'video/mp4',
+            model: job.model,
+            raw: job.raw
+          )
+        end
+
+        private
+
+        def render_video_extension(source)
+          if source.is_a?(Video) && source.raw && (video = generated_video(source.raw)) && video['uri']
+            return { uri: video['uri'] }
+          end
+
+          video = video_extension_attachment(source)
+          return { uri: video.source.to_s } if video.url?
+          return { uri: video.provider_file_uri } if video.provider_file?
+
+          raise ArgumentError, 'Gemini extends generated Veo videos; pass the returned Video or its URI'
+        end
+
+        def render_video_image(image)
+          { inlineData: { mimeType: image.mime_type, data: image.encoded } }
+        end
+
+        def validate_animate_inputs!(with:)
+          raise Error, 'Veo takes a single reference image' if with.size > 1
+
+          with.each do |attachment|
+            raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
+          end
+        end
+
+        def generated_video(body)
+          body.dig('response', 'generateVideoResponse', 'generatedSamples', 0, 'video')
+        end
+
+        def filtered_video_failure(body)
+          reasons = body.dig('response', 'generateVideoResponse', 'raiMediaFilteredReasons')
+          error = Array(reasons).join(' ')
+          error = 'Gemini returned no video' if error.empty?
+          { status: :failed, raw: body, error: error }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/gemini.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    # The Google Gemini generateContent API.
+    class Gemini < Protocol
+      include Gemini::Caches
+      include Gemini::Chat
+      include Gemini::Embeddings
+      include Gemini::Images
+      include Gemini::Videos
+      include Gemini::Media
+      include Gemini::Models
+      include Gemini::Streaming
+      include Gemini::Tools
+      include Gemini::Speech
+      include Gemini::Transcription
+
+      # Gemini nests each tool's options inside its key, so aliases are
+      # lambdas placing the options there.
+      SERVER_TOOL_ALIASES = %i[
+        google_search url_context code_execution file_search google_maps
+      ].to_h do |tool_key|
+        [tool_key, ->(options) { { tool: { tool_key => Support::Utils.deep_symbolize_keys(options) } } }]
+      end.merge(
+        web_search: ->(options) { { tool: { google_search: Support::Utils.deep_symbolize_keys(options) } } },
+        web_fetch: ->(options) { { tool: { url_context: Support::Utils.deep_symbolize_keys(options) } } }
+      ).freeze
+
+      def server_tool_aliases
+        SERVER_TOOL_ALIASES
+      end
+    end
+  end
+end