ruby_llm_swarm 1.9.1

data/lib/ruby_llm/providers/gemini/streaming.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Gemini
+      # Streaming methods for the Gemini API implementation
+      module Streaming
+        def stream_url
+          "models/#{@model}:streamGenerateContent?alt=sse"
+        end
+
+        def build_chunk(data)
+          Chunk.new(
+            role: :assistant,
+            model_id: extract_model_id(data),
+            content: extract_content(data),
+            input_tokens: extract_input_tokens(data),
+            output_tokens: extract_output_tokens(data),
+            tool_calls: extract_tool_calls(data)
+          )
+        end
+
+        private
+
+        def extract_model_id(data)
+          data['modelVersion']
+        end
+
+        def extract_content(data)
+          return nil unless data['candidates']&.any?
+
+          candidate = data['candidates'][0]
+          parts = candidate.dig('content', 'parts')
+          return nil unless parts
+
+          text_parts = parts.select { |p| p['text'] }
+          text_parts.map { |p| p['text'] }.join if text_parts.any?
+        end
+
+        def extract_input_tokens(data)
+          data.dig('usageMetadata', 'promptTokenCount')
+        end
+
+        def extract_output_tokens(data)
+          candidates = data.dig('usageMetadata', 'candidatesTokenCount') || 0
+          thoughts = data.dig('usageMetadata', 'thoughtsTokenCount') || 0
+          total = candidates + thoughts
+          total.positive? ? total : nil
+        end
+
+        def parse_streaming_error(data)
+          error_data = JSON.parse(data)
+          [error_data['error']['code'], error_data['error']['message']]
+        rescue JSON::ParserError => e
+          RubyLLM.logger.debug "Failed to parse streaming error: #{e.message}"
+          [500, "Failed to parse error: #{data}"]
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gemini/tools.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Gemini
+      # Tools methods for the Gemini API implementation
+      module Tools
+        def format_tools(tools)
+          return [] if tools.empty?
+
+          [{
+            functionDeclarations: tools.values.map { |tool| function_declaration_for(tool) }
+          }]
+        end
+
+        def format_tool_call(msg)
+          parts = []
+
+          if msg.content && !(msg.content.respond_to?(:empty?) && msg.content.empty?)
+            formatted_content = Media.format_content(msg.content)
+            parts.concat(formatted_content.is_a?(Array) ? formatted_content : [formatted_content])
+          end
+
+          msg.tool_calls.each_value do |tool_call|
+            parts << {
+              functionCall: {
+                name: tool_call.name,
+                args: tool_call.arguments
+              }
+            }
+          end
+
+          parts
+        end
+
+        def format_tool_result(msg, function_name = nil)
+          function_name ||= msg.tool_call_id
+
+          [{
+            functionResponse: {
+              name: function_name,
+              response: {
+                name: function_name,
+                content: Media.format_content(msg.content)
+              }
+            }
+          }]
+        end
+
+        def extract_tool_calls(data) # rubocop:disable Metrics/PerceivedComplexity
+          return nil unless data
+
+          candidate = data.is_a?(Hash) ? data.dig('candidates', 0) : nil
+          return nil unless candidate
+
+          parts = candidate.dig('content', 'parts')
+          return nil unless parts.is_a?(Array)
+
+          tool_calls = parts.each_with_object({}) do |part, result|
+            function_data = part['functionCall']
+            next unless function_data
+
+            id = SecureRandom.uuid
+
+            result[id] = ToolCall.new(
+              id:,
+              name: function_data['name'],
+              arguments: function_data['args'] || {}
+            )
+          end
+
+          tool_calls.empty? ? nil : tool_calls
+        end
+
+        private
+
+        def function_declaration_for(tool)
+          parameters_schema = tool.params_schema ||
+                              RubyLLM::Tool::SchemaDefinition.from_parameters(tool.parameters)&.json_schema
+
+          declaration = {
+            name: tool.name,
+            description: tool.description
+          }
+
+          declaration[:parameters] = convert_tool_schema_to_gemini(parameters_schema) if parameters_schema
+
+          return declaration if tool.provider_params.empty?
+
+          RubyLLM::Utils.deep_merge(declaration, tool.provider_params)
+        end
+
+        def convert_tool_schema_to_gemini(schema)
+          return nil unless schema
+
+          schema = RubyLLM::Utils.deep_stringify_keys(schema)
+
+          raise ArgumentError, 'Gemini tool parameters must be objects' unless schema['type'] == 'object'
+
+          {
+            type: 'OBJECT',
+            properties: schema.fetch('properties', {}).transform_values { |property| convert_property(property) },
+            required: (schema['required'] || []).map(&:to_s)
+          }
+        end
+
+        def convert_property(property_schema) # rubocop:disable Metrics/PerceivedComplexity
+          normalized_schema = normalize_any_of_schema(property_schema)
+          working_schema = normalized_schema || property_schema
+
+          type = param_type_for_gemini(working_schema['type'])
+
+          property = {
+            type: type
+          }
+
+          copy_common_attributes(property, property_schema)
+          copy_common_attributes(property, working_schema)
+
+          case type
+          when 'ARRAY'
+            items_schema = working_schema['items'] || property_schema['items'] || { 'type' => 'string' }
+            property[:items] = convert_property(items_schema)
+            copy_tool_attributes(property, working_schema, %w[minItems maxItems])
+            copy_tool_attributes(property, property_schema, %w[minItems maxItems])
+          when 'OBJECT'
+            nested_properties = working_schema.fetch('properties', {}).transform_values do |child|
+              convert_property(child)
+            end
+            property[:properties] = nested_properties
+            required = working_schema['required'] || property_schema['required']
+            property[:required] = required.map(&:to_s) if required
+          end
+
+          property
+        end
+
+        def copy_common_attributes(target, source)
+          copy_tool_attributes(target, source, %w[description enum format nullable maximum minimum multipleOf])
+        end
+
+        def copy_tool_attributes(target, source, attributes)
+          attributes.each do |attribute|
+            value = schema_value(source, attribute)
+            next if value.nil?
+
+            target[attribute.to_sym] = value
+          end
+        end
+
+        def normalize_any_of_schema(schema) # rubocop:disable Metrics/PerceivedComplexity
+          any_of = schema['anyOf'] || schema[:anyOf]
+          return nil unless any_of.is_a?(Array) && any_of.any?
+
+          null_entries, non_null_entries = any_of.partition { |entry| schema_type(entry).to_s == 'null' }
+
+          if non_null_entries.size == 1 && null_entries.any?
+            normalized = RubyLLM::Utils.deep_dup(non_null_entries.first)
+            normalized['nullable'] = true
+            normalized
+          elsif non_null_entries.any?
+            RubyLLM::Utils.deep_dup(non_null_entries.first)
+          else
+            { 'type' => 'string', 'nullable' => true }
+          end
+        end
+
+        def schema_type(schema)
+          schema['type'] || schema[:type]
+        end
+
+        def schema_value(source, attribute) # rubocop:disable Metrics/PerceivedComplexity
+          case attribute
+          when 'multipleOf'
+            source['multipleOf'] || source[:multipleOf] || source['multiple_of'] || source[:multiple_of]
+          when 'minItems'
+            source['minItems'] || source[:minItems] || source['min_items'] || source[:min_items]
+          when 'maxItems'
+            source['maxItems'] || source[:maxItems] || source['max_items'] || source[:max_items]
+          else
+            source[attribute] || source[attribute.to_sym]
+          end
+        end
+
+        def param_type_for_gemini(type)
+          case type.to_s.downcase
+          when 'integer' then 'INTEGER'
+          when 'number', 'float', 'double' then 'NUMBER'
+          when 'boolean' then 'BOOLEAN'
+          when 'array' then 'ARRAY'
+          when 'object' then 'OBJECT'
+          else 'STRING'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gemini/transcription.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Gemini
+      # Audio transcription helpers for the Gemini API implementation
+      module Transcription
+        DEFAULT_PROMPT = 'Transcribe the provided audio and respond with only the transcript text.'
+
+        def transcribe(audio_file, model:, language:, **options)
+          attachment = Attachment.new(audio_file)
+          payload = render_transcription_payload(attachment, language:, **options)
+          response = @connection.post(transcription_url(model), payload)
+          parse_transcription_response(response, model:)
+        end
+
+        private
+
+        def transcription_url(model)
+          "models/#{model}:generateContent"
+        end
+
+        def render_transcription_payload(attachment, language:, **options)
+          prompt = build_prompt(options[:prompt], language)
+          audio_part = format_audio_part(attachment)
+
+          raise UnsupportedAttachmentError, attachment.mime_type unless attachment.audio?
+
+          payload = {
+            contents: [
+              {
+                role: 'user',
+                parts: [
+                  { text: prompt },
+                  audio_part
+                ]
+              }
+            ]
+          }
+
+          generation_config = build_generation_config(options)
+          payload[:generationConfig] = generation_config unless generation_config.empty?
+          payload[:safetySettings] = options[:safety_settings] if options[:safety_settings]
+
+          payload
+        end
+
+        def build_generation_config(options)
+          config = {}
+          response_mime_type = options.fetch(:response_mime_type, 'text/plain')
+
+          config[:responseMimeType] = response_mime_type if response_mime_type
+          config[:temperature] = options[:temperature] if options.key?(:temperature)
+          config[:maxOutputTokens] = options[:max_output_tokens] if options[:max_output_tokens]
+
+          config
+        end
+
+        def build_prompt(custom_prompt, language)
+          prompt = DEFAULT_PROMPT
+          prompt += " Respond in the #{language} language." if language
+          prompt += " #{custom_prompt}" if custom_prompt
+          prompt
+        end
+
+        def format_audio_part(attachment)
+          {
+            inline_data: {
+              mime_type: attachment.mime_type,
+              data: attachment.encoded
+            }
+          }
+        end
+
+        def parse_transcription_response(response, model:)
+          data = response.body
+          text = extract_text(data)
+
+          usage = extract_usage(data)
+
+          RubyLLM::Transcription.new(
+            text: text,
+            model: model,
+            input_tokens: usage[:input_tokens],
+            output_tokens: usage[:output_tokens]
+          )
+        end
+
+        def extract_text(data)
+          candidate = data.is_a?(Hash) ? data.dig('candidates', 0) : nil
+          return unless candidate
+
+          parts = candidate.dig('content', 'parts') || []
+          texts = parts.filter_map { |part| part['text'] }
+          texts.join if texts.any?
+        end
+
+        def extract_usage(data)
+          metadata = data.is_a?(Hash) ? data['usageMetadata'] : nil
+          return { input_tokens: nil, output_tokens: nil } unless metadata
+
+          {
+            input_tokens: metadata['promptTokenCount'],
+            output_tokens: sum_output_tokens(metadata)
+          }
+        end
+
+        def sum_output_tokens(metadata)
+          candidates = metadata['candidatesTokenCount'] || 0
+          thoughts = metadata['thoughtsTokenCount'] || 0
+          candidates + thoughts
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gemini.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    # Native Gemini API implementation
+    class Gemini < Provider
+      include Gemini::Chat
+      include Gemini::Embeddings
+      include Gemini::Images
+      include Gemini::Models
+      include Gemini::Transcription
+      include Gemini::Streaming
+      include Gemini::Tools
+      include Gemini::Media
+
+      def api_base
+        @config.gemini_api_base || 'https://generativelanguage.googleapis.com/v1beta'
+      end
+
+      def headers
+        {
+          'x-goog-api-key' => @config.gemini_api_key
+        }
+      end
+
+      class << self
+        def capabilities
+          Gemini::Capabilities
+        end
+
+        def configuration_requirements
+          %i[gemini_api_key]
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gpustack/chat.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class GPUStack
+      # Chat methods of the GPUStack API integration
+      module Chat
+        module_function
+
+        def format_messages(messages)
+          messages.map do |msg|
+            {
+              role: format_role(msg.role),
+              content: GPUStack::Media.format_content(msg.content),
+              tool_calls: format_tool_calls(msg.tool_calls),
+              tool_call_id: msg.tool_call_id
+            }.compact
+          end
+        end
+
+        def format_role(role)
+          role.to_s
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gpustack/media.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class GPUStack
+      # Handles formatting of media content (images, audio) for GPUStack APIs
+      module Media
+        extend OpenAI::Media
+
+        module_function
+
+        def format_content(content)
+          return content.value if content.is_a?(RubyLLM::Content::Raw)
+          return content.to_json if content.is_a?(Hash) || content.is_a?(Array)
+          return content unless content.is_a?(Content)
+
+          parts = []
+          parts << format_text(content.text) if content.text
+
+          content.attachments.each do |attachment|
+            case attachment.type
+            when :image
+              parts << GPUStack::Media.format_image(attachment)
+            when :text
+              parts << format_text_file(attachment)
+            else
+              raise UnsupportedAttachmentError, attachment.mime_type
+            end
+          end
+
+          parts
+        end
+
+        def format_image(image)
+          {
+            type: 'image_url',
+            image_url: {
+              url: image.for_llm,
+              detail: 'auto'
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gpustack/models.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class GPUStack
+      # Models methods of the GPUStack API integration
+      module Models
+        module_function
+
+        def models_url
+          'models'
+        end
+
+        def parse_list_models_response(response, slug, _capabilities)
+          items = response.body['items'] || []
+          items.map do |model|
+            Model::Info.new(
+              id: model['name'],
+              name: model['name'],
+              created_at: model['created_at'] ? Time.parse(model['created_at']) : nil,
+              provider: slug,
+              family: 'gpustack',
+              metadata: {
+                description: model['description'],
+                source: model['source'],
+                huggingface_repo_id: model['huggingface_repo_id'],
+                ollama_library_model_name: model['ollama_library_model_name'],
+                backend: model['backend'],
+                meta: model['meta'],
+                categories: model['categories']
+              },
+              context_window: model.dig('meta', 'n_ctx'),
+              max_output_tokens: model.dig('meta', 'n_ctx'),
+              capabilities: build_capabilities(model),
+              modalities: build_modalities(model),
+              pricing: {}
+            )
+          end
+        end
+
+        private
+
+        def determine_model_type(model)
+          return 'embedding' if model['categories']&.include?('embedding')
+          return 'chat' if model['categories']&.include?('llm')
+
+          'other'
+        end
+
+        def build_capabilities(model) # rubocop:disable Metrics/PerceivedComplexity
+          capabilities = []
+
+          # Add streaming by default for LLM models
+          capabilities << 'streaming' if model['categories']&.include?('llm')
+
+          # Map GPUStack metadata to standard capabilities
+          capabilities << 'function_calling' if model.dig('meta', 'support_tool_calls')
+          capabilities << 'vision' if model.dig('meta', 'support_vision')
+          capabilities << 'reasoning' if model.dig('meta', 'support_reasoning')
+
+          # GPUStack models generally support structured output and json mode
+          capabilities << 'structured_output' if model['categories']&.include?('llm')
+          capabilities << 'json_mode' if model['categories']&.include?('llm')
+
+          capabilities
+        end
+
+        def build_modalities(model)
+          input_modalities = []
+          output_modalities = []
+
+          if model['categories']&.include?('llm')
+            input_modalities << 'text'
+            input_modalities << 'image' if model.dig('meta', 'support_vision')
+            input_modalities << 'audio' if model.dig('meta', 'support_audio')
+            output_modalities << 'text'
+          elsif model['categories']&.include?('embedding')
+            input_modalities << 'text'
+            output_modalities << 'embeddings'
+          end
+
+          {
+            input: input_modalities,
+            output: output_modalities
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gpustack.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    # GPUStack API integration based on Ollama.
+    class GPUStack < OpenAI
+      include GPUStack::Chat
+      include GPUStack::Models
+      include GPUStack::Media
+
+      def api_base
+        @config.gpustack_api_base
+      end
+
+      def headers
+        return {} unless @config.gpustack_api_key
+
+        {
+          'Authorization' => "Bearer #{@config.gpustack_api_key}"
+        }
+      end
+
+      class << self
+        def local?
+          true
+        end
+
+        def configuration_requirements
+          %i[gpustack_api_base]
+        end
+      end
+    end
+  end
+end