dify_llm 1.8.1 → 1.9.0

data/lib/ruby_llm/providers/gemini/chat.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'set'
+require 'rubygems/version'
+
 module RubyLLM
   module Providers
     class Gemini
@@ -20,10 +23,7 @@ module RubyLLM
 
           payload[:generationConfig][:temperature] = temperature unless temperature.nil?
 
-          if schema
-            payload[:generationConfig][:responseMimeType] = 'application/json'
-            payload[:generationConfig][:responseSchema] = convert_schema_to_gemini(schema)
-          end
+          payload[:generationConfig].merge!(structured_output_config(schema, model)) if schema
 
           payload[:tools] = format_tools(tools) if tools.any?
           payload
@@ -32,40 +32,29 @@ module RubyLLM
         private
 
         def format_messages(messages)
-          messages.map do |msg|
-            {
-              role: format_role(msg.role),
-              parts: format_parts(msg)
-            }
-          end
+          formatter = MessageFormatter.new(
+            messages,
+            format_role: method(:format_role),
+            format_parts: method(:format_parts),
+            format_tool_result: method(:format_tool_result)
+          )
+          formatter.format
         end
 
         def format_role(role)
           case role
           when :assistant then 'model'
-          when :system, :tool then 'user'
+          when :system then 'user'
+          when :tool then 'function'
           else role.to_s
           end
         end
 
         def format_parts(msg)
           if msg.tool_call?
-            [{
-              functionCall: {
-                name: msg.tool_calls.values.first.name,
-                args: msg.tool_calls.values.first.arguments
-              }
-            }]
+            format_tool_call(msg)
           elsif msg.tool_result?
-            [{
-              functionResponse: {
-                name: msg.tool_call_id,
-                response: {
-                  name: msg.tool_call_id,
-                  content: Media.format_content(msg.content)
-                }
-              }
-            }]
+            format_tool_result(msg)
           else
             Media.format_content(msg.content)
           end
@@ -77,7 +66,7 @@ module RubyLLM
 
           Message.new(
             role: :assistant,
-            content: extract_content(data),
+            content: parse_content(data),
             tool_calls: tool_calls,
             input_tokens: data.dig('usageMetadata', 'promptTokenCount'),
             output_tokens: calculate_output_tokens(data),
@@ -89,23 +78,19 @@ module RubyLLM
         def convert_schema_to_gemini(schema)
           return nil unless schema
 
-          build_base_schema(schema).tap do |result|
-            result[:description] = schema[:description] if schema[:description]
-            apply_type_specific_attributes(result, schema)
-          end
+          GeminiSchema.new(schema).to_h
         end
 
-        def extract_content(data)
+        def parse_content(data)
           candidate = data.dig('candidates', 0)
           return '' unless candidate
 
           return '' if function_call?(candidate)
 
           parts = candidate.dig('content', 'parts')
-          text_parts = parts&.select { |p| p['text'] }
-          return '' unless text_parts&.any?
+          return '' unless parts&.any?
 
-          text_parts.map { |p| p['text'] }.join
+          build_response_content(parts)
         end
 
         def function_call?(candidate)
@@ -119,50 +104,348 @@ module RubyLLM
           candidates + thoughts
         end
 
-        def build_base_schema(schema)
-          case schema[:type]
-          when 'object'
-            build_object_schema(schema)
-          when 'array'
-            { type: 'ARRAY', items: schema[:items] ? convert_schema_to_gemini(schema[:items]) : { type: 'STRING' } }
-          when 'number'
-            { type: 'NUMBER' }
-          when 'integer'
-            { type: 'INTEGER' }
-          when 'boolean'
-            { type: 'BOOLEAN' }
-          else
-            { type: 'STRING' }
+        def response_json_schema_supported?(model)
+          version = gemini_version(model)
+          version && version >= Gem::Version.new('2.5')
+        end
+
+        def build_json_schema(schema)
+          normalized = RubyLLM::Utils.deep_dup(schema)
+          normalized.delete(:strict)
+          normalized.delete('strict')
+          RubyLLM::Utils.deep_stringify_keys(normalized)
+        end
+
+        def gemini_version(model)
+          return nil unless model
+
+          candidates = [
+            safe_string(model.id),
+            safe_string(model.respond_to?(:family) ? model.family : nil),
+            safe_string(model_metadata_value(model, :version)),
+            safe_string(model_metadata_value(model, 'version')),
+            safe_string(model_metadata_value(model, :description))
+          ].compact
+
+          candidates.each do |candidate|
+            version = extract_version(candidate)
+            return version if version
           end
+
+          nil
         end
 
-        def build_object_schema(schema)
+        def model_metadata_value(model, key)
+          return unless model.respond_to?(:metadata)
+
+          metadata = model.metadata
+          return unless metadata.is_a?(Hash)
+
+          metadata[key] || metadata[key.to_s]
+        end
+
+        def safe_string(value)
+          value&.to_s
+        end
+
+        def extract_version(text)
+          return nil unless text
+
+          match = text.match(/(\d+\.\d+|\d+)/)
+          return nil unless match
+
+          Gem::Version.new(match[1])
+        rescue ArgumentError
+          nil
+        end
+
+        def structured_output_config(schema, model)
           {
-            type: 'OBJECT',
-            properties: (schema[:properties] || {}).transform_values { |prop| convert_schema_to_gemini(prop) },
-            required: schema[:required] || []
-          }.tap do |object|
-            object[:propertyOrdering] = schema[:propertyOrdering] if schema[:propertyOrdering]
-            object[:nullable] = schema[:nullable] if schema.key?(:nullable)
+            responseMimeType: 'application/json'
+          }.tap do |config|
+            if response_json_schema_supported?(model)
+              config[:responseJsonSchema] = build_json_schema(schema)
+            else
+              config[:responseSchema] = convert_schema_to_gemini(schema)
+            end
           end
         end
 
-        def apply_type_specific_attributes(result, schema)
-          case schema[:type]
-          when 'string'
-            copy_attributes(result, schema, :enum, :format, :nullable)
-          when 'number', 'integer'
-            copy_attributes(result, schema, :format, :minimum, :maximum, :enum, :nullable)
-          when 'array'
-            copy_attributes(result, schema, :minItems, :maxItems, :nullable)
-          when 'boolean'
-            copy_attributes(result, schema, :nullable)
+        # formats a message
+        class MessageFormatter
+          def initialize(messages, format_role:, format_parts:, format_tool_result:)
+            @messages = messages
+            @index = 0
+            @tool_call_names = {}
+            @format_role = format_role
+            @format_parts = format_parts
+            @format_tool_result = format_tool_result
+          end
+
+          def format
+            formatted = []
+
+            while current_message
+              if tool_message?(current_message)
+                tool_parts, next_index = collect_tool_parts
+                formatted << build_tool_response(tool_parts)
+                @index = next_index
+              else
+                remember_tool_calls if current_message.tool_call?
+                formatted << build_standard_message(current_message)
+                @index += 1
+              end
+            end
+
+            formatted
+          end
+
+          private
+
+          def current_message
+            @messages[@index]
+          end
+
+          def tool_message?(message)
+            message&.role == :tool
+          end
+
+          def collect_tool_parts
+            parts = []
+            index = @index
+
+            while tool_message?(@messages[index])
+              tool_message = @messages[index]
+              tool_name = @tool_call_names.delete(tool_message.tool_call_id)
+              parts.concat(format_tool_result(tool_message, tool_name))
+              index += 1
+            end
+
+            [parts, index]
+          end
+
+          def build_tool_response(parts)
+            { role: 'function', parts: parts }
+          end
+
+          def remember_tool_calls
+            current_message.tool_calls.each do |tool_call_id, tool_call|
+              @tool_call_names[tool_call_id] = tool_call.name
+            end
+          end
+
+          def build_standard_message(message)
+            {
+              role: @format_role.call(message.role),
+              parts: @format_parts.call(message)
+            }
+          end
+
+          def format_tool_result(message, tool_name)
+            @format_tool_result.call(message, tool_name)
           end
         end
 
-        def copy_attributes(target, source, *attributes)
-          attributes.each do |attr|
-            target[attr] = source[attr] if attr == :nullable ? source.key?(attr) : source[attr]
+        # converts json schema to gemini
+        class GeminiSchema
+          def initialize(schema)
+            @raw_schema = RubyLLM::Utils.deep_dup(schema)
+            @definitions = {}
+          end
+
+          def to_h
+            return nil unless @raw_schema
+
+            symbolized = symbolize_and_extract_definitions(@raw_schema)
+            convert(symbolized, Set.new)
+          end
+
+          private
+
+          attr_reader :definitions
+
+          def symbolize_and_extract_definitions(value)
+            case value
+            when Hash
+              value.each_with_object({}) do |(key, val), hash|
+                key_sym = begin
+                  key.to_sym
+                rescue StandardError
+                  key
+                end
+
+                if definition_key?(key_sym)
+                  merge_definitions(val)
+                else
+                  hash[key_sym] = symbolize_and_extract_definitions(val)
+                end
+              end
+            when Array
+              value.map { |item| symbolize_and_extract_definitions(item) }
+            else
+              value
+            end
+          end
+
+          def definition_key?(key)
+            %i[$defs definitions].include?(key)
+          end
+
+          def merge_definitions(raw_defs)
+            return unless raw_defs
+
+            symbolized = symbolize_and_extract_definitions(raw_defs)
+            @definitions = if definitions.empty?
+                             symbolized
+                           else
+                             RubyLLM::Utils.deep_merge(definitions, symbolized)
+                           end
+          end
+
+          def convert(schema, visited_refs)
+            return default_string_schema unless schema.is_a?(Hash)
+
+            schema = strip_unsupported_keys(schema)
+
+            if schema[:$ref]
+              resolved = resolve_reference(schema, visited_refs)
+              return resolved if resolved
+            end
+
+            schema = normalize_any_of(schema)
+
+            result = case schema[:type].to_s
+                     when 'object'
+                       build_object(schema, visited_refs)
+                     when 'array'
+                       build_array(schema, visited_refs)
+                     when 'number'
+                       build_scalar('NUMBER', schema, %i[format minimum maximum enum nullable multipleOf])
+                     when 'integer'
+                       build_scalar('INTEGER', schema, %i[format minimum maximum enum nullable multipleOf])
+                     when 'boolean'
+                       build_scalar('BOOLEAN', schema, %i[nullable])
+                     else
+                       build_scalar('STRING', schema, %i[enum format nullable])
+                     end
+
+            apply_description(result, schema)
+            result
+          end
+
+          def strip_unsupported_keys(schema)
+            schema.dup.tap do |copy|
+              copy.delete(:strict)
+              copy.delete(:additionalProperties)
+            end
+          end
+
+          def resolve_reference(schema, visited_refs)
+            ref = schema[:$ref]
+            return unless ref
+            return if visited_refs.include?(ref)
+
+            referenced = lookup_definition(ref)
+            return unless referenced
+
+            overrides = schema.except(:$ref)
+            visited_refs.add(ref)
+            merged = RubyLLM::Utils.deep_merge(referenced, overrides)
+            convert(merged, visited_refs)
+          ensure
+            visited_refs.delete(ref)
+          end
+
+          def lookup_definition(ref) # rubocop:disable Metrics/PerceivedComplexity
+            segments = ref.to_s.split('/').reject(&:empty?)
+            return nil if segments.empty?
+
+            segments.shift if segments.first == '#'
+            segments.shift if %w[$defs definitions].include?(segments.first)
+
+            current = definitions
+
+            segments.each do |segment|
+              break current = nil unless current.is_a?(Hash)
+
+              key = begin
+                segment.to_sym
+              rescue StandardError
+                segment
+              end
+              current = current[key]
+            end
+
+            current ? RubyLLM::Utils.deep_dup(current) : nil
+          end
+
+          def normalize_any_of(schema)
+            any_of = schema[:anyOf]
+            return schema unless any_of
+
+            options = Array(any_of).map { |option| RubyLLM::Utils.deep_symbolize_keys(option) }
+            nullables, non_null = options.partition { |option| schema_type(option) == 'null' }
+
+            base = RubyLLM::Utils.deep_symbolize_keys(non_null.first || { type: 'string' })
+            base[:nullable] = true if nullables.any?
+
+            without_any_of = schema.each_with_object({}) do |(key, value), result|
+              result[key] = value unless key == :anyOf
+            end
+
+            without_any_of.merge(base)
+          end
+
+          def schema_type(option)
+            (option[:type] || option['type']).to_s.downcase
+          end
+
+          def build_object(schema, visited_refs)
+            properties = schema.fetch(:properties, {}).transform_values do |child|
+              convert(child, visited_refs)
+            end
+
+            {
+              type: 'OBJECT',
+              properties: properties
+            }.tap do |object|
+              required = Array(schema[:required]).map(&:to_s).uniq
+              object[:required] = required if required.any?
+              object[:propertyOrdering] = schema[:propertyOrdering] if schema[:propertyOrdering]
+              copy_attribute(object, schema, :nullable)
+            end
+          end
+
+          def build_array(schema, visited_refs)
+            items_schema = schema[:items] ? convert(schema[:items], visited_refs) : default_string_schema
+
+            {
+              type: 'ARRAY',
+              items: items_schema
+            }.tap do |array|
+              copy_attribute(array, schema, :minItems)
+              copy_attribute(array, schema, :maxItems)
+              copy_attribute(array, schema, :nullable)
+            end
+          end
+
+          def build_scalar(type, schema, allowed_keys)
+            { type: type }.tap do |result|
+              allowed_keys.each { |key| copy_attribute(result, schema, key) }
+            end
+          end
+
+          def apply_description(target, schema)
+            description = schema[:description]
+            target[:description] = description if description
+          end
+
+          def copy_attribute(target, source, key)
+            target[key] = source[key] if source.key?(key)
+          end
+
+          def default_string_schema
+            { type: 'STRING' }
           end
         end
       end

data/lib/ruby_llm/providers/gemini/media.rb

@@ -2,12 +2,13 @@
 
 module RubyLLM
   module Providers
-    class Gemini
+    class Gemini # rubocop:disable Style/Documentation
       # Media handling methods for the Gemini API integration
       module Media
         module_function
 
         def format_content(content)
+          return content.value if content.is_a?(RubyLLM::Content::Raw)
           return [format_text(content.to_json)] if content.is_a?(Hash) || content.is_a?(Array)
           return [format_text(content)] unless content.is_a?(Content)
 
@@ -49,6 +50,63 @@ module RubyLLM
           }
         end
       end
+
+      def build_response_content(parts) # rubocop:disable Metrics/PerceivedComplexity
+        text = []
+        attachments = []
+
+        parts.each_with_index do |part, index|
+          if part['text']
+            text << part['text']
+          elsif part['inlineData']
+            attachment = build_inline_attachment(part['inlineData'], index)
+            attachments << attachment if attachment
+          elsif part['fileData']
+            attachment = build_file_attachment(part['fileData'], index)
+            attachments << attachment if attachment
+          end
+        end
+
+        text = text.join
+        text = nil if text.empty?
+        return text if attachments.empty?
+
+        Content.new(text:, attachments:)
+      end
+
+      def build_inline_attachment(inline_data, index)
+        encoded = inline_data['data']
+        return unless encoded
+
+        mime_type = inline_data['mimeType']
+        decoded = Base64.decode64(encoded)
+        io = StringIO.new(decoded)
+        io.set_encoding(Encoding::BINARY) if io.respond_to?(:set_encoding)
+
+        filename = attachment_filename(mime_type, index)
+        RubyLLM::Attachment.new(io, filename:)
+      rescue ArgumentError => e
+        RubyLLM.logger.warn "Failed to decode Gemini inline data attachment: #{e.message}"
+        nil
+      end
+
+      def build_file_attachment(file_data, index)
+        uri = file_data['fileUri']
+        return unless uri
+
+        filename = file_data['filename'] || attachment_filename(file_data['mimeType'], index)
+        RubyLLM::Attachment.new(uri, filename:)
+      end
+
+      def attachment_filename(mime_type, index)
+        return "gemini_attachment_#{index + 1}" unless mime_type
+
+        extension = mime_type.split('/').last.to_s
+        extension = 'jpg' if extension == 'jpeg'
+        extension = 'txt' if extension == 'plain'
+        extension = extension.tr('+', '.')
+        "gemini_attachment_#{index + 1}.#{extension}"
+      end
     end
   end
 end