ruby_llm_community 0.0.6 → 1.0.0

data/lib/ruby_llm/providers/openai/response.rb

@@ -13,7 +13,7 @@ module RubyLLM
 
         def render_response_payload(messages, tools:, temperature:, model:, cache_prompts:, stream: false, schema: nil) # rubocop:disable Metrics/ParameterLists,Lint/UnusedMethodArgument
           payload = {
-            model: model,
+            model: model.id,
             input: format_input(messages),
             stream: stream
           }
@@ -40,39 +40,72 @@ module RubyLLM
           payload
         end
 
-        def format_input(messages) # rubocop:disable Metrics/PerceivedComplexity
+        def format_input(messages)
           all_tool_calls = messages.flat_map do |m|
             m.tool_calls&.values || []
           end
-          messages.flat_map do |msg|
-            if msg.tool_call?
-              msg.tool_calls.map do |_, tc|
-                {
-                  type: 'function_call',
-                  call_id: tc.id,
-                  name: tc.name,
-                  arguments: JSON.generate(tc.arguments),
-                  status: 'completed'
-                }
-              end
-            elsif msg.role == :tool
-              {
-                type: 'function_call_output',
-                call_id: all_tool_calls.detect { |tc| tc.id == msg.tool_call_id }&.id,
-                output: msg.content,
-                status: 'completed'
-              }
-            else
-              {
-                type: 'message',
-                role: format_role(msg.role),
-                content: ResponseMedia.format_content(msg.content),
-                status: 'completed'
-              }.compact
-            end
+          messages.flat_map { |msg| format_message_input(msg, all_tool_calls) }.flatten
+        end
+
+        def format_message_input(msg, all_tool_calls)
+          if msg.tool_call?
+            format_tool_call_message(msg)
+          elsif msg.role == :tool
+            format_tool_response_message(msg, all_tool_calls)
+          elsif assistant_message_with_image_attachment?(msg)
+            format_image_generation_message(msg)
+          else
+            format_regular_message(msg)
           end
         end
 
+        def format_tool_call_message(msg)
+          msg.tool_calls.map do |_, tc|
+            {
+              type: 'function_call',
+              call_id: tc.id,
+              name: tc.name,
+              arguments: JSON.generate(tc.arguments),
+              status: 'completed'
+            }
+          end
+        end
+
+        def format_tool_response_message(msg, all_tool_calls)
+          {
+            type: 'function_call_output',
+            call_id: all_tool_calls.detect { |tc| tc.id == msg.tool_call_id }&.id,
+            output: msg.content,
+            status: 'completed'
+          }
+        end
+
+        def format_image_generation_message(msg)
+          items = []
+          image_attachment = msg.content.attachments.first
+          if image_attachment.reasoning_id
+            items << {
+              type: 'reasoning',
+              id: image_attachment.reasoning_id,
+              summary: []
+            }
+          end
+          items << {
+            type: 'image_generation_call',
+            id: image_attachment.id
+          }
+          items
+        end
+
+        def format_regular_message(msg)
+          {
+            type: 'message',
+            role: format_role(msg.role),
+            content: ResponseMedia.format_content(msg.content),
+            status: 'completed'
+          }.compact
+        end
+
         def format_role(role)
           case role
           when :system
@@ -93,16 +126,62 @@ module RubyLLM
 
           Message.new(
             role: :assistant,
-            content: all_output_text(outputs),
+            content: all_output_content(outputs),
             tool_calls: parse_response_tool_calls(outputs),
             input_tokens: data['usage']['input_tokens'],
             output_tokens: data['usage']['output_tokens'],
             cached_tokens: data.dig('usage', 'input_tokens_details', 'cached_tokens'),
             model_id: data['model'],
+            reasoning_id: extract_reasoning_id(outputs),
             raw: response
           )
         end
 
+        def all_output_content(outputs)
+          @current_outputs = outputs
+          text_content = extract_text_content(outputs)
+          image_outputs = outputs.select { |o| o['type'] == 'image_generation_call' }
+
+          return text_content unless image_outputs.any?
+
+          build_content_with_images(text_content, image_outputs)
+        end
+
+        private
+
+        def extract_text_content(outputs)
+          outputs.select { |o| o['type'] == 'message' }.flat_map do |o|
+            o['content'].filter_map do |c|
+              c['type'] == 'output_text' && c['text']
+            end
+          end.join("\n")
+        end
+
+        def build_content_with_images(text_content, image_outputs)
+          content = RubyLLM::Content.new(text_content)
+          reasoning_id = extract_reasoning_id(@current_outputs)
+          image_outputs.each do |output|
+            attach_image_to_content(content, output, reasoning_id)
+          end
+          content
+        end
+
+        def attach_image_to_content(content, output, reasoning_id)
+          image_data = output['result']
+          output_format = output['output_format'] || 'png'
+          mime_type = "image/#{output_format}"
+
+          content.attach(
+            RubyLLM::ImageAttachment.new(
+              data: image_data,
+              mime_type: mime_type,
+              model_id: nil,
+              id: output['id'],
+              reasoning_id: reasoning_id
+            )
+          )
+        end
+
         def all_output_text(outputs)
           outputs.select { |o| o['type'] == 'message' }.flat_map do |o|
             o['content'].filter_map do |c|
@@ -110,6 +189,18 @@ module RubyLLM
             end
           end.join("\n")
         end
+
+        def assistant_message_with_image_attachment?(msg)
+          msg.role == :assistant &&
+            msg.content.is_a?(RubyLLM::Content) &&
+            msg.content.attachments.any? &&
+            msg.content.attachments.first.is_a?(RubyLLM::ImageAttachment)
+        end
+
+        def extract_reasoning_id(outputs)
+          reasoning_item = outputs.find { |o| o['type'] == 'reasoning' }
+          reasoning_item&.dig('id')
+        end
       end
     end
   end

data/lib/ruby_llm/providers/openai/response_media.rb

@@ -7,12 +7,12 @@ module RubyLLM
       module ResponseMedia
         module_function
 
-        def format_content(content)
+        def format_content(content) # rubocop:disable Metrics/PerceivedComplexity
           return content.to_json if content.is_a?(Hash) || content.is_a?(Array)
           return content unless content.is_a?(Content)
 
           parts = []
-          parts << format_text(content.text) if content.text
+          parts << format_text(content.text) if content.text && !content.text.empty?
 
           content.attachments.each do |attachment|
             case attachment.type

data/lib/ruby_llm/providers/openai/streaming.rb

@@ -26,60 +26,65 @@ module RubyLLM
 
         def build_responses_chunk(data)
           case data['type']
-          when 'response.text.delta'
-            # Text content delta - deprecated format
-            Chunk.new(
-              role: :assistant,
-              model_id: data.dig('response', 'model'),
-              content: data['delta'],
-              tool_calls: nil,
-              input_tokens: nil,
-              output_tokens: nil
-            )
           when 'response.output_text.delta'
-            # Text content delta - new format
-            Chunk.new(
-              role: :assistant,
-              model_id: nil, # Model is in the completion event
-              content: data['delta'],
-              tool_calls: nil,
-              input_tokens: nil,
-              output_tokens: nil
-            )
+            build_text_delta_chunk(data)
           when 'response.function_call_arguments.delta'
-            # Tool call arguments delta - handled by accumulator
-            # We need to track these deltas to build up the complete tool call
             build_tool_call_delta_chunk(data)
+          when 'response.image_generation_call.partial_image'
+            build_partial_image_chunk(data)
           when 'response.output_item.added'
-            # New tool call or message starting
-            if data.dig('item', 'type') == 'function_call'
-              build_tool_call_start_chunk(data)
-            else
-              build_empty_chunk(data)
-            end
+            handle_output_item_added(data)
           when 'response.output_item.done'
-            # Tool call or message completed
-            if data.dig('item', 'type') == 'function_call'
-              build_tool_call_complete_chunk(data)
-            else
-              build_empty_chunk(data)
-            end
+            handle_output_item_done(data)
           when 'response.completed'
-            # Final response with usage stats
-            Chunk.new(
-              role: :assistant,
-              model_id: data.dig('response', 'model'),
-              content: nil,
-              tool_calls: nil,
-              input_tokens: data.dig('response', 'usage', 'input_tokens'),
-              output_tokens: data.dig('response', 'usage', 'output_tokens')
-            )
+            build_completion_chunk(data)
+          else
+            build_empty_chunk(data)
+          end
+        end
+
+        def build_text_delta_chunk(data)
+          Chunk.new(
+            role: :assistant,
+            model_id: nil,
+            content: data['delta'],
+            tool_calls: nil,
+            input_tokens: nil,
+            output_tokens: nil
+          )
+        end
+
+        def handle_output_item_added(data)
+          if data.dig('item', 'type') == 'function_call'
+            build_tool_call_start_chunk(data)
+          elsif data.dig('item', 'type') == 'reasoning'
+            build_reasoning_chunk(data)
           else
-            # Other event types (response.created, response.in_progress, etc.)
             build_empty_chunk(data)
           end
         end
 
+        def handle_output_item_done(data)
+          if data.dig('item', 'type') == 'function_call'
+            build_tool_call_complete_chunk(data)
+          elsif data.dig('item', 'type') == 'image_generation_call'
+            build_completed_image_chunk(data)
+          else
+            build_empty_chunk(data)
+          end
+        end
+
+        def build_completion_chunk(data)
+          Chunk.new(
+            role: :assistant,
+            model_id: data.dig('response', 'model'),
+            content: nil,
+            tool_calls: nil,
+            input_tokens: data.dig('response', 'usage', 'input_tokens'),
+            output_tokens: data.dig('response', 'usage', 'output_tokens')
+          )
+        end
+
         def build_chat_completions_chunk(data)
           Chunk.new(
             role: :assistant,
@@ -93,8 +98,6 @@ module RubyLLM
         end
 
         def build_tool_call_delta_chunk(data)
-          # For tool call argument deltas, we need to create a partial tool call
-          # The accumulator will handle building up the complete arguments
           tool_call_data = {
             'id' => data['item_id'],
             'function' => {
@@ -153,10 +156,10 @@ module RubyLLM
           )
         end
 
-        def build_empty_chunk(data)
+        def build_empty_chunk(_data)
           Chunk.new(
             role: :assistant,
-            model_id: data.dig('response', 'model'),
+            model_id: nil,
             content: nil,
             tool_calls: nil,
             input_tokens: nil,
@@ -164,6 +167,63 @@ module RubyLLM
           )
         end
 
+        def build_partial_image_chunk(data)
+          content = build_image_content(data['partial_image_b64'], 'image/png', nil, nil)
+
+          Chunk.new(
+            role: :assistant,
+            model_id: nil,
+            content: content,
+            tool_calls: nil,
+            input_tokens: nil,
+            output_tokens: nil
+          )
+        end
+
+        def build_completed_image_chunk(data)
+          item = data['item']
+          image_data = item['result']
+          output_format = item['output_format'] || 'png'
+          mime_type = "image/#{output_format}"
+          revised_prompt = item['revised_prompt']
+
+          content = build_image_content(image_data, mime_type, nil, revised_prompt)
+
+          Chunk.new(
+            role: :assistant,
+            model_id: nil,
+            content: content,
+            tool_calls: nil,
+            input_tokens: nil,
+            output_tokens: nil
+          )
+        end
+
+        def build_reasoning_chunk(data)
+          Chunk.new(
+            role: :assistant,
+            model_id: nil,
+            content: nil,
+            tool_calls: nil,
+            input_tokens: nil,
+            output_tokens: nil,
+            reasoning_id: data.dig('item', 'id')
+          )
+        end
+
+        def build_image_content(base64_data, mime_type, model_id, revised_prompt = nil)
+          text_content = revised_prompt || ''
+          content = RubyLLM::Content.new(text_content)
+          content.attach(
+            RubyLLM::ImageAttachment.new(
+              data: base64_data,
+              mime_type: mime_type,
+              model_id: model_id
+            )
+          )
+          content
+        end
+
         def create_streaming_tool_call(tool_call_data)
           ToolCall.new(
             id: tool_call_data['id'],

data/lib/ruby_llm/providers/openai/tools.rb

@@ -83,7 +83,7 @@ module RubyLLM
 
         def parse_response_tool_calls(outputs)
           # TODO: implement the other & built-in tools
-          # 'web_search_call', 'file_search_call', 'image_generation_call',
+          # 'web_search_call', 'file_search_call',
           # 'code_interpreter_call', 'local_shell_call', 'mcp_call',
           # 'mcp_list_tools', 'mcp_approval_request'
           outputs.select { |o| o['type'] == 'function_call' }.to_h do |o|

data/lib/ruby_llm/providers/openai.rb

@@ -2,9 +2,7 @@
 
 module RubyLLM
   module Providers
-    # OpenAI API integration using the new Responses API. Handles response generation,
-    # function calling, and OpenAI's unique streaming format. Supports GPT-4, GPT-3.5,
-    # and other OpenAI models.
+    # OpenAI API integration.
     class OpenAI < OpenAIBase
       include OpenAI::Response
       include OpenAI::ResponseMedia

data/lib/ruby_llm/providers/openai_base.rb

@@ -26,8 +26,8 @@ module RubyLLM
         }.compact
       end
 
-      def maybe_normalize_temperature(temperature, model_id)
-        OpenAI::Capabilities.normalize_temperature(temperature, model_id)
+      def maybe_normalize_temperature(temperature, model)
+        OpenAI::Capabilities.normalize_temperature(temperature, model.id)
       end
 
       class << self

data/lib/ruby_llm/providers/openrouter/models.rb

@@ -13,13 +13,11 @@ module RubyLLM
 
         def parse_list_models_response(response, slug, _capabilities)
           Array(response.body['data']).map do |model_data| # rubocop:disable Metrics/BlockLength
-            # Extract modalities directly from architecture
             modalities = {
               input: Array(model_data.dig('architecture', 'input_modalities')),
               output: Array(model_data.dig('architecture', 'output_modalities'))
             }
 
-            # Construct pricing from API data, only adding non-zero values
             pricing = { text_tokens: { standard: {} } }
 
             pricing_types = {
@@ -34,7 +32,6 @@ module RubyLLM
               pricing[:text_tokens][:standard][target_key] = value * 1_000_000 if value.positive?
             end
 
-            # Convert OpenRouter's supported parameters to our capability format
             capabilities = supported_parameters_to_capabilities(model_data['supported_parameters'])
 
             Model::Info.new(
@@ -63,23 +60,11 @@ module RubyLLM
           return [] unless params
 
           capabilities = []
-
-          # Standard capabilities mapping
-          capabilities << 'streaming' # Assume all OpenRouter models support streaming
-
-          # Function calling capability
+          capabilities << 'streaming'
           capabilities << 'function_calling' if params.include?('tools') || params.include?('tool_choice')
-
-          # Structured output capability
           capabilities << 'structured_output' if params.include?('response_format')
-
-          # Batch capability
           capabilities << 'batch' if params.include?('batch')
-
-          # Additional mappings based on params
-          # Handles advanced model capabilities that might be inferred from supported params
           capabilities << 'predicted_outputs' if params.include?('logit_bias') && params.include?('top_k')
-
           capabilities
         end
       end

data/lib/ruby_llm/providers/perplexity/capabilities.rb

@@ -106,7 +106,6 @@ module RubyLLM
           }
         end
 
-        # Pricing information for Perplexity models (USD per 1M tokens)
         PRICES = {
           sonar: {
             input: 1.0,

data/lib/ruby_llm/providers/perplexity/chat.rb

@@ -8,7 +8,6 @@ module RubyLLM
         module_function
 
         def format_role(role)
-          # Perplexity doesn't use the new OpenAI convention for system prompts
           role.to_s
         end
       end

data/lib/ruby_llm/providers/perplexity.rb

@@ -34,17 +34,13 @@ module RubyLLM
 
         # If response is HTML (Perplexity returns HTML for auth errors)
         if body.include?('<html>') && body.include?('<title>')
-          # Extract title content
           title_match = body.match(%r{<title>(.+?)</title>})
           if title_match
-            # Clean up the title - remove status code if present
             message = title_match[1]
-            message = message.sub(/^\d+\s+/, '') # Remove leading digits and space
+            message = message.sub(/^\d+\s+/, '')
             return message
           end
         end
-
-        # Fall back to parent's implementation
         super
       end
     end

data/lib/ruby_llm/providers/vertexai/chat.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class VertexAI
+      # Chat methods for the Vertex AI implementation
+      module Chat
+        def completion_url
+          "projects/#{@config.vertexai_project_id}/locations/#{@config.vertexai_location}/publishers/google/models/#{@model}:generateContent" # rubocop:disable Layout/LineLength
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/vertexai/embeddings.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class VertexAI
+      # Embeddings methods for the Vertex AI implementation
+      module Embeddings
+        module_function
+
+        def embedding_url(model:)
+          "projects/#{@config.vertexai_project_id}/locations/#{@config.vertexai_location}/publishers/google/models/#{model}:predict" # rubocop:disable Layout/LineLength
+        end
+
+        def render_embedding_payload(text, model:, dimensions:) # rubocop:disable Lint/UnusedMethodArgument
+          {
+            instances: [text].flatten.map { |t| { content: t.to_s } }
+          }.tap do |payload|
+            payload[:parameters] = { outputDimensionality: dimensions } if dimensions
+          end
+        end
+
+        def parse_embedding_response(response, model:, text:)
+          predictions = response.body['predictions']
+          vectors = predictions&.map { |p| p.dig('embeddings', 'values') }
+          vectors = vectors.first if vectors&.length == 1 && !text.is_a?(Array)
+
+          Embedding.new(vectors:, model:, input_tokens: 0)
+        end
+      end
+    end
+  end
+end