llm_conductor 1.1.2 → 1.3.0

data/examples/gpt_vision_usage.rb

@@ -0,0 +1,156 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/llm_conductor'
+
+# This example demonstrates using GPT-4o vision capabilities
+# Set your OpenAI API key: export OPENAI_API_KEY='your-key-here'
+
+puts '=' * 80
+puts 'GPT-4o Vision Usage Examples'
+puts '=' * 80
+puts
+
+# Check for API key
+api_key = ENV['OPENAI_API_KEY']
+if api_key.nil? || api_key.empty?
+  puts 'ERROR: OPENAI_API_KEY environment variable is not set!'
+  puts
+  puts 'Please set your OpenAI API key:'
+  puts '  export OPENAI_API_KEY="your-key-here"'
+  puts
+  puts 'You can get an API key from: https://platform.openai.com/api-keys'
+  exit 1
+end
+
+# Configure the client
+LlmConductor.configure do |config|
+  config.openai(api_key:)
+end
+
+# Example 1: Single Image Analysis
+puts "\n1. Single Image Analysis"
+puts '-' * 80
+
+response = LlmConductor.generate(
+  model: 'gpt-4o',
+  vendor: :openai,
+  prompt: {
+    text: 'What is in this image? Please describe it in detail.',
+    images: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'
+  }
+)
+
+puts "Response: #{response.output}"
+puts "Tokens: #{response.input_tokens} input, #{response.output_tokens} output"
+
+# Example 2: Multiple Images Comparison
+puts "\n2. Multiple Images Comparison"
+puts '-' * 80
+
+response = LlmConductor.generate(
+  model: 'gpt-4o',
+  vendor: :openai,
+  prompt: {
+    text: 'Compare these two images. What are the main differences?',
+    images: [
+      'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/1024px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg',
+      'https://upload.wikimedia.org/wikipedia/commons/thumb/3/3f/Placeholder_view_vector.svg/1024px-Placeholder_view_vector.svg.png'
+    ]
+  }
+)
+
+puts "Response: #{response.output}"
+puts "Tokens: #{response.input_tokens} input, #{response.output_tokens} output"
+
+# Example 3: Image with Detail Level - High Resolution
+puts "\n3. Image with Detail Level - High Resolution"
+puts '-' * 80
+
+response = LlmConductor.generate(
+  model: 'gpt-4o',
+  vendor: :openai,
+  prompt: {
+    text: 'Analyze this high-resolution image in detail. What are all the elements you can see?',
+    images: [
+      { url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg', detail: 'high' }
+    ]
+  }
+)
+
+puts "Response: #{response.output}"
+puts "Tokens: #{response.input_tokens} input, #{response.output_tokens} output"
+
+# Example 4: Image with Detail Level - Low (Faster, Cheaper)
+puts "\n4. Image with Detail Level - Low (Faster, Cheaper)"
+puts '-' * 80
+
+response = LlmConductor.generate(
+  model: 'gpt-4o',
+  vendor: :openai,
+  prompt: {
+    text: 'Give me a quick description of this image.',
+    images: [
+      { url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/1024px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg', detail: 'low' }
+    ]
+  }
+)
+
+puts "Response: #{response.output}"
+puts "Tokens: #{response.input_tokens} input, #{response.output_tokens} output"
+
+# Example 5: Raw Format (Advanced)
+puts "\n5. Raw Format (Advanced)"
+puts '-' * 80
+
+response = LlmConductor.generate(
+  model: 'gpt-4o',
+  vendor: :openai,
+  prompt: [
+    { type: 'text', text: 'What is in this image?' },
+    { type: 'image_url',
+      image_url: { url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/1024px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg' } },
+    { type: 'text', text: 'Describe the weather conditions.' }
+  ]
+)
+
+puts "Response: #{response.output}"
+puts "Tokens: #{response.input_tokens} input, #{response.output_tokens} output"
+
+# Example 6: Text-Only Request (Backward Compatible)
+puts "\n6. Text-Only Request (Backward Compatible)"
+puts '-' * 80
+
+response = LlmConductor.generate(
+  model: 'gpt-4o',
+  vendor: :openai,
+  prompt: 'What is the capital of France?'
+)
+
+puts "Response: #{response.output}"
+puts "Tokens: #{response.input_tokens} input, #{response.output_tokens} output"
+
+# Example 7: Multiple Images with Mixed Detail Levels
+puts "\n7. Multiple Images with Mixed Detail Levels"
+puts '-' * 80
+
+response = LlmConductor.generate(
+  model: 'gpt-4o',
+  vendor: :openai,
+  prompt: {
+    text: 'Compare these images at different detail levels.',
+    images: [
+      {
+        url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/1024px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg', detail: 'high'
+      },
+      { url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/3/3f/Placeholder_view_vector.svg/1024px-Placeholder_view_vector.svg.png', detail: 'low' }
+    ]
+  }
+)
+
+puts "Response: #{response.output}"
+puts "Tokens: #{response.input_tokens} input, #{response.output_tokens} output"
+
+puts "\n#{'=' * 80}"
+puts 'All examples completed successfully!'
+puts '=' * 80

data/examples/zai_usage.rb

@@ -0,0 +1,163 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example of Z.ai GLM model usage including multimodal/vision capabilities
+require_relative '../lib/llm_conductor'
+
+# Configure Z.ai
+LlmConductor.configure do |config|
+  config.zai(
+    api_key: ENV['ZAI_API_KEY']
+  )
+end
+
+# Example 1: Simple text-only request with GLM-4-plus
+puts '=== Example 1: Text-only request with GLM-4-plus ==='
+response = LlmConductor.generate(
+  model: 'glm-4-plus',
+  vendor: :zai,
+  prompt: 'What is the capital of France? Please answer in one sentence.'
+)
+puts response.output
+puts "Tokens used: #{response.total_tokens}\n\n"
+
+# Example 2: Text request with GLM-4.5V (vision model, text-only mode)
+puts '=== Example 2: Text-only request with GLM-4.5V ==='
+response = LlmConductor.generate(
+  model: 'glm-4.5v',
+  vendor: :zai,
+  prompt: 'Explain the concept of machine learning in simple terms.'
+)
+puts response.output
+puts "Tokens used: #{response.total_tokens}\n\n"
+
+# Example 3: Vision request with a single image
+puts '=== Example 3: Single image analysis with GLM-4.5V ==='
+response = LlmConductor.generate(
+  model: 'glm-4.5v',
+  vendor: :zai,
+  prompt: {
+    text: 'What do you see in this image? Please describe it in detail.',
+    images: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'
+  }
+)
+puts response.output
+puts "Tokens used: #{response.total_tokens}\n\n"
+
+# Example 4: Vision request with multiple images
+puts '=== Example 4: Multiple images comparison with GLM-4.5V ==='
+response = LlmConductor.generate(
+  model: 'glm-4.5v',
+  vendor: :zai,
+  prompt: {
+    text: 'Compare these two images and describe the differences you observe.',
+    images: [
+      'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg',
+      'https://upload.wikimedia.org/wikipedia/commons/thumb/3/3f/Placeholder_view_vector.svg/681px-Placeholder_view_vector.svg.png'
+    ]
+  }
+)
+puts response.output
+puts "Tokens used: #{response.total_tokens}\n\n"
+
+# Example 5: Image with detail level specification
+puts '=== Example 5: Image with detail level ==='
+response = LlmConductor.generate(
+  model: 'glm-4.5v',
+  vendor: :zai,
+  prompt: {
+    text: 'Describe this image in detail, including colors, objects, and atmosphere.',
+    images: [
+      {
+        url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg',
+        detail: 'high'
+      }
+    ]
+  }
+)
+puts response.output
+puts "Tokens used: #{response.total_tokens}\n\n"
+
+# Example 6: Using raw array format (advanced)
+puts '=== Example 6: Raw array format ==='
+response = LlmConductor.generate(
+  model: 'glm-4.5v',
+  vendor: :zai,
+  prompt: [
+    { type: 'text', text: 'What objects can you identify in this image?' },
+    {
+      type: 'image_url',
+      image_url: {
+        url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'
+      }
+    }
+  ]
+)
+puts response.output
+puts "Tokens used: #{response.total_tokens}\n\n"
+
+# Example 7: Base64 encoded image (for local images)
+puts '=== Example 7: Using base64 encoded image ==='
+# NOTE: In real usage, you would read and encode a local file
+# require 'base64'
+# image_data = Base64.strict_encode64(File.read('path/to/image.jpg'))
+# image_url = "data:image/jpeg;base64,#{image_data}"
+
+# For this example, we'll use a URL
+response = LlmConductor.generate(
+  model: 'glm-4.5v',
+  vendor: :zai,
+  prompt: {
+    text: 'Analyze this image and extract any text you can see.',
+    images: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'
+  }
+)
+puts response.output
+puts "Tokens used: #{response.total_tokens}\n\n"
+
+# Example 8: Error handling
+puts '=== Example 8: Error handling ==='
+begin
+  response = LlmConductor.generate(
+    model: 'glm-4.5v',
+    vendor: :zai,
+    prompt: {
+      text: 'Analyze this image',
+      images: 'invalid-url'
+    }
+  )
+
+  if response.success?
+    puts response.output
+  else
+    puts "Error: #{response.metadata[:error]}"
+  end
+rescue StandardError => e
+  puts "Exception: #{e.message}"
+end
+
+# Example 9: Document understanding (OCR)
+puts "\n=== Example 9: Document understanding ==="
+response = LlmConductor.generate(
+  model: 'glm-4.5v',
+  vendor: :zai,
+  prompt: {
+    text: 'Please read any text visible in this image and transcribe it.',
+    images: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'
+  }
+)
+puts response.output
+puts "Tokens used: #{response.total_tokens}\n\n"
+
+# Example 10: Complex reasoning with image
+puts '=== Example 10: Complex reasoning with image ==='
+response = LlmConductor.generate(
+  model: 'glm-4.5v',
+  vendor: :zai,
+  prompt: {
+    text: 'Based on this image, what time of day do you think it is? Explain your reasoning.',
+    images: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'
+  }
+)
+puts response.output
+puts "Tokens used: #{response.total_tokens}\n\n"

data/lib/llm_conductor/client_factory.rb

@@ -19,7 +19,8 @@ module LlmConductor
         ollama: Clients::OllamaClient,
         gemini: Clients::GeminiClient,
         google: Clients::GeminiClient,
-        groq: Clients::GroqClient
+        groq: Clients::GroqClient,
+        zai: Clients::ZaiClient
       }
 
       client_classes.fetch(vendor) do
@@ -35,6 +36,8 @@ module LlmConductor
         :openai
       when /^gemini/i
         :gemini
+      when /^glm/i
+        :zai
       when /^(llama|mixtral|gemma|qwen)/i
         :groq
       else

data/lib/llm_conductor/clients/anthropic_client.rb

@@ -1,18 +1,23 @@
 # frozen_string_literal: true
 
 require 'anthropic'
+require_relative 'concerns/vision_support'
 
 module LlmConductor
   module Clients
     # Anthropic Claude client implementation for accessing Claude models via Anthropic API
+    # Supports both text-only and multimodal (vision) requests
     class AnthropicClient < BaseClient
+      include Concerns::VisionSupport
+
       private
 
       def generate_content(prompt)
+        content = format_content(prompt)
         response = client.messages.create(
           model:,
           max_tokens: 4096,
-          messages: [{ role: 'user', content: prompt }]
+          messages: [{ role: 'user', content: }]
         )
 
         response.content.first.text
@@ -20,6 +25,28 @@ module LlmConductor
         raise StandardError, "Anthropic API error: #{e.message}"
       end
 
+      # Anthropic uses a different image format than OpenAI
+      # Format: { type: 'image', source: { type: 'url', url: '...' } }
+      def format_image_url(url)
+        { type: 'image', source: { type: 'url', url: } }
+      end
+
+      def format_image_hash(image_hash)
+        # Anthropic doesn't have a 'detail' parameter like OpenAI
+        {
+          type: 'image',
+          source: {
+            type: 'url',
+            url: image_hash[:url] || image_hash['url']
+          }
+        }
+      end
+
+      # Anthropic recommends placing images before text
+      def images_before_text?
+        true
+      end
+
       def client
         @client ||= begin
           config = LlmConductor.configuration.provider_config(:anthropic)

data/lib/llm_conductor/clients/concerns/vision_support.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module LlmConductor
+  module Clients
+    module Concerns
+      # Shared module for vision/multimodal support across different LLM clients
+      # Provides common functionality for formatting images and text content
+      module VisionSupport
+        private
+
+        # Override token calculation to handle multimodal content
+        def calculate_tokens(content)
+          case content
+          when String then super(content)
+          when Hash then calculate_tokens_from_hash(content)
+          when Array then calculate_tokens_from_array(content)
+          else super(content.to_s)
+          end
+        end
+
+        # Calculate tokens from a hash containing text and/or images
+        # @param content_hash [Hash] Hash with :text and/or :images keys
+        # @return [Integer] Token count for text portion
+        def calculate_tokens_from_hash(content_hash)
+          text = content_hash[:text] || content_hash['text'] || ''
+          # Call the parent class's calculate_tokens with the extracted text
+          method(:calculate_tokens).super_method.call(text)
+        end
+
+        # Calculate tokens from an array of content parts
+        # @param content_array [Array] Array of content parts with type and text
+        # @return [Integer] Token count for all text parts
+        def calculate_tokens_from_array(content_array)
+          text_parts = extract_text_from_array(content_array)
+          # Call the parent class's calculate_tokens with the joined text
+          method(:calculate_tokens).super_method.call(text_parts)
+        end
+
+        # Extract and join text from array of content parts
+        # @param content_array [Array] Array of content parts
+        # @return [String] Joined text from all text parts
+        def extract_text_from_array(content_array)
+          content_array
+            .select { |part| text_part?(part) }
+            .map { |part| extract_text_from_part(part) }
+            .join(' ')
+        end
+
+        # Check if a content part is a text part
+        # @param part [Hash] Content part
+        # @return [Boolean] true if part is a text type
+        def text_part?(part)
+          part[:type] == 'text' || part['type'] == 'text'
+        end
+
+        # Extract text from a content part
+        # @param part [Hash] Content part with text
+        # @return [String] Text content
+        def extract_text_from_part(part)
+          part[:text] || part['text'] || ''
+        end
+
+        # Format content based on whether it's a simple string or multimodal content
+        # @param prompt [String, Hash, Array] The prompt content
+        # @return [String, Array] Formatted content for the API
+        def format_content(prompt)
+          case prompt
+          when Hash
+            # Handle hash with text and/or images
+            format_multimodal_hash(prompt)
+          when Array
+            # Already formatted as array of content parts
+            prompt
+          else
+            # Simple string prompt
+            prompt.to_s
+          end
+        end
+
+        # Format a hash containing text and/or images into multimodal content array
+        # @param prompt_hash [Hash] Hash with :text and/or :images keys
+        # @return [Array] Array of content parts for the API
+        def format_multimodal_hash(prompt_hash)
+          content_parts = []
+
+          # Add image parts (order depends on provider)
+          images = prompt_hash[:images] || prompt_hash['images'] || []
+          images = [images] unless images.is_a?(Array)
+
+          if images_before_text?
+            # Anthropic recommends images before text
+            images.each { |image| content_parts << format_image_part(image) }
+            add_text_part(content_parts, prompt_hash)
+          else
+            # OpenAI/most others: text before images
+            add_text_part(content_parts, prompt_hash)
+            images.each { |image| content_parts << format_image_part(image) }
+          end
+
+          content_parts
+        end
+
+        # Add text part to content array if present
+        # @param content_parts [Array] The content parts array
+        # @param prompt_hash [Hash] Hash with :text key
+        def add_text_part(content_parts, prompt_hash)
+          return unless prompt_hash[:text] || prompt_hash['text']
+
+          text = prompt_hash[:text] || prompt_hash['text']
+          content_parts << { type: 'text', text: }
+        end
+
+        # Format an image into the appropriate API structure
+        # This method should be overridden by clients that need different formats
+        # @param image [String, Hash] Image URL or hash with url/detail keys
+        # @return [Hash] Formatted image part for the API
+        def format_image_part(image)
+          case image
+          when String
+            format_image_url(image)
+          when Hash
+            format_image_hash(image)
+          end
+        end
+
+        # Format a simple image URL string
+        # Override this in subclasses for provider-specific format
+        # @param url [String] Image URL
+        # @return [Hash] Formatted image part
+        def format_image_url(url)
+          # Default: OpenAI format
+          { type: 'image_url', image_url: { url: } }
+        end
+
+        # Format an image hash with url and optional detail
+        # Override this in subclasses for provider-specific format
+        # @param image_hash [Hash] Hash with url and optional detail keys
+        # @return [Hash] Formatted image part
+        def format_image_hash(image_hash)
+          # Default: OpenAI format with detail support
+          {
+            type: 'image_url',
+            image_url: {
+              url: image_hash[:url] || image_hash['url'],
+              detail: image_hash[:detail] || image_hash['detail']
+            }.compact
+          }
+        end
+
+        # Whether to place images before text in the content array
+        # Override this in subclasses if needed (e.g., Anthropic recommends images first)
+        # @return [Boolean] true if images should come before text
+        def images_before_text?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/llm_conductor/clients/gpt_client.rb

@@ -1,13 +1,19 @@
 # frozen_string_literal: true
 
+require_relative 'concerns/vision_support'
+
 module LlmConductor
   module Clients
     # OpenAI GPT client implementation for accessing GPT models via OpenAI API
+    # Supports both text-only and multimodal (vision) requests
     class GptClient < BaseClient
+      include Concerns::VisionSupport
+
       private
 
       def generate_content(prompt)
-        client.chat(parameters: { model:, messages: [{ role: 'user', content: prompt }] })
+        content = format_content(prompt)
+        client.chat(parameters: { model:, messages: [{ role: 'user', content: }] })
               .dig('choices', 0, 'message', 'content')
       end
 

data/lib/llm_conductor/clients/openrouter_client.rb

@@ -1,32 +1,15 @@
 # frozen_string_literal: true
 
+require_relative 'concerns/vision_support'
+
 module LlmConductor
   module Clients
     # OpenRouter client implementation for accessing various LLM providers through OpenRouter API
     # Supports both text-only and multimodal (vision) requests
     class OpenrouterClient < BaseClient
-      private
+      include Concerns::VisionSupport
 
-      # Override token calculation to handle multimodal content
-      def calculate_tokens(content)
-        case content
-        when String
-          super(content)
-        when Hash
-          # For multimodal content, count tokens only for text part
-          # Note: This is an approximation as images have variable token counts
-          text = content[:text] || content['text'] || ''
-          super(text)
-        when Array
-          # For pre-formatted arrays, extract and count text parts
-          text_parts = content.select { |part| part[:type] == 'text' || part['type'] == 'text' }
-                              .map { |part| part[:text] || part['text'] || '' }
-                              .join(' ')
-          super(text_parts)
-        else
-          super(content.to_s)
-        end
-      end
+      private
 
       def generate_content(prompt)
         content = format_content(prompt)
@@ -61,66 +44,6 @@ module LlmConductor
         end
       end
 
-      # Format content based on whether it's a simple string or multimodal content
-      # @param prompt [String, Hash, Array] The prompt content
-      # @return [String, Array] Formatted content for the API
-      def format_content(prompt)
-        case prompt
-        when Hash
-          # Handle hash with text and/or images
-          format_multimodal_hash(prompt)
-        when Array
-          # Already formatted as array of content parts
-          prompt
-        else
-          # Simple string prompt
-          prompt.to_s
-        end
-      end
-
-      # Format a hash containing text and/or images into multimodal content array
-      # @param prompt_hash [Hash] Hash with :text and/or :images keys
-      # @return [Array] Array of content parts for the API
-      def format_multimodal_hash(prompt_hash)
-        content_parts = []
-
-        # Add text part if present
-        if prompt_hash[:text] || prompt_hash['text']
-          text = prompt_hash[:text] || prompt_hash['text']
-          content_parts << { type: 'text', text: }
-        end
-
-        # Add image parts if present
-        images = prompt_hash[:images] || prompt_hash['images'] || []
-        images = [images] unless images.is_a?(Array)
-
-        images.each do |image|
-          content_parts << format_image_part(image)
-        end
-
-        content_parts
-      end
-
-      # Format an image into the appropriate API structure
-      # @param image [String, Hash] Image URL or hash with url/detail keys
-      # @return [Hash] Formatted image part for the API
-      def format_image_part(image)
-        case image
-        when String
-          # Simple URL string
-          { type: 'image_url', image_url: { url: image } }
-        when Hash
-          # Hash with url and optional detail level
-          {
-            type: 'image_url',
-            image_url: {
-              url: image[:url] || image['url'],
-              detail: image[:detail] || image['detail']
-            }.compact
-          }
-        end
-      end
-
       def client
         @client ||= begin
           config = LlmConductor.configuration.provider_config(:openrouter)