ruby_llm-agents 0.5.0 → 1.0.0

data/lib/ruby_llm/agents/image/analyzer/execution.rb

@@ -0,0 +1,402 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require_relative "../concerns/image_operation_execution"
+
+module RubyLLM
+  module Agents
+    class ImageAnalyzer
+      # Execution logic for image analyzers
+      #
+      # Handles image validation, budget tracking, caching,
+      # analysis execution, and result building.
+      #
+      module Execution
+        include Concerns::ImageOperationExecution
+
+        # Execute the image analysis pipeline
+        #
+        # @return [ImageAnalysisResult] The result containing analysis data
+        def execute
+          started_at = Time.current
+
+          resolve_tenant_context!
+          check_budget! if budget_tracking_enabled?
+          validate_image!
+
+          # Check cache
+          cached = check_cache(ImageAnalysisResult) if cache_enabled?
+          return cached if cached
+
+          # Analyze image
+          analysis_data = analyze_image
+
+          # Build result
+          result = build_result(
+            analysis: analysis_data,
+            started_at: started_at,
+            completed_at: Time.current
+          )
+
+          # Cache result
+          write_cache(result) if cache_enabled?
+
+          # Track execution
+          record_execution(result) if execution_tracking_enabled?
+
+          result
+        rescue StandardError => e
+          record_failed_execution(e, started_at) if execution_tracking_enabled?
+          build_error_result(e, started_at)
+        end
+
+        private
+
+        def execution_type
+          "image_analysis"
+        end
+
+        def validate_image!
+          raise ArgumentError, "Image cannot be blank" if image.nil?
+
+          # Validate image exists if it's a path
+          if image.is_a?(String) && !image.start_with?("http")
+            unless File.exist?(image)
+              raise ArgumentError, "Image file does not exist: #{image}"
+            end
+          end
+        end
+
+        def analyze_image
+          # Build the analysis prompt based on configuration
+          prompt = build_analysis_prompt
+
+          # Use RubyLLM's vision capabilities
+          response = call_vision_model(prompt)
+
+          # Parse the response into structured data
+          parse_analysis_response(response)
+        end
+
+        def build_analysis_prompt
+          return resolve_custom_prompt if resolve_custom_prompt
+
+          parts = []
+          analysis = resolve_analysis_type
+
+          case analysis
+          when :caption
+            parts << "Provide a brief, single-sentence caption for this image."
+          when :detailed
+            parts << "Provide a detailed description of this image including:"
+            parts << "- A brief caption (1 sentence)"
+            parts << "- A detailed description (2-3 paragraphs)"
+            parts << "- Key tags/keywords (up to #{resolve_max_tags})"
+          when :tags
+            parts << "List up to #{resolve_max_tags} relevant tags/keywords for this image as a comma-separated list."
+          when :objects
+            parts << "Identify all objects visible in this image. For each object provide:"
+            parts << "- Name of the object"
+            parts << "- Approximate location (top-left, center, bottom-right, etc.)"
+            parts << "- Confidence level (high, medium, low)"
+          when :colors
+            parts << "Identify the dominant colors in this image. For each color provide:"
+            parts << "- Hex color code"
+            parts << "- Color name"
+            parts << "- Approximate percentage of the image"
+          when :all
+            parts << "Analyze this image comprehensively. Provide:"
+            parts << "1. A brief caption (1 sentence)"
+            parts << "2. A detailed description (2-3 paragraphs)"
+            parts << "3. Key tags/keywords (up to #{resolve_max_tags})"
+            parts << "4. Objects detected with locations"
+            parts << "5. Dominant colors with hex codes and percentages"
+          end
+
+          # Add conditional extraction requests
+          if resolve_extract_colors && analysis != :colors && analysis != :all
+            parts << "\nAlso identify the dominant colors (hex, name, percentage)."
+          end
+
+          if resolve_detect_objects && analysis != :objects && analysis != :all
+            parts << "\nAlso identify objects visible with their locations."
+          end
+
+          if resolve_extract_text
+            parts << "\nAlso extract any visible text (OCR)."
+          end
+
+          parts << "\nFormat your response as JSON with the following structure:"
+          parts << build_json_schema
+
+          parts.join("\n")
+        end
+
+        def build_json_schema
+          schema = {
+            caption: "Brief caption string",
+            description: "Detailed description string (if applicable)",
+            tags: ["array", "of", "tag", "strings"],
+            objects: [{ name: "object name", location: "position", confidence: "high/medium/low" }],
+            colors: [{ hex: "#RRGGBB", name: "color name", percentage: 25 }],
+            text: "Extracted text if any"
+          }
+
+          JSON.pretty_generate(schema)
+        end
+
+        def call_vision_model(prompt)
+          model = resolve_model
+
+          # Prepare image for the vision model
+          image_content = prepare_image_content
+
+          # Use RubyLLM chat with vision
+          chat = RubyLLM.chat(model: model)
+          chat.ask(prompt, with: { image: image_content })
+        end
+
+        def prepare_image_content
+          if image.is_a?(String)
+            if image.start_with?("http")
+              # URL - pass directly
+              image
+            else
+              # File path - read and encode as base64
+              require "base64"
+              mime_type = detect_mime_type(image)
+              data = Base64.strict_encode64(File.binread(image))
+              "data:#{mime_type};base64,#{data}"
+            end
+          elsif image.respond_to?(:read)
+            # IO object - read and encode
+            require "base64"
+            content = image.read
+            image.rewind if image.respond_to?(:rewind)
+            data = Base64.strict_encode64(content)
+            "data:image/png;base64,#{data}"
+          else
+            image.to_s
+          end
+        end
+
+        def detect_mime_type(path)
+          ext = File.extname(path).downcase
+          case ext
+          when ".jpg", ".jpeg" then "image/jpeg"
+          when ".png" then "image/png"
+          when ".gif" then "image/gif"
+          when ".webp" then "image/webp"
+          when ".bmp" then "image/bmp"
+          else "image/png"
+          end
+        end
+
+        def parse_analysis_response(response)
+          content = response.content
+
+          # Try to extract JSON from the response
+          json_match = content.match(/\{[\s\S]*\}/m)
+          if json_match
+            begin
+              parsed = JSON.parse(json_match[0], symbolize_names: true)
+              normalize_analysis_data(parsed)
+            rescue JSON::ParserError
+              # Fall back to text parsing
+              parse_text_response(content)
+            end
+          else
+            parse_text_response(content)
+          end
+        end
+
+        def normalize_analysis_data(data)
+          {
+            caption: data[:caption]&.to_s,
+            description: data[:description]&.to_s,
+            tags: normalize_tags(data[:tags]),
+            objects: normalize_objects(data[:objects]),
+            colors: normalize_colors(data[:colors]),
+            text: data[:text]&.to_s,
+            raw_response: data
+          }
+        end
+
+        def normalize_tags(tags)
+          return [] unless tags.is_a?(Array)
+
+          tags.map(&:to_s).map(&:strip).reject(&:empty?).first(resolve_max_tags)
+        end
+
+        def normalize_objects(objects)
+          return [] unless objects.is_a?(Array)
+
+          objects.map do |obj|
+            next unless obj.is_a?(Hash)
+            {
+              name: obj[:name]&.to_s,
+              location: obj[:location]&.to_s,
+              confidence: obj[:confidence]&.to_s&.downcase,
+              bbox: obj[:bbox] || obj[:bounding_box]
+            }.compact
+          end.compact
+        end
+
+        def normalize_colors(colors)
+          return [] unless colors.is_a?(Array)
+
+          colors.map do |color|
+            next unless color.is_a?(Hash)
+            {
+              hex: color[:hex]&.to_s,
+              name: color[:name]&.to_s,
+              percentage: color[:percentage]&.to_f
+            }.compact
+          end.compact
+        end
+
+        def parse_text_response(content)
+          # Basic text parsing when JSON extraction fails
+          {
+            caption: extract_caption_from_text(content),
+            description: content,
+            tags: extract_tags_from_text(content),
+            objects: [],
+            colors: [],
+            text: nil,
+            raw_response: content
+          }
+        end
+
+        def extract_caption_from_text(content)
+          # Try to extract first sentence as caption
+          first_sentence = content.split(/[.!?]/).first&.strip
+          first_sentence&.slice(0, 200)
+        end
+
+        def extract_tags_from_text(content)
+          # Extract potential tags from comma-separated lists or bullet points
+          tags = []
+
+          # Look for bullet point lists
+          content.scan(/[-*•]\s*(\w+(?:\s+\w+)?)/).each do |match|
+            tags << match[0].strip.downcase
+          end
+
+          # Look for comma-separated items after "tags:" or "keywords:"
+          if content =~ /(?:tags|keywords)[:\s]*(.+?)(?:\n|$)/i
+            Regexp.last_match(1).split(/[,;]/).each do |tag|
+              tags << tag.strip.downcase
+            end
+          end
+
+          tags.uniq.first(resolve_max_tags)
+        end
+
+        def build_result(analysis:, started_at:, completed_at:)
+          ImageAnalysisResult.new(
+            image: image,
+            model_id: resolve_model,
+            analysis_type: resolve_analysis_type,
+            caption: analysis[:caption],
+            description: analysis[:description],
+            tags: analysis[:tags],
+            objects: analysis[:objects],
+            colors: analysis[:colors],
+            text: analysis[:text],
+            raw_response: analysis[:raw_response],
+            started_at: started_at,
+            completed_at: completed_at,
+            tenant_id: @tenant_id,
+            analyzer_class: self.class.name
+          )
+        end
+
+        def build_error_result(error, started_at)
+          ImageAnalysisResult.new(
+            image: image,
+            model_id: resolve_model,
+            analysis_type: resolve_analysis_type,
+            caption: nil,
+            description: nil,
+            tags: [],
+            objects: [],
+            colors: [],
+            text: nil,
+            raw_response: nil,
+            started_at: started_at,
+            completed_at: Time.current,
+            tenant_id: @tenant_id,
+            analyzer_class: self.class.name,
+            error_class: error.class.name,
+            error_message: error.message
+          )
+        end
+
+        # Resolution methods
+
+        def resolve_analysis_type
+          options[:analysis_type] || self.class.analysis_type
+        end
+
+        def resolve_extract_colors
+          options.fetch(:extract_colors, self.class.extract_colors)
+        end
+
+        def resolve_detect_objects
+          options.fetch(:detect_objects, self.class.detect_objects)
+        end
+
+        def resolve_extract_text
+          options.fetch(:extract_text, self.class.extract_text)
+        end
+
+        def resolve_custom_prompt
+          options[:custom_prompt] || self.class.custom_prompt
+        end
+
+        def resolve_max_tags
+          options[:max_tags] || self.class.max_tags
+        end
+
+        # Cache key components
+        def cache_key_components
+          [
+            "image_analyzer",
+            self.class.name,
+            self.class.version,
+            resolve_model,
+            resolve_analysis_type.to_s,
+            resolve_extract_colors.to_s,
+            resolve_detect_objects.to_s,
+            resolve_extract_text.to_s,
+            Digest::SHA256.hexdigest(image_digest)
+          ]
+        end
+
+        def image_digest
+          if image.is_a?(String) && File.exist?(image)
+            File.read(image)
+          elsif image.respond_to?(:read)
+            content = image.read
+            image.rewind if image.respond_to?(:rewind)
+            content
+          else
+            image.to_s
+          end
+        end
+
+        def build_execution_metadata(result)
+          {
+            analysis_type: result.analysis_type,
+            tags_count: result.tags.size,
+            objects_count: result.objects.size,
+            colors_count: result.colors.size,
+            has_text: result.text.present?
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/image/analyzer.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative "analyzer/dsl"
+require_relative "analyzer/execution"
+
+module RubyLLM
+  module Agents
+    # Image analyzer for understanding and captioning images
+    #
+    # Analyzes images using vision models to extract captions, tags,
+    # descriptions, detected objects, and color information.
+    #
+    # @example Basic usage
+    #   result = RubyLLM::Agents::ImageAnalyzer.call(image: "path/to/photo.jpg")
+    #   result.caption     # => "A sunset over mountains"
+    #   result.tags        # => [:nature, :sunset, :mountains]
+    #   result.description # => "A detailed description..."
+    #
+    # @example Custom analyzer class
+    #   class ProductAnalyzer < RubyLLM::Agents::ImageAnalyzer
+    #     model "gpt-4o"
+    #     analysis_type :detailed
+    #     extract_colors true
+    #     detect_objects true
+    #
+    #     description "Analyzes product photos"
+    #   end
+    #
+    #   result = ProductAnalyzer.call(image: product_photo)
+    #   result.objects  # => [{name: "laptop", confidence: 0.98, bbox: [...]}]
+    #   result.colors   # => [{hex: "#C0C0C0", name: "silver", percentage: 45}]
+    #
+    class ImageAnalyzer
+      extend DSL
+      include Execution
+
+      class << self
+        # Execute image analysis
+        #
+        # @param image [String, IO] Path, URL, or IO object of the image to analyze
+        # @param options [Hash] Additional options (model, analysis_type, etc.)
+        # @return [ImageAnalysisResult] The result containing analysis data
+        def call(image:, **options)
+          new(image: image, **options).call
+        end
+
+        # Ensure subclasses inherit DSL settings
+        def inherited(subclass)
+          super
+          subclass.instance_variable_set(:@model, @model)
+          subclass.instance_variable_set(:@analysis_type, @analysis_type)
+          subclass.instance_variable_set(:@extract_colors, @extract_colors)
+          subclass.instance_variable_set(:@detect_objects, @detect_objects)
+          subclass.instance_variable_set(:@extract_text, @extract_text)
+          subclass.instance_variable_set(:@custom_prompt, @custom_prompt)
+          subclass.instance_variable_set(:@max_tags, @max_tags)
+          subclass.instance_variable_set(:@version, @version)
+          subclass.instance_variable_set(:@description, @description)
+          subclass.instance_variable_set(:@cache_ttl, @cache_ttl)
+        end
+      end
+
+      attr_reader :image, :options, :tenant_id
+
+      # Initialize a new image analyzer instance
+      #
+      # @param image [String, IO] Image to analyze (path, URL, or IO object)
+      # @param options [Hash] Additional options
+      # @option options [String] :model Model to use
+      # @option options [Symbol] :analysis_type Type of analysis (:caption, :detailed, :tags, :objects)
+      # @option options [Boolean] :extract_colors Whether to extract color information
+      # @option options [Boolean] :detect_objects Whether to detect objects
+      # @option options [Boolean] :extract_text Whether to extract text (OCR)
+      # @option options [String] :custom_prompt Custom analysis prompt
+      # @option options [Object] :tenant Tenant for multi-tenancy
+      def initialize(image:, **options)
+        @image = image
+        @options = options
+        @tenant_id = nil
+      end
+
+      # Execute the image analysis
+      #
+      # @return [ImageAnalysisResult] The result containing analysis data
+      def call
+        execute
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/image/background_remover/dsl.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require_relative "../concerns/image_operation_dsl"
+
+module RubyLLM
+  module Agents
+    class BackgroundRemover
+      # DSL for configuring background removers
+      #
+      # Provides class-level methods to configure model, output format,
+      # and edge refinement options.
+      #
+      # @example
+      #   class ProductBackgroundRemover < RubyLLM::Agents::BackgroundRemover
+      #     model "rembg"
+      #     output_format :png
+      #     refine_edges true
+      #     alpha_matting true
+      #   end
+      #
+      module DSL
+        include Concerns::ImageOperationDSL
+
+        VALID_OUTPUT_FORMATS = %i[png webp].freeze
+
+        # Set or get the output format
+        #
+        # @param value [Symbol, nil] Output format (:png, :webp)
+        # @return [Symbol] The output format
+        def output_format(value = nil)
+          if value
+            unless VALID_OUTPUT_FORMATS.include?(value)
+              raise ArgumentError, "Output format must be one of: #{VALID_OUTPUT_FORMATS.join(', ')}"
+            end
+            @output_format = value
+          else
+            @output_format || inherited_or_default(:output_format, :png)
+          end
+        end
+
+        # Set or get whether to refine edges
+        #
+        # When enabled, applies additional processing to smooth
+        # and refine the edges of the extracted subject.
+        #
+        # @param value [Boolean, nil] Enable edge refinement
+        # @return [Boolean] Whether edge refinement is enabled
+        def refine_edges(value = nil)
+          if value.nil?
+            result = @refine_edges
+            result = inherited_or_default(:refine_edges, false) if result.nil?
+            result
+          else
+            @refine_edges = value
+          end
+        end
+
+        # Set or get whether to use alpha matting
+        #
+        # Alpha matting produces better results for hair, fur,
+        # and semi-transparent elements but is slower.
+        #
+        # @param value [Boolean, nil] Enable alpha matting
+        # @return [Boolean] Whether alpha matting is enabled
+        def alpha_matting(value = nil)
+          if value.nil?
+            result = @alpha_matting
+            result = inherited_or_default(:alpha_matting, false) if result.nil?
+            result
+          else
+            @alpha_matting = value
+          end
+        end
+
+        # Set or get the foreground threshold
+        #
+        # Pixels with confidence above this threshold are considered
+        # foreground. Lower values include more pixels.
+        #
+        # @param value [Float, nil] Threshold (0.0-1.0)
+        # @return [Float] The foreground threshold
+        def foreground_threshold(value = nil)
+          if value
+            unless value.is_a?(Numeric) && value.between?(0.0, 1.0)
+              raise ArgumentError, "Foreground threshold must be between 0.0 and 1.0"
+            end
+            @foreground_threshold = value.to_f
+          else
+            @foreground_threshold || inherited_or_default(:foreground_threshold, 0.5)
+          end
+        end
+
+        # Set or get the background threshold
+        #
+        # Pixels with confidence below this threshold are considered
+        # background. Higher values exclude more pixels.
+        #
+        # @param value [Float, nil] Threshold (0.0-1.0)
+        # @return [Float] The background threshold
+        def background_threshold(value = nil)
+          if value
+            unless value.is_a?(Numeric) && value.between?(0.0, 1.0)
+              raise ArgumentError, "Background threshold must be between 0.0 and 1.0"
+            end
+            @background_threshold = value.to_f
+          else
+            @background_threshold || inherited_or_default(:background_threshold, 0.5)
+          end
+        end
+
+        # Set or get the erode size
+        #
+        # Size of morphological erosion applied to shrink the mask
+        # slightly to avoid edge artifacts.
+        #
+        # @param value [Integer, nil] Erode size in pixels
+        # @return [Integer] The erode size
+        def erode_size(value = nil)
+          if value
+            unless value.is_a?(Integer) && value >= 0
+              raise ArgumentError, "Erode size must be a non-negative integer"
+            end
+            @erode_size = value
+          else
+            @erode_size || inherited_or_default(:erode_size, 0)
+          end
+        end
+
+        # Set or get whether to return the mask
+        #
+        # When enabled, the result will include the segmentation mask
+        # in addition to the extracted foreground.
+        #
+        # @param value [Boolean, nil] Return segmentation mask
+        # @return [Boolean] Whether to return the mask
+        def return_mask(value = nil)
+          if value.nil?
+            result = @return_mask
+            result = inherited_or_default(:return_mask, false) if result.nil?
+            result
+          else
+            @return_mask = value
+          end
+        end
+
+        private
+
+        def default_model
+          config.default_background_remover_model || "rembg"
+        end
+      end
+    end
+  end
+end