ragdoll 0.1.1 → 0.1.8

data/app/services/ragdoll/image_description_service.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+# Image description service using RubyLLM
+
+require "ruby_llm"
+require "base64"
+require "rmagick"
+
+module Ragdoll
+  class ImageDescriptionService
+    class DescriptionError < StandardError; end
+
+    DEFAULT_OPTIONS = {
+      model: "gemma3",
+      provider: :ollama,
+      assume_model_exists: true, # Bypass registry check
+      temperature: 0.4,
+      prompt: "Describe the image in detail."
+    }.freeze
+
+    DEFAULT_FALLBACK_OPTIONS = {
+      model: "smollm2",
+      provider: :ollama,
+      assume_model_exists: true, # Bypass LLM registry check
+      temperature: 0.6
+    }.freeze
+
+    def initialize(primary: DEFAULT_OPTIONS, fallback: DEFAULT_FALLBACK_OPTIONS)
+      puts "🚀 ImageDescriptionService: Initializing with primary: #{primary.inspect}"
+      puts "🚀 ImageDescriptionService: Initializing with fallback: #{fallback.inspect}"
+
+      # Configure RubyLLM using the same pattern as the working example
+      configure_ruby_llm_globally
+
+      # Duplicate hashes to avoid modifying frozen constants
+      primary_opts = primary.dup
+      fallback_opts = fallback.dup
+
+      primary_temp    = primary_opts.delete(:temperature) || DEFAULT_OPTIONS[:temperature]
+      @primary_prompt = primary_opts.delete(:prompt) || DEFAULT_OPTIONS[:prompt]
+      fallback_temp   = fallback_opts.delete(:temperature) || DEFAULT_FALLBACK_OPTIONS[:temperature]
+
+      puts "🤖 ImageDescriptionService: Attempting to create primary model..."
+      begin
+        @primary = RubyLLM.chat(**primary_opts).with_temperature(primary_temp)
+        puts "✅ ImageDescriptionService: Primary model created successfully: #{@primary.class}"
+      rescue StandardError => e
+        puts "❌ ImageDescriptionService: Primary model creation failed: #{e.message}"
+        @primary = nil
+      end
+
+      puts "🔄 ImageDescriptionService: Attempting to create fallback model..."
+      begin
+        @fallback = RubyLLM.chat(**fallback_opts).with_temperature(fallback_temp)
+        puts "✅ ImageDescriptionService: Fallback model created successfully: #{@fallback.class}"
+      rescue StandardError => e
+        puts "❌ ImageDescriptionService: Fallback model creation failed: #{e.message}"
+        @fallback = nil
+      end
+
+      return unless @primary.nil? && @fallback.nil?
+
+      puts "⚠️  ImageDescriptionService: WARNING - No models available! Service will return placeholders only."
+    end
+
+    # Generate a description for a local image file.
+    # path (String) - absolute path to the image
+    def generate_description(path)
+      puts "🔍 ImageDescriptionService: Starting description generation for #{path}"
+      start_time = Time.now
+
+      @image_path = path
+      return "" unless @image_path && File.exist?(@image_path) && image_file?
+
+      # Attempt to read image and prepare data; on failure return placeholder
+      data = nil
+      begin
+        puts "📸 ImageDescriptionService: Reading image with Magick..."
+        @image = Magick::Image.read(@image_path).first
+        data = prepare_image_data
+        puts "✅ ImageDescriptionService: Image data prepared (#{data.length} chars base64)"
+      rescue StandardError => e
+        puts "❌ ImageDescriptionService: Failed to read image: #{e.message}"
+        return "[Image file: #{File.basename(@image_path)}]"
+      end
+      return "" unless data
+
+      # Attempt vision model call if client available
+      if @primary
+        puts "🤖 ImageDescriptionService: Attempting primary model (#{@primary.inspect})"
+        begin
+          @primary.add_message(
+            role: "user",
+            content: [
+              { type: "text", text: @primary_prompt },
+              { type: "image_url", image_url: { url: "data:#{@image.mime_type};base64,#{data}" } }
+            ]
+          )
+          puts "📤 ImageDescriptionService: Calling primary model complete()..."
+          response = @primary.complete
+          puts "📥 ImageDescriptionService: Primary model response received: #{response.inspect}"
+          desc = extract_description(response)
+          if desc && !desc.empty?
+            elapsed = Time.now - start_time
+            puts "✅ ImageDescriptionService: Primary model success! Description: '#{desc[0..100]}...' (#{elapsed.round(2)}s)"
+            return desc
+          end
+        rescue StandardError => e
+          puts "❌ ImageDescriptionService: Primary model failed: #{e.message}"
+        end
+      else
+        puts "⚠️  ImageDescriptionService: No primary model available"
+      end
+
+      # Attempt fallback if available
+      if @fallback
+        puts "🔄 ImageDescriptionService: Attempting fallback model (#{@fallback.inspect})"
+        begin
+          fallback_response = @fallback.ask(fallback_prompt).content
+          elapsed = Time.now - start_time
+          puts "✅ ImageDescriptionService: Fallback model success! Description: '#{fallback_response[0..100]}...' (#{elapsed.round(2)}s)"
+          return fallback_response
+        rescue StandardError => e
+          puts "❌ ImageDescriptionService: Fallback model failed: #{e.message}"
+        end
+      else
+        puts "⚠️  ImageDescriptionService: No fallback model available"
+      end
+
+      # Default placeholder when LLM unavailable
+      elapsed = Time.now - start_time
+      puts "🔚 ImageDescriptionService: Returning placeholder after #{elapsed.round(2)}s"
+      "[Image file: #{File.basename(@image_path)}]"
+    end
+
+    private
+
+    def configure_ruby_llm_globally
+      puts "⚙️  ImageDescriptionService: Configuring RubyLLM globally..."
+
+      # Get Ragdoll configuration or use defaults
+      ragdoll_config = begin
+        Ragdoll.configuration
+      rescue StandardError
+        nil
+      end
+      # FIXME: ollama_url is not in current config structure, should use ruby_llm_config[:ollama][:endpoint]
+      ollama_endpoint = ragdoll_config&.ruby_llm_config&.dig(:ollama, :endpoint) || ENV.fetch("OLLAMA_API_BASE", ENV.fetch("OLLAMA_ENDPOINT", "http://localhost:11434"))
+
+      puts "🔗 ImageDescriptionService: Using ollama endpoint: #{ollama_endpoint}"
+
+      # Follow the exact pattern from the working example
+      RubyLLM.configure do |config|
+        # Set all provider configs like the working example
+        config.openai_api_key         = ENV.fetch("OPENAI_API_KEY", nil)
+        config.openai_organization_id = ENV.fetch("OPENAI_ORGANIZATION_ID", nil)
+        config.openai_project_id      = ENV.fetch("OPENAI_PROJECT_ID", nil)
+        config.anthropic_api_key      = ENV.fetch("ANTHROPIC_API_KEY", nil)
+        config.gemini_api_key         = ENV.fetch("GEMINI_API_KEY", nil)
+        config.deepseek_api_key       = ENV.fetch("DEEPSEEK_API_KEY", nil)
+        config.openrouter_api_key     = ENV.fetch("OPENROUTER_API_KEY", nil)
+        config.bedrock_api_key        = ENV.fetch("BEDROCK_ACCESS_KEY_ID", nil)
+        config.bedrock_secret_key     = ENV.fetch("BEDROCK_SECRET_ACCESS_KEY", nil)
+        config.bedrock_region         = ENV.fetch("BEDROCK_REGION", nil)
+        config.bedrock_session_token  = ENV.fetch("BEDROCK_SESSION_TOKEN", nil)
+
+        # Key: Use the exact same method name as the working example
+        config.ollama_api_base        = ollama_endpoint
+        config.openai_api_base        = ENV.fetch("OPENAI_API_BASE", nil)
+        config.log_level              = :error
+      end
+
+      puts "✅ ImageDescriptionService: RubyLLM configured successfully with global settings"
+    rescue StandardError => e
+      puts "❌ ImageDescriptionService: Failed to configure RubyLLM: #{e.message}"
+    end
+
+    def image_file?
+      %w[.jpg .jpeg .png .gif .bmp .webp .svg .ico .tiff
+         .tif].include?(File.extname(@image_path).downcase)
+    end
+
+    def prepare_image_data
+      Base64.strict_encode64(File.binread(@image_path))
+    rescue StandardError
+      nil
+    end
+
+    def extract_description(response)
+      text = if response.respond_to?(:content)
+               response.content
+             elsif response.is_a?(Hash) && response.dig("choices", 0, "message", "content")
+               response["choices"][0]["message"]["content"]
+             end
+      clean_description(text)
+    end
+
+    def clean_description(description)
+      return unless description.is_a?(String)
+
+      cleaned = description
+                .strip
+                .sub(/\ADescription:?:?\s*/i, "")
+                .gsub(/\s+/, " ")
+                .gsub(@image_path, "")
+                .strip
+      cleaned << "." unless cleaned =~ /[.!?]\z/
+      cleaned
+    end
+
+    def fallback_prompt
+      <<~PROMPT
+        You are a text-based AI tasked with generating a descriptive guess about an image based on its physical characteristics and the absolute pathname provided.
+
+        Please consider the following details:
+
+        1. **Absolute Pathname:** #{@image_path}
+        2. **Image Characteristics:**
+           - **Width:** #{@image.columns}
+           - **Height:** #{@image.rows}
+           - **MIME/Type:** #{@image.mime_type}
+           - **File Size:** #{@image.filesize} bytes
+           - **Number of Colors:** #{@image.number_colors}
+
+        Based on the above information, please make your best guess about what the image might depict. Consider common uses for the file format, the aspect ratio, and any hints from the pathname itself. Provide provide your best guess as a brief description that includes potential subjects, themes, or contexts of the image.
+
+      PROMPT
+    end
+  end
+end

data/app/services/ragdoll/metadata_generator.rb

@@ -0,0 +1,329 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  # Service for generating structured metadata using LLM providers
+  # Leverages structured output capabilities to ensure consistent metadata schemas
+  class MetadataGenerator
+    def initialize(llm_client: nil)
+      @llm_client = llm_client || default_llm_client
+    end
+
+    # Generate metadata for a document based on its content and type
+    def generate_for_document(document)
+      case document.document_type
+      when "text", "markdown", "html"
+        generate_text_metadata(document)
+      when "image"
+        generate_image_metadata(document)
+      when "audio"
+        generate_audio_metadata(document)
+      when "pdf", "docx"
+        generate_pdf_metadata(document)
+      when "mixed"
+        generate_mixed_metadata(document)
+      else
+        generate_text_metadata(document) # fallback
+      end
+    end
+
+    # Generate metadata for text content
+    def generate_text_metadata(document)
+      # Combine all text content from the document
+      text_content = document.text_contents.map(&:content).join("\n\n")
+      return {} if text_content.blank?
+
+      schema = Ragdoll::MetadataSchemas::TEXT_SCHEMA
+      prompt = build_text_analysis_prompt(text_content)
+
+      generate_structured_metadata(prompt, schema)
+    end
+
+    # Generate metadata for image content
+    def generate_image_metadata(document)
+      # For images, we need to use vision-capable models
+      image_content = document.image_contents.first
+      return {} unless image_content&.image_attached?
+
+      schema = Ragdoll::MetadataSchemas::IMAGE_SCHEMA
+      prompt = build_image_analysis_prompt(image_content)
+
+      # This would use a vision model like GPT-4V, Claude 3, etc.
+      generate_structured_metadata(prompt, schema, content_type: "image", image: image_content.image)
+    end
+
+    # Generate metadata for audio content
+    def generate_audio_metadata(document)
+      audio_content = document.audio_contents.first
+      return {} unless audio_content
+
+      schema = Ragdoll::MetadataSchemas::AUDIO_SCHEMA
+
+      # Use transcript if available, otherwise analyze audio directly
+      prompt = if audio_content.transcript.present?
+                 build_audio_transcript_analysis_prompt(audio_content.transcript, audio_content.duration)
+               else
+                 # This would require audio-capable models or speech-to-text preprocessing
+                 build_audio_analysis_prompt(audio_content)
+               end
+
+      generate_structured_metadata(prompt, schema)
+    end
+
+    # Generate metadata for PDF content
+    def generate_pdf_metadata(document)
+      text_content = document.text_contents.map(&:content).join("\n\n")
+      return {} if text_content.blank?
+
+      schema = Ragdoll::MetadataSchemas::PDF_SCHEMA
+      prompt = build_pdf_analysis_prompt(text_content, document.file_metadata)
+
+      generate_structured_metadata(prompt, schema)
+    end
+
+    # Generate metadata for mixed/multi-modal content
+    def generate_mixed_metadata(document)
+      schema = Ragdoll::MetadataSchemas::MIXED_SCHEMA
+
+      # Combine analysis from all content types
+      content_summaries = []
+
+      document.text_contents.each do |text|
+        content_summaries << { type: "text", content: text.content[0..500] }
+      end
+
+      document.image_contents.each do |image|
+        content_summaries << { type: "image", description: image.description || "Image content" }
+      end
+
+      document.audio_contents.each do |audio|
+        content_summaries << { type: "audio", transcript: audio.transcript || "Audio content" }
+      end
+
+      prompt = build_mixed_analysis_prompt(content_summaries)
+      generate_structured_metadata(prompt, schema)
+    end
+
+    private
+
+    # Core method for generating structured metadata using LLM
+    def generate_structured_metadata(prompt, schema, content_type: "text", image: nil)
+      case @llm_client&.provider
+      when "openai"
+        generate_with_openai(prompt, schema, content_type, image)
+      when "anthropic"
+        generate_with_anthropic(prompt, schema, content_type, image)
+      when "ollama"
+        generate_with_ollama(prompt, schema)
+      else
+        # Fallback to basic LLM call without structured output
+        generate_with_fallback(prompt, schema)
+      end
+    rescue StandardError => e
+      Rails.logger.error "Metadata generation failed: #{e.message}" if defined?(Rails)
+      puts "Metadata generation failed: #{e.message}"
+      {}
+    end
+
+    # OpenAI structured output
+    def generate_with_openai(prompt, schema, content_type, image)
+      messages = build_messages(prompt, content_type, image)
+
+      response = @llm_client.chat(
+        model: "gpt-4o", # Use latest model with structured output
+        messages: messages,
+        response_format: {
+          type: "json_schema",
+          json_schema: {
+            name: "document_metadata",
+            schema: schema
+          }
+        },
+        temperature: 0.1
+      )
+
+      JSON.parse(response.dig("choices", 0, "message", "content") || "{}")
+    end
+
+    # Anthropic structured output (using XML format)
+    def generate_with_anthropic(prompt, schema, content_type, image)
+      # Anthropic doesn't have native JSON schema support yet
+      # Use XML format with clear instructions
+      structured_prompt = "#{prompt}\n\nPlease respond with a JSON object that follows this exact schema:\n#{schema.to_json}\n\nRespond only with valid JSON, no explanations."
+
+      messages = build_messages(structured_prompt, content_type, image)
+
+      response = @llm_client.chat(
+        model: "claude-3-5-sonnet-20241022",
+        messages: messages,
+        temperature: 0.1
+      )
+
+      # Extract JSON from response
+      content = response.dig("content", 0, "text") || "{}"
+      JSON.parse(content.match(/\{.*\}/m)&.to_s || "{}")
+    end
+
+    # Ollama structured output (using Llama 3.1 or similar)
+    def generate_with_ollama(prompt, schema)
+      structured_prompt = "#{prompt}\n\nRespond with valid JSON matching this schema:\n#{schema.to_json}"
+
+      response = @llm_client.generate(
+        model: "llama3.1:8b",
+        prompt: structured_prompt,
+        format: "json",
+        options: { temperature: 0.1 }
+      )
+
+      JSON.parse(response["response"] || "{}")
+    end
+
+    # Fallback for any LLM provider
+    def generate_with_fallback(prompt, schema)
+      structured_prompt = "#{prompt}\n\nPlease respond with a JSON object that includes these fields: #{schema[:required]&.join(', ')}\n\nRespond only with valid JSON."
+
+      response = @llm_client.generate(prompt: structured_prompt)
+      JSON.parse(response || "{}")
+    rescue JSON::ParserError
+      {}
+    end
+
+    # Build messages array for chat-based APIs
+    def build_messages(prompt, content_type, image)
+      messages = [
+        {
+          role: "system",
+          content: "You are an expert document analyzer. Generate structured metadata as valid JSON following the provided schema exactly."
+        }
+      ]
+
+      messages << if content_type == "image" && image
+                    {
+                      role: "user",
+                      content: [
+                        { type: "text", text: prompt },
+                        { type: "image_url", image_url: { url: image_url_for(image) } }
+                      ]
+                    }
+                  else
+                    { role: "user", content: prompt }
+                  end
+
+      messages
+    end
+
+    # Build analysis prompts for different content types
+    def build_text_analysis_prompt(text_content)
+      <<~PROMPT
+        Analyze the following text document and extract structured metadata:
+
+        #{text_content[0..2000]}#{text_content.length > 2000 ? '...' : ''}
+
+        Please analyze this text and provide comprehensive metadata including:
+        - A concise summary (2-3 sentences)
+        - Relevant keywords and topics
+        - Document classification and complexity level
+        - Sentiment analysis
+        - Estimated reading time
+        - Language detection
+      PROMPT
+    end
+
+    def build_image_analysis_prompt(image_content)
+      existing_description = image_content.description || image_content.alt_text
+      base_prompt = <<~PROMPT
+        Analyze this image and provide comprehensive metadata including:
+        - Detailed description of what's shown
+        - Objects, people, or elements visible
+        - Scene type and visual style
+        - Dominant colors and mood
+        - Any visible text content
+        - Relevant keywords for search
+      PROMPT
+
+      if existing_description.present?
+        "#{base_prompt}\n\nExisting description: #{existing_description}"
+      else
+        base_prompt
+      end
+    end
+
+    def build_audio_transcript_analysis_prompt(transcript, duration)
+      <<~PROMPT
+        Analyze the following audio transcript and provide metadata:
+
+        Duration: #{duration} seconds
+        Transcript: #{transcript[0..1500]}#{transcript.length > 1500 ? '...' : ''}
+
+        Please analyze this audio content and provide:
+        - Summary of the spoken content
+        - Content type (speech, music, podcast, etc.)
+        - Topics discussed
+        - Number of speakers
+        - Language and mood
+        - Key quotes or important phrases
+      PROMPT
+    end
+
+    def build_audio_analysis_prompt(audio_content)
+      <<~PROMPT
+        Analyze this audio file and provide metadata:
+
+        Duration: #{audio_content.duration} seconds
+        Sample Rate: #{audio_content.sample_rate} Hz
+
+        Please determine:
+        - Type of audio content (speech, music, sound effects, etc.)
+        - If music: genre, mood, instruments
+        - If speech: estimated number of speakers, formality level
+        - Overall audio characteristics
+      PROMPT
+    end
+
+    def build_pdf_analysis_prompt(text_content, file_metadata)
+      <<~PROMPT
+        Analyze this PDF document and provide structured metadata:
+
+        File info: #{file_metadata}
+        Content preview: #{text_content[0..2000]}#{text_content.length > 2000 ? '...' : ''}
+
+        Please analyze this PDF and provide:
+        - Document type and classification
+        - Summary of content
+        - Topics and keywords
+        - Document structure analysis
+        - Complexity level
+        - Estimated reading time
+      PROMPT
+    end
+
+    def build_mixed_analysis_prompt(content_summaries)
+      content_desc = content_summaries.map { |c| "#{c[:type]}: #{c.values[1]}" }.join("\n\n")
+
+      <<~PROMPT
+        Analyze this multi-modal document containing different types of content:
+
+        #{content_desc}
+
+        Please provide comprehensive metadata for this mixed-content document:
+        - Overall summary combining all content types
+        - How the different content types relate to each other
+        - Primary vs secondary content types
+        - Keywords spanning all content
+        - Classification for the complete document
+      PROMPT
+    end
+
+    # Utility methods
+    def default_llm_client
+      # This would integrate with your LLM client
+      # Could use ruby_llm or direct API clients
+      nil
+    end
+
+    def image_url_for(image)
+      # Convert Shrine attachment to URL for vision APIs
+      # This would need proper implementation based on your Shrine setup
+      image.url if image.respond_to?(:url)
+    end
+  end
+end

data/app/services/ragdoll/model_resolver.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  # Service for resolving models with provider/model parsing and inheritance
+  class ModelResolver
+    def initialize(config_service = nil)
+      @config_service = config_service || Ragdoll::ConfigurationService.new
+    end
+
+    # Resolve model for a task, returns Model object
+    def resolve_for_task(task_type, content_type = :text)
+      model_string = @config_service.resolve_model(task_type, content_type)
+
+      raise Ragdoll::Core::ConfigurationError, "No model configured for task '#{task_type}'" if model_string.nil?
+
+      Ragdoll::Core::Model.new(model_string)
+    end
+
+    # Resolve embedding model for content type, returns Model object with metadata
+    def resolve_embedding(content_type = :text)
+      embedding_config = @config_service.config.models[:embedding]
+      model_string = embedding_config[content_type]
+
+      raise Ragdoll::Core::ConfigurationError, "No embedding model configured for content type '#{content_type}'" if model_string.nil?
+
+      model = Ragdoll::Core::Model.new(model_string)
+
+      # Return object with model and embedding-specific metadata
+      OpenStruct.new(
+        model: model,
+        provider_type: embedding_config[:provider],
+        max_dimensions: embedding_config[:max_dimensions],
+        cache_embeddings: embedding_config[:cache_embeddings]
+      )
+    end
+
+    # Get provider credentials for a Model object
+    def provider_credentials_for_model(model)
+      provider = model.provider
+
+      if provider.nil?
+        # Use default provider if none specified
+        provider = @config_service.config.llm_providers[:default_provider]
+      end
+
+      @config_service.provider_credentials(provider)
+    end
+
+    # Resolve all models for debugging/introspection
+    def resolve_all_models
+      {
+        text_generation: {
+          default: resolve_for_task(:default),
+          summary: resolve_for_task(:summary),
+          keywords: resolve_for_task(:keywords)
+        },
+        embedding: {
+          text: resolve_embedding(:text),
+          image: resolve_embedding(:image),
+          audio: resolve_embedding(:audio)
+        }
+      }
+    rescue Ragdoll::Core::ConfigurationError => e
+      # Return partial results with error information
+      { error: e.message, partial: true }
+    end
+
+    private
+
+    attr_reader :config_service
+  end
+end

data/app/services/ragdoll/search_engine.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# FIXME: This is crap.  It does not focus on search.
+
+module Ragdoll
+  class SearchEngine
+    def initialize(embedding_service, config_service: nil)
+      @embedding_service = embedding_service
+      @config_service = config_service || Ragdoll::ConfigurationService.new
+    end
+
+    def search_documents(query, options = {})
+      search_config = @config_service.search_config
+      limit = options[:limit] || search_config[:max_results]
+      threshold = options[:threshold] || search_config[:similarity_threshold]
+      filters = options[:filters] || {}
+
+      # Generate embedding for the query
+      query_embedding = @embedding_service.generate_embedding(query)
+      return [] if query_embedding.nil?
+
+      # Search using ActiveRecord models
+      Ragdoll::Embedding.search_similar(query_embedding,
+                                       limit: limit,
+                                       threshold: threshold,
+                                       filters: filters)
+    end
+
+    def search_similar_content(query_or_embedding, options = {})
+      search_config = @config_service.search_config
+      limit = options[:limit] || search_config[:max_results]
+      threshold = options[:threshold] || search_config[:similarity_threshold]
+      filters = options[:filters] || {}
+
+      if query_or_embedding.is_a?(Array)
+        # It's already an embedding
+        query_embedding = query_or_embedding
+      else
+        # It's a query string, generate embedding
+        query_embedding = @embedding_service.generate_embedding(query_or_embedding)
+        return [] if query_embedding.nil?
+      end
+
+      # Search using ActiveRecord models
+      Ragdoll::Embedding.search_similar(query_embedding,
+                                       limit: limit,
+                                       threshold: threshold,
+                                       filters: filters)
+    end
+  end
+end