ragdoll 0.1.1 → 0.1.3

data/app/services/ragdoll/document_management.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class DocumentManagement
+    class << self
+      def add_document(location, content, metadata = {})
+        # Ensure location is an absolute path if it's a file path
+        absolute_location = location.start_with?("http") || location.start_with?("ftp") ? location : File.expand_path(location)
+
+        # Get file modification time if it's a file path
+        file_modified_at = if File.exist?(absolute_location) && !absolute_location.start_with?("http")
+                             File.mtime(absolute_location)
+                           else
+                             Time.current
+                           end
+
+        # Check if document already exists with same location and file_modified_at
+        existing_document = Ragdoll::Document.find_by(
+          location: absolute_location,
+          file_modified_at: file_modified_at
+        )
+
+        # Return existing document ID if found (skip duplicate)
+        return existing_document.id.to_s if existing_document
+
+        document = Ragdoll::Document.create!(
+          location: absolute_location,
+          title: metadata[:title] || metadata["title"] || extract_title_from_location(location),
+          document_type: metadata[:document_type] || metadata["document_type"] || "text",
+          metadata: metadata.is_a?(Hash) ? metadata : {},
+          status: "pending",
+          file_modified_at: file_modified_at
+        )
+
+        # Set content using the model's setter to trigger TextContent creation
+        document.content = content if content.present?
+
+        document.id.to_s
+      end
+
+      def get_document(id)
+        document = Ragdoll::Document.find_by(id: id)
+        return nil unless document
+
+        hash = document.to_hash
+        hash[:content] = document.content
+        hash
+      end
+
+      def update_document(id, **updates)
+        document = Ragdoll::Document.find_by(id: id)
+        return nil unless document
+
+        # Only update allowed fields
+        allowed_updates = updates.slice(:title, :metadata, :status, :document_type)
+        document.update!(allowed_updates) if allowed_updates.any?
+
+        document.to_hash
+      end
+
+      def delete_document(id)
+        document = Ragdoll::Document.find_by(id: id)
+        return nil unless document
+
+        document.destroy!
+        true
+      end
+
+      def list_documents(options = {})
+        limit = options[:limit] || 100
+        offset = options[:offset] || 0
+
+        Ragdoll::Document.offset(offset).limit(limit).recent.map(&:to_hash)
+      end
+
+      def get_document_stats
+        Ragdoll::Document.stats
+      end
+
+      # FIXME: should this be here?
+
+      def add_embedding(embeddable_id, chunk_index, embedding_vector, metadata = {})
+        # The embeddable_type should be the actual STI subclass, not the base class
+        embeddable_type = if metadata[:embeddable_type]
+                           metadata[:embeddable_type]
+                         else
+                           # Look up the actual STI type from the content record
+                           content = Ragdoll::Content.find(embeddable_id)
+                           content.class.name
+                         end
+        
+        Ragdoll::Embedding.create!(
+          embeddable_id: embeddable_id,
+          embeddable_type: embeddable_type,
+          chunk_index: chunk_index,
+          embedding_vector: embedding_vector,
+          content: metadata[:content] || ""
+        ).id.to_s
+      end
+
+      private
+
+      def extract_title_from_location(location)
+        File.basename(location, File.extname(location))
+      end
+    end
+  end
+end

data/app/services/ragdoll/document_processor.rb

@@ -0,0 +1,342 @@
+# frozen_string_literal: true
+
+require "pdf-reader"
+require "docx"
+require "rmagick"
+# Image description service is auto-loaded from app/services
+
+module Ragdoll
+  class DocumentProcessor
+    class ParseError < Ragdoll::Core::DocumentError; end
+    class UnsupportedFormatError < ParseError; end
+
+    def self.parse(file_path)
+      new(file_path).parse
+    end
+
+    # Parse from Shrine attached file
+    def self.parse_attachment(attached_file)
+      attached_file.open do |tempfile|
+        new(tempfile.path, attached_file).parse
+      end
+    end
+
+    # Create document from file path
+    def self.create_document_from_file(file_path, **options)
+      parsed = parse(file_path)
+
+      # Get file modification time
+      file_modified_at = File.exist?(file_path) ? File.mtime(file_path) : Time.current
+
+      document = Ragdoll::Document.create!(
+        location: File.expand_path(file_path),
+        title: parsed[:title] || File.basename(file_path, File.extname(file_path)),
+        content: parsed[:content],
+        document_type: determine_document_type(file_path),
+        metadata: parsed[:metadata] || {},
+        status: "processed",
+        file_modified_at: file_modified_at,
+        **options
+      )
+
+      # Attach the file if it exists
+      document.file = File.open(file_path) if File.exist?(file_path)
+
+      document
+    end
+
+    # Create document from uploaded file (Shrine compatible)
+    def self.create_document_from_upload(uploaded_file, **options)
+      # Create document first
+      document = Ragdoll::Document.create!(
+        location: uploaded_file.original_filename || "uploaded_file",
+        title: options[:title] || File.basename(uploaded_file.original_filename || "uploaded_file",
+                                                File.extname(uploaded_file.original_filename || "")),
+        content: "", # Will be extracted after file attachment
+        document_type: determine_document_type_from_content_type(uploaded_file.mime_type),
+        status: "processing",
+        metadata: options[:metadata] || {},
+        file_modified_at: Time.current
+      )
+
+      # Attach the file
+      document.file = uploaded_file
+
+      # Extract content from attached file
+      if document.file.present?
+        parsed = parse_attachment(document.file)
+        document.update!(
+          content: parsed[:content],
+          title: parsed[:title] || document.title,
+          metadata: document.metadata.merge(parsed[:metadata] || {}),
+          status: "processed"
+        )
+      end
+
+      document
+    end
+
+    def initialize(file_path, attached_file = nil)
+      @file_path = file_path
+      @attached_file = attached_file
+      @file_extension = File.extname(file_path).downcase
+    end
+
+    def parse
+      case @file_extension
+      when ".pdf"
+        parse_pdf
+      when ".docx"
+        parse_docx
+      when ".txt", ".md", ".markdown"
+        parse_text
+      when ".html", ".htm"
+        parse_html
+      when ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".tif"
+        parse_image
+      else
+        parse_text # Default to text parsing for unknown formats
+      end
+    rescue StandardError => e # StandardError => e
+      raise ParseError, "#{__LINE__} Failed to parse #{@file_path}: #{e.message}"
+    end
+
+    private
+
+    def parse_pdf
+      content = ""
+      metadata = {}
+
+      begin
+        PDF::Reader.open(@file_path) do |reader|
+          # Extract metadata
+          if reader.info
+            metadata[:title] = reader.info[:Title] if reader.info[:Title]
+            metadata[:author] = reader.info[:Author] if reader.info[:Author]
+            metadata[:subject] = reader.info[:Subject] if reader.info[:Subject]
+            metadata[:creator] = reader.info[:Creator] if reader.info[:Creator]
+            metadata[:producer] = reader.info[:Producer] if reader.info[:Producer]
+            metadata[:creation_date] = reader.info[:CreationDate] if reader.info[:CreationDate]
+            metadata[:modification_date] = reader.info[:ModDate] if reader.info[:ModDate]
+          end
+
+          metadata[:page_count] = reader.page_count
+
+          # Extract text from all pages
+          reader.pages.each_with_index do |page, index|
+            page_text = page.text.strip
+            next if page_text.empty?
+
+            content += "\n\n--- Page #{index + 1} ---\n\n" if content.length.positive?
+            content += page_text
+          end
+        end
+      rescue PDF::Reader::MalformedPDFError => e
+        raise ParseError, "Malformed PDF: #{e.message}"
+      rescue PDF::Reader::UnsupportedFeatureError => e
+        raise ParseError, "Unsupported PDF feature: #{e.message}"
+      end
+
+      {
+        content: content.strip,
+        metadata: metadata,
+        document_type: "pdf"
+      }
+    end
+
+    def parse_docx
+      content = ""
+      metadata = {}
+
+      begin
+        doc = Docx::Document.open(@file_path)
+
+        # Extract core properties
+        if doc.core_properties
+          metadata[:title] = doc.core_properties.title if doc.core_properties.title
+          metadata[:author] = doc.core_properties.creator if doc.core_properties.creator
+          metadata[:subject] = doc.core_properties.subject if doc.core_properties.subject
+          metadata[:description] = doc.core_properties.description if doc.core_properties.description
+          metadata[:keywords] = doc.core_properties.keywords if doc.core_properties.keywords
+          metadata[:created] = doc.core_properties.created if doc.core_properties.created
+          metadata[:modified] = doc.core_properties.modified if doc.core_properties.modified
+          if doc.core_properties.last_modified_by
+            metadata[:last_modified_by] =
+              doc.core_properties.last_modified_by
+          end
+        end
+
+        # Extract text from paragraphs
+        doc.paragraphs.each do |paragraph|
+          paragraph_text = paragraph.text.strip
+          next if paragraph_text.empty?
+
+          content += "#{paragraph_text}\n\n"
+        end
+
+        # Extract text from tables
+        doc.tables.each_with_index do |table, table_index|
+          content += "\n--- Table #{table_index + 1} ---\n\n"
+
+          table.rows.each do |row|
+            row_text = row.cells.map(&:text).join(" | ")
+            content += "#{row_text}\n" unless row_text.strip.empty?
+          end
+
+          content += "\n"
+        end
+
+        metadata[:paragraph_count] = doc.paragraphs.count
+        metadata[:table_count] = doc.tables.count
+      rescue StandardError => e # StandardError => e
+        raise ParseError, "#{__LINE__} Failed to parse DOCX: #{e.message}"
+      end
+
+      {
+        content: content.strip,
+        metadata: metadata,
+        document_type: "docx"
+      }
+    end
+
+    def parse_text
+      content = File.read(@file_path, encoding: "UTF-8")
+      metadata = {
+        file_size: File.size(@file_path),
+        encoding: "UTF-8"
+      }
+
+      document_type = case @file_extension
+                      when ".md", ".markdown" then "markdown"
+                      when ".txt" then "text"
+                      else "text"
+                      end
+
+      {
+        content: content,
+        metadata: metadata,
+        document_type: document_type
+      }
+    rescue Encoding::InvalidByteSequenceError
+      # Try with different encoding
+      content = File.read(@file_path, encoding: "ISO-8859-1")
+      metadata = {
+        file_size: File.size(@file_path),
+        encoding: "ISO-8859-1"
+      }
+
+      {
+        content: content,
+        metadata: metadata,
+        document_type: "text"
+      }
+    end
+
+    def parse_html
+      content = File.read(@file_path, encoding: "UTF-8")
+
+      # Basic HTML tag stripping (for more advanced parsing, consider using Nokogiri)
+      clean_content = content
+                      .gsub(%r{<script[^>]*>.*?</script>}mi, "") # Remove script tags
+                      .gsub(%r{<style[^>]*>.*?</style>}mi, "")   # Remove style tags
+                      .gsub(/<[^>]+>/, " ")                     # Remove all HTML tags
+                      .gsub(/\s+/, " ")                         # Normalize whitespace
+                      .strip
+
+      metadata = {
+        file_size: File.size(@file_path),
+        original_format: "html"
+      }
+
+      {
+        content: clean_content,
+        metadata: metadata,
+        document_type: "html"
+      }
+    end
+
+    def parse_image
+      puts "🖼️  DocumentProcessor: Starting image parsing for #{@file_path}"
+
+      metadata = {
+        file_size: File.size(@file_path),
+        file_type: @file_extension.sub(".", ""),
+        original_filename: File.basename(@file_path)
+      }
+
+      # Extract image dimensions
+      begin
+        img = Magick::Image.read(@file_path).first
+        metadata[:width]  = img.columns
+        metadata[:height] = img.rows
+        puts "📏 DocumentProcessor: Image dimensions: #{img.columns}x#{img.rows}"
+      rescue StandardError => e # StandardError
+        puts "❌ DocumentProcessor: Failed to get image dimensions: #{e.message}"
+        metadata[:width]  = nil
+        metadata[:height] = nil
+      end
+
+      puts "🤖 DocumentProcessor: Creating ImageDescriptionService and calling generate_description..."
+      desc = Ragdoll::ImageDescriptionService.new.generate_description(@file_path)
+
+      puts "📝 DocumentProcessor: Received description: '#{desc}'"
+
+      metadata[:description] = desc if desc && !desc.empty?
+
+      # Use AI-generated description or fallback placeholder
+      content = desc && !desc.empty? ? desc : "Image file: #{File.basename(@file_path)}"
+
+      puts "✅ DocumentProcessor: Image parsing complete. Content: '#{content[0..100]}...'"
+
+      {
+        content: content,
+        metadata: metadata,
+        document_type: "image"
+      }
+    end
+
+    # Helper methods for document type determination
+    def self.determine_document_type(file_path)
+      case File.extname(file_path).downcase
+      when ".pdf" then "pdf"
+      when ".docx" then "docx"
+      when ".txt" then "text"
+      when ".md", ".markdown" then "markdown"
+      when ".html", ".htm" then "html"
+      when ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".tif" then "image"
+      else "text"
+      end
+    end
+
+    def self.determine_document_type_from_content_type(content_type)
+      case content_type
+      when "application/pdf" then "pdf"
+      when "application/vnd.openxmlformats-officedocument.wordprocessingml.document" then "docx"
+      when "text/plain" then "text"
+      when "text/markdown" then "markdown"
+      when "text/html" then "html"
+      when %r{^image/} then "image"
+      else "text"
+      end
+    end
+
+    def self.determine_content_type(file_path)
+      case File.extname(file_path).downcase
+      when ".pdf" then "application/pdf"
+      when ".docx" then "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+      when ".txt" then "text/plain"
+      when ".md", ".markdown" then "text/markdown"
+      when ".html", ".htm" then "text/html"
+      when ".jpg", ".jpeg" then "image/jpeg"
+      when ".png" then "image/png"
+      when ".gif" then "image/gif"
+      when ".webp" then "image/webp"
+      when ".bmp" then "image/bmp"
+      when ".svg" then "image/svg+xml"
+      when ".ico" then "image/x-icon"
+      when ".tiff", ".tif" then "image/tiff"
+      else "application/octet-stream"
+      end
+    end
+  end
+end

data/app/services/ragdoll/embedding_service.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+
+module Ragdoll
+  class EmbeddingService
+    def initialize(client: nil, config_service: nil, model_resolver: nil)
+      @client = client
+      @config_service = config_service || Ragdoll::ConfigurationService.new
+      @model_resolver = model_resolver || Ragdoll::ModelResolver.new(@config_service)
+      configure_ruby_llm unless @client
+    end
+
+    def generate_embedding(text)
+      return nil if text.nil? || text.strip.empty?
+
+      # Clean and prepare text
+      cleaned_text = clean_text(text)
+
+      begin
+        if @client
+          # Use custom client for testing
+          embedding_config = @model_resolver.resolve_embedding(:text)
+          response = @client.embed(
+            input: cleaned_text,
+            model: embedding_config.model.to_s
+          )
+
+          if response && response["embeddings"]&.first
+            response["embeddings"].first
+          elsif response && response["data"]&.first && response["data"].first["embedding"]
+            response["data"].first["embedding"]
+          else
+            raise Ragdoll::Core::EmbeddingError, "Invalid response format from embedding API"
+          end
+        else
+          # Use RubyLLM for real embedding generation
+          embedding_config = @model_resolver.resolve_embedding(:text)
+          # Use just the model name for RubyLLM
+          model = embedding_config.model.model
+
+          begin
+            response = RubyLLM.embed(cleaned_text, model: model)
+
+            # Extract the embedding vector from RubyLLM::Embedding object
+            return generate_fallback_embedding unless response.respond_to?(:instance_variable_get)
+
+            vectors = response.instance_variable_get(:@vectors)
+            return generate_fallback_embedding unless vectors && vectors.is_a?(Array)
+
+            vectors
+          rescue StandardError
+            # If RubyLLM fails, use fallback
+            generate_fallback_embedding
+          end
+        end
+      rescue StandardError => e
+        # Only use fallback if no client was provided (RubyLLM failures)
+        # If a client was provided, we should raise the error for proper test behavior
+        raise Ragdoll::Core::EmbeddingError, "Failed to generate embedding: #{e.message}" if @client
+
+        # No client - this is a RubyLLM configuration issue, use fallback
+        puts "Warning: Embedding generation failed (#{e.message}), using fallback"
+        generate_fallback_embedding
+      end
+    end
+
+    def generate_embeddings_batch(texts)
+      return [] if texts.empty?
+
+      # Clean all texts
+      cleaned_texts = texts.map { |text| clean_text(text) }.reject { |t| t.nil? || t.strip.empty? }
+      return [] if cleaned_texts.empty?
+
+      begin
+        if @client
+          # Use custom client for testing
+          embedding_config = @model_resolver.resolve_embedding(:text)
+          response = @client.embed(
+            input: cleaned_texts,
+            model: embedding_config.model.to_s
+          )
+
+          if response && response["embeddings"]
+            response["embeddings"]
+          elsif response && response["data"]
+            response["data"].map { |item| item["embedding"] }
+          else
+            raise Ragdoll::Core::EmbeddingError, "Invalid response format from embedding API"
+          end
+        else
+          # Use RubyLLM for real embedding generation (batch mode)
+          embedding_config = @model_resolver.resolve_embedding(:text)
+          # Use just the model name for RubyLLM
+          model = embedding_config.model.model
+
+          cleaned_texts.map do |text|
+            response = RubyLLM.embed(text, model: model)
+
+            # Extract the embedding vector from RubyLLM::Embedding object
+            next generate_fallback_embedding unless response.respond_to?(:instance_variable_get)
+
+            vectors = response.instance_variable_get(:@vectors)
+            next generate_fallback_embedding unless vectors && vectors.is_a?(Array)
+
+            vectors
+          rescue StandardError
+            # If RubyLLM fails, use fallback
+            generate_fallback_embedding
+          end
+        end
+      rescue StandardError => e
+        # Only use fallback if no client was provided (RubyLLM failures)
+        # If a client was provided, we should raise the error for proper test behavior
+        raise Ragdoll::Core::EmbeddingError, "Failed to generate embeddings: #{e.message}" if @client
+
+        # No client - this is a RubyLLM configuration issue, use fallback
+        puts "Warning: Batch embedding generation failed (#{e.message}), using fallback"
+        texts.map { generate_fallback_embedding }
+      end
+    end
+
+    def cosine_similarity(embedding1, embedding2)
+      return 0.0 if embedding1.nil? || embedding2.nil?
+      return 0.0 if embedding1.length != embedding2.length
+
+      dot_product = embedding1.zip(embedding2).sum { |a, b| a * b }
+      magnitude1 = Math.sqrt(embedding1.sum { |a| a * a })
+      magnitude2 = Math.sqrt(embedding2.sum { |a| a * a })
+
+      return 0.0 if magnitude1 == 0.0 || magnitude2 == 0.0
+
+      dot_product / (magnitude1 * magnitude2)
+    end
+
+    private
+
+    def configure_ruby_llm
+      # Configure ruby_llm based on Ragdoll configuration
+      provider = @config_service.config.llm_providers[:default_provider]
+      config = @config_service.provider_credentials(provider)
+
+      RubyLLM.configure do |ruby_llm_config|
+        case provider
+        when :openai
+          ruby_llm_config.openai_api_key = config[:api_key]
+          # Set organization and project if methods exist
+          if config[:organization] && ruby_llm_config.respond_to?(:openai_organization=)
+            ruby_llm_config.openai_organization = config[:organization]
+          end
+          ruby_llm_config.openai_project = config[:project] if config[:project] && ruby_llm_config.respond_to?(:openai_project=)
+        when :anthropic
+          ruby_llm_config.anthropic_api_key = config[:api_key] if ruby_llm_config.respond_to?(:anthropic_api_key=)
+        when :google
+          ruby_llm_config.google_api_key = config[:api_key] if ruby_llm_config.respond_to?(:google_api_key=)
+          if config[:project_id] && ruby_llm_config.respond_to?(:google_project_id=)
+            ruby_llm_config.google_project_id = config[:project_id]
+          end
+        when :azure
+          ruby_llm_config.azure_api_key = config[:api_key] if ruby_llm_config.respond_to?(:azure_api_key=)
+          ruby_llm_config.azure_endpoint = config[:endpoint] if config[:endpoint] && ruby_llm_config.respond_to?(:azure_endpoint=)
+          if config[:api_version] && ruby_llm_config.respond_to?(:azure_api_version=)
+            ruby_llm_config.azure_api_version = config[:api_version]
+          end
+        when :ollama
+          if config[:endpoint] && ruby_llm_config.respond_to?(:ollama_endpoint=)
+            ruby_llm_config.ollama_endpoint = config[:endpoint]
+          end
+        when :huggingface
+          ruby_llm_config.huggingface_api_key = config[:api_key] if ruby_llm_config.respond_to?(:huggingface_api_key=)
+        when :openrouter
+          ruby_llm_config.openrouter_api_key = config[:api_key] if ruby_llm_config.respond_to?(:openrouter_api_key=)
+        else
+          # Don't raise error for unsupported providers in case RubyLLM doesn't support them yet
+          puts "Warning: Unsupported embedding provider: #{provider}"
+        end
+      end
+    end
+
+    def clean_text(text)
+      return "" if text.nil?
+
+      # Remove excessive whitespace and normalize
+      cleaned = text.strip
+                    .gsub(/\s+/, " ")              # Multiple spaces to single space
+                    .gsub(/\n+/, "\n")             # Multiple newlines to single newline
+                    .gsub(/\t+/, " ")              # Tabs to spaces
+
+      # Truncate if too long (most embedding models have token limits)
+      max_chars = 8000 # Conservative limit for most embedding models
+      cleaned.length > max_chars ? cleaned[0, max_chars] : cleaned
+    end
+
+    # Generate a fallback embedding for testing/development when LLM services are unavailable
+    def generate_fallback_embedding(dimensions = 1536)
+      # Generate deterministic pseudo-random embeddings based on the object_id
+      # This ensures consistent results for testing while providing different embeddings for different instances
+      rng = Random.new(object_id)
+      dimensions.times.map { rng.rand * 2.0 - 1.0 }
+    end
+  end
+end