ragdoll 0.1.0 → 0.1.1

data/lib/ragdoll/core/embedding_service.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+
+module Ragdoll
+  module Core
+    class EmbeddingService
+      def initialize(client: nil)
+        @client = client
+        configure_ruby_llm unless @client
+      end
+
+      def generate_embedding(text)
+        return nil if text.nil? || text.strip.empty?
+
+        # Clean and prepare text
+        cleaned_text = clean_text(text)
+
+        begin
+          if @client
+            # Use custom client for testing
+            # FIXME: embedding_model is not in current config structure
+            response = @client.embed(
+              input: cleaned_text,
+              model: Ragdoll.config.models[:embedding][:text]
+            )
+
+            if response && response["embeddings"]&.first
+              response["embeddings"].first
+            elsif response && response["data"]&.first && response["data"].first["embedding"]
+              response["data"].first["embedding"]
+            else
+              raise EmbeddingError, "Invalid response format from embedding API"
+            end
+          else
+            # Use RubyLLM for real embedding generation
+            model = Ragdoll.config.models[:embedding][:text] || 'text-embedding-3-small'
+            response = RubyLLM.embed(cleaned_text, model: model)
+            
+            # Extract the embedding vector from RubyLLM::Embedding object
+            if response.respond_to?(:instance_variable_get)
+              vectors = response.instance_variable_get(:@vectors)
+              if vectors && vectors.is_a?(Array)
+                vectors
+              else
+                raise EmbeddingError, "No vectors found in RubyLLM response"
+              end
+            else
+              raise EmbeddingError, "Unexpected response type from RubyLLM: #{response.class}"
+            end
+          end
+        rescue StandardError => e
+          raise EmbeddingError, "Failed to generate embedding: #{e.message}"
+        end
+      end
+
+      def generate_embeddings_batch(texts)
+        return [] if texts.empty?
+
+        # Clean all texts
+        cleaned_texts = texts.map { |text| clean_text(text) }.reject { |t| t.nil? || t.strip.empty? }
+        return [] if cleaned_texts.empty?
+
+        begin
+          if @client
+            # Use custom client for testing
+            # FIXME: embedding_model is not in current config structure
+            response = @client.embed(
+              input: cleaned_texts,
+              model: Ragdoll.config.models[:embedding][:text]
+            )
+
+            if response && response["embeddings"]
+              response["embeddings"]
+            elsif response && response["data"]
+              response["data"].map { |item| item["embedding"] }
+            else
+              raise EmbeddingError, "Invalid response format from embedding API"
+            end
+          else
+            # Use RubyLLM for real embedding generation (batch mode)
+            model = Ragdoll.config.models[:embedding][:text] || 'text-embedding-3-small'
+            
+            cleaned_texts.map do |text|
+              response = RubyLLM.embed(text, model: model)
+              
+              # Extract the embedding vector from RubyLLM::Embedding object
+              if response.respond_to?(:instance_variable_get)
+                vectors = response.instance_variable_get(:@vectors)
+                if vectors && vectors.is_a?(Array)
+                  vectors
+                else
+                  raise EmbeddingError, "No vectors found in RubyLLM response"
+                end
+              else
+                raise EmbeddingError, "Unexpected response type from RubyLLM: #{response.class}"
+              end
+            end
+          end
+        rescue StandardError => e
+          raise EmbeddingError, "Failed to generate embeddings: #{e.message}"
+        end
+      end
+
+      def cosine_similarity(embedding1, embedding2)
+        return 0.0 if embedding1.nil? || embedding2.nil?
+        return 0.0 if embedding1.length != embedding2.length
+
+        dot_product = embedding1.zip(embedding2).sum { |a, b| a * b }
+        magnitude1 = Math.sqrt(embedding1.sum { |a| a * a })
+        magnitude2 = Math.sqrt(embedding2.sum { |a| a * a })
+
+        return 0.0 if magnitude1 == 0.0 || magnitude2 == 0.0
+
+        dot_product / (magnitude1 * magnitude2)
+      end
+
+      private
+
+      def configure_ruby_llm
+        # Configure ruby_llm based on Ragdoll configuration
+        # FIXME: embedding_provider and llm_provider are not in current config structure
+        # FIXME: llm_config is not in current config structure, should use ruby_llm_config directly
+        provider = :openai # Default provider
+        config = Ragdoll.config.ruby_llm_config[provider] || {}
+
+        RubyLLM.configure do |ruby_llm_config|
+          case provider
+          when :openai
+            ruby_llm_config.openai_api_key = config[:api_key]
+            # Set organization and project if methods exist
+            if config[:organization] && ruby_llm_config.respond_to?(:openai_organization=)
+              ruby_llm_config.openai_organization = config[:organization]
+            end
+            if config[:project] && ruby_llm_config.respond_to?(:openai_project=)
+              ruby_llm_config.openai_project = config[:project]
+            end
+          when :anthropic
+            ruby_llm_config.anthropic_api_key = config[:api_key] if ruby_llm_config.respond_to?(:anthropic_api_key=)
+          when :google
+            ruby_llm_config.google_api_key = config[:api_key] if ruby_llm_config.respond_to?(:google_api_key=)
+            if config[:project_id] && ruby_llm_config.respond_to?(:google_project_id=)
+              ruby_llm_config.google_project_id = config[:project_id]
+            end
+          when :azure
+            ruby_llm_config.azure_api_key = config[:api_key] if ruby_llm_config.respond_to?(:azure_api_key=)
+            if config[:endpoint] && ruby_llm_config.respond_to?(:azure_endpoint=)
+              ruby_llm_config.azure_endpoint = config[:endpoint]
+            end
+            if config[:api_version] && ruby_llm_config.respond_to?(:azure_api_version=)
+              ruby_llm_config.azure_api_version = config[:api_version]
+            end
+          when :ollama
+            if config[:endpoint] && ruby_llm_config.respond_to?(:ollama_endpoint=)
+              ruby_llm_config.ollama_endpoint = config[:endpoint]
+            end
+          when :huggingface
+            ruby_llm_config.huggingface_api_key = config[:api_key] if ruby_llm_config.respond_to?(:huggingface_api_key=)
+          when :openrouter
+            ruby_llm_config.openrouter_api_key = config[:api_key] if ruby_llm_config.respond_to?(:openrouter_api_key=)
+          else
+            # Don't raise error for unsupported providers in case RubyLLM doesn't support them yet
+            puts "Warning: Unsupported embedding provider: #{provider}"
+          end
+        end
+      end
+
+      def clean_text(text)
+        return "" if text.nil?
+
+        # Remove excessive whitespace and normalize
+        cleaned = text.strip
+                      .gsub(/\s+/, " ")              # Multiple spaces to single space
+                      .gsub(/\n+/, "\n")             # Multiple newlines to single newline
+                      .gsub(/\t+/, " ")              # Tabs to spaces
+
+        # Truncate if too long (most embedding models have token limits)
+        max_chars = 8000 # Conservative limit for most embedding models
+        cleaned.length > max_chars ? cleaned[0, max_chars] : cleaned
+      end
+    end
+  end
+end

data/lib/ragdoll/core/errors.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Core
+    class Error < StandardError; end
+    class EmbeddingError < Error; end
+    class SearchError < Error; end
+    class DocumentError < Error; end
+    class ConfigurationError < Error; end
+  end
+end

data/lib/ragdoll/core/jobs/extract_keywords.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Ragdoll
+  module Core
+    module Jobs
+      class ExtractKeywords < ActiveJob::Base
+        queue_as :default
+
+        def perform(document_id)
+          document = Models::Document.find(document_id)
+          return unless document.content.present?
+          return if document.keywords.present?
+
+          text_service = TextGenerationService.new
+          keywords_array = text_service.extract_keywords(document.content)
+
+          if keywords_array.present?
+            keywords_string = keywords_array.join(", ")
+            document.update!(keywords: keywords_string)
+          end
+        rescue ActiveRecord::RecordNotFound
+          # Document was deleted, nothing to do
+        rescue StandardError => e
+          Rails.logger.error "Failed to generate keywords for document #{document_id}: #{e.message}" if defined?(Rails)
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/ragdoll/core/jobs/extract_text.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Ragdoll
+  module Core
+    module Jobs
+      class ExtractText < ActiveJob::Base
+        queue_as :default
+
+        def perform(document_id)
+          document = Models::Document.find(document_id)
+          return unless document.file_attached?
+          return if document.content.present?
+
+          document.update!(status: "processing")
+
+          extracted_content = document.extract_text_from_file
+
+          if extracted_content.present?
+            document.update!(
+              content: extracted_content,
+              status: "processed"
+            )
+
+            # Queue follow-up jobs
+            GenerateSummaryJob.perform_later(document_id)
+            GenerateKeywordsJob.perform_later(document_id)
+            GenerateEmbeddingsJob.perform_later(document_id)
+          else
+            document.update!(status: "error")
+          end
+        rescue ActiveRecord::RecordNotFound
+          # Document was deleted, nothing to do
+        rescue StandardError => e
+          document&.update!(status: "error")
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/ragdoll/core/jobs/generate_embeddings.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Ragdoll
+  module Core
+    module Jobs
+      class GenerateEmbeddings < ActiveJob::Base
+        queue_as :default
+
+        def perform(document_id, chunk_size: nil, chunk_overlap: nil)
+          document = Models::Document.find(document_id)
+          return unless document.content.present?
+          return if document.all_embeddings.exists?
+
+          # Process all content records using their own generate_embeddings! methods
+          document.contents.each(&:generate_embeddings!)
+
+          # Update document status to processed
+          document.update!(status: "processed")
+        rescue ActiveRecord::RecordNotFound
+          # Document was deleted, nothing to do
+        rescue StandardError => e
+          if defined?(Rails)
+            Rails.logger.error "Failed to generate embeddings for document #{document_id}: #{e.message}"
+          end
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/ragdoll/core/jobs/generate_summary.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Ragdoll
+  module Core
+    module Jobs
+      class GenerateSummary < ActiveJob::Base
+        queue_as :default
+
+        def perform(document_id)
+          document = Models::Document.find(document_id)
+          return unless document.content.present?
+          return if document.summary.present?
+
+          text_service = TextGenerationService.new
+          summary = text_service.generate_summary(document.content)
+
+          document.update!(summary: summary) if summary.present?
+        rescue ActiveRecord::RecordNotFound
+          # Document was deleted, nothing to do
+        rescue StandardError => e
+          Rails.logger.error "Failed to generate summary for document #{document_id}: #{e.message}" if defined?(Rails)
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/ragdoll/core/metadata_schemas.rb

@@ -0,0 +1,334 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Core
+    # Document metadata schemas for LLM structured output
+    # Each document type has a specific schema that guides LLM generation
+    module MetadataSchemas
+      # Text document metadata schema
+      TEXT_SCHEMA = {
+        type: "object",
+        properties: {
+          summary: {
+            type: "string",
+            description: "Concise summary of the text content (2-3 sentences)"
+          },
+          keywords: {
+            type: "array",
+            items: { type: "string" },
+            description: "Relevant keywords and phrases extracted from the text",
+            maxItems: 10
+          },
+          classification: {
+            type: "string",
+            enum: %w[research article blog documentation technical legal financial marketing other],
+            description: "Document classification category"
+          },
+          topics: {
+            type: "array",
+            items: { type: "string" },
+            description: "Main topics discussed in the document",
+            maxItems: 5
+          },
+          sentiment: {
+            type: "string",
+            enum: %w[positive negative neutral mixed],
+            description: "Overall sentiment of the text"
+          },
+          reading_time_minutes: {
+            type: "integer",
+            description: "Estimated reading time in minutes"
+          },
+          language: {
+            type: "string",
+            description: "Primary language of the text (ISO 639-1 code)"
+          },
+          complexity_level: {
+            type: "string",
+            enum: %w[beginner intermediate advanced expert],
+            description: "Complexity/difficulty level of the content"
+          },
+          tags: {
+            type: "array",
+            items: { type: "string" },
+            description: "User-defined or AI-suggested tags for organization"
+          }
+        },
+        required: %w[summary keywords classification]
+      }.freeze
+
+      # Image document metadata schema
+      IMAGE_SCHEMA = {
+        type: "object",
+        properties: {
+          description: {
+            type: "string",
+            description: "Detailed description of what is shown in the image"
+          },
+          summary: {
+            type: "string",
+            description: "Brief summary of the image content (1 sentence)"
+          },
+          objects: {
+            type: "array",
+            items: { type: "string" },
+            description: "List of objects, people, or items visible in the image",
+            maxItems: 15
+          },
+          scene_type: {
+            type: "string",
+            enum: %w[indoor outdoor portrait landscape diagram chart screenshot artwork photo other],
+            description: "Type of scene or image category"
+          },
+          colors: {
+            type: "array",
+            items: { type: "string" },
+            description: "Dominant colors in the image",
+            maxItems: 5
+          },
+          style: {
+            type: "string",
+            enum: %w[photograph illustration diagram chart screenshot artwork technical drawing other],
+            description: "Visual style or format of the image"
+          },
+          mood: {
+            type: "string",
+            enum: %w[professional casual formal technical artistic dramatic serene energetic other],
+            description: "Overall mood or tone of the image"
+          },
+          text_content: {
+            type: "string",
+            description: "Any visible text in the image (OCR extracted)"
+          },
+          keywords: {
+            type: "array",
+            items: { type: "string" },
+            description: "Relevant keywords for image search and categorization",
+            maxItems: 10
+          },
+          classification: {
+            type: "string",
+            enum: %w[technical diagram photo artwork chart screenshot document other],
+            description: "Image classification category"
+          },
+          tags: {
+            type: "array",
+            items: { type: "string" },
+            description: "User-defined or AI-suggested tags for organization"
+          }
+        },
+        required: %w[description summary scene_type classification]
+      }.freeze
+
+      # Audio document metadata schema
+      AUDIO_SCHEMA = {
+        type: "object",
+        properties: {
+          summary: {
+            type: "string",
+            description: "Summary of audio content (speech transcript summary or music description)"
+          },
+          content_type: {
+            type: "string",
+            enum: %w[speech music podcast interview lecture presentation sound_effect other],
+            description: "Type of audio content"
+          },
+          keywords: {
+            type: "array",
+            items: { type: "string" },
+            description: "Relevant keywords extracted from transcript or describing music",
+            maxItems: 10
+          },
+          classification: {
+            type: "string",
+            enum: %w[educational entertainment business technical musical interview podcast other],
+            description: "Audio content classification"
+          },
+          topics: {
+            type: "array",
+            items: { type: "string" },
+            description: "Main topics discussed (for speech) or musical elements (for music)",
+            maxItems: 5
+          },
+          language: {
+            type: "string",
+            description: "Language of speech content (ISO 639-1 code) or N/A for music"
+          },
+          speakers: {
+            type: "array",
+            items: { type: "string" },
+            description: "Number or names of speakers (for speech content)",
+            maxItems: 10
+          },
+          mood: {
+            type: "string",
+            enum: %w[formal casual energetic calm professional educational entertaining informative other],
+            description: "Overall mood or tone of the audio"
+          },
+          genre: {
+            type: "string",
+            description: "Music genre (for musical content) or speech type (for spoken content)"
+          },
+          key_quotes: {
+            type: "array",
+            items: { type: "string" },
+            description: "Important quotes or phrases from speech content",
+            maxItems: 3
+          },
+          tags: {
+            type: "array",
+            items: { type: "string" },
+            description: "User-defined or AI-suggested tags for organization"
+          }
+        },
+        required: %w[summary content_type classification]
+      }.freeze
+
+      # PDF document metadata schema (combines text analysis with document structure)
+      PDF_SCHEMA = {
+        type: "object",
+        properties: {
+          summary: {
+            type: "string",
+            description: "Summary of the PDF document content"
+          },
+          document_type: {
+            type: "string",
+            enum: %w[research_paper report manual presentation legal financial technical academic other],
+            description: "Type of PDF document"
+          },
+          keywords: {
+            type: "array",
+            items: { type: "string" },
+            description: "Keywords extracted from the document text",
+            maxItems: 15
+          },
+          classification: {
+            type: "string",
+            enum: %w[academic business legal technical manual report presentation other],
+            description: "Document classification category"
+          },
+          topics: {
+            type: "array",
+            items: { type: "string" },
+            description: "Main topics covered in the document",
+            maxItems: 8
+          },
+          structure: {
+            type: "object",
+            properties: {
+              has_table_of_contents: { type: "boolean" },
+              has_bibliography: { type: "boolean" },
+              has_figures: { type: "boolean" },
+              has_tables: { type: "boolean" },
+              estimated_pages: { type: "integer" }
+            }
+          },
+          reading_time_minutes: {
+            type: "integer",
+            description: "Estimated reading time in minutes"
+          },
+          complexity_level: {
+            type: "string",
+            enum: %w[beginner intermediate advanced expert],
+            description: "Complexity level of the content"
+          },
+          language: {
+            type: "string",
+            description: "Primary language of the document"
+          },
+          tags: {
+            type: "array",
+            items: { type: "string" },
+            description: "User-defined or AI-suggested tags for organization"
+          }
+        },
+        required: %w[summary document_type classification]
+      }.freeze
+
+      # Mixed/multi-modal document metadata schema
+      MIXED_SCHEMA = {
+        type: "object",
+        properties: {
+          summary: {
+            type: "string",
+            description: "Overall summary combining all content types in the document"
+          },
+          content_types: {
+            type: "array",
+            items: { type: "string", enum: %w[text image audio] },
+            description: "Types of content present in this multi-modal document"
+          },
+          primary_content_type: {
+            type: "string",
+            enum: %w[text image audio],
+            description: "The primary or dominant content type"
+          },
+          keywords: {
+            type: "array",
+            items: { type: "string" },
+            description: "Keywords extracted from all content types",
+            maxItems: 15
+          },
+          classification: {
+            type: "string",
+            enum: %w[multimedia_presentation research educational marketing technical training other],
+            description: "Multi-modal document classification"
+          },
+          topics: {
+            type: "array",
+            items: { type: "string" },
+            description: "Main topics across all content types",
+            maxItems: 8
+          },
+          cohesion_analysis: {
+            type: "string",
+            description: "How well the different content types work together"
+          },
+          tags: {
+            type: "array",
+            items: { type: "string" },
+            description: "User-defined or AI-suggested tags for organization"
+          }
+        },
+        required: %w[summary content_types primary_content_type classification]
+      }.freeze
+
+      # Get schema for document type
+      def self.schema_for(document_type)
+        case document_type.to_s.downcase
+        when "text", "markdown", "html"
+          TEXT_SCHEMA
+        when "image"
+          IMAGE_SCHEMA
+        when "audio"
+          AUDIO_SCHEMA
+        when "pdf", "docx"
+          PDF_SCHEMA
+        when "mixed"
+          MIXED_SCHEMA
+        else
+          TEXT_SCHEMA # fallback
+        end
+      end
+
+      # Get required fields for document type
+      def self.required_fields_for(document_type)
+        schema_for(document_type)[:required] || []
+      end
+
+      # Validate metadata against schema
+      def self.validate_metadata(document_type, metadata)
+        schema = schema_for(document_type)
+        required_fields = schema[:required] || []
+
+        errors = []
+        required_fields.each do |field|
+          errors << "Missing required field: #{field}" unless metadata.key?(field)
+        end
+
+        errors
+      end
+    end
+  end
+end