ragdoll 0.1.0 → 0.1.1

data/lib/ragdoll/core/models/document.rb

@@ -0,0 +1,678 @@
+# frozen_string_literal: true
+
+require "active_record"
+require_relative "../metadata_schemas"
+
+# == Schema Information
+#
+# Table name: ragdoll_documents
+#
+#  id                                                                                         :bigint           not null, primary key
+#  document_type(Document format type (text, image, audio, pdf, docx, html, markdown, mixed)) :string           default("text"), not null
+#  (no file_data column - file data stored in content models via STI)
+#  file_metadata(File properties and processing metadata, separate from AI-generated content) :json
+#  location(Source location of document (file path, URL, or identifier))                      :string           not null
+#  metadata(LLM-generated structured metadata using document-type-specific schemas)           :json
+#  status(Document processing status: pending, processing, processed, error)                  :string           default("pending"), not null
+#  title(Human-readable document title for display and search)                                :string           not null
+#  file_modified_at(Timestamp when the source file was last modified)                        :datetime         not null
+#  created_at(Standard creation and update timestamps)                                        :datetime         not null
+#  updated_at(Standard creation and update timestamps)                                        :datetime         not null
+#
+# Indexes
+#
+#  index_ragdoll_documents_on_created_at                (created_at)
+#  index_ragdoll_documents_on_document_type             (document_type)
+#  index_ragdoll_documents_on_document_type_and_status  (document_type,status)
+#  index_ragdoll_documents_on_fulltext_search           (to_tsvector('english'::regconfig, (((((((COALESCE(title, ''::character varying))::text || ' '::text) || COALESCE((metadata ->> 'summary'::text), ''::text)) || ' '::text) || COALESCE((metadata ->> 'keywords'::text), ''::text)) || ' '::text) || COALESCE((metadata ->> 'description'::text), ''::text)))) USING gin
+#  index_ragdoll_documents_on_location                  (location) UNIQUE
+#  index_ragdoll_documents_on_metadata_classification   (((metadata ->> 'classification'::text)))
+#  index_ragdoll_documents_on_metadata_type             (((metadata ->> 'document_type'::text)))
+#  index_ragdoll_documents_on_status                    (status)
+#  index_ragdoll_documents_on_title                     (title)
+#
+
+module Ragdoll
+  module Core
+    module Models
+      class Document < ActiveRecord::Base
+        self.table_name = "ragdoll_documents"
+
+        # PostgreSQL full-text search on summary and keywords
+        # Uses PostgreSQL's built-in full-text search capabilities
+
+        # File handling moved to content models - no Shrine attachment at document level
+
+        # Multi-modal content relationships using STI
+        has_many :contents,
+                 class_name: "Ragdoll::Core::Models::Content",
+                 foreign_key: "document_id",
+                 dependent: :destroy
+
+        has_many :text_contents,
+                 -> { where(type: "Ragdoll::Core::Models::TextContent") },
+                 class_name: "Ragdoll::Core::Models::TextContent",
+                 foreign_key: "document_id"
+
+        has_many :image_contents,
+                 -> { where(type: "Ragdoll::Core::Models::ImageContent") },
+                 class_name: "Ragdoll::Core::Models::ImageContent",
+                 foreign_key: "document_id"
+
+        has_many :audio_contents,
+                 -> { where(type: "Ragdoll::Core::Models::AudioContent") },
+                 class_name: "Ragdoll::Core::Models::AudioContent",
+                 foreign_key: "document_id"
+
+        # All embeddings across content types
+        has_many :text_embeddings, through: :text_contents, source: :embeddings
+        has_many :image_embeddings, through: :image_contents, source: :embeddings
+        has_many :audio_embeddings, through: :audio_contents, source: :embeddings
+
+        validates :location, presence: true
+        validates :title, presence: true
+        validates :document_type, presence: true,
+                                  inclusion: { in: %w[text image audio pdf docx html markdown mixed] }
+        validates :summary, presence: false  # Allow empty summaries initially
+        validates :keywords, presence: false  # Allow empty keywords initially
+        validates :status, inclusion: { in: %w[pending processing processed error] }
+        validates :file_modified_at, presence: true
+
+        # Ensure location is always an absolute path for file paths
+        before_validation :normalize_location
+        before_validation :set_default_file_modified_at
+
+        # JSON columns are handled natively by PostgreSQL - no serialization needed
+
+        scope :processed, -> { where(status: "processed") }
+        scope :by_type, ->(type) { where(document_type: type) }
+        scope :recent, -> { order(created_at: :desc) }
+        scope :with_content, -> { joins(:contents).distinct }
+        scope :without_content, -> { left_joins(:contents).where(contents: { id: nil }) }
+
+        # Callbacks to process content
+        after_commit :create_content_from_pending, on: %i[create update],
+                                                   if: :has_pending_content?
+
+        def processed?
+          status == "processed"
+        end
+
+        # Multi-modal content type detection
+        def multi_modal?
+          content_types.length > 1
+        end
+
+        def content_types
+          types = []
+          types << "text" if text_contents.any?
+          types << "image" if image_contents.any?
+          types << "audio" if audio_contents.any?
+          types
+        end
+
+        def primary_content_type
+          return document_type if %w[text image audio].include?(document_type)
+          return content_types.first if content_types.any?
+
+          "text" # default
+        end
+
+        # Dynamic content method that forwards to appropriate content table
+        def content
+          case primary_content_type
+          when "text"
+            # Return the combined content from all text_contents
+            text_contents.pluck(:content).compact.join("\n\n")
+          when "image"
+            # Return the combined descriptions from all image_contents (content field stores description)
+            image_contents.pluck(:content).compact.join("\n\n")
+          when "audio"
+            # Return the combined transcripts from all audio_contents (content field stores transcript)
+            audio_contents.pluck(:content).compact.join("\n\n")
+          else
+            # Fallback: try to get any available content
+            contents.pluck(:content).compact.join("\n\n")
+          end
+        end
+
+        # Set content method for backwards compatibility
+        def content=(value)
+          # Store the content to be created after save
+          @pending_content = value
+
+          # If document is already persisted, create the content immediately
+          if persisted?
+            create_content_from_pending
+          end
+        end
+
+        # Content statistics
+        def total_word_count
+          text_contents.sum { |tc| tc.word_count }
+        end
+
+        def total_character_count
+          text_contents.sum { |tc| tc.character_count }
+        end
+
+        def total_embedding_count
+          text_embeddings.count + image_embeddings.count + audio_embeddings.count
+        end
+
+        def embeddings_by_type
+          {
+            text: text_embeddings.count,
+            image: image_embeddings.count,
+            audio: audio_embeddings.count
+          }
+        end
+
+        # Document metadata methods - now using dedicated columns
+        def has_summary?
+          summary.present?
+        end
+
+        def has_keywords?
+          keywords.present?
+        end
+
+        def keywords_array
+          return [] unless keywords.present?
+
+          case keywords
+          when Array
+            keywords
+          when String
+            keywords.split(",").map(&:strip).reject(&:empty?)
+          else
+            []
+          end
+        end
+
+        def add_keyword(keyword)
+          current_keywords = keywords_array
+          return if current_keywords.include?(keyword.strip)
+
+          current_keywords << keyword.strip
+          self.keywords = current_keywords.join(", ")
+        end
+
+        def remove_keyword(keyword)
+          current_keywords = keywords_array
+          current_keywords.delete(keyword.strip)
+          self.keywords = current_keywords.join(", ")
+        end
+
+        # Metadata accessors for common fields
+        def description
+          metadata["description"]
+        end
+
+        def description=(value)
+          self.metadata = metadata.merge("description" => value)
+        end
+
+        def classification
+          metadata["classification"]
+        end
+
+        def classification=(value)
+          self.metadata = metadata.merge("classification" => value)
+        end
+
+        def tags
+          metadata["tags"] || []
+        end
+
+        def tags=(value)
+          self.metadata = metadata.merge("tags" => Array(value))
+        end
+
+        # File-related helper methods - now delegated to content models
+        def has_files?
+          contents.any? { |c| c.data.present? }
+        end
+
+        def total_file_size
+          # Could be implemented by summing file sizes from content metadata
+          contents.sum { |c| c.metadata.dig('file_size') || 0 }
+        end
+
+        def primary_file_type
+          # Return the document_type as the primary file type
+          document_type
+        end
+
+        # Content processing for multi-modal documents
+        def process_content!
+          # Content processing is now handled by individual content models
+          # This method orchestrates the overall processing
+          
+          # Generate embeddings for all content
+          generate_embeddings_for_all_content!
+
+          # Generate structured metadata using LLM
+          generate_metadata!
+
+          update!(status: "processed")
+        end
+
+        # Generate embeddings for all content types
+        def generate_embeddings_for_all_content!
+          text_contents.each(&:generate_embeddings!)
+          image_contents.each(&:generate_embeddings!)
+          audio_contents.each(&:generate_embeddings!)
+        end
+
+        # Generate structured metadata using LLM
+        def generate_metadata!
+          require_relative "../services/metadata_generator"
+
+          generator = Services::MetadataGenerator.new
+          generated_metadata = generator.generate_for_document(self)
+
+          # Validate metadata against schema
+          errors = MetadataSchemas.validate_metadata(document_type, generated_metadata)
+          if errors.any?
+            Rails.logger.warn "Metadata validation errors: #{errors.join(', ')}" if defined?(Rails)
+            puts "Metadata validation errors: #{errors.join(', ')}"
+          end
+
+          # Merge with existing metadata (preserving user-set values)
+          self.metadata = metadata.merge(generated_metadata)
+          save!
+        rescue StandardError => e
+          Rails.logger.error "Metadata generation failed: #{e.message}" if defined?(Rails)
+          puts "Metadata generation failed: #{e.message}"
+        end
+
+        # PostgreSQL full-text search on metadata fields
+        def self.search_content(query, **options)
+          return none if query.blank?
+
+          # Use PostgreSQL's built-in full-text search across metadata fields
+          where(
+            "to_tsvector('english', COALESCE(title, '') || ' ' || COALESCE(metadata->>'summary', '') || ' ' || COALESCE(metadata->>'keywords', '') || ' ' || COALESCE(metadata->>'description', '')) @@ plainto_tsquery('english', ?)",
+            query
+          ).limit(options[:limit] || 20)
+        end
+
+        # Faceted search by metadata fields
+        def self.faceted_search(query: nil, keywords: [], classification: nil, tags: [], **options)
+          scope = all
+
+          # Filter by keywords if provided
+          if keywords.any?
+            keywords.each do |keyword|
+              scope = scope.where("metadata->>'keywords' ILIKE ?", "%#{keyword}%")
+            end
+          end
+
+          # Filter by classification
+          scope = scope.where("metadata->>'classification' = ?", classification) if classification.present?
+
+          # Filter by tags
+          if tags.any?
+            tags.each do |tag|
+              scope = scope.where("metadata ? 'tags' AND metadata->'tags' @> ?", [tag].to_json)
+            end
+          end
+
+          # Apply PostgreSQL full-text search if query provided
+          if query.present?
+            scope = scope.where(
+              "to_tsvector('english', COALESCE(title, '') || ' ' || COALESCE(metadata->>'summary', '') || ' ' || COALESCE(metadata->>'keywords', '') || ' ' || COALESCE(metadata->>'description', '')) @@ plainto_tsquery('english', ?)",
+              query
+            )
+          end
+
+          scope.limit(options[:limit] || 20)
+        end
+
+        # Get all unique keywords from metadata
+        def self.all_keywords
+          keywords = []
+          where("metadata ? 'keywords'").pluck(:metadata).each do |meta|
+            case meta["keywords"]
+            when Array
+              keywords.concat(meta["keywords"])
+            when String
+              keywords.concat(meta["keywords"].split(",").map(&:strip))
+            end
+          end
+          keywords.uniq.sort
+        end
+
+        # Get all unique classifications
+        def self.all_classifications
+          where("metadata ? 'classification'").distinct.pluck("metadata->>'classification'").compact.sort
+        end
+
+        # Get all unique tags
+        def self.all_tags
+          tags = []
+          where("metadata ? 'tags'").pluck(:metadata).each do |meta|
+            tags.concat(Array(meta["tags"]))
+          end
+          tags.uniq.sort
+        end
+
+        # Get keyword frequencies for faceted search
+        def self.keyword_frequencies
+          frequencies = Hash.new(0)
+          where("metadata ? 'keywords'").pluck(:metadata).each do |meta|
+            case meta["keywords"]
+            when Array
+              meta["keywords"].each { |k| frequencies[k] += 1 }
+            when String
+              meta["keywords"].split(",").map(&:strip).each { |k| frequencies[k] += 1 }
+            end
+          end
+          frequencies.sort_by { |_k, v| -v }.to_h
+        end
+
+        # Hybrid search combining semantic and PostgreSQL full-text search
+        def self.hybrid_search(query, query_embedding: nil, **options)
+          limit = options[:limit] || 20
+          semantic_weight = options[:semantic_weight] || 0.7
+          text_weight = options[:text_weight] || 0.3
+
+          results = []
+
+          # Get semantic search results if embedding provided
+          if query_embedding
+            semantic_results = embeddings_search(query_embedding, limit: limit)
+            results.concat(semantic_results.map do |result|
+              result.merge(
+                search_type: "semantic",
+                weighted_score: result[:combined_score] * semantic_weight
+              )
+            end)
+          end
+
+          # Get PostgreSQL full-text search results
+          text_results = search_content(query, limit: limit)
+          text_results.each_with_index do |doc, index|
+            score = (limit - index).to_f / limit * text_weight
+            results << {
+              document_id: doc.id.to_s,
+              document_title: doc.title,
+              document_location: doc.location,
+              content: doc.content[0..500], # Preview
+              search_type: "full_text",
+              weighted_score: score,
+              document: doc
+            }
+          end
+
+          # Combine and deduplicate by document_id
+          combined = results.group_by { |r| r[:document_id] }
+                            .map do |_doc_id, doc_results|
+            best_result = doc_results.max_by { |r| r[:weighted_score] }
+            total_score = doc_results.sum { |r| r[:weighted_score] }
+            search_types = doc_results.map { |r| r[:search_type] }.uniq
+
+            best_result.merge(
+              combined_score: total_score,
+              search_types: search_types
+            )
+          end
+
+          combined.sort_by { |r| -r[:combined_score] }.take(limit)
+        end
+
+        # Extract keywords from query string (words > 4 characters)
+        def self.extract_keywords(query:)
+          return [] if query.nil? || query.strip.empty?
+
+          query.split(/\s+/)
+               .map(&:strip)
+               .reject(&:empty?)
+               .select { |word| word.length > 4 }
+        end
+
+        # Get search data for indexing
+        def search_data
+          data = {
+            title: title,
+            document_type: document_type,
+            location: location,
+            status: status,
+            total_word_count: total_word_count,
+            total_character_count: total_character_count,
+            total_embedding_count: total_embedding_count,
+            content_types: content_types,
+            multi_modal: multi_modal?
+          }
+
+          # Add document metadata
+          data.merge!(metadata.transform_keys { |k| "metadata_#{k}" }) if metadata.present?
+
+          # Add file metadata
+          data.merge!(file_metadata.transform_keys { |k| "file_#{k}" }) if file_metadata.present?
+
+          data
+        end
+
+        def all_embeddings(content_type: nil)
+          content_ids = []
+
+          # If content_type is specified, only get IDs for that type
+          if content_type
+            case content_type.to_s
+            when 'text'
+              content_ids.concat(text_contents.pluck(:id)) if text_contents.any?
+            when 'image'
+              content_ids.concat(image_contents.pluck(:id)) if image_contents.any?
+            when 'audio'
+              content_ids.concat(audio_contents.pluck(:id)) if audio_contents.any?
+            end
+          else
+            # Collect all content IDs across all content types
+            content_ids.concat(text_contents.pluck(:id)) if text_contents.any?
+            content_ids.concat(image_contents.pluck(:id)) if image_contents.any?
+            content_ids.concat(audio_contents.pluck(:id)) if audio_contents.any?
+          end
+
+          return Ragdoll::Core::Models::Embedding.none if content_ids.empty?
+
+          # Use the base STI class name 'Ragdoll::Core::Models::Content' as that's what's stored
+          # in polymorphic associations with STI
+          Ragdoll::Core::Models::Embedding.where(
+            embeddable_type: 'Ragdoll::Core::Models::Content',
+            embeddable_id: content_ids
+          )
+        end
+
+        private
+
+
+        def has_pending_content?
+          @pending_content.present?
+        end
+
+        def create_content_from_pending
+          return unless @pending_content.present?
+
+          value = @pending_content
+          @pending_content = nil
+
+          case primary_content_type
+          when "text"
+            # Create or update the first text_content
+            if text_contents.any?
+              text_contents.first.update!(content: value)
+            else
+              text_contents.create!(
+                content: value,
+                embedding_model: default_text_model,
+                metadata: { manually_set: true }
+              )
+            end
+          when "image"
+            # For images, set the description (stored in content field)
+            if image_contents.any?
+              image_contents.first.update!(content: value)  # content field stores description
+            else
+              image_contents.create!(
+                content: value,  # content field stores description
+                embedding_model: default_image_model,
+                metadata: { manually_set: true }
+              )
+            end
+          when "audio"
+            # For audio, set the transcript (stored in content field)
+            if audio_contents.any?
+              audio_contents.first.update!(content: value)  # content field stores transcript
+            else
+              audio_contents.create!(
+                content: value,  # content field stores transcript
+                embedding_model: default_audio_model,
+                metadata: { manually_set: true }
+              )
+            end
+          else
+            # Default to text content
+            text_contents.create!(
+              content: value,
+              embedding_model: default_text_model,
+              metadata: { manually_set: true }
+            )
+          end
+        end
+
+
+        def self.embeddings_search(query_embedding, **options)
+          Ragdoll::Core::Models::Embedding.search_similar(query_embedding, **options)
+        end
+
+        # File processing is now handled by DocumentProcessor and content models
+        # These methods are no longer needed at the document level
+
+        # Default model names for each content type
+        def default_text_model
+          "text-embedding-3-large"
+        end
+
+        def default_image_model
+          "clip-vit-large-patch14"
+        end
+
+        def default_audio_model
+          "whisper-embedding-v1"
+        end
+
+        # File extraction is now handled by DocumentProcessor
+        # Content-specific extraction is handled by individual content models
+
+        # Get document statistics
+        def self.stats
+          {
+            total_documents: count,
+            by_status: group(:status).count,
+            by_type: group(:document_type).count,
+            multi_modal_documents: joins(:text_contents, :image_contents).distinct.count +
+              joins(:text_contents, :audio_contents).distinct.count +
+              joins(:image_contents, :audio_contents).distinct.count,
+            total_text_contents: joins(:text_contents).count,
+            total_image_contents: joins(:image_contents).count,
+            total_audio_contents: joins(:audio_contents).count,
+            total_embeddings: {
+              text: joins(:text_embeddings).count,
+              image: joins(:image_embeddings).count,
+              audio: joins(:audio_embeddings).count
+            },
+            storage_type: "activerecord_polymorphic"
+          }
+        end
+
+        public
+
+        # Convert document to hash representation for API responses
+        def to_hash(include_content: false)
+          {
+            id: id.to_s,
+            title: title,
+            location: location,
+            document_type: document_type,
+            status: status,
+            content_length: content&.length || 0,
+            file_modified_at: file_modified_at&.iso8601,
+            created_at: created_at&.iso8601,
+            updated_at: updated_at&.iso8601,
+            metadata: metadata || {},
+            content_summary: {
+              text_contents: text_contents.count,
+              image_contents: image_contents.count,
+              audio_contents: audio_contents.count,
+              embeddings_count: total_embeddings_count,
+              embeddings_ready: status == "processed"
+            }
+          }.tap do |hash|
+            if include_content
+              hash[:content_details] = {
+                text_content: text_contents.map(&:content),
+                image_descriptions: image_contents.map(&:description),
+                audio_transcripts: audio_contents.map(&:transcript)
+              }
+            end
+          end
+        end
+
+        private
+
+        def total_embeddings_count
+          # Count embeddings through polymorphic associations
+          embedding_count = 0
+
+          # Count embeddings for text contents
+          text_contents.each do |content|
+            embedding_count += content.embeddings.count
+          end
+
+          # Count embeddings for image contents
+          image_contents.each do |content|
+            embedding_count += content.embeddings.count
+          end
+
+          # Count embeddings for audio contents
+          audio_contents.each do |content|
+            embedding_count += content.embeddings.count
+          end
+
+          embedding_count
+        end
+
+        # Normalize location to absolute path for file paths
+        def normalize_location
+          return if location.blank?
+
+          # Don't normalize URLs or other non-file protocols
+          return if location.start_with?("http://", "https://", "ftp://", "sftp://")
+
+          # Convert relative file paths to absolute paths
+          self.location = File.expand_path(location)
+        end
+
+        # Set default file_modified_at if not provided
+        def set_default_file_modified_at
+          return if file_modified_at.present?
+
+          # If location is a file path that exists, use file mtime
+          if location.present? && !location.start_with?("http://", "https://", "ftp://", "sftp://")
+            expanded_location = File.expand_path(location)
+            self.file_modified_at = if File.exist?(expanded_location)
+                                      File.mtime(expanded_location)
+                                    else
+                                      Time.current
+                                    end
+          else
+            # For URLs or non-file locations, use current time
+            self.file_modified_at = Time.current
+          end
+        end
+      end
+    end
+  end
+end