ragdoll 0.0.2

data/lib/ragdoll/core/database.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "logger"
+
+module Ragdoll
+  module Core
+    class Database
+      def self.setup(config = {})
+        database_config = default_config.merge(config)
+
+        # Set up ActiveRecord connection
+        ActiveRecord::Base.establish_connection(database_config)
+
+        # Set up logging if specified
+        ActiveRecord::Base.logger = database_config[:logger] if database_config[:logger]
+
+        # Auto-migrate if specified
+        return unless database_config[:auto_migrate]
+
+        migrate!
+      end
+
+      def self.migrate!
+        # Get the path to the gem root directory
+        # Current file is lib/ragdoll/core/database.rb, so go up 3 levels to get to gem root
+        gem_root = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", ".."))
+        migration_paths = [
+          File.join(gem_root, "db", "migrate")
+        ]
+
+        ActiveRecord::Migration.verbose = true
+
+        # Ensure schema_migrations table exists first
+        unless ActiveRecord::Base.connection.table_exists?("schema_migrations")
+          ActiveRecord::Base.connection.create_table("schema_migrations", id: false) do |t|
+            t.string :version, null: false
+          end
+          ActiveRecord::Base.connection.add_index("schema_migrations", :version, unique: true)
+        end
+
+        # Debug migration path (silenced for clean test output)
+        # puts "Migration path: #{migration_paths.first}" if ActiveRecord::Migration.verbose
+        migration_files = Dir[File.join(migration_paths.first, "*.rb")].sort
+        # puts "Found #{migration_files.length} migration files" if ActiveRecord::Migration.verbose
+
+        # Load and run each migration manually since ActiveRecord migration context seems broken
+        migration_files.each do |migration_file|
+          # Extract version from filename
+          version = File.basename(migration_file, ".rb").split("_").first
+
+          # Skip if already migrated
+          next if ActiveRecord::Base.connection.select_values(
+            "SELECT version FROM schema_migrations WHERE version = '#{version}'"
+          ).any?
+
+          # Load the migration file to define the class
+          require migration_file
+
+          # Get the migration class - convert snake_case to CamelCase
+          filename_parts = File.basename(migration_file, ".rb").split("_")[1..]
+          migration_class_name = filename_parts.map(&:capitalize).join
+
+          begin
+            migration_class = Object.const_get(migration_class_name)
+          rescue NameError
+            puts "Warning: Could not find migration class #{migration_class_name} in #{migration_file}"
+            next
+          end
+
+          # Run the migration quietly
+          old_verbose = ActiveRecord::Migration.verbose
+          ActiveRecord::Migration.verbose = false
+          migration_class.migrate(:up)
+          ActiveRecord::Migration.verbose = old_verbose
+
+          # Record the migration
+          ActiveRecord::Base.connection.insert(
+            "INSERT INTO schema_migrations (version) VALUES ('#{version}')"
+          )
+
+          # Silenced migration progress - uncomment for debugging
+          # puts "Migrated #{migration_class_name}" if ActiveRecord::Migration.verbose
+        end
+      end
+
+      def self.reset!
+        ActiveRecord::Migration.verbose = false
+
+        # Drop all tables in correct order (respecting foreign key constraints)
+        # Order: dependent tables first, then parent tables
+        tables_to_drop = %w[
+          ragdoll_embeddings
+          ragdoll_text_contents
+          ragdoll_image_contents
+          ragdoll_audio_contents
+          ragdoll_documents
+          schema_migrations
+        ]
+
+        tables_to_drop.each do |table|
+          if ActiveRecord::Base.connection.table_exists?(table)
+            # For PostgreSQL, we can use CASCADE to drop dependent objects
+            if ActiveRecord::Base.connection.adapter_name.downcase.include?("postgresql")
+              ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS #{table} CASCADE")
+            else
+              ActiveRecord::Base.connection.drop_table(table)
+            end
+          end
+        end
+
+        migrate!
+      end
+
+      def self.connected?
+        ActiveRecord::Base.connected?
+      end
+
+      def self.disconnect!
+        ActiveRecord::Base.clear_all_connections!
+      end
+
+      def self.default_config
+        {
+          adapter: "postgresql",
+          database: "ragdoll_development",
+          username: "ragdoll",
+          password: ENV["RAGDOLL_DATABASE_PASSWORD"],
+          host: "localhost",
+          port: 5432,
+          auto_migrate: true,
+          logger: Logger.new($stdout, level: Logger::WARN)
+        }
+      end
+
+      def self.migration_paths
+        [File.join(File.dirname(__FILE__), "..", "..", "..", "..", "db", "migrate")]
+      end
+    end
+  end
+end

data/lib/ragdoll/core/document_management.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Core
+    class DocumentManagement
+      class << self
+        def add_document(location, content, metadata = {})
+          # Ensure location is an absolute path if it's a file path
+          absolute_location = location.start_with?("http") || location.start_with?("ftp") ? location : File.expand_path(location)
+
+          # Get file modification time if it's a file path
+          file_modified_at = if File.exist?(absolute_location) && !absolute_location.start_with?("http")
+                               File.mtime(absolute_location)
+                             else
+                               Time.current
+                             end
+
+          # Check if document already exists with same location and file_modified_at
+          existing_document = Models::Document.find_by(
+            location: absolute_location,
+            file_modified_at: file_modified_at
+          )
+
+          # Return existing document ID if found (skip duplicate)
+          return existing_document.id.to_s if existing_document
+
+          document = Models::Document.create!(
+            location: absolute_location,
+            title: metadata[:title] || metadata["title"] || extract_title_from_location(location),
+            document_type: metadata[:document_type] || metadata["document_type"] || "text",
+            metadata: metadata.is_a?(Hash) ? metadata : {},
+            status: "pending",
+            file_modified_at: file_modified_at
+          )
+
+          # Set content using the model's setter to trigger TextContent creation
+          document.content = content if content.present?
+
+          document.id.to_s
+        end
+
+        def get_document(id)
+          document = Models::Document.find_by(id: id)
+          return nil unless document
+
+          hash = document.to_hash
+          hash[:content] = document.content
+          hash
+        end
+
+        def update_document(id, **updates)
+          document = Models::Document.find_by(id: id)
+          return nil unless document
+
+          # Only update allowed fields
+          allowed_updates = updates.slice(:title, :metadata, :status, :document_type)
+          document.update!(allowed_updates) if allowed_updates.any?
+
+          document.to_hash
+        end
+
+        def delete_document(id)
+          document = Models::Document.find_by(id: id)
+          return nil unless document
+
+          document.destroy!
+          true
+        end
+
+        def list_documents(options = {})
+          limit = options[:limit] || 100
+          offset = options[:offset] || 0
+
+          Models::Document.offset(offset).limit(limit).recent.map(&:to_hash)
+        end
+
+        def get_document_stats
+          Models::Document.stats
+        end
+
+        # FIXME: should this be here?
+
+        def add_embedding(embeddable_id, chunk_index, embedding_vector, metadata = {})
+          # The embeddable_type should be the actual STI subclass, not the base class
+          embeddable_type = if metadata[:embeddable_type]
+                             metadata[:embeddable_type]
+                           else
+                             # Look up the actual STI type from the content record
+                             content = Models::Content.find(embeddable_id)
+                             content.class.name
+                           end
+          
+          Models::Embedding.create!(
+            embeddable_id: embeddable_id,
+            embeddable_type: embeddable_type,
+            chunk_index: chunk_index,
+            embedding_vector: embedding_vector,
+            content: metadata[:content] || ""
+          ).id.to_s
+        end
+
+        private
+
+        def extract_title_from_location(location)
+          File.basename(location, File.extname(location))
+        end
+      end
+    end
+  end
+end

data/lib/ragdoll/core/document_processor.rb

@@ -0,0 +1,344 @@
+# frozen_string_literal: true
+
+require "pdf-reader"
+require "docx"
+require "rmagick"
+require_relative "services/image_description_service"
+
+module Ragdoll
+  module Core
+    class DocumentProcessor
+      class ParseError < DocumentError; end
+      class UnsupportedFormatError < ParseError; end
+
+      def self.parse(file_path)
+        new(file_path).parse
+      end
+
+      # Parse from Shrine attached file
+      def self.parse_attachment(attached_file)
+        attached_file.open do |tempfile|
+          new(tempfile.path, attached_file).parse
+        end
+      end
+
+      # Create document from file path
+      def self.create_document_from_file(file_path, **options)
+        parsed = parse(file_path)
+
+        # Get file modification time
+        file_modified_at = File.exist?(file_path) ? File.mtime(file_path) : Time.current
+
+        document = Models::Document.create!(
+          location: File.expand_path(file_path),
+          title: parsed[:title] || File.basename(file_path, File.extname(file_path)),
+          content: parsed[:content],
+          document_type: determine_document_type(file_path),
+          metadata: parsed[:metadata] || {},
+          status: "processed",
+          file_modified_at: file_modified_at,
+          **options
+        )
+
+        # Attach the file if it exists
+        document.file = File.open(file_path) if File.exist?(file_path)
+
+        document
+      end
+
+      # Create document from uploaded file (Shrine compatible)
+      def self.create_document_from_upload(uploaded_file, **options)
+        # Create document first
+        document = Models::Document.create!(
+          location: uploaded_file.original_filename || "uploaded_file",
+          title: options[:title] || File.basename(uploaded_file.original_filename || "uploaded_file",
+                                                  File.extname(uploaded_file.original_filename || "")),
+          content: "", # Will be extracted after file attachment
+          document_type: determine_document_type_from_content_type(uploaded_file.mime_type),
+          status: "processing",
+          metadata: options[:metadata] || {},
+          file_modified_at: Time.current
+        )
+
+        # Attach the file
+        document.file = uploaded_file
+
+        # Extract content from attached file
+        if document.file.present?
+          parsed = parse_attachment(document.file)
+          document.update!(
+            content: parsed[:content],
+            title: parsed[:title] || document.title,
+            metadata: document.metadata.merge(parsed[:metadata] || {}),
+            status: "processed"
+          )
+        end
+
+        document
+      end
+
+      def initialize(file_path, attached_file = nil)
+        @file_path = file_path
+        @attached_file = attached_file
+        @file_extension = File.extname(file_path).downcase
+      end
+
+      def parse
+        case @file_extension
+        when ".pdf"
+          parse_pdf
+        when ".docx"
+          parse_docx
+        when ".txt", ".md", ".markdown"
+          parse_text
+        when ".html", ".htm"
+          parse_html
+        when ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".tif"
+          parse_image
+        else
+          parse_text # Default to text parsing for unknown formats
+        end
+      rescue StandardError => e # StandardError => e
+        raise ParseError, "#{__LINE__} Failed to parse #{@file_path}: #{e.message}"
+      end
+
+      private
+
+      def parse_pdf
+        content = ""
+        metadata = {}
+
+        begin
+          PDF::Reader.open(@file_path) do |reader|
+            # Extract metadata
+            if reader.info
+              metadata[:title] = reader.info[:Title] if reader.info[:Title]
+              metadata[:author] = reader.info[:Author] if reader.info[:Author]
+              metadata[:subject] = reader.info[:Subject] if reader.info[:Subject]
+              metadata[:creator] = reader.info[:Creator] if reader.info[:Creator]
+              metadata[:producer] = reader.info[:Producer] if reader.info[:Producer]
+              metadata[:creation_date] = reader.info[:CreationDate] if reader.info[:CreationDate]
+              metadata[:modification_date] = reader.info[:ModDate] if reader.info[:ModDate]
+            end
+
+            metadata[:page_count] = reader.page_count
+
+            # Extract text from all pages
+            reader.pages.each_with_index do |page, index|
+              page_text = page.text.strip
+              next if page_text.empty?
+
+              content += "\n\n--- Page #{index + 1} ---\n\n" if content.length.positive?
+              content += page_text
+            end
+          end
+        rescue PDF::Reader::MalformedPDFError => e
+          raise ParseError, "Malformed PDF: #{e.message}"
+        rescue PDF::Reader::UnsupportedFeatureError => e
+          raise ParseError, "Unsupported PDF feature: #{e.message}"
+        end
+
+        {
+          content: content.strip,
+          metadata: metadata,
+          document_type: "pdf"
+        }
+      end
+
+      def parse_docx
+        content = ""
+        metadata = {}
+
+        begin
+          doc = Docx::Document.open(@file_path)
+
+          # Extract core properties
+          if doc.core_properties
+            metadata[:title] = doc.core_properties.title if doc.core_properties.title
+            metadata[:author] = doc.core_properties.creator if doc.core_properties.creator
+            metadata[:subject] = doc.core_properties.subject if doc.core_properties.subject
+            metadata[:description] = doc.core_properties.description if doc.core_properties.description
+            metadata[:keywords] = doc.core_properties.keywords if doc.core_properties.keywords
+            metadata[:created] = doc.core_properties.created if doc.core_properties.created
+            metadata[:modified] = doc.core_properties.modified if doc.core_properties.modified
+            if doc.core_properties.last_modified_by
+              metadata[:last_modified_by] =
+                doc.core_properties.last_modified_by
+            end
+          end
+
+          # Extract text from paragraphs
+          doc.paragraphs.each do |paragraph|
+            paragraph_text = paragraph.text.strip
+            next if paragraph_text.empty?
+
+            content += "#{paragraph_text}\n\n"
+          end
+
+          # Extract text from tables
+          doc.tables.each_with_index do |table, table_index|
+            content += "\n--- Table #{table_index + 1} ---\n\n"
+
+            table.rows.each do |row|
+              row_text = row.cells.map(&:text).join(" | ")
+              content += "#{row_text}\n" unless row_text.strip.empty?
+            end
+
+            content += "\n"
+          end
+
+          metadata[:paragraph_count] = doc.paragraphs.count
+          metadata[:table_count] = doc.tables.count
+        rescue StandardError => e # StandardError => e
+          raise ParseError, "#{__LINE__} Failed to parse DOCX: #{e.message}"
+        end
+
+        {
+          content: content.strip,
+          metadata: metadata,
+          document_type: "docx"
+        }
+      end
+
+      def parse_text
+        content = File.read(@file_path, encoding: "UTF-8")
+        metadata = {
+          file_size: File.size(@file_path),
+          encoding: "UTF-8"
+        }
+
+        document_type = case @file_extension
+                        when ".md", ".markdown" then "markdown"
+                        when ".txt" then "text"
+                        else "text"
+                        end
+
+        {
+          content: content,
+          metadata: metadata,
+          document_type: document_type
+        }
+      rescue Encoding::InvalidByteSequenceError
+        # Try with different encoding
+        content = File.read(@file_path, encoding: "ISO-8859-1")
+        metadata = {
+          file_size: File.size(@file_path),
+          encoding: "ISO-8859-1"
+        }
+
+        {
+          content: content,
+          metadata: metadata,
+          document_type: "text"
+        }
+      end
+
+      def parse_html
+        content = File.read(@file_path, encoding: "UTF-8")
+
+        # Basic HTML tag stripping (for more advanced parsing, consider using Nokogiri)
+        clean_content = content
+                        .gsub(%r{<script[^>]*>.*?</script>}mi, "") # Remove script tags
+                        .gsub(%r{<style[^>]*>.*?</style>}mi, "")   # Remove style tags
+                        .gsub(/<[^>]+>/, " ")                     # Remove all HTML tags
+                        .gsub(/\s+/, " ")                         # Normalize whitespace
+                        .strip
+
+        metadata = {
+          file_size: File.size(@file_path),
+          original_format: "html"
+        }
+
+        {
+          content: clean_content,
+          metadata: metadata,
+          document_type: "html"
+        }
+      end
+
+      def parse_image
+        puts "🖼️  DocumentProcessor: Starting image parsing for #{@file_path}"
+
+        metadata = {
+          file_size: File.size(@file_path),
+          file_type: @file_extension.sub(".", ""),
+          original_filename: File.basename(@file_path)
+        }
+
+        # Extract image dimensions
+        begin
+          img = Magick::Image.read(@file_path).first
+          metadata[:width]  = img.columns
+          metadata[:height] = img.rows
+          puts "📏 DocumentProcessor: Image dimensions: #{img.columns}x#{img.rows}"
+        rescue StandardError => e # StandardError
+          puts "❌ DocumentProcessor: Failed to get image dimensions: #{e.message}"
+          metadata[:width]  = nil
+          metadata[:height] = nil
+        end
+
+        puts "🤖 DocumentProcessor: Creating ImageDescriptionService and calling generate_description..."
+        desc = Services::ImageDescriptionService.new.generate_description(@file_path)
+
+        puts "📝 DocumentProcessor: Received description: '#{desc}'"
+
+        metadata[:description] = desc if desc && !desc.empty?
+
+        # Use AI-generated description or fallback placeholder
+        content = desc && !desc.empty? ? desc : "Image file: #{File.basename(@file_path)}"
+
+        puts "✅ DocumentProcessor: Image parsing complete. Content: '#{content[0..100]}...'"
+
+        {
+          content: content,
+          metadata: metadata,
+          document_type: "image"
+        }
+      end
+
+      # Helper methods for document type determination
+      def self.determine_document_type(file_path)
+        case File.extname(file_path).downcase
+        when ".pdf" then "pdf"
+        when ".docx" then "docx"
+        when ".txt" then "text"
+        when ".md", ".markdown" then "markdown"
+        when ".html", ".htm" then "html"
+        when ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".tif" then "image"
+        else "text"
+        end
+      end
+
+      def self.determine_document_type_from_content_type(content_type)
+        case content_type
+        when "application/pdf" then "pdf"
+        when "application/vnd.openxmlformats-officedocument.wordprocessingml.document" then "docx"
+        when "text/plain" then "text"
+        when "text/markdown" then "markdown"
+        when "text/html" then "html"
+        when %r{^image/} then "image"
+        else "text"
+        end
+      end
+
+      def self.determine_content_type(file_path)
+        case File.extname(file_path).downcase
+        when ".pdf" then "application/pdf"
+        when ".docx" then "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+        when ".txt" then "text/plain"
+        when ".md", ".markdown" then "text/markdown"
+        when ".html", ".htm" then "text/html"
+        when ".jpg", ".jpeg" then "image/jpeg"
+        when ".png" then "image/png"
+        when ".gif" then "image/gif"
+        when ".webp" then "image/webp"
+        when ".bmp" then "image/bmp"
+        when ".svg" then "image/svg+xml"
+        when ".ico" then "image/x-icon"
+        when ".tiff", ".tif" then "image/tiff"
+        else "application/octet-stream"
+        end
+      end
+    end
+  end
+end