ragdoll 0.1.1 → 0.1.3

data/lib/ragdoll/core.rb

@@ -7,26 +7,44 @@ $DEBUG_ME = true
 
 # require_relative "../extensions/openstruct_merge"  # File doesn't exist
 
+# Add app/models, app/jobs, app/services, and app/lib to the load path
+$LOAD_PATH.unshift(File.expand_path("../../app/models", __dir__))
+$LOAD_PATH.unshift(File.expand_path("../../app/jobs", __dir__))
+$LOAD_PATH.unshift(File.expand_path("../../app/services", __dir__))
+$LOAD_PATH.unshift(File.expand_path("../../app/lib", __dir__))
+
 require_relative "core/version"
 require_relative "core/errors"
+require_relative "core/model"
 require_relative "core/configuration"
+# Require services from app/services/ragdoll
+require "ragdoll/configuration_service"
+require "ragdoll/model_resolver"
 require_relative "core/database"
 require_relative "core/shrine_config"
-require_relative "core/models/document"
-require_relative "core/models/embedding"
-require_relative "core/models/text_content"
-require_relative "core/models/audio_content"
-require_relative "core/models/image_content"
-require_relative "core/document_processor"
-require_relative "core/document_management"
-require_relative "core/text_chunker"
-require_relative "core/embedding_service"
-require_relative "core/text_generation_service"
-require_relative "core/search_engine"
-require_relative "core/services/image_description_service"
-require_relative "core/jobs/generate_embeddings"
-require_relative "core/jobs/generate_summary"
-require_relative "core/jobs/extract_keywords"
+
+# Require models from app/models/ragdoll
+require "ragdoll/document"
+require "ragdoll/embedding"
+require "ragdoll/content"
+require "ragdoll/text_content"
+require "ragdoll/audio_content"
+require "ragdoll/image_content"
+require "ragdoll/document_processor"
+require "ragdoll/document_management"
+require "ragdoll/text_chunker"
+require "ragdoll/embedding_service"
+require "ragdoll/text_generation_service"
+require "ragdoll/search_engine"
+require "ragdoll/image_description_service"
+require "ragdoll/metadata_generator"
+# Require from app/lib/ragdoll
+require "ragdoll/metadata_schemas"
+# Require jobs from app/jobs/ragdoll
+require "ragdoll/generate_embeddings_job"
+require "ragdoll/generate_summary_job"
+require "ragdoll/extract_keywords_job"
+require "ragdoll/extract_text_job"
 require_relative "core/client"
 
 module Ragdoll
@@ -56,8 +74,8 @@ module Ragdoll
     end
 
     # Factory method for creating clients
-    def self.client(config = nil)
-      Client.new(config)
+    def self.client(_config = nil)
+      Client.new
     end
 
     # Delegate high-level API methods to default client

data/lib/ragdoll.rb

@@ -137,7 +137,7 @@ module Ragdoll
     #   all_docs.each { |doc| puts doc.title }
     # @return [ActiveRecord::Relation] a relation of all documents.
     def documents
-      Ragdoll::Core::Models::Document.all
+      Ragdoll::Document.all
     end
     alias_method :docs, :documents
 

data/lib/tasks/annotate.rake

@@ -18,7 +18,7 @@ task :environment do
                                     adapter: "postgresql",
                                     database: "ragdoll_development",
                                     username: "ragdoll",
-                                    password: ENV["RAGDOLL_DATABASE_PASSWORD"] || ENV["DATABASE_PASSWORD"],
+                                    password: ENV.fetch("RAGDOLL_DATABASE_PASSWORD", ENV["DATABASE_PASSWORD"]),
                                     host: "localhost",
                                     port: 5432,
                                     auto_migrate: false # Don't auto-migrate during annotation

data/lib/tasks/db.rake

@@ -18,7 +18,7 @@ namespace :db do
       ActiveRecord::Base.establish_connection(
         adapter: 'postgresql',
         database: 'postgres', # Connect to postgres database initially
-        username: ENV['POSTGRES_SUPERUSER'] || 'postgres',
+        username: ENV.fetch('POSTGRES_SUPERUSER', 'postgres'),
         password: ENV['POSTGRES_SUPERUSER_PASSWORD'],
         host: config.database_config[:host] || 'localhost',
         port: config.database_config[:port] || 5432
@@ -60,7 +60,7 @@ namespace :db do
       ActiveRecord::Base.establish_connection(
         adapter: 'postgresql',
         database: 'ragdoll_development',
-        username: ENV['POSTGRES_SUPERUSER'] || 'postgres',
+        username: ENV.fetch('POSTGRES_SUPERUSER', 'postgres'),
         password: ENV['POSTGRES_SUPERUSER_PASSWORD'],
         host: config.database_config[:host] || 'localhost',
         port: config.database_config[:port] || 5432

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ragdoll
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Dewayne VanHoozer
@@ -18,6 +18,27 @@ extra_rdoc_files: []
 files:
 - README.md
 - Rakefile
+- app/jobs/ragdoll/extract_keywords_job.rb
+- app/jobs/ragdoll/extract_text_job.rb
+- app/jobs/ragdoll/generate_embeddings_job.rb
+- app/jobs/ragdoll/generate_summary_job.rb
+- app/lib/ragdoll/metadata_schemas.rb
+- app/models/ragdoll/audio_content.rb
+- app/models/ragdoll/content.rb
+- app/models/ragdoll/document.rb
+- app/models/ragdoll/embedding.rb
+- app/models/ragdoll/image_content.rb
+- app/models/ragdoll/text_content.rb
+- app/services/ragdoll/configuration_service.rb
+- app/services/ragdoll/document_management.rb
+- app/services/ragdoll/document_processor.rb
+- app/services/ragdoll/embedding_service.rb
+- app/services/ragdoll/image_description_service.rb
+- app/services/ragdoll/metadata_generator.rb
+- app/services/ragdoll/model_resolver.rb
+- app/services/ragdoll/search_engine.rb
+- app/services/ragdoll/text_chunker.rb
+- app/services/ragdoll/text_generation_service.rb
 - db/migrate/001_enable_postgresql_extensions.rb
 - db/migrate/004_create_ragdoll_documents.rb
 - db/migrate/005_create_ragdoll_embeddings.rb
@@ -28,27 +49,9 @@ files:
 - lib/ragdoll/core/client.rb
 - lib/ragdoll/core/configuration.rb
 - lib/ragdoll/core/database.rb
-- lib/ragdoll/core/document_management.rb
-- lib/ragdoll/core/document_processor.rb
-- lib/ragdoll/core/embedding_service.rb
 - lib/ragdoll/core/errors.rb
-- lib/ragdoll/core/jobs/extract_keywords.rb
-- lib/ragdoll/core/jobs/extract_text.rb
-- lib/ragdoll/core/jobs/generate_embeddings.rb
-- lib/ragdoll/core/jobs/generate_summary.rb
-- lib/ragdoll/core/metadata_schemas.rb
-- lib/ragdoll/core/models/audio_content.rb
-- lib/ragdoll/core/models/content.rb
-- lib/ragdoll/core/models/document.rb
-- lib/ragdoll/core/models/embedding.rb
-- lib/ragdoll/core/models/image_content.rb
-- lib/ragdoll/core/models/text_content.rb
-- lib/ragdoll/core/search_engine.rb
-- lib/ragdoll/core/services/image_description_service.rb
-- lib/ragdoll/core/services/metadata_generator.rb
+- lib/ragdoll/core/model.rb
 - lib/ragdoll/core/shrine_config.rb
-- lib/ragdoll/core/text_chunker.rb
-- lib/ragdoll/core/text_generation_service.rb
 - lib/ragdoll/core/version.rb
 - lib/tasks/annotate.rake
 - lib/tasks/db.rake
@@ -63,6 +66,7 @@ metadata:
 rdoc_options: []
 require_paths:
 - lib
+- app/models
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/ragdoll/core/document_management.rb

@@ -1,110 +0,0 @@
-# frozen_string_literal: true
-
-module Ragdoll
-  module Core
-    class DocumentManagement
-      class << self
-        def add_document(location, content, metadata = {})
-          # Ensure location is an absolute path if it's a file path
-          absolute_location = location.start_with?("http") || location.start_with?("ftp") ? location : File.expand_path(location)
-
-          # Get file modification time if it's a file path
-          file_modified_at = if File.exist?(absolute_location) && !absolute_location.start_with?("http")
-                               File.mtime(absolute_location)
-                             else
-                               Time.current
-                             end
-
-          # Check if document already exists with same location and file_modified_at
-          existing_document = Models::Document.find_by(
-            location: absolute_location,
-            file_modified_at: file_modified_at
-          )
-
-          # Return existing document ID if found (skip duplicate)
-          return existing_document.id.to_s if existing_document
-
-          document = Models::Document.create!(
-            location: absolute_location,
-            title: metadata[:title] || metadata["title"] || extract_title_from_location(location),
-            document_type: metadata[:document_type] || metadata["document_type"] || "text",
-            metadata: metadata.is_a?(Hash) ? metadata : {},
-            status: "pending",
-            file_modified_at: file_modified_at
-          )
-
-          # Set content using the model's setter to trigger TextContent creation
-          document.content = content if content.present?
-
-          document.id.to_s
-        end
-
-        def get_document(id)
-          document = Models::Document.find_by(id: id)
-          return nil unless document
-
-          hash = document.to_hash
-          hash[:content] = document.content
-          hash
-        end
-
-        def update_document(id, **updates)
-          document = Models::Document.find_by(id: id)
-          return nil unless document
-
-          # Only update allowed fields
-          allowed_updates = updates.slice(:title, :metadata, :status, :document_type)
-          document.update!(allowed_updates) if allowed_updates.any?
-
-          document.to_hash
-        end
-
-        def delete_document(id)
-          document = Models::Document.find_by(id: id)
-          return nil unless document
-
-          document.destroy!
-          true
-        end
-
-        def list_documents(options = {})
-          limit = options[:limit] || 100
-          offset = options[:offset] || 0
-
-          Models::Document.offset(offset).limit(limit).recent.map(&:to_hash)
-        end
-
-        def get_document_stats
-          Models::Document.stats
-        end
-
-        # FIXME: should this be here?
-
-        def add_embedding(embeddable_id, chunk_index, embedding_vector, metadata = {})
-          # The embeddable_type should be the actual STI subclass, not the base class
-          embeddable_type = if metadata[:embeddable_type]
-                             metadata[:embeddable_type]
-                           else
-                             # Look up the actual STI type from the content record
-                             content = Models::Content.find(embeddable_id)
-                             content.class.name
-                           end
-          
-          Models::Embedding.create!(
-            embeddable_id: embeddable_id,
-            embeddable_type: embeddable_type,
-            chunk_index: chunk_index,
-            embedding_vector: embedding_vector,
-            content: metadata[:content] || ""
-          ).id.to_s
-        end
-
-        private
-
-        def extract_title_from_location(location)
-          File.basename(location, File.extname(location))
-        end
-      end
-    end
-  end
-end

data/lib/ragdoll/core/document_processor.rb

@@ -1,344 +0,0 @@
-# frozen_string_literal: true
-
-require "pdf-reader"
-require "docx"
-require "rmagick"
-require_relative "services/image_description_service"
-
-module Ragdoll
-  module Core
-    class DocumentProcessor
-      class ParseError < DocumentError; end
-      class UnsupportedFormatError < ParseError; end
-
-      def self.parse(file_path)
-        new(file_path).parse
-      end
-
-      # Parse from Shrine attached file
-      def self.parse_attachment(attached_file)
-        attached_file.open do |tempfile|
-          new(tempfile.path, attached_file).parse
-        end
-      end
-
-      # Create document from file path
-      def self.create_document_from_file(file_path, **options)
-        parsed = parse(file_path)
-
-        # Get file modification time
-        file_modified_at = File.exist?(file_path) ? File.mtime(file_path) : Time.current
-
-        document = Models::Document.create!(
-          location: File.expand_path(file_path),
-          title: parsed[:title] || File.basename(file_path, File.extname(file_path)),
-          content: parsed[:content],
-          document_type: determine_document_type(file_path),
-          metadata: parsed[:metadata] || {},
-          status: "processed",
-          file_modified_at: file_modified_at,
-          **options
-        )
-
-        # Attach the file if it exists
-        document.file = File.open(file_path) if File.exist?(file_path)
-
-        document
-      end
-
-      # Create document from uploaded file (Shrine compatible)
-      def self.create_document_from_upload(uploaded_file, **options)
-        # Create document first
-        document = Models::Document.create!(
-          location: uploaded_file.original_filename || "uploaded_file",
-          title: options[:title] || File.basename(uploaded_file.original_filename || "uploaded_file",
-                                                  File.extname(uploaded_file.original_filename || "")),
-          content: "", # Will be extracted after file attachment
-          document_type: determine_document_type_from_content_type(uploaded_file.mime_type),
-          status: "processing",
-          metadata: options[:metadata] || {},
-          file_modified_at: Time.current
-        )
-
-        # Attach the file
-        document.file = uploaded_file
-
-        # Extract content from attached file
-        if document.file.present?
-          parsed = parse_attachment(document.file)
-          document.update!(
-            content: parsed[:content],
-            title: parsed[:title] || document.title,
-            metadata: document.metadata.merge(parsed[:metadata] || {}),
-            status: "processed"
-          )
-        end
-
-        document
-      end
-
-      def initialize(file_path, attached_file = nil)
-        @file_path = file_path
-        @attached_file = attached_file
-        @file_extension = File.extname(file_path).downcase
-      end
-
-      def parse
-        case @file_extension
-        when ".pdf"
-          parse_pdf
-        when ".docx"
-          parse_docx
-        when ".txt", ".md", ".markdown"
-          parse_text
-        when ".html", ".htm"
-          parse_html
-        when ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".tif"
-          parse_image
-        else
-          parse_text # Default to text parsing for unknown formats
-        end
-      rescue StandardError => e # StandardError => e
-        raise ParseError, "#{__LINE__} Failed to parse #{@file_path}: #{e.message}"
-      end
-
-      private
-
-      def parse_pdf
-        content = ""
-        metadata = {}
-
-        begin
-          PDF::Reader.open(@file_path) do |reader|
-            # Extract metadata
-            if reader.info
-              metadata[:title] = reader.info[:Title] if reader.info[:Title]
-              metadata[:author] = reader.info[:Author] if reader.info[:Author]
-              metadata[:subject] = reader.info[:Subject] if reader.info[:Subject]
-              metadata[:creator] = reader.info[:Creator] if reader.info[:Creator]
-              metadata[:producer] = reader.info[:Producer] if reader.info[:Producer]
-              metadata[:creation_date] = reader.info[:CreationDate] if reader.info[:CreationDate]
-              metadata[:modification_date] = reader.info[:ModDate] if reader.info[:ModDate]
-            end
-
-            metadata[:page_count] = reader.page_count
-
-            # Extract text from all pages
-            reader.pages.each_with_index do |page, index|
-              page_text = page.text.strip
-              next if page_text.empty?
-
-              content += "\n\n--- Page #{index + 1} ---\n\n" if content.length.positive?
-              content += page_text
-            end
-          end
-        rescue PDF::Reader::MalformedPDFError => e
-          raise ParseError, "Malformed PDF: #{e.message}"
-        rescue PDF::Reader::UnsupportedFeatureError => e
-          raise ParseError, "Unsupported PDF feature: #{e.message}"
-        end
-
-        {
-          content: content.strip,
-          metadata: metadata,
-          document_type: "pdf"
-        }
-      end
-
-      def parse_docx
-        content = ""
-        metadata = {}
-
-        begin
-          doc = Docx::Document.open(@file_path)
-
-          # Extract core properties
-          if doc.core_properties
-            metadata[:title] = doc.core_properties.title if doc.core_properties.title
-            metadata[:author] = doc.core_properties.creator if doc.core_properties.creator
-            metadata[:subject] = doc.core_properties.subject if doc.core_properties.subject
-            metadata[:description] = doc.core_properties.description if doc.core_properties.description
-            metadata[:keywords] = doc.core_properties.keywords if doc.core_properties.keywords
-            metadata[:created] = doc.core_properties.created if doc.core_properties.created
-            metadata[:modified] = doc.core_properties.modified if doc.core_properties.modified
-            if doc.core_properties.last_modified_by
-              metadata[:last_modified_by] =
-                doc.core_properties.last_modified_by
-            end
-          end
-
-          # Extract text from paragraphs
-          doc.paragraphs.each do |paragraph|
-            paragraph_text = paragraph.text.strip
-            next if paragraph_text.empty?
-
-            content += "#{paragraph_text}\n\n"
-          end
-
-          # Extract text from tables
-          doc.tables.each_with_index do |table, table_index|
-            content += "\n--- Table #{table_index + 1} ---\n\n"
-
-            table.rows.each do |row|
-              row_text = row.cells.map(&:text).join(" | ")
-              content += "#{row_text}\n" unless row_text.strip.empty?
-            end
-
-            content += "\n"
-          end
-
-          metadata[:paragraph_count] = doc.paragraphs.count
-          metadata[:table_count] = doc.tables.count
-        rescue StandardError => e # StandardError => e
-          raise ParseError, "#{__LINE__} Failed to parse DOCX: #{e.message}"
-        end
-
-        {
-          content: content.strip,
-          metadata: metadata,
-          document_type: "docx"
-        }
-      end
-
-      def parse_text
-        content = File.read(@file_path, encoding: "UTF-8")
-        metadata = {
-          file_size: File.size(@file_path),
-          encoding: "UTF-8"
-        }
-
-        document_type = case @file_extension
-                        when ".md", ".markdown" then "markdown"
-                        when ".txt" then "text"
-                        else "text"
-                        end
-
-        {
-          content: content,
-          metadata: metadata,
-          document_type: document_type
-        }
-      rescue Encoding::InvalidByteSequenceError
-        # Try with different encoding
-        content = File.read(@file_path, encoding: "ISO-8859-1")
-        metadata = {
-          file_size: File.size(@file_path),
-          encoding: "ISO-8859-1"
-        }
-
-        {
-          content: content,
-          metadata: metadata,
-          document_type: "text"
-        }
-      end
-
-      def parse_html
-        content = File.read(@file_path, encoding: "UTF-8")
-
-        # Basic HTML tag stripping (for more advanced parsing, consider using Nokogiri)
-        clean_content = content
-                        .gsub(%r{<script[^>]*>.*?</script>}mi, "") # Remove script tags
-                        .gsub(%r{<style[^>]*>.*?</style>}mi, "")   # Remove style tags
-                        .gsub(/<[^>]+>/, " ")                     # Remove all HTML tags
-                        .gsub(/\s+/, " ")                         # Normalize whitespace
-                        .strip
-
-        metadata = {
-          file_size: File.size(@file_path),
-          original_format: "html"
-        }
-
-        {
-          content: clean_content,
-          metadata: metadata,
-          document_type: "html"
-        }
-      end
-
-      def parse_image
-        puts "🖼️  DocumentProcessor: Starting image parsing for #{@file_path}"
-
-        metadata = {
-          file_size: File.size(@file_path),
-          file_type: @file_extension.sub(".", ""),
-          original_filename: File.basename(@file_path)
-        }
-
-        # Extract image dimensions
-        begin
-          img = Magick::Image.read(@file_path).first
-          metadata[:width]  = img.columns
-          metadata[:height] = img.rows
-          puts "📏 DocumentProcessor: Image dimensions: #{img.columns}x#{img.rows}"
-        rescue StandardError => e # StandardError
-          puts "❌ DocumentProcessor: Failed to get image dimensions: #{e.message}"
-          metadata[:width]  = nil
-          metadata[:height] = nil
-        end
-
-        puts "🤖 DocumentProcessor: Creating ImageDescriptionService and calling generate_description..."
-        desc = Services::ImageDescriptionService.new.generate_description(@file_path)
-
-        puts "📝 DocumentProcessor: Received description: '#{desc}'"
-
-        metadata[:description] = desc if desc && !desc.empty?
-
-        # Use AI-generated description or fallback placeholder
-        content = desc && !desc.empty? ? desc : "Image file: #{File.basename(@file_path)}"
-
-        puts "✅ DocumentProcessor: Image parsing complete. Content: '#{content[0..100]}...'"
-
-        {
-          content: content,
-          metadata: metadata,
-          document_type: "image"
-        }
-      end
-
-      # Helper methods for document type determination
-      def self.determine_document_type(file_path)
-        case File.extname(file_path).downcase
-        when ".pdf" then "pdf"
-        when ".docx" then "docx"
-        when ".txt" then "text"
-        when ".md", ".markdown" then "markdown"
-        when ".html", ".htm" then "html"
-        when ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".tif" then "image"
-        else "text"
-        end
-      end
-
-      def self.determine_document_type_from_content_type(content_type)
-        case content_type
-        when "application/pdf" then "pdf"
-        when "application/vnd.openxmlformats-officedocument.wordprocessingml.document" then "docx"
-        when "text/plain" then "text"
-        when "text/markdown" then "markdown"
-        when "text/html" then "html"
-        when %r{^image/} then "image"
-        else "text"
-        end
-      end
-
-      def self.determine_content_type(file_path)
-        case File.extname(file_path).downcase
-        when ".pdf" then "application/pdf"
-        when ".docx" then "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-        when ".txt" then "text/plain"
-        when ".md", ".markdown" then "text/markdown"
-        when ".html", ".htm" then "text/html"
-        when ".jpg", ".jpeg" then "image/jpeg"
-        when ".png" then "image/png"
-        when ".gif" then "image/gif"
-        when ".webp" then "image/webp"
-        when ".bmp" then "image/bmp"
-        when ".svg" then "image/svg+xml"
-        when ".ico" then "image/x-icon"
-        when ".tiff", ".tif" then "image/tiff"
-        else "application/octet-stream"
-        end
-      end
-    end
-  end
-end