ragdoll 0.1.0 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bea4621e2b802db79d78f8b1d0679cf2f81ed35b91d35683ce0afcb83ddc54e1
-  data.tar.gz: ec12fb975b154f77a42d54fb3c716d523e1b90e4e0122b3576c5aac15a957340
+  metadata.gz: 2016536d66d295c1fe5054aedb77526271692d7562131df9de9e1ad756309459
+  data.tar.gz: 725a221ab132fd9ce77f623114c034d675c626428c9d5d8c72e45e275b08feea
 SHA512:
-  metadata.gz: 3702308d3b772dfc0ebf429a26bae0f0378456e9d6c48357b8e2a4cdeb3744e78b43fd610ef308ff6348f1bec28bb37d51bd5e335d78583574b3212c8f544a33
-  data.tar.gz: 9beebfebafe1ed2e815a3042949b68e5f208d8c555a92228eb4098fa99900f07d2985654766e2907de3069f5d10b6cf3e65fb7a3b431ee9db836e6111f1e27f2
+  metadata.gz: 221c7d3408a9ec1b4c2f735bf733ae40aab896fdc07858b69de0866acb684c1eb65a3fb054342a6d20cd8a6e0b4e3f0c866f1df3a5bd8e5a475d6c3d72062b1a
+  data.tar.gz: 3228762fd152ff2a2fd5c0f514ae39e11e483dba698b8139f6c0696437a70209fb0576d67fb271eed45c4c7a2c08247dcbd68a2eab8f19cda144c01d38c2299f

data/README.md

@@ -1,75 +1,353 @@
-# Ragdoll
+<div align="center" style="background-color: yellow; color: black; padding: 20px; margin: 20px 0; border: 2px solid black; font-size: 48px; font-weight: bold;">
+  ⚠️ CAUTION ⚠️<br />
+  Software Under Development by a Crazy Man
+</div>
+<br />
+<div align="center">
+  <table>
+    <tr>
+      <td width="50%">
+        <a href="https://research.ibm.com/blog/retrieval-augmented-generation-RAG" target="_blank">
+          <img src="ragdoll.png" alt="Ragdoll" width="800">
+        </a>
+      </td>
+      <td width="50%" valign="top">
+        <p>Multi-modal RAG (Retrieval-Augmented Generation) is an architecture that integrates multiple data types (such as text, images, and audio) to enhance AI response generation. It combines retrieval-based methods, which fetch relevant information from a knowledge base, with generative large language models (LLMs) that create coherent and contextually appropriate outputs. This approach allows for more comprehensive and engaging user interactions, such as chatbots that respond with both text and images or educational tools that incorporate visual aids into learning materials. By leveraging various modalities, multi-modal RAG systems improve context understanding and user experience.</p>
+      </td>
+    </tr>
+  </table>
+</div>
 
-Ragdoll is a Rails Engine designed for document ingestion and search. It allows you to import documents, vectorize them, and perform searches using vector representations.
+# Ragdoll::Core
 
-## Installation as a Rails Engine
+Database-oriented multi-modal RAG (Retrieval-Augmented Generation) library built on ActiveRecord. Features PostgreSQL + pgvector for high-performance semantic search, polymorphic content architecture, and dual metadata design for sophisticated document analysis.
 
-To use Ragdoll as a Rails Engine, add this line to your application's Gemfile:
+## Quick Start
 
-```bash
-bundle add ragdoll
+```ruby
+require 'ragdoll'
+
+# Configure with PostgreSQL + pgvector
+Ragdoll::Core.configure do |config|
+  # Database configuration (PostgreSQL only)
+  config.database_config = {
+    adapter: 'postgresql',
+    database: 'ragdoll_production',
+    username: 'ragdoll',
+    password: ENV['DATABASE_PASSWORD'],
+    host: 'localhost',
+    port: 5432,
+    auto_migrate: true
+  }
+
+  # Ruby LLM configuration
+  config.ruby_llm_config[:openai][:api_key] = ENV['OPENAI_API_KEY']
+  config.ruby_llm_config[:openai][:organization] = ENV['OPENAI_ORGANIZATION']
+  config.ruby_llm_config[:openai][:project] = ENV['OPENAI_PROJECT']
+
+  # Model configuration
+  config.models[:default] = 'openai/gpt-4o'
+  config.models[:embedding][:text] = 'text-embedding-3-small'
+
+  # Logging configuration
+  config.logging_config[:log_level] = :warn
+  config.logging_config[:log_filepath] = File.join(Dir.home, '.ragdoll', 'ragdoll.log')
+end
+
+# Add documents - returns detailed result
+result = Ragdoll::Core.add_document(path: 'research_paper.pdf')
+puts result[:message]  # "Document 'research_paper' added successfully with ID 123"
+doc_id = result[:document_id]
+
+# Check document status
+status = Ragdoll::Core.document_status(id: doc_id)
+puts status[:message]  # Shows processing status and embeddings count
+
+# Search across content
+results = Ragdoll::Core.search(query: 'neural networks')
+
+# Get detailed document information
+document = Ragdoll::Core.get_document(id: doc_id)
 ```
 
-And then execute:
+## High-Level API
 
-```bash
-bundle install
+The `Ragdoll` module provides a convenient high-level API for common operations:
+
+### Document Management
+
+```ruby
+# Add single document - returns detailed result hash
+result = Ragdoll::Core.add_document(path: 'document.pdf')
+puts result[:success]         # true
+puts result[:document_id]     # "123"
+puts result[:message]         # "Document 'document' added successfully with ID 123"
+puts result[:embeddings_queued] # true
+
+# Check document processing status
+status = Ragdoll::Core.document_status(id: result[:document_id])
+puts status[:status]          # "processed"
+puts status[:embeddings_count] # 15
+puts status[:embeddings_ready] # true
+puts status[:message]         # "Document processed successfully with 15 embeddings"
+
+# Get detailed document information
+document = Ragdoll::Core.get_document(id: result[:document_id])
+puts document[:title]         # "document"
+puts document[:status]        # "processed"
+puts document[:embeddings_count] # 15
+puts document[:content_length]   # 5000
+
+# Update document metadata
+Ragdoll::Core.update_document(id: result[:document_id], title: 'New Title')
+
+# Delete document
+Ragdoll::Core.delete_document(id: result[:document_id])
+
+# List all documents
+documents = Ragdoll::Core.list_documents(limit: 10)
+
+# System statistics
+stats = Ragdoll::Core.stats
+puts stats[:total_documents]  # 50
+puts stats[:total_embeddings] # 1250
 ```
 
-Or install it yourself as:
+### Search and Retrieval
 
-```bash
-gem install ragdoll
+```ruby
+# Semantic search across all content types
+results = Ragdoll::Core.search(query: 'artificial intelligence')
+
+# Search specific content types
+text_results = Ragdoll::Core.search(query: 'machine learning', content_type: 'text')
+image_results = Ragdoll::Core.search(query: 'neural network diagram', content_type: 'image')
+audio_results = Ragdoll::Core.search(query: 'AI discussion', content_type: 'audio')
+
+# Advanced search with metadata filters
+results = Ragdoll::Core.search(
+  query: 'deep learning',
+  classification: 'research',
+  keywords: ['AI', 'neural networks'],
+  tags: ['technical']
+)
+
+# Get context for RAG applications
+context = Ragdoll::Core.get_context(query: 'machine learning', limit: 5)
+
+# Enhanced prompt with context
+enhanced = Ragdoll::Core.enhance_prompt(
+  prompt: 'What is machine learning?',
+  context_limit: 5
+)
+
+# Hybrid search combining semantic and full-text
+results = Ragdoll::Core.hybrid_search(
+  query: 'neural networks',
+  semantic_weight: 0.7,
+  text_weight: 0.3
+)
 ```
 
-## Usage as a Rails Engine
+### System Operations
 
-### Importing Documents
+```ruby
+# Get system statistics
+stats = Ragdoll::Core.stats
+# Returns information about documents, content types, embeddings, etc.
 
-To import documents from a file, glob, or directory, use the following command:
+# Health check
+healthy = Ragdoll::Core.healthy?
 
-```bash
-ragdoll import PATH
+# Get configuration
+config = Ragdoll::Core.configuration
+
+# Reset configuration (useful for testing)
+Ragdoll::Core.reset_configuration!
 ```
 
-- `PATH`: The path to the file or directory to import.
-- Use the `-r` or `--recursive` option to import files recursively from directories.
-- Use the `-j` or `--jobs` option to specify the number of concurrent import jobs.
+### Configuration
 
-### Managing Jobs
+```ruby
+# Configure the system
+Ragdoll::Core.configure do |config|
+  # Database configuration (PostgreSQL only - REQUIRED)
+  config.database_config = {
+    adapter: 'postgresql',
+    database: 'ragdoll_production',
+    username: 'ragdoll',
+    password: ENV['DATABASE_PASSWORD'],
+    host: 'localhost',
+    port: 5432,
+    auto_migrate: true
+  }
 
-To manage import jobs, use the following command:
+  # Ruby LLM configuration for multiple providers
+  config.ruby_llm_config[:openai][:api_key] = ENV['OPENAI_API_KEY']
+  config.ruby_llm_config[:openai][:organization] = ENV['OPENAI_ORGANIZATION']
+  config.ruby_llm_config[:openai][:project] = ENV['OPENAI_PROJECT']
 
-```bash
-ragdoll jobs [JOB_ID]
+  config.ruby_llm_config[:anthropic][:api_key] = ENV['ANTHROPIC_API_KEY']
+  config.ruby_llm_config[:google][:api_key] = ENV['GOOGLE_API_KEY']
+
+  # Model configuration
+  config.models[:default] = 'openai/gpt-4o'
+  config.models[:summary] = 'openai/gpt-4o'
+  config.models[:keywords] = 'openai/gpt-4o'
+  config.models[:embedding][:text] = 'text-embedding-3-small'
+  config.models[:embedding][:image] = 'image-embedding-3-small'
+  config.models[:embedding][:audio] = 'audio-embedding-3-small'
+
+  # Logging configuration
+  config.logging_config[:log_level] = :warn  # :debug, :info, :warn, :error, :fatal
+  config.logging_config[:log_filepath] = File.join(Dir.home, '.ragdoll', 'ragdoll.log')
+
+  # Processing settings
+  config.chunking[:text][:max_tokens] = 1000
+  config.chunking[:text][:overlap] = 200
+  config.search[:similarity_threshold] = 0.7
+  config.search[:max_results] = 10
+end
 ```
 
-- `JOB_ID`: The ID of a specific job to manage.
-- Use `--stop`, `--pause`, or `--resume` to control a specific job.
-- Use `--stop-all`, `--pause-all`, or `--resume-all` to control all jobs.
+## Current Implementation Status
+
+### ✅ **Fully Implemented**
+- **Text document processing**: PDF, DOCX, HTML, Markdown, plain text files
+- **Embedding generation**: Text chunking and vector embedding creation
+- **Database schema**: Multi-modal polymorphic architecture with PostgreSQL + pgvector
+- **Dual metadata architecture**: Separate LLM-generated content analysis and file properties
+- **Search functionality**: Semantic search with cosine similarity and usage analytics
+- **Document management**: Add, update, delete, list operations
+- **Background processing**: ActiveJob integration for async embedding generation
+- **LLM metadata generation**: AI-powered structured content analysis with schema validation
+- **Logging**: Configurable file-based logging with multiple levels
+
+### 🚧 **In Development**
+- **Image processing**: Framework exists but vision AI integration needs completion
+- **Audio processing**: Framework exists but speech-to-text integration needs completion
+- **Hybrid search**: Combining semantic and full-text search capabilities
+
+### 📋 **Planned Features**
+- **Multi-modal search**: Search across text, image, and audio content types
+- **Content-type specific embedding models**: Different models for text, image, audio
+- **Enhanced metadata schemas**: Domain-specific metadata templates
+
+## Architecture Highlights
+
+### Dual Metadata Design
+
+Ragdoll uses a sophisticated dual metadata architecture to separate concerns:
+
+- **`metadata` (JSON)**: LLM-generated content analysis including summary, keywords, classification, topics, sentiment, and domain-specific insights
+- **`file_metadata` (JSON)**: System-generated file properties including size, MIME type, dimensions, processing parameters, and technical characteristics
+
+This separation enables both semantic search operations on content meaning and efficient file management operations.
+
+### Polymorphic Multi-Modal Architecture
+
+The database schema uses polymorphic associations to elegantly support multiple content types:
+
+- **Documents**: Central entity with dual metadata columns
+- **Content Types**: Specialized tables for `text_contents`, `image_contents`, `audio_contents`
+- **Embeddings**: Unified vector storage via polymorphic `embeddable` associations
+
+## Text Document Processing (Current)
+
+Currently, Ragdoll processes text documents through:
+
+1. **Content Extraction**: Extracts text from PDF, DOCX, HTML, Markdown, and plain text
+2. **Metadata Generation**: AI-powered analysis creates structured content metadata
+3. **Text Chunking**: Splits content into manageable chunks with configurable size/overlap
+4. **Embedding Generation**: Creates vector embeddings using OpenAI or other providers
+5. **Database Storage**: Stores in polymorphic multi-modal architecture with dual metadata
+6. **Search**: Semantic search using cosine similarity with usage analytics
 
-### Searching Documents
+### Example Usage
+```ruby
+# Add a text document
+result = Ragdoll::Core.add_document(path: 'document.pdf')
 
-To search the database with a prompt, use the following command:
+# Check processing status
+status = Ragdoll::Core.document_status(id: result[:document_id])
+
+# Search the content
+results = Ragdoll::Core.search(query: 'machine learning')
+```
+
+## PostgreSQL + pgvector Configuration
+
+### Database Setup
 
 ```bash
-ragdoll search PROMPT
+# Install PostgreSQL and pgvector
+brew install postgresql pgvector  # macOS
+# or
+apt-get install postgresql postgresql-contrib  # Ubuntu
+
+# Create database and enable pgvector extension
+createdb ragdoll_production
+psql -d ragdoll_production -c "CREATE EXTENSION IF NOT EXISTS vector;"
+```
+
+### Configuration Example
+
+```ruby
+Ragdoll::Core.configure do |config|
+  config.database_config = {
+    adapter: 'postgresql',
+    database: 'ragdoll_production',
+    username: 'ragdoll',
+    password: ENV['DATABASE_PASSWORD'],
+    host: 'localhost',
+    port: 5432,
+    pool: 20,
+    auto_migrate: true
+  }
+end
 ```
 
-- `PROMPT`: The search prompt as a string or use the `-p` option to specify a file containing the prompt text.
-- Use the `--max_count` option to specify the maximum number of results to return.
-- Use the `--rerank` option to rerank results using keyword search.
+## Performance Features
 
-## Development and Contribution
+- **Native pgvector**: Hardware-accelerated similarity search
+- **IVFFlat indexing**: Fast approximate nearest neighbor search
+- **Polymorphic embeddings**: Unified search across content types
+- **Batch processing**: Efficient bulk operations
+- **Background jobs**: Asynchronous document processing
+- **Connection pooling**: High-concurrency support
+
+## Installation
+
+```bash
+# Install system dependencies
+brew install postgresql pgvector  # macOS
+# or
+apt-get install postgresql postgresql-contrib  # Ubuntu
+
+# Install gem
+gem install ragdoll
+
+# Or add to Gemfile
+gem 'ragdoll'
+```
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+## Requirements
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+- **Ruby**: 3.2+
+- **PostgreSQL**: 12+ with pgvector extension (REQUIRED - no other databases supported)
+- **Dependencies**: activerecord, pg, pgvector, neighbor, ruby_llm, pdf-reader, docx, rubyzip, shrine, rmagick, opensearch-ruby, searchkick, ruby-progressbar
 
-## Contributing
+## Related Projects
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/ragdoll.
+- **ragdoll-cli**: Standalone CLI application using ragdoll
+- **ragdoll-rails**: Rails engine with web interface for ragdoll
 
-## License
+## Key Design Principles
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+1. **Database-Oriented**: Built on ActiveRecord with PostgreSQL + pgvector for production performance
+2. **Multi-Modal First**: Text, image, and audio content as first-class citizens via polymorphic architecture
+3. **Dual Metadata Design**: Separates LLM-generated content analysis from file properties
+4. **LLM-Enhanced**: Structured metadata generation with schema validation using latest AI capabilities
+5. **High-Level API**: Simple, intuitive interface for complex operations
+6. **Scalable**: Designed for production workloads with background processing and proper indexing
+7. **Extensible**: Easy to add new content types and embedding models through polymorphic design

data/Rakefile

@@ -1,10 +1,72 @@
-# This file defines the Rake tasks for the Ragdoll gem, including tasks for testing.
-
 # frozen_string_literal: true
 
+require "simplecov"
+SimpleCov.start
+
+# Suppress bundler/rubygems warnings
+$VERBOSE = nil
+
 require "bundler/gem_tasks"
-require "minitest/test_task"
+require "rake/testtask"
+
+def ci_environment?
+  ENV["CI"] == "true" || ENV["RAGDOLL_SKIP_DATABASE_TESTS"] == "true"
+end
+
+desc "Setup test database"
+task :setup_test_db do
+  require_relative "lib/ragdoll-core"
+
+  # Database configuration for tests
+  test_db_config = {
+    adapter: "postgresql",
+    database: "ragdoll_test",
+    username: ENV.fetch("RAGDOLL_POSTGRES_USER", "postgres"),
+    password: ENV.fetch("RAGDOLL_POSTGRES_PASSWORD", ""),
+    host: ENV.fetch("RAGDOLL_POSTGRES_HOST", "localhost"),
+    port: ENV.fetch("RAGDOLL_POSTGRES_PORT", 5432)
+  }
+
+  # Ensure database exists
+  begin
+    # Try to connect to the database
+    ActiveRecord::Base.establish_connection(test_db_config)
+    ActiveRecord::Base.connection.execute("SELECT 1")
+  rescue ActiveRecord::NoDatabaseError
+    # Database doesn't exist, create it
+    puts "Creating ragdoll_test database..."
+    admin_config = test_db_config.merge(database: "postgres")
+    ActiveRecord::Base.establish_connection(admin_config)
+    ActiveRecord::Base.connection.execute("CREATE DATABASE ragdoll_test")
+    ActiveRecord::Base.establish_connection(test_db_config)
+  rescue PG::ConnectionBad => e
+    puts "Error connecting to PostgreSQL: #{e.message}"
+    puts "Please ensure PostgreSQL is running and accessible"
+    exit 1
+  end
+
+  # Ensure pgvector extension is installed
+  begin
+    ActiveRecord::Base.connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
+  rescue StandardError => e
+    puts "Warning: Could not install pgvector extension: #{e.message}"
+  end
+
+  # Run migrations
+  Ragdoll::Core::Database.setup(test_db_config.merge(auto_migrate: true, logger: nil))
+  puts "Test database setup complete"
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+# Make test task depend on database setup only if not skipping database tests
+task test: :setup_test_db unless ci_environment?
 
-Minitest::TestTask.create
+# Load annotate tasks
+Dir.glob("lib/tasks/*.rake").each { |r| load r }
 
 task default: :test

data/app/jobs/ragdoll/extract_keywords_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Ragdoll
+  class ExtractKeywordsJob < ActiveJob::Base
+    queue_as :default
+
+    def perform(document_id)
+      document = Ragdoll::Document.find(document_id)
+      return unless document.content.present?
+      return if document.keywords.present?
+
+      text_service = Ragdoll::TextGenerationService.new
+      keywords_array = text_service.extract_keywords(document.content)
+
+      if keywords_array.present?
+        keywords_string = keywords_array.join(", ")
+        document.update!(keywords: keywords_string)
+      end
+    rescue ActiveRecord::RecordNotFound
+      # Document was deleted, nothing to do
+    rescue StandardError => e
+      Rails.logger.error "Failed to generate keywords for document #{document_id}: #{e.message}" if defined?(Rails)
+      raise e
+    end
+  end
+end

data/app/jobs/ragdoll/extract_text_job.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Ragdoll
+  class ExtractTextJob < ActiveJob::Base
+    queue_as :default
+
+    def perform(document_id)
+      document = Ragdoll::Document.find(document_id)
+      return unless document.file_attached?
+      return if document.content.present?
+
+      document.update!(status: "processing")
+
+      extracted_content = document.extract_text_from_file
+
+      if extracted_content.present?
+        document.update!(
+          content: extracted_content,
+          status: "processed"
+        )
+
+        # Queue follow-up jobs
+        Ragdoll::GenerateSummaryJob.perform_later(document_id)
+        Ragdoll::ExtractKeywordsJob.perform_later(document_id)
+        Ragdoll::GenerateEmbeddingsJob.perform_later(document_id)
+      else
+        document.update!(status: "error")
+      end
+    rescue ActiveRecord::RecordNotFound
+      # Document was deleted, nothing to do
+    rescue StandardError => e
+      document&.update!(status: "error")
+      raise e
+    end
+  end
+end

data/app/jobs/ragdoll/generate_embeddings_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Ragdoll
+  class GenerateEmbeddingsJob < ActiveJob::Base
+    queue_as :default
+
+    def perform(document_id, chunk_size: nil, chunk_overlap: nil)
+      document = Ragdoll::Document.find(document_id)
+      return unless document.content.present?
+      return if document.all_embeddings.exists?
+
+      # Process all content records using their own generate_embeddings! methods
+      document.contents.each(&:generate_embeddings!)
+
+      # Update document status to processed
+      document.update!(status: "processed")
+    rescue ActiveRecord::RecordNotFound
+      # Document was deleted, nothing to do
+    rescue StandardError => e
+      if defined?(Rails)
+        Rails.logger.error "Failed to generate embeddings for document #{document_id}: #{e.message}"
+      end
+      raise e
+    end
+  end
+end

data/app/jobs/ragdoll/generate_summary_job.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Ragdoll
+  class GenerateSummaryJob < ActiveJob::Base
+    queue_as :default
+
+    def perform(document_id)
+      document = Ragdoll::Document.find(document_id)
+      return unless document.content.present?
+      return if document.summary.present?
+
+      text_service = Ragdoll::TextGenerationService.new
+      summary = text_service.generate_summary(document.content)
+
+      document.update!(summary: summary) if summary.present?
+    rescue ActiveRecord::RecordNotFound
+      # Document was deleted, nothing to do
+    rescue StandardError => e
+      Rails.logger.error "Failed to generate summary for document #{document_id}: #{e.message}" if defined?(Rails)
+      raise e
+    end
+  end
+end