ragdoll 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d5c41105ebbbb39c2c38db7518607f45d446e6cd3b024abde75beb434ff3d2b
-  data.tar.gz: 2d9714a078bd7b9a8adda80869af3f983e3cb637356cba64e1550f0fe77911ee
+  metadata.gz: bea4621e2b802db79d78f8b1d0679cf2f81ed35b91d35683ce0afcb83ddc54e1
+  data.tar.gz: ec12fb975b154f77a42d54fb3c716d523e1b90e4e0122b3576c5aac15a957340
 SHA512:
-  metadata.gz: bf742b9919e1d542b45325e11e197bc9aca313b892c34567ea51d4efd7e7815ecc8081165af5cfef12e2453b86ee215d3a92a807a3c98d486b3797ac3dbaf214
-  data.tar.gz: da808c5780e3fecd02ef2ab0c4414b0f70cf3eef6a69ba08311c473ed19f98f3d3d4d0b5912c294dc0441a03f58a3ff510be126736d47881190cf56841e6233d
+  metadata.gz: 3702308d3b772dfc0ebf429a26bae0f0378456e9d6c48357b8e2a4cdeb3744e78b43fd610ef308ff6348f1bec28bb37d51bd5e335d78583574b3212c8f544a33
+  data.tar.gz: 9beebfebafe1ed2e815a3042949b68e5f208d8c555a92228eb4098fa99900f07d2985654766e2907de3069f5d10b6cf3e65fb7a3b431ee9db836e6111f1e27f2

data/README.md

@@ -1,353 +1,75 @@
-<div align="center" style="background-color: yellow; color: black; padding: 20px; margin: 20px 0; border: 2px solid black; font-size: 48px; font-weight: bold;">
-  ⚠️ CAUTION ⚠️<br />
-  Software Under Development by a Crazy Man
-</div>
-<br />
-<div align="center">
-  <table>
-    <tr>
-      <td width="50%">
-        <a href="https://research.ibm.com/blog/retrieval-augmented-generation-RAG" target="_blank">
-          <img src="rag_doll.png" alt="Ragdoll" width="800">
-        </a>
-      </td>
-      <td width="50%" valign="top">
-        <p>Multi-modal RAG (Retrieval-Augmented Generation) is an architecture that integrates multiple data types (such as text, images, and audio) to enhance AI response generation. It combines retrieval-based methods, which fetch relevant information from a knowledge base, with generative large language models (LLMs) that create coherent and contextually appropriate outputs. This approach allows for more comprehensive and engaging user interactions, such as chatbots that respond with both text and images or educational tools that incorporate visual aids into learning materials. By leveraging various modalities, multi-modal RAG systems improve context understanding and user experience.</p>
-      </td>
-    </tr>
-  </table>
-</div>
+# Ragdoll
 
-# Ragdoll::Core
+Ragdoll is a Rails Engine designed for document ingestion and search. It allows you to import documents, vectorize them, and perform searches using vector representations.
 
-Database-oriented multi-modal RAG (Retrieval-Augmented Generation) library built on ActiveRecord. Features PostgreSQL + pgvector for high-performance semantic search, polymorphic content architecture, and dual metadata design for sophisticated document analysis.
+## Installation as a Rails Engine
 
-## Quick Start
+To use Ragdoll as a Rails Engine, add this line to your application's Gemfile:
 
-```ruby
-require 'ragdoll'
-
-# Configure with PostgreSQL + pgvector
-Ragdoll::Core.configure do |config|
-  # Database configuration (PostgreSQL only)
-  config.database_config = {
-    adapter: 'postgresql',
-    database: 'ragdoll_production',
-    username: 'ragdoll',
-    password: ENV['DATABASE_PASSWORD'],
-    host: 'localhost',
-    port: 5432,
-    auto_migrate: true
-  }
-
-  # Ruby LLM configuration
-  config.ruby_llm_config[:openai][:api_key] = ENV['OPENAI_API_KEY']
-  config.ruby_llm_config[:openai][:organization] = ENV['OPENAI_ORGANIZATION']
-  config.ruby_llm_config[:openai][:project] = ENV['OPENAI_PROJECT']
-
-  # Model configuration
-  config.models[:default] = 'openai/gpt-4o'
-  config.models[:embedding][:text] = 'text-embedding-3-small'
-
-  # Logging configuration
-  config.logging_config[:log_level] = :warn
-  config.logging_config[:log_filepath] = File.join(Dir.home, '.ragdoll', 'ragdoll.log')
-end
-
-# Add documents - returns detailed result
-result = Ragdoll::Core.add_document(path: 'research_paper.pdf')
-puts result[:message]  # "Document 'research_paper' added successfully with ID 123"
-doc_id = result[:document_id]
-
-# Check document status
-status = Ragdoll::Core.document_status(id: doc_id)
-puts status[:message]  # Shows processing status and embeddings count
-
-# Search across content
-results = Ragdoll::Core.search(query: 'neural networks')
-
-# Get detailed document information
-document = Ragdoll::Core.get_document(id: doc_id)
-```
-
-## High-Level API
-
-The `Ragdoll` module provides a convenient high-level API for common operations:
-
-### Document Management
-
-```ruby
-# Add single document - returns detailed result hash
-result = Ragdoll::Core.add_document(path: 'document.pdf')
-puts result[:success]         # true
-puts result[:document_id]     # "123"
-puts result[:message]         # "Document 'document' added successfully with ID 123"
-puts result[:embeddings_queued] # true
-
-# Check document processing status
-status = Ragdoll::Core.document_status(id: result[:document_id])
-puts status[:status]          # "processed"
-puts status[:embeddings_count] # 15
-puts status[:embeddings_ready] # true
-puts status[:message]         # "Document processed successfully with 15 embeddings"
-
-# Get detailed document information
-document = Ragdoll::Core.get_document(id: result[:document_id])
-puts document[:title]         # "document"
-puts document[:status]        # "processed"
-puts document[:embeddings_count] # 15
-puts document[:content_length]   # 5000
-
-# Update document metadata
-Ragdoll::Core.update_document(id: result[:document_id], title: 'New Title')
-
-# Delete document
-Ragdoll::Core.delete_document(id: result[:document_id])
-
-# List all documents
-documents = Ragdoll::Core.list_documents(limit: 10)
-
-# System statistics
-stats = Ragdoll::Core.stats
-puts stats[:total_documents]  # 50
-puts stats[:total_embeddings] # 1250
+```bash
+bundle add ragdoll
 ```
 
-### Search and Retrieval
-
-```ruby
-# Semantic search across all content types
-results = Ragdoll::Core.search(query: 'artificial intelligence')
-
-# Search specific content types
-text_results = Ragdoll::Core.search(query: 'machine learning', content_type: 'text')
-image_results = Ragdoll::Core.search(query: 'neural network diagram', content_type: 'image')
-audio_results = Ragdoll::Core.search(query: 'AI discussion', content_type: 'audio')
-
-# Advanced search with metadata filters
-results = Ragdoll::Core.search(
-  query: 'deep learning',
-  classification: 'research',
-  keywords: ['AI', 'neural networks'],
-  tags: ['technical']
-)
+And then execute:
 
-# Get context for RAG applications
-context = Ragdoll::Core.get_context(query: 'machine learning', limit: 5)
-
-# Enhanced prompt with context
-enhanced = Ragdoll::Core.enhance_prompt(
-  prompt: 'What is machine learning?',
-  context_limit: 5
-)
-
-# Hybrid search combining semantic and full-text
-results = Ragdoll::Core.hybrid_search(
-  query: 'neural networks',
-  semantic_weight: 0.7,
-  text_weight: 0.3
-)
+```bash
+bundle install
 ```
 
-### System Operations
-
-```ruby
-# Get system statistics
-stats = Ragdoll::Core.stats
-# Returns information about documents, content types, embeddings, etc.
+Or install it yourself as:
 
-# Health check
-healthy = Ragdoll::Core.healthy?
-
-# Get configuration
-config = Ragdoll::Core.configuration
-
-# Reset configuration (useful for testing)
-Ragdoll::Core.reset_configuration!
+```bash
+gem install ragdoll
 ```
 
-### Configuration
-
-```ruby
-# Configure the system
-Ragdoll::Core.configure do |config|
-  # Database configuration (PostgreSQL only - REQUIRED)
-  config.database_config = {
-    adapter: 'postgresql',
-    database: 'ragdoll_production',
-    username: 'ragdoll',
-    password: ENV['DATABASE_PASSWORD'],
-    host: 'localhost',
-    port: 5432,
-    auto_migrate: true
-  }
-
-  # Ruby LLM configuration for multiple providers
-  config.ruby_llm_config[:openai][:api_key] = ENV['OPENAI_API_KEY']
-  config.ruby_llm_config[:openai][:organization] = ENV['OPENAI_ORGANIZATION']
-  config.ruby_llm_config[:openai][:project] = ENV['OPENAI_PROJECT']
-
-  config.ruby_llm_config[:anthropic][:api_key] = ENV['ANTHROPIC_API_KEY']
-  config.ruby_llm_config[:google][:api_key] = ENV['GOOGLE_API_KEY']
+## Usage as a Rails Engine
 
-  # Model configuration
-  config.models[:default] = 'openai/gpt-4o'
-  config.models[:summary] = 'openai/gpt-4o'
-  config.models[:keywords] = 'openai/gpt-4o'
-  config.models[:embedding][:text] = 'text-embedding-3-small'
-  config.models[:embedding][:image] = 'image-embedding-3-small'
-  config.models[:embedding][:audio] = 'audio-embedding-3-small'
+### Importing Documents
 
-  # Logging configuration
-  config.logging_config[:log_level] = :warn  # :debug, :info, :warn, :error, :fatal
-  config.logging_config[:log_filepath] = File.join(Dir.home, '.ragdoll', 'ragdoll.log')
+To import documents from a file, glob, or directory, use the following command:
 
-  # Processing settings
-  config.chunking[:text][:max_tokens] = 1000
-  config.chunking[:text][:overlap] = 200
-  config.search[:similarity_threshold] = 0.7
-  config.search[:max_results] = 10
-end
+```bash
+ragdoll import PATH
 ```
 
-## Current Implementation Status
-
-### ✅ **Fully Implemented**
-- **Text document processing**: PDF, DOCX, HTML, Markdown, plain text files
-- **Embedding generation**: Text chunking and vector embedding creation
-- **Database schema**: Multi-modal polymorphic architecture with PostgreSQL + pgvector
-- **Dual metadata architecture**: Separate LLM-generated content analysis and file properties
-- **Search functionality**: Semantic search with cosine similarity and usage analytics
-- **Document management**: Add, update, delete, list operations
-- **Background processing**: ActiveJob integration for async embedding generation
-- **LLM metadata generation**: AI-powered structured content analysis with schema validation
-- **Logging**: Configurable file-based logging with multiple levels
-
-### 🚧 **In Development**
-- **Image processing**: Framework exists but vision AI integration needs completion
-- **Audio processing**: Framework exists but speech-to-text integration needs completion
-- **Hybrid search**: Combining semantic and full-text search capabilities
-
-### 📋 **Planned Features**
-- **Multi-modal search**: Search across text, image, and audio content types
-- **Content-type specific embedding models**: Different models for text, image, audio
-- **Enhanced metadata schemas**: Domain-specific metadata templates
-
-## Architecture Highlights
-
-### Dual Metadata Design
-
-Ragdoll uses a sophisticated dual metadata architecture to separate concerns:
-
-- **`metadata` (JSON)**: LLM-generated content analysis including summary, keywords, classification, topics, sentiment, and domain-specific insights
-- **`file_metadata` (JSON)**: System-generated file properties including size, MIME type, dimensions, processing parameters, and technical characteristics
-
-This separation enables both semantic search operations on content meaning and efficient file management operations.
-
-### Polymorphic Multi-Modal Architecture
-
-The database schema uses polymorphic associations to elegantly support multiple content types:
+- `PATH`: The path to the file or directory to import.
+- Use the `-r` or `--recursive` option to import files recursively from directories.
+- Use the `-j` or `--jobs` option to specify the number of concurrent import jobs.
 
-- **Documents**: Central entity with dual metadata columns
-- **Content Types**: Specialized tables for `text_contents`, `image_contents`, `audio_contents`
-- **Embeddings**: Unified vector storage via polymorphic `embeddable` associations
+### Managing Jobs
 
-## Text Document Processing (Current)
-
-Currently, Ragdoll processes text documents through:
-
-1. **Content Extraction**: Extracts text from PDF, DOCX, HTML, Markdown, and plain text
-2. **Metadata Generation**: AI-powered analysis creates structured content metadata
-3. **Text Chunking**: Splits content into manageable chunks with configurable size/overlap
-4. **Embedding Generation**: Creates vector embeddings using OpenAI or other providers
-5. **Database Storage**: Stores in polymorphic multi-modal architecture with dual metadata
-6. **Search**: Semantic search using cosine similarity with usage analytics
-
-### Example Usage
-```ruby
-# Add a text document
-result = Ragdoll::Core.add_document(path: 'document.pdf')
-
-# Check processing status
-status = Ragdoll::Core.document_status(id: result[:document_id])
-
-# Search the content
-results = Ragdoll::Core.search(query: 'machine learning')
-```
-
-## PostgreSQL + pgvector Configuration
-
-### Database Setup
+To manage import jobs, use the following command:
 
 ```bash
-# Install PostgreSQL and pgvector
-brew install postgresql pgvector  # macOS
-# or
-apt-get install postgresql postgresql-contrib  # Ubuntu
-
-# Create database and enable pgvector extension
-createdb ragdoll_production
-psql -d ragdoll_production -c "CREATE EXTENSION IF NOT EXISTS vector;"
-```
-
-### Configuration Example
-
-```ruby
-Ragdoll::Core.configure do |config|
-  config.database_config = {
-    adapter: 'postgresql',
-    database: 'ragdoll_production',
-    username: 'ragdoll',
-    password: ENV['DATABASE_PASSWORD'],
-    host: 'localhost',
-    port: 5432,
-    pool: 20,
-    auto_migrate: true
-  }
-end
+ragdoll jobs [JOB_ID]
 ```
 
-## Performance Features
+- `JOB_ID`: The ID of a specific job to manage.
+- Use `--stop`, `--pause`, or `--resume` to control a specific job.
+- Use `--stop-all`, `--pause-all`, or `--resume-all` to control all jobs.
 
-- **Native pgvector**: Hardware-accelerated similarity search
-- **IVFFlat indexing**: Fast approximate nearest neighbor search
-- **Polymorphic embeddings**: Unified search across content types
-- **Batch processing**: Efficient bulk operations
-- **Background jobs**: Asynchronous document processing
-- **Connection pooling**: High-concurrency support
+### Searching Documents
 
-## Installation
+To search the database with a prompt, use the following command:
 
 ```bash
-# Install system dependencies
-brew install postgresql pgvector  # macOS
-# or
-apt-get install postgresql postgresql-contrib  # Ubuntu
+ragdoll search PROMPT
+```
 
-# Install gem
-gem install ragdoll
+- `PROMPT`: The search prompt as a string or use the `-p` option to specify a file containing the prompt text.
+- Use the `--max_count` option to specify the maximum number of results to return.
+- Use the `--rerank` option to rerank results using keyword search.
 
-# Or add to Gemfile
-gem 'ragdoll'
-```
+## Development and Contribution
 
-## Requirements
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-- **Ruby**: 3.2+
-- **PostgreSQL**: 12+ with pgvector extension (REQUIRED - no other databases supported)
-- **Dependencies**: activerecord, pg, pgvector, neighbor, ruby_llm, pdf-reader, docx, rubyzip, shrine, rmagick, opensearch-ruby, searchkick, ruby-progressbar
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
-## Related Projects
+## Contributing
 
-- **ragdoll-cli**: Standalone CLI application using ragdoll
-- **ragdoll-rails**: Rails engine with web interface for ragdoll
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/ragdoll.
 
-## Key Design Principles
+## License
 
-1. **Database-Oriented**: Built on ActiveRecord with PostgreSQL + pgvector for production performance
-2. **Multi-Modal First**: Text, image, and audio content as first-class citizens via polymorphic architecture
-3. **Dual Metadata Design**: Separates LLM-generated content analysis from file properties
-4. **LLM-Enhanced**: Structured metadata generation with schema validation using latest AI capabilities
-5. **High-Level API**: Simple, intuitive interface for complex operations
-6. **Scalable**: Designed for production workloads with background processing and proper indexing
-7. **Extensible**: Easy to add new content types and embedding models through polymorphic design
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -1,21 +1,10 @@
-# frozen_string_literal: true
-
-require 'simplecov'
-SimpleCov.start
+# This file defines the Rake tasks for the Ragdoll gem, including tasks for testing.
 
-# Suppress bundler/rubygems warnings
-$VERBOSE = nil
+# frozen_string_literal: true
 
 require "bundler/gem_tasks"
-require "rake/testtask"
-
-Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList["test/**/*_test.rb"]
-end
+require "minitest/test_task"
 
-# Load annotate tasks
-Dir.glob("lib/tasks/*.rake").each { |r| load r }
+Minitest::TestTask.create
 
 task default: :test

data/app/models/ragdoll/document.rb

@@ -0,0 +1,9 @@
+# This file defines the Document model for the Ragdoll gem.
+
+# frozen_string_literal: true
+
+module Ragdoll
+  class Document < ApplicationRecord
+    has_many :embeddings, dependent: :destroy
+  end
+end

data/app/models/ragdoll/embedding.rb

@@ -0,0 +1,9 @@
+# This file defines the Embedding model for the Ragdoll gem.
+
+# frozen_string_literal: true
+
+module Ragdoll
+  class Embedding < ApplicationRecord
+    belongs_to :document
+  end
+end

data/config/initializers/ragdoll.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Initializer for Ragdoll engine
+Ragdoll.configure do |config|
+  # Set configuration options here
+end

data/config/routes.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Ragdoll::Engine.routes.draw do
+  # Define your engine routes here
+end

data/db/migrate/20250218123456_create_documents.rb

@@ -0,0 +1,20 @@
+# This migration creates the documents table with necessary extensions for PostgreSQL.
+
+module Ragdoll
+  class CreateDocuments < ActiveRecord::Migration[7.0]
+  def change
+    enable_extension 'pg_trgm'
+    enable_extension 'fuzzystrmatch'
+
+    create_table :documents do |t|
+      t.string :location
+      t.string :summary
+      t.string :type
+      t.datetime :processing_started_at
+      t.datetime :processing_finished_at
+
+      t.timestamps
+    end
+  end
+  end
+end

data/lib/config/database.yml

@@ -0,0 +1,28 @@
+# This file contains the database configuration for the Ragdoll gem, using environment variables.
+
+default: &default
+  adapter: postgresql
+  encoding: unicode
+  pool: <%= ENV.fetch("RAGDOLL_POOL", 5) %>
+  timeout: <%= ENV.fetch("RAGDOLL_TIMEOUT", 5000) %>
+
+development:
+  <<: *default
+  host: <%= ENV.fetch("RAGDOLL_HOST", "localhost") %>
+  database: <%= ENV.fetch("RAGDOLL_DATABASE", "ragdoll_development") %>
+  username: <%= ENV.fetch("RAGDOLL_USER", "user") %>
+  password: <%= ENV.fetch("RAGDOLL_PASSWORD", "password") %>
+
+test:
+  <<: *default
+  host: <%= ENV.fetch("RAGDOLL_HOST", "localhost") %>
+  database: <%= ENV.fetch("RAGDOLL_DATABASE", "ragdoll_test") %>
+  username: <%= ENV.fetch("RAGDOLL_USER", "user") %>
+  password: <%= ENV.fetch("RAGDOLL_PASSWORD", "password") %>
+
+production:
+  <<: *default
+  host: <%= ENV.fetch("RAGDOLL_HOST") %>
+  database: <%= ENV.fetch("RAGDOLL_DATABASE") %>
+  username: <%= ENV.fetch("RAGDOLL_USER") %>
+  password: <%= ENV.fetch("RAGDOLL_PASSWORD") %>

data/lib/config/ragdoll.yml

@@ -0,0 +1,31 @@
+# This file contains the default configuration settings for the Ragdoll gem, including database configurations.
+
+default: &default
+  database:
+    host: localhost
+    database: ragdoll_development
+    user: user
+    password: password
+    pool: 5
+    timeout: 5000
+
+  llm:
+    embeddings_model: "llama-2-7b"
+    reranking_model: "llama-2-13b"
+    chat_model: "llama-2-70b"
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+  database:
+    database: ragdoll_test
+
+production:
+  <<: *default
+  database:
+    host: <%= ENV.fetch("RAGDOLL_HOST") %>
+    database: <%= ENV.fetch("RAGDOLL_DATABASE") %>
+    user: <%= ENV.fetch("RAGDOLL_USER") %>
+    password: <%= ENV.fetch("RAGDOLL_PASSWORD") %>

data/lib/ragdoll/engine.rb

@@ -0,0 +1,16 @@
+# This file defines the Ragdoll engine, which integrates the gem with Rails applications.
+
+# frozen_string_literal: true
+
+require "rails/engine"
+
+module Ragdoll
+  class Engine < ::Rails::Engine
+    isolate_namespace Ragdoll
+    config.generators do |g|
+      g.test_framework :minitest
+      g.fixture_replacement :factory_bot
+      g.factory_bot dir: 'test/factories'
+    end
+  end
+end

data/lib/ragdoll/import_job.rb

@@ -0,0 +1,15 @@
+# This file defines the ImportJob class for handling document import tasks in the background.
+
+# frozen_string_literal: true
+
+module Ragdoll
+  class ImportJob < SolidJob::Base
+    def perform(file)
+      document = File.read(file)
+      ingestion = Ragdoll::Ingestion.new(document)
+      vectorized_chunks = ingestion.chunk_and_vectorize
+      ingestion.store_in_database
+      puts "Imported #{file} successfully."
+    end
+  end
+end

data/lib/ragdoll/ingestion.rb

@@ -0,0 +1,30 @@
+# This file contains the Ingestion class responsible for processing documents by chunking and vectorizing them.
+
+# frozen_string_literal: true
+
+module Ragdoll
+  class Ingestion
+    def initialize(document)
+      @document = document
+    end
+
+    def chunk_and_vectorize
+      # Example logic for chunking and vectorization
+      chunks = @document.split("\n\n") # Split document into paragraphs
+      vectorized_chunks = chunks.map { |chunk| vectorize(chunk) }
+      vectorized_chunks
+    end
+
+    def store_in_database
+      # Implement logic to store vectorized data in the database
+    end
+
+    private
+
+    def vectorize(chunk)
+      # Placeholder for vectorization logic
+      # Convert chunk to a vector representation
+      chunk.split.map(&:downcase) # Simple example: split words and downcase
+    end
+  end
+end

data/lib/ragdoll/search.rb

@@ -0,0 +1,18 @@
+# This file contains the Search class responsible for querying the database with a prompt.
+
+# frozen_string_literal: true
+
+module Ragdoll
+  class Search
+    def initialize(prompt)
+      @prompt = prompt
+    end
+
+    def search_database(max_count)
+      # Example logic for searching the database
+      # This is a placeholder for actual database search logic
+      results = [] # Placeholder for actual database query results
+      results.select { |entry| entry.include?(@prompt) }
+    end
+  end
+end

data/lib/ragdoll/version.rb

@@ -0,0 +1,7 @@
+# This file defines the version number for the Ragdoll gem.
+
+# frozen_string_literal: true
+
+module Ragdoll
+  VERSION = "0.1.0"
+end