ragdoll 0.1.0 → 0.1.1

data/lib/tasks/db.rake

@@ -0,0 +1,338 @@
+# frozen_string_literal: true
+
+require "rake"
+
+namespace :db do
+  desc "Create the database"
+  task :create do
+    require_relative "../ragdoll-core"
+
+    config = Ragdoll::Core.configuration
+    puts "Creating database with config: #{config.database_config.inspect}"
+
+    case config.database_config[:adapter]
+    when "postgresql"
+      puts "PostgreSQL database setup - running as superuser to create database and role..."
+      
+      # Connect as superuser to create database and role
+      ActiveRecord::Base.establish_connection(
+        adapter: 'postgresql',
+        database: 'postgres', # Connect to postgres database initially
+        username: ENV['POSTGRES_SUPERUSER'] || 'postgres',
+        password: ENV['POSTGRES_SUPERUSER_PASSWORD'],
+        host: config.database_config[:host] || 'localhost',
+        port: config.database_config[:port] || 5432
+      )
+      
+      # Run individual SQL commands to avoid transaction block issues
+      begin
+        ActiveRecord::Base.connection.execute("DROP DATABASE IF EXISTS ragdoll_development")
+      rescue => e
+        puts "Note: #{e.message}" if e.message.include?("does not exist")
+      end
+      
+      begin
+        ActiveRecord::Base.connection.execute("DROP ROLE IF EXISTS ragdoll")
+      rescue => e
+        puts "Note: #{e.message}" if e.message.include?("does not exist")
+      end
+      
+      begin
+        ActiveRecord::Base.connection.execute("CREATE ROLE ragdoll WITH LOGIN CREATEDB")
+      rescue => e
+        puts "Note: Role already exists, continuing..." if e.message.include?("already exists")
+      end
+      
+      begin
+        ActiveRecord::Base.connection.execute <<-SQL
+          CREATE DATABASE ragdoll_development
+            WITH OWNER = ragdoll
+            ENCODING = 'UTF8'
+            CONNECTION LIMIT = -1
+        SQL
+      rescue => e
+        puts "Note: Database already exists, continuing..." if e.message.include?("already exists")
+      end
+      
+      ActiveRecord::Base.connection.execute("GRANT ALL PRIVILEGES ON DATABASE ragdoll_development TO ragdoll")
+      
+      # Connect to the new database to set schema privileges
+      ActiveRecord::Base.establish_connection(
+        adapter: 'postgresql',
+        database: 'ragdoll_development',
+        username: ENV['POSTGRES_SUPERUSER'] || 'postgres',
+        password: ENV['POSTGRES_SUPERUSER_PASSWORD'],
+        host: config.database_config[:host] || 'localhost',
+        port: config.database_config[:port] || 5432
+      )
+      
+      ActiveRecord::Base.connection.execute <<-SQL
+        -- Grant schema privileges (must be done while connected to the database)
+        GRANT ALL PRIVILEGES ON SCHEMA public TO ragdoll;
+        GRANT CREATE ON SCHEMA public TO ragdoll;
+        
+        -- Set default privileges for future objects
+        ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO ragdoll;
+        ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO ragdoll;
+        ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO ragdoll;
+        
+        -- Enable pgvector extension
+        CREATE EXTENSION IF NOT EXISTS vector;
+      SQL
+      
+      puts "Database and role created successfully"
+    when "mysql2"
+      # For MySQL, we'd typically create the database here
+      puts "MySQL database creation - ensure the database exists on your server"
+    end
+
+    puts "Database creation completed"
+  end
+
+  desc "Drop the database"
+  task :drop do
+    require_relative "../ragdoll-core"
+
+    config = Ragdoll::Core.configuration
+    puts "Dropping database with config: #{config.database_config.inspect}"
+
+    case config.database_config[:adapter]
+    when "postgresql", "mysql2"
+      puts "For #{config.database_config[:adapter]}, please drop the database manually on your server"
+    end
+
+    puts "Database drop completed"
+  end
+
+  desc "Setup the database (create and migrate)"
+  task setup: %i[create migrate]
+
+  desc "Reset the database (drop, create, and migrate)"
+  task reset: %i[drop create migrate]
+
+  desc "Run pending migrations"
+  task :migrate do
+    require_relative "../ragdoll-core"
+
+    puts "Running migrations..."
+    Ragdoll::Core::Database.setup({
+                                    auto_migrate: false
+                                  })
+
+    Ragdoll::Core::Database.migrate!
+    puts "Migrations completed"
+  end
+
+  desc "Rollback the database by one migration"
+  task :rollback do
+    require_relative "../ragdoll-core"
+
+    puts "Rolling back migrations..."
+    # For now, we'll implement a simple reset since our manual migration doesn't support rollback
+    puts "Note: Rollback not yet implemented, use db:reset to start over"
+  end
+
+  desc "Show migration status"
+  task :migrate_status do
+    require_relative "../ragdoll-core"
+
+    Ragdoll::Core::Database.setup({
+                                    auto_migrate: false
+                                  })
+
+    puts "\nMigration Status:"
+    puts "=================="
+
+    # Get migration files
+    migration_paths = [File.join(File.dirname(__FILE__), "..", "..", "db", "migrate")]
+    migration_files = Dir[File.join(migration_paths.first, "*.rb")].sort
+
+    # Get applied migrations
+    applied_versions = []
+    if ActiveRecord::Base.connection.table_exists?("schema_migrations")
+      applied_versions = ActiveRecord::Base.connection.select_values(
+        "SELECT version FROM schema_migrations ORDER BY version"
+      )
+    end
+
+    puts format("%-8s %-20s %s", "Status", "Migration ID", "Migration Name")
+    puts "-" * 60
+
+    migration_files.each do |migration_file|
+      version = File.basename(migration_file, ".rb").split("_").first
+      name = File.basename(migration_file, ".rb").split("_")[1..].join("_")
+      status = applied_versions.include?(version) ? "up" : "down"
+
+      puts format("%-8s %-20s %s", status, version, name)
+    end
+
+    puts "\nTotal migrations: #{migration_files.length}"
+    puts "Applied migrations: #{applied_versions.length}"
+    puts "Pending migrations: #{migration_files.length - applied_versions.length}"
+  end
+
+  desc "Show database schema information"
+  task :schema do
+    require_relative "../ragdoll-core"
+
+    Ragdoll::Core::Database.setup({
+                                    auto_migrate: false
+                                  })
+
+    puts "\nDatabase Schema:"
+    puts "================"
+    puts "Adapter: #{ActiveRecord::Base.connection.adapter_name}"
+
+    if ActiveRecord::Base.connection.tables.any?
+      ActiveRecord::Base.connection.tables.sort.each do |table|
+        puts "\nTable: #{table}"
+        columns = ActiveRecord::Base.connection.columns(table)
+        columns.each do |column|
+          puts "  #{column.name}: #{column.type} (#{column.sql_type})#{unless column.null
+                                                                         ' NOT NULL'
+                                                                       end}#{if column.default
+                                                                               " DEFAULT #{column.default.inspect}"
+                                                                             end}"
+        end
+
+        # Show indexes
+        indexes = ActiveRecord::Base.connection.indexes(table)
+        next unless indexes.any?
+
+        puts "  Indexes:"
+        indexes.each do |index|
+          unique_text = index.unique ? " (unique)" : ""
+          puts "    #{index.name}: [#{index.columns.join(', ')}]#{unique_text}"
+        end
+      end
+    else
+      puts "No tables found. Run 'rake db:migrate' to create tables."
+    end
+  end
+
+  desc "Open database console"
+  task :console do
+    require_relative "../ragdoll-core"
+
+    config = Ragdoll::Core.configuration
+
+    case config.database_config[:adapter]
+    when "postgresql"
+      db_config = config.database_config
+      psql_cmd = "psql"
+      psql_cmd += " -h #{db_config[:host]}" if db_config[:host]
+      psql_cmd += " -p #{db_config[:port]}" if db_config[:port]
+      psql_cmd += " -U #{db_config[:username]}" if db_config[:username]
+      psql_cmd += " #{db_config[:database]}"
+      puts "Opening PostgreSQL console..."
+      system(psql_cmd)
+    when "mysql2"
+      db_config = config.database_config
+      mysql_cmd = "mysql"
+      mysql_cmd += " -h #{db_config[:host]}" if db_config[:host]
+      mysql_cmd += " -P #{db_config[:port]}" if db_config[:port]
+      mysql_cmd += " -u #{db_config[:username]}" if db_config[:username]
+      mysql_cmd += " -p" if db_config[:password]
+      mysql_cmd += " #{db_config[:database]}"
+      puts "Opening MySQL console..."
+      system(mysql_cmd)
+    else
+      puts "Console not supported for adapter: #{config.database_config[:adapter]}"
+    end
+  end
+
+  desc "Show database statistics"
+  task :stats do
+    require_relative "../ragdoll-core"
+
+    Ragdoll::Core::Database.setup({
+                                    auto_migrate: false
+                                  })
+
+    puts "\nDatabase Statistics:"
+    puts "==================="
+
+    if ActiveRecord::Base.connection.table_exists?("ragdoll_documents")
+      doc_count = ActiveRecord::Base.connection.select_value("SELECT COUNT(*) FROM ragdoll_documents")
+      puts "Documents: #{doc_count}"
+
+      if doc_count.positive?
+        doc_types = ActiveRecord::Base.connection.select_rows(
+          "SELECT document_type, COUNT(*) FROM ragdoll_documents GROUP BY document_type"
+        )
+        puts "Document types:"
+        doc_types.each { |type, count| puts "  #{type}: #{count}" }
+
+        statuses = ActiveRecord::Base.connection.select_rows(
+          "SELECT status, COUNT(*) FROM ragdoll_documents GROUP BY status"
+        )
+        puts "Document statuses:"
+        statuses.each { |status, count| puts "  #{status}: #{count}" }
+      end
+    else
+      puts "Documents table does not exist"
+    end
+
+    if ActiveRecord::Base.connection.table_exists?("ragdoll_embeddings")
+      embedding_count = ActiveRecord::Base.connection.select_value("SELECT COUNT(*) FROM ragdoll_embeddings")
+      puts "Embeddings: #{embedding_count}"
+
+      if embedding_count.positive?
+        models = ActiveRecord::Base.connection.select_rows(
+          "SELECT model_name, COUNT(*) FROM ragdoll_embeddings GROUP BY model_name"
+        )
+        puts "Embedding models:"
+        models.each { |model, count| puts "  #{model}: #{count}" }
+
+        usage_stats = ActiveRecord::Base.connection.select_one(
+          "SELECT AVG(usage_count) as avg_usage, MAX(usage_count) as max_usage FROM ragdoll_embeddings"
+        )
+        puts "Usage statistics:"
+        puts "  Average usage: #{usage_stats['avg_usage'].to_f.round(2)}"
+        puts "  Max usage: #{usage_stats['max_usage']}"
+      end
+    else
+      puts "Embeddings table does not exist"
+    end
+  end
+
+  desc "Truncate all tables (remove all data but keep structure)"
+  task :truncate do
+    require_relative "../ragdoll-core"
+
+    Ragdoll::Core::Database.setup({
+                                    auto_migrate: false
+                                  })
+
+    puts "Truncating all tables..."
+
+    # Disable foreign key checks temporarily
+    case ActiveRecord::Base.connection.adapter_name.downcase
+    when "postgresql"
+      ActiveRecord::Base.connection.execute("SET session_replication_role = 'replica'")
+    when "mysql"
+      ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS = 0")
+    end
+
+    # Truncate tables in correct order (dependent tables first)
+    %w[ragdoll_embeddings ragdoll_documents].each do |table|
+      if ActiveRecord::Base.connection.table_exists?(table)
+        ActiveRecord::Base.connection.execute("DELETE FROM #{table}")
+        puts "Truncated #{table}"
+      end
+    end
+
+    # Re-enable foreign key checks
+    case ActiveRecord::Base.connection.adapter_name.downcase
+    when "postgresql"
+      ActiveRecord::Base.connection.execute("SET session_replication_role = 'origin'")
+    when "mysql"
+      ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS = 1")
+    end
+
+    puts "All tables truncated"
+  end
+end
+
+# Make db tasks available as top-level commands
+task db: "db:migrate_status"

metadata

@@ -1,28 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ragdoll
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Dewayne VanHoozer
 bindir: bin
 cert_chain: []
-date: 2025-02-19 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '7.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '7.1'
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
 description: Under development.  Contributors welcome.
 email:
 - dvanhoozer@gmail.com
@@ -32,30 +18,47 @@ extra_rdoc_files: []
 files:
 - README.md
 - Rakefile
-- app/models/ragdoll/document.rb
-- app/models/ragdoll/embedding.rb
-- config/initializers/ragdoll.rb
-- config/routes.rb
-- db/migrate/20250218123456_create_documents.rb
-- lib/config/database.yml
-- lib/config/ragdoll.yml
+- db/migrate/001_enable_postgresql_extensions.rb
+- db/migrate/004_create_ragdoll_documents.rb
+- db/migrate/005_create_ragdoll_embeddings.rb
+- db/migrate/006_create_ragdoll_contents.rb
+- lib/ragdoll-core.rb
 - lib/ragdoll.rb
-- lib/ragdoll/engine.rb
-- lib/ragdoll/import_job.rb
-- lib/ragdoll/ingestion.rb
-- lib/ragdoll/search.rb
-- lib/ragdoll/version.rb
-- lib/tasks/import_task.thor
-- lib/tasks/jobs_task.thor
-- lib/tasks/ragdoll_tasks.thor
-- lib/tasks/search_task.thor
+- lib/ragdoll/core.rb
+- lib/ragdoll/core/client.rb
+- lib/ragdoll/core/configuration.rb
+- lib/ragdoll/core/database.rb
+- lib/ragdoll/core/document_management.rb
+- lib/ragdoll/core/document_processor.rb
+- lib/ragdoll/core/embedding_service.rb
+- lib/ragdoll/core/errors.rb
+- lib/ragdoll/core/jobs/extract_keywords.rb
+- lib/ragdoll/core/jobs/extract_text.rb
+- lib/ragdoll/core/jobs/generate_embeddings.rb
+- lib/ragdoll/core/jobs/generate_summary.rb
+- lib/ragdoll/core/metadata_schemas.rb
+- lib/ragdoll/core/models/audio_content.rb
+- lib/ragdoll/core/models/content.rb
+- lib/ragdoll/core/models/document.rb
+- lib/ragdoll/core/models/embedding.rb
+- lib/ragdoll/core/models/image_content.rb
+- lib/ragdoll/core/models/text_content.rb
+- lib/ragdoll/core/search_engine.rb
+- lib/ragdoll/core/services/image_description_service.rb
+- lib/ragdoll/core/services/metadata_generator.rb
+- lib/ragdoll/core/shrine_config.rb
+- lib/ragdoll/core/text_chunker.rb
+- lib/ragdoll/core/text_generation_service.rb
+- lib/ragdoll/core/version.rb
+- lib/tasks/annotate.rake
+- lib/tasks/db.rake
 homepage: https://github.com/MadBomber/ragdoll
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/MadBomber/ragdoll
-  source_code_uri: https://github.com/MadBomber/ragdoll
+  source_code_uri: https://github.com/MadBomber/ragdoll/blob/main
   changelog_uri: https://github.com/MadBomber/ragdoll/blob/main/CHANGELOG.md
 rdoc_options: []
 require_paths:
@@ -64,14 +67,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.3
+rubygems_version: 3.7.1
 specification_version: 4
-summary: Ruby on Rails Engine
+summary: Multi-Modal Retrieval Augmented Generation
 test_files: []

data/app/models/ragdoll/document.rb

@@ -1,9 +0,0 @@
-# This file defines the Document model for the Ragdoll gem.
-
-# frozen_string_literal: true
-
-module Ragdoll
-  class Document < ApplicationRecord
-    has_many :embeddings, dependent: :destroy
-  end
-end

data/app/models/ragdoll/embedding.rb

@@ -1,9 +0,0 @@
-# This file defines the Embedding model for the Ragdoll gem.
-
-# frozen_string_literal: true
-
-module Ragdoll
-  class Embedding < ApplicationRecord
-    belongs_to :document
-  end
-end

data/config/initializers/ragdoll.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-# Initializer for Ragdoll engine
-Ragdoll.configure do |config|
-  # Set configuration options here
-end

data/config/routes.rb

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-Ragdoll::Engine.routes.draw do
-  # Define your engine routes here
-end

data/db/migrate/20250218123456_create_documents.rb

@@ -1,20 +0,0 @@
-# This migration creates the documents table with necessary extensions for PostgreSQL.
-
-module Ragdoll
-  class CreateDocuments < ActiveRecord::Migration[7.0]
-  def change
-    enable_extension 'pg_trgm'
-    enable_extension 'fuzzystrmatch'
-
-    create_table :documents do |t|
-      t.string :location
-      t.string :summary
-      t.string :type
-      t.datetime :processing_started_at
-      t.datetime :processing_finished_at
-
-      t.timestamps
-    end
-  end
-  end
-end

data/lib/config/database.yml

@@ -1,28 +0,0 @@
-# This file contains the database configuration for the Ragdoll gem, using environment variables.
-
-default: &default
-  adapter: postgresql
-  encoding: unicode
-  pool: <%= ENV.fetch("RAGDOLL_POOL", 5) %>
-  timeout: <%= ENV.fetch("RAGDOLL_TIMEOUT", 5000) %>
-
-development:
-  <<: *default
-  host: <%= ENV.fetch("RAGDOLL_HOST", "localhost") %>
-  database: <%= ENV.fetch("RAGDOLL_DATABASE", "ragdoll_development") %>
-  username: <%= ENV.fetch("RAGDOLL_USER", "user") %>
-  password: <%= ENV.fetch("RAGDOLL_PASSWORD", "password") %>
-
-test:
-  <<: *default
-  host: <%= ENV.fetch("RAGDOLL_HOST", "localhost") %>
-  database: <%= ENV.fetch("RAGDOLL_DATABASE", "ragdoll_test") %>
-  username: <%= ENV.fetch("RAGDOLL_USER", "user") %>
-  password: <%= ENV.fetch("RAGDOLL_PASSWORD", "password") %>
-
-production:
-  <<: *default
-  host: <%= ENV.fetch("RAGDOLL_HOST") %>
-  database: <%= ENV.fetch("RAGDOLL_DATABASE") %>
-  username: <%= ENV.fetch("RAGDOLL_USER") %>
-  password: <%= ENV.fetch("RAGDOLL_PASSWORD") %>

data/lib/config/ragdoll.yml

@@ -1,31 +0,0 @@
-# This file contains the default configuration settings for the Ragdoll gem, including database configurations.
-
-default: &default
-  database:
-    host: localhost
-    database: ragdoll_development
-    user: user
-    password: password
-    pool: 5
-    timeout: 5000
-
-  llm:
-    embeddings_model: "llama-2-7b"
-    reranking_model: "llama-2-13b"
-    chat_model: "llama-2-70b"
-
-development:
-  <<: *default
-
-test:
-  <<: *default
-  database:
-    database: ragdoll_test
-
-production:
-  <<: *default
-  database:
-    host: <%= ENV.fetch("RAGDOLL_HOST") %>
-    database: <%= ENV.fetch("RAGDOLL_DATABASE") %>
-    user: <%= ENV.fetch("RAGDOLL_USER") %>
-    password: <%= ENV.fetch("RAGDOLL_PASSWORD") %>

data/lib/ragdoll/engine.rb

@@ -1,16 +0,0 @@
-# This file defines the Ragdoll engine, which integrates the gem with Rails applications.
-
-# frozen_string_literal: true
-
-require "rails/engine"
-
-module Ragdoll
-  class Engine < ::Rails::Engine
-    isolate_namespace Ragdoll
-    config.generators do |g|
-      g.test_framework :minitest
-      g.fixture_replacement :factory_bot
-      g.factory_bot dir: 'test/factories'
-    end
-  end
-end

data/lib/ragdoll/import_job.rb

@@ -1,15 +0,0 @@
-# This file defines the ImportJob class for handling document import tasks in the background.
-
-# frozen_string_literal: true
-
-module Ragdoll
-  class ImportJob < SolidJob::Base
-    def perform(file)
-      document = File.read(file)
-      ingestion = Ragdoll::Ingestion.new(document)
-      vectorized_chunks = ingestion.chunk_and_vectorize
-      ingestion.store_in_database
-      puts "Imported #{file} successfully."
-    end
-  end
-end

data/lib/ragdoll/ingestion.rb

@@ -1,30 +0,0 @@
-# This file contains the Ingestion class responsible for processing documents by chunking and vectorizing them.
-
-# frozen_string_literal: true
-
-module Ragdoll
-  class Ingestion
-    def initialize(document)
-      @document = document
-    end
-
-    def chunk_and_vectorize
-      # Example logic for chunking and vectorization
-      chunks = @document.split("\n\n") # Split document into paragraphs
-      vectorized_chunks = chunks.map { |chunk| vectorize(chunk) }
-      vectorized_chunks
-    end
-
-    def store_in_database
-      # Implement logic to store vectorized data in the database
-    end
-
-    private
-
-    def vectorize(chunk)
-      # Placeholder for vectorization logic
-      # Convert chunk to a vector representation
-      chunk.split.map(&:downcase) # Simple example: split words and downcase
-    end
-  end
-end

data/lib/ragdoll/search.rb

@@ -1,18 +0,0 @@
-# This file contains the Search class responsible for querying the database with a prompt.
-
-# frozen_string_literal: true
-
-module Ragdoll
-  class Search
-    def initialize(prompt)
-      @prompt = prompt
-    end
-
-    def search_database(max_count)
-      # Example logic for searching the database
-      # This is a placeholder for actual database search logic
-      results = [] # Placeholder for actual database query results
-      results.select { |entry| entry.include?(@prompt) }
-    end
-  end
-end

data/lib/ragdoll/version.rb

@@ -1,7 +0,0 @@
-# This file defines the version number for the Ragdoll gem.
-
-# frozen_string_literal: true
-
-module Ragdoll
-  VERSION = "0.1.0"
-end