ragdoll 0.0.2 → 0.1.0

data/lib/ragdoll.rb

@@ -1,249 +1,12 @@
+# This file is the main entry point for the Ragdoll gem, requiring all necessary components.
+
 # frozen_string_literal: true
 
-require "debug_me"
-include DebugMe
-$DEBUG_ME = true
+# frozen_string_literal: true
 
-require "delegate"
-require_relative "ragdoll/core"
+require "ragdoll/version"
+require "ragdoll/engine"
 
 module Ragdoll
-  class << self
-
-    #################
-    # Configuration #
-    #################
-
-    # Retrieve the current configuration.
-    # @example
-    #   config = Ragdoll.config
-    #   puts config.database_config[:adapter]
-    # @example
-    #   current_config = Ragdoll.configuration
-    #   puts current_config.models[:default]
-    # @return [Ragdoll::Core::Configuration] the current configuration instance.
-    def config
-      Core.config
-    end
-
-    # Configure the Ragdoll module.
-    # @yieldparam config [Ragdoll::Core::Configuration] the configuration instance to modify.
-    # @example
-    #   Ragdoll.configure do |config|
-    #     config.database_config[:adapter] = "postgres"
-    #   end
-    # @yield [Ragdoll::Core::Configuration] yields the configuration instance for modification.
-    def configure(*args, **kwargs, &block)
-      Ragdoll::Core.configure(*args, **kwargs, &block)
-    end
-
-    # Access the current configuration.
-    # @param args [Array] additional arguments for configuration.
-    # @param kwargs [Hash] keyword arguments for configuration.
-    # @return [Ragdoll::Core::Configuration] the current configuration instance.
-    def configuration(*args, **kwargs)
-      Ragdoll::Core.configuration(*args, **kwargs)
-    end
-
-    # @example
-    #   Ragdoll.reset_configuration!
-    #   puts Ragdoll.config.models[:default]
-    def reset_configuration!(*args, **kwargs)
-      Ragdoll::Core.reset_configuration!(*args, **kwargs)
-    end
-
-
-    #######################
-    # Document Management #
-    #######################
-
-    # Add a directory of documents to the system.
-    # @param path [String] the path to the directory containing documents.
-    # @example
-    #   Ragdoll.add_directory(path: "/path/to/documents", recursive: true)
-    # @param recursive [Boolean] whether to add documents from subdirectories.
-    def add_directory(*args, **kwargs)
-      Ragdoll::Core.add_directory(*args, **kwargs)
-    end
-
-    # Add a single document to the system.
-    # @example
-    #   Ragdoll.add_document(path: "/path/to/document.txt")
-    # @param path [String] the file path of the document to add.
-    def add_document(*args, **kwargs)
-      Ragdoll::Core.add_document(*args, **kwargs)
-    end
-    alias_method :add, :add_document
-
-    # Retrieve a document by its identifier.
-    # @param id [String] the identifier of the document to retrieve.
-    # @example
-    #   document = Ragdoll.get_document(id: "123")
-    #   puts document[:title] if document
-    # @return [Hash, nil] the document data or nil if not found.
-    def get_document(*args, **kwargs)
-      Ragdoll::Core.get_document(*args, **kwargs)
-    end
-    alias_method :get, :get_document
-
-    # List all documents in the system.
-    # @param options [Hash] options for listing documents, such as limit and offset.
-    # @example
-    #   documents = Ragdoll.list_documents(limit: 10)
-    #   documents.each { |doc| puts doc[:title] }
-    # @return [Array<Hash>] an array of document data.
-    def list_documents(*args, **kwargs)
-      Ragdoll::Core.list_documents(*args, **kwargs)
-    end
-    alias_method :list, :list_documents
-
-    # Delete a document by its identifier.
-    # @param id [String] the identifier of the document to delete.
-    # @example
-    #   success = Ragdoll.delete_document(id: "123")
-    #   puts "Deleted" if success
-    # @return [Boolean] true if the document was successfully deleted.
-    def delete_document(*args, **kwargs)
-      Ragdoll::Core.delete_document(*args, **kwargs)
-    end
-    alias_method :delete, :delete_document
-
-    # Get the status of a document.
-    # @param id [String] the identifier of the document to check status.
-    # @example
-    #   status = Ragdoll.document_status(id: "123")
-    #   puts status[:status]
-    # @return [Hash] the status information of the document.
-    def document_status(*args, **kwargs)
-      Ragdoll::Core.document_status(*args, **kwargs)
-    end
-    alias_method :status, :document_status
-
-    # Update a document's information.
-    # @param id [String] the identifier of the document to update.
-    # @param updates [Hash] the fields to update in the document.
-    # @example
-    #   updated_doc = Ragdoll.update_document(id: "123", title: "New Title")
-    #   puts updated_doc[:title]
-    # @return [Hash] the updated document data.
-    def update_document(*args, **kwargs)
-      Ragdoll::Core.update_document(*args, **kwargs)
-    end
-    alias_method :update, :update_document
-
-    # Retrieve all documents.
-    # @example
-    #   all_docs = Ragdoll.documents
-    #   all_docs.each { |doc| puts doc.title }
-    # @return [ActiveRecord::Relation] a relation of all documents.
-    def documents
-      Ragdoll::Core::Models::Document.all
-    end
-    alias_method :docs, :documents
-
-    #############
-    # Retrieval #
-    #############
-
-    # FIXME: This high-level API method should be able to take a query that is
-    #        a string or a file.  If its a file, then the downstream Process will
-    #        be responsible for reading the file and passing the contents to the
-    #        search method based upon whether the content is text, image or audio.
-
-    # Perform a search for documents based on a query.
-    # @param query [String] the search query string.
-    # @param options [Hash] additional search options, such as filters and limits.
-    # @example
-    #   response = Ragdoll.search(query: "example search")
-    #   response[:results].each { |result| puts result[:document_title] }
-    # @return [Hash] the search results.
-    def search(*args, **kwargs)
-      Ragdoll::Core.search(*args, **kwargs)
-    end
-
-    # Enhance a prompt with additional context.
-    # @param prompt [String] the original prompt to enhance.
-    # @param context_limit [Integer] the number of context chunks to include.
-    # @param options [Hash] additional options for enhancing the prompt.
-    # @example
-    #   enhanced = Ragdoll.enhance_prompt(prompt: "What is AI?", context_limit: 3)
-    #   puts enhanced[:enhanced_prompt]
-    # @return [Hash] the enhanced prompt data.
-    def enhance_prompt(*args, **kwargs)
-      Ragdoll::Core.enhance_prompt(*args, **kwargs)
-    end
-
-    # Retrieve context for a given query.
-    # @param query [String] the query to retrieve context for.
-    # @param limit [Integer] the number of context chunks to retrieve.
-    # @param options [Hash] additional options for context retrieval.
-    # @example
-    #   context = Ragdoll.get_context(query: "AI", limit: 5)
-    #   puts context[:combined_context]
-    # @return [Hash] the context data.
-    def get_context(*args, **kwargs)
-      Ragdoll::Core.get_context(*args, **kwargs)
-    end
-
-    # Search for content similar to a given query.
-    # @param query [String] the query to find similar content for.
-    # @param options [Hash] additional options for the search, such as filters and limits.
-    # @example
-    #   similar_content = Ragdoll.search_similar_content(query: "AI")
-    #   similar_content.each { |content| puts content[:document_title] }
-    # @return [Array<Hash>] an array of similar content data.
-    def search_similar_content(*args, **kwargs)
-      Ragdoll::Core.search_similar_content(*args, **kwargs)
-    end
-
-
-    ###############
-    # Misc. Stuff #
-    ###############
-
-    # Retrieve statistics about the system.
-    # @example
-    #   stats = Ragdoll.stats
-    #   puts stats[:total_documents]
-    # @return [Hash] the system statistics.
-    def stats(*args, **kwargs)
-      Ragdoll::Core.stats(*args, **kwargs)
-    end
-
-    # Check if the system is healthy.
-    # @example
-    #   puts "System is healthy" if Ragdoll.healthy?
-    # @return [Boolean] true if the system is healthy.
-    def healthy?(*args, **kwargs)
-      Ragdoll::Core.healthy?(*args, **kwargs)
-    end
-
-    # Retrieve the client instance.
-    # @example
-    #   client = Ragdoll.client
-    #   puts client.inspect
-    # @return [Ragdoll::Core::Client] the client instance.
-    def client(*args, **kwargs)
-      Ragdoll::Core.client(*args, **kwargs)
-    end
-
-    # Retrieve the version information of the Ragdoll modules.
-    # @example
-    #   versions = Ragdoll.version
-    #   versions.each { |version| puts version }
-    # @return [Array<String>] an array of version strings for each module.
-    def version
-      versions = []
-
-      ObjectSpace.each_object(Module) do |mod|
-        if mod.name =~ /^Ragdoll::\w+$/
-          if defined?(mod::VERSION) && mod::VERSION.is_a?(String)
-            versions << "#{mod.name}: #{mod::VERSION}"
-          end
-        end
-      end
-
-      versions
-    end
-  end
+  class Error < StandardError; end
 end

data/lib/tasks/import_task.thor

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'thor'
+require_relative '../ragdoll/import_job'
+
+module Ragdoll
+  class ImportTask < Thor
+    desc "import PATH", "Import documents from a file, glob, or directory"
+    method_option :recursive, aliases: "-r", type: :boolean, default: false, desc: "Recursively import files from directories"
+    method_option :jobs, aliases: ["-j", "--jobs"], type: :numeric, default: 1, desc: "Number of concurrent import jobs"
+    def import(path)
+      queue = SolidQueue.new(concurrency: options[:jobs])
+      files = if File.directory?(path)
+                if options[:recursive]
+                  Dir.glob("#{path}/**/*")
+                else
+                  Dir.glob("#{path}/*")
+                end
+              else
+                [path]
+              end
+
+      files.each do |file|
+        next unless File.file?(file)
+
+        queue.push(file) do |file|
+          Ragdoll::ImportJob.perform_async(file)
+        end
+      end
+    end
+  end
+end

data/lib/tasks/jobs_task.thor

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'thor'
+
+module Ragdoll
+  class JobsTask < Thor
+    desc "jobs [JOB_ID]", "Report the status of all running and queued import jobs, or a specific job if JOB_ID is provided"
+    method_option :stop_all, type: :boolean, default: false, desc: "Stop all running and queued jobs"
+    method_option :pause_all, type: :boolean, default: false, desc: "Pause all running jobs"
+    method_option :resume_all, type: :boolean, default: false, desc: "Resume all paused jobs"
+    method_option :stop, type: :boolean, default: false, desc: "Stop a specific job"
+    method_option :pause, type: :boolean, default: false, desc: "Pause a specific job"
+    method_option :resume, type: :boolean, default: false, desc: "Resume a specific job"
+    def jobs(job_id = nil)
+      if job_id
+        if options[:stop]
+          puts "Stopping job ID: #{job_id}..."
+        elsif options[:pause]
+          puts "Pausing job ID: #{job_id}..."
+        elsif options[:resume]
+          puts "Resuming job ID: #{job_id}..."
+        else
+          puts "Fetching status for job ID: #{job_id}..."
+        end
+      else
+        if options[:stop_all]
+          puts "Stopping all jobs..."
+        elsif options[:pause_all]
+          puts "Pausing all running jobs..."
+        elsif options[:resume_all]
+          puts "Resuming all paused jobs..."
+        else
+          puts "Fetching status of all running and queued import jobs..."
+          puts "Job ID: 12345, Status: Running, File: document1.txt"
+          puts "Job ID: 12346, Status: Running, File: document2.txt"
+        end
+      end
+    end
+  end
+end

data/lib/tasks/ragdoll_tasks.thor

@@ -0,0 +1,7 @@
+require 'thor'
+
+module Ragdoll
+  class Tasks < Thor
+    # Move your existing CLI tasks here
+  end
+end

data/lib/tasks/search_task.thor

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'thor'
+require_relative '../ragdoll/search'
+
+module Ragdoll
+  class SearchTask < Thor
+    desc "search PROMPT", "Search the database with a prompt"
+    method_option :prompt, aliases: ["-p", "--prompt"], type: :string, desc: "File path containing the prompt text"
+    method_option :max_count, type: :numeric, default: 10, desc: "Maximum number of results to return"
+    method_option :rerank, type: :boolean, default: false, desc: "Rerank results using keyword search"
+    def search(prompt = nil)
+      if options[:prompt]
+        prompt = File.read(options[:prompt])
+      end
+
+      unless prompt
+        puts "Please provide a prompt as a string or with the -p option."
+        return
+      end
+
+      keywords = extract_keywords(prompt)
+      vectorized_prompt = vectorize_prompt(prompt)
+      search_instance = Ragdoll::Search.new(vectorized_prompt)
+      results = search_instance.search_database(options[:max_count])
+
+      if options[:rerank]
+        results = rerank_results(results, keywords)
+      end
+
+      results.each do |result|
+        puts "Source: #{result[:source]}"
+        puts "Metadata: #{result[:metadata]}"
+        puts "--------------------------------"
+      end
+    end
+
+    private
+
+    def rerank_results(results, keywords)
+      results.sort_by do |result|
+        content = result[:source].downcase
+        keywords.count { |keyword| content.include?(keyword) }
+      end.reverse
+    end
+
+    def extract_keywords(prompt)
+      prompt.split.map(&:downcase).uniq
+    end
+
+    def vectorize_prompt(prompt)
+      prompt.split.map(&:downcase)
+    end
+  end
+end

metadata

@@ -1,14 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: ragdoll
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Dewayne VanHoozer
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
-dependencies: []
+date: 2025-02-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.1'
 description: Under development.  Contributors welcome.
 email:
 - dvanhoozer@gmail.com
@@ -18,47 +32,30 @@ extra_rdoc_files: []
 files:
 - README.md
 - Rakefile
-- db/migrate/001_enable_postgresql_extensions.rb
-- db/migrate/004_create_ragdoll_documents.rb
-- db/migrate/005_create_ragdoll_embeddings.rb
-- db/migrate/006_create_ragdoll_contents.rb
-- lib/ragdoll-core.rb
+- app/models/ragdoll/document.rb
+- app/models/ragdoll/embedding.rb
+- config/initializers/ragdoll.rb
+- config/routes.rb
+- db/migrate/20250218123456_create_documents.rb
+- lib/config/database.yml
+- lib/config/ragdoll.yml
 - lib/ragdoll.rb
-- lib/ragdoll/core.rb
-- lib/ragdoll/core/client.rb
-- lib/ragdoll/core/configuration.rb
-- lib/ragdoll/core/database.rb
-- lib/ragdoll/core/document_management.rb
-- lib/ragdoll/core/document_processor.rb
-- lib/ragdoll/core/embedding_service.rb
-- lib/ragdoll/core/errors.rb
-- lib/ragdoll/core/jobs/extract_keywords.rb
-- lib/ragdoll/core/jobs/extract_text.rb
-- lib/ragdoll/core/jobs/generate_embeddings.rb
-- lib/ragdoll/core/jobs/generate_summary.rb
-- lib/ragdoll/core/metadata_schemas.rb
-- lib/ragdoll/core/models/audio_content.rb
-- lib/ragdoll/core/models/content.rb
-- lib/ragdoll/core/models/document.rb
-- lib/ragdoll/core/models/embedding.rb
-- lib/ragdoll/core/models/image_content.rb
-- lib/ragdoll/core/models/text_content.rb
-- lib/ragdoll/core/search_engine.rb
-- lib/ragdoll/core/services/image_description_service.rb
-- lib/ragdoll/core/services/metadata_generator.rb
-- lib/ragdoll/core/shrine_config.rb
-- lib/ragdoll/core/text_chunker.rb
-- lib/ragdoll/core/text_generation_service.rb
-- lib/ragdoll/core/version.rb
-- lib/tasks/annotate.rake
-- lib/tasks/db.rake
+- lib/ragdoll/engine.rb
+- lib/ragdoll/import_job.rb
+- lib/ragdoll/ingestion.rb
+- lib/ragdoll/search.rb
+- lib/ragdoll/version.rb
+- lib/tasks/import_task.thor
+- lib/tasks/jobs_task.thor
+- lib/tasks/ragdoll_tasks.thor
+- lib/tasks/search_task.thor
 homepage: https://github.com/MadBomber/ragdoll
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/MadBomber/ragdoll
-  source_code_uri: https://github.com/MadBomber/ragdoll/blob/main
+  source_code_uri: https://github.com/MadBomber/ragdoll
   changelog_uri: https://github.com/MadBomber/ragdoll/blob/main/CHANGELOG.md
 rdoc_options: []
 require_paths:
@@ -67,14 +64,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.1
+rubygems_version: 3.6.3
 specification_version: 4
-summary: Multi-Modal Retrieval Augmented Generation
+summary: Ruby on Rails Engine
 test_files: []

data/db/migrate/001_enable_postgresql_extensions.rb

@@ -1,23 +0,0 @@
-class EnablePostgresqlExtensions < ActiveRecord::Migration[7.0]
-  def up
-    # This migration is now handled by the db:create rake task
-    # Just ensure required extensions are available
-    
-    # Vector similarity search (required for embeddings)
-    execute "CREATE EXTENSION IF NOT EXISTS vector"
-    
-    # Useful optional extensions for text processing and search
-    execute "CREATE EXTENSION IF NOT EXISTS unaccent"  # Remove accents from text
-    execute "CREATE EXTENSION IF NOT EXISTS pg_trgm"   # Trigram matching for fuzzy search
-    
-    # UUID support (useful for generating unique identifiers)
-    execute "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\""
-  end
-
-  def down
-    execute <<-SQL
-      DROP DATABASE IF EXISTS ragdoll_development;
-      DROP ROLE IF EXISTS ragdoll;
-    SQL
-  end
-end

data/db/migrate/004_create_ragdoll_documents.rb

@@ -1,70 +0,0 @@
-class CreateRagdollDocuments < ActiveRecord::Migration[7.0]
-  def change
-    create_table :ragdoll_documents,
-      comment: "Core documents table with LLM-generated structured metadata" do |t|
-
-      t.string :location, null: false,
-        comment: "Source location of document (file path, URL, or identifier)"
-
-      t.string :title, null: false,
-        comment: "Human-readable document title for display and search"
-
-      t.text :summary, null: false, default: "",
-        comment: "LLM-generated summary of document content"
-
-      t.text :keywords , null: false, default: "",
-        comment: "LLM-generated comma-separated keywords of document"
-
-      t.string :document_type, null: false, default: "text",
-        comment: "Document format type"
-
-      t.string :status, null: false, default: "pending",
-        comment: "Document processing status"
-
-      t.json :metadata, default: {},
-        comment: "LLM-generated structured metadata about the file"
-
-      t.timestamp :file_modified_at, null: false, default: -> { "CURRENT_TIMESTAMP" },
-        comment: "Timestamp when the source file was last modified"
-
-      t.timestamps null: false,
-        comment: "Standard creation and update timestamps"
-
-      ###########
-      # Indexes #
-      ###########
-
-      t.index :location, unique: true,
-        comment: "Unique index for document source lookup"
-
-      t.index :title,
-        comment: "Index for title-based search"
-
-      t.index :document_type,
-        comment: "Index for filtering by document type"
-
-      t.index :status,
-        comment: "Index for filtering by processing status"
-
-      t.index :created_at,
-        comment: "Index for chronological sorting"
-
-      t.index %i[document_type status],
-        comment: "Composite index for type+status filtering"
-
-      t.index "to_tsvector('english', COALESCE(title, '') ||
-        ' ' ||
-        COALESCE(metadata->>'summary', '') ||
-        ' ' || COALESCE(metadata->>'keywords', '') ||
-        ' ' || COALESCE(metadata->>'description', ''))",
-        using: :gin, name: "index_ragdoll_documents_on_fulltext_search",
-        comment: "Full-text search across title and metadata fields"
-
-      t.index "(metadata->>'document_type')", name: "index_ragdoll_documents_on_metadata_type",
-        comment: "Index for filtering by document type"
-
-      t.index "(metadata->>'classification')", name: "index_ragdoll_documents_on_metadata_classification",
-        comment: "Index for filtering by document classification"
-    end
-  end
-end

data/db/migrate/005_create_ragdoll_embeddings.rb

@@ -1,41 +0,0 @@
-class CreateRagdollEmbeddings < ActiveRecord::Migration[7.0]
-  def change
-    create_table :ragdoll_embeddings,
-      comment: "Polymorphic vector embeddings storage for semantic similarity search" do |t|
-
-        t.references :embeddable, polymorphic: true, null: false,
-        comment: "Polymorphic reference to embeddable content"
-
-      t.text :content, null: false, default: "",
-        comment: "Original text content that was embedded"
-
-      t.vector :embedding_vector, limit: 1536, null: false,
-        comment: "Vector embedding using pgvector"
-
-      t.integer :chunk_index, null: false,
-        comment: "Chunk index for ordering embeddings"
-
-      t.integer :usage_count, default: 0,
-        comment: "Number of times used in similarity searches"
-
-      t.datetime :returned_at,
-        comment: "Timestamp of most recent usage"
-
-      t.json :metadata, default: {},
-        comment: "Embedding-specific metadata (positions, processing info)"
-
-      t.timestamps null: false,
-        comment: "Standard creation and update timestamps"
-
-      ###########
-      # Indexes #
-      ###########
-
-      t.index %i[embeddable_type embeddable_id],
-        comment: "Index for finding embeddings by embeddable content"
-
-      t.index :embedding_vector, using: :ivfflat, opclass: :vector_cosine_ops, name: "index_ragdoll_embeddings_on_embedding_vector_cosine",
-        comment: "IVFFlat index for fast cosine similarity search"
-    end
-  end
-end