ragdoll-cli 0.1.10 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4794fa5ad365db7598b0cb87530391c6d8ca24c1ea431149e614e4997a4d5a6
-  data.tar.gz: 1a3fb4eb495a8ed99028b57925c6ddead351c4bc031e02b6b2c075c024c869aa
+  metadata.gz: 95d02e9e2a6f2ed5278406deae5fb36a8b55aaa1fa9228e3ca4bc8ad602daaf8
+  data.tar.gz: 34e38890fb8829a213057bb4f3142ce81c620e1e2463b70b8b3d2d0692beae5a
 SHA512:
-  metadata.gz: bd2009f1c320bc2cad1666b9f154f4e9c4899e9e703e89bded35cd3bc2a679d8f9a5726ebd3000546bab6c4a3fbe1bd79f2a98486ac85234fbdffde9a59479b6
-  data.tar.gz: dfa5ec3f7be52723c6850714ec7b3ebec1e9fd3cfd929fb970ab5bc1df2b77233924b2286f8ef3788a864e2d70ae817b7acc6ed361ac9278288d31d45087b40d
+  metadata.gz: 5767875a5559078f341466140788711c61b45f9cd52bc77fd0844b43574a84c9462367abbe29c88d9c997df86f6e6996f116fbe2c4bb5201321f13d8ccac6eb5
+  data.tar.gz: 68d51ae3e7061e95a9a2f74f55d269afdc3af9bb377edd6edb5758eae7a49938a4f3032cb3c358a4e82d367b1a28ff26882183b6f7714fa4cf04eb556a988c8f

data/README.md

@@ -1,7 +1,7 @@
-<div align="center" style="background-color: yellow; color: black; padding: 20px; margin: 20px 0; border: 2px solid black; font-size: 48px; font-weight: bold;">
-  ⚠️ CAUTION ⚠️<br />
-  Software Under Development by a Crazy Man
-</div>
+> [!CAUTION]
+> **Software Under Development by a Crazy Man**
+>
+> Evolved from multi-modal to unified text-based RAG architecture.
 <br />
 <div align="center">
   <table>
@@ -12,7 +12,7 @@
         </a>
       </td>
       <td width="50%" valign="top">
-        <p>Multi-modal RAG (Retrieval-Augmented Generation) is an architecture that integrates multiple data types (such as text, images, and audio) to enhance AI response generation. It combines retrieval-based methods, which fetch relevant information from a knowledge base, with generative large language models (LLMs) that create coherent and contextually appropriate outputs. This approach allows for more comprehensive and engaging user interactions, such as chatbots that respond with both text and images or educational tools that incorporate visual aids into learning materials. By leveraging various modalities, multi-modal RAG systems improve context understanding and user experience.</p>
+        <p>Unified Text-Based RAG converts all media types—images, audio, documents—into comprehensive text representations before vectorization. This approach enables powerful cross-modal search where you can find images through AI-generated descriptions, audio through transcripts, and all content through a single, unified text-based search index. The system combines intelligent text conversion with retrieval-based methods and generative large language models for enhanced AI response generation.</p>
       </td>
     </tr>
   </table>
@@ -20,7 +20,7 @@
 
 # Ragdoll::CLI
 
-Standalone command-line interface for the Ragdoll RAG (Retrieval-Augmented Generation) system. Provides document import, search, and management capabilities through a simple CLI.
+Standalone command-line interface for the Ragdoll unified text-based RAG system. Converts all media types to searchable text and provides powerful cross-modal search capabilities through a simple CLI.
 
 ## Installation
 
@@ -42,9 +42,9 @@ This will install the `ragdoll` command-line tool.
    export OPENAI_API_KEY=your_api_key_here
    ```
 
-3. **Import documents:**
+3. **Add documents:**
    ```bash
-   ragdoll import "docs/*.pdf" --recursive
+   ragdoll add docs/*.pdf --recursive
    ```
 
 4. **Search for content:**
@@ -77,21 +77,57 @@ ragdoll config path
 ragdoll config database
 ```
 
-### Document Import
+### Document Management
 
 ```bash
-# Import files matching a pattern
-ragdoll import "documents/*.pdf"
+# Add a single document
+ragdoll add document.pdf
 
-# Import recursively from directory
-ragdoll import "docs/**/*" --recursive
+# Add multiple documents and directories
+ragdoll add file1.pdf file2.txt ../docs
+
+# Add files matching a pattern
+ragdoll add "documents/*.pdf"
+
+# Add recursively from directory (default: true)
+ragdoll add "docs/" --recursive
 
 # Filter by document type
-ragdoll import "files/*" --type pdf
+ragdoll add "files/*" --type pdf
+
+# Available types: pdf, docx, txt, md, html
+
+# Skip confirmation prompts
+ragdoll add docs/ --skip-confirmation
+
+# Force addition of duplicate documents
+ragdoll add document.pdf --force-duplicate
 
 # Available types: pdf, docx, txt, md, html
 ```
 
+#### Duplicate Detection
+
+Ragdoll automatically detects and prevents duplicate documents from being processed:
+
+```bash
+# Normal behavior - duplicates are detected and skipped
+ragdoll add document.pdf
+ragdoll add document.pdf  # Skipped (duplicate detected)
+
+# Force addition of duplicates when needed
+ragdoll add document.pdf --force-duplicate  # Creates new document despite duplicate
+
+# Batch processing safely handles mixed new/duplicate files
+ragdoll add docs/*.pdf  # Only processes new files, skips duplicates
+```
+
+**Duplicate Detection Features:**
+- **File-based detection**: Compares file location, modification time, and SHA256 hash
+- **Content-based detection**: Compares extracted text content and metadata
+- **Smart similarity**: Detects duplicates even with minor differences (5% tolerance)
+- **Performance optimized**: Uses database indexes for fast duplicate lookups
+
 ### Search
 
 ```bash
@@ -136,12 +172,9 @@ ragdoll search "backpropagation algorithm" --search-type fulltext
 ragdoll search "transformer architecture" --search-type hybrid --semantic-weight 0.7 --text-weight 0.3
 ```
 
-### Document Management
+### Document Operations
 
 ```bash
-# Add a single document
-ragdoll add <path>
-
 # List all documents
 ragdoll list
 
@@ -155,6 +188,10 @@ ragdoll list --format plain
 # Check document status
 ragdoll status <id>
 
+# Show detailed document information
+ragdoll show <id>
+ragdoll show <id> --format json
+
 # Update document metadata
 ragdoll update <id> --title "New Title"
 
@@ -164,8 +201,6 @@ ragdoll delete <id> --force  # Bypass confirmation
 
 # Show system statistics
 ragdoll stats
-ragdoll stats --format json
-ragdoll stats --format plain
 ```
 
 ### Retrieval Utilities

data/lib/ragdoll/cli/commands/search.rb

@@ -10,6 +10,11 @@ module Ragdoll
 
         puts "Searching for: #{query}"
         puts "Search type: #{options[:search_type] || 'semantic'}"
+
+        # Show deprecation warning for content_type
+        if options[:content_type]
+          puts "⚠️  DEPRECATED: --content_type option is deprecated. Unified text-based system converts all media to searchable text."
+        end
         
         # Show hybrid search weights if applicable
         if options[:search_type] == 'hybrid'

data/lib/ragdoll/cli/configuration_loader.rb

@@ -22,7 +22,9 @@ module Ragdoll
 
         default_config = {
           'llm_provider' => 'openai',
-          'embedding_model' => 'text-embedding-3-small',
+          'embedding_model' => 'text-embedding-3-large',
+          'embedding_provider' => 'openai',
+          'use_unified_content' => true,
           'chunk_size' => 1000,
           'chunk_overlap' => 200,
           'search_similarity_threshold' => 0.7,
@@ -72,7 +74,13 @@ module Ragdoll
             ragdoll_config.llm_provider = config['llm_provider']&.to_sym || :openai
           end
           if ragdoll_config.respond_to?(:embedding_model=)
-            ragdoll_config.embedding_model = config['embedding_model'] || 'text-embedding-3-small'
+            ragdoll_config.embedding_model = config['embedding_model'] || 'text-embedding-3-large'
+          end
+          if ragdoll_config.respond_to?(:embedding_provider=)
+            ragdoll_config.embedding_provider = config['embedding_provider']&.to_sym || :openai
+          end
+          if ragdoll_config.respond_to?(:use_unified_content=)
+            ragdoll_config.use_unified_content = config['use_unified_content'] != false
           end
 
           # Processing settings

data/lib/ragdoll/cli/standalone_client.rb

@@ -6,7 +6,13 @@ module Ragdoll
       include DebugMe
 
       def add_document(path, **options)
-        Ragdoll.add_document(path: path, **options)
+        # Map force_duplicate option to force parameter for core library
+        core_options = options.dup
+        if core_options.key?(:force_duplicate)
+          core_options[:force] = core_options.delete(:force_duplicate)
+        end
+        
+        Ragdoll.add_document(path: path, **core_options)
       end
 
 

data/lib/ragdoll/cli/version.rb

@@ -3,6 +3,6 @@
 
 module Ragdoll
   module CLI
-    VERSION = "0.1.10"
+    VERSION = "0.1.12"
   end
 end

data/lib/ragdoll/cli.rb

@@ -42,7 +42,7 @@ module Ragdoll
       method_option :threshold, type: :numeric,
                                desc: 'Similarity threshold (0.0-1.0, lower = more results)'
       method_option :content_type, type: :string, aliases: '-c',
-                                   desc: 'Filter by content type (text, image, audio)'
+                                   desc: 'DEPRECATED: Content type filtering (unified text-based system converts all media to text)'
       method_option :classification, type: :string, aliases: '-C',
                                      desc: 'Filter by classification'
       method_option :keywords, type: :string, aliases: '-k',
@@ -285,17 +285,22 @@ module Ragdoll
                                 desc: 'Recursively process subdirectories (default: true)'
       method_option :type, type: :string, aliases: '-t',
                            desc: 'Filter by document type (pdf, docx, txt, md, html)'
-      method_option :force, type: :boolean, default: false, aliases: '-f',
+      method_option :skip_confirmation, type: :boolean, default: false, aliases: '-y',
                             desc: 'Skip confirmation prompts'
+      method_option :force_duplicate, type: :boolean, default: false, aliases: '-f',
+                            desc: 'Force addition of duplicate documents (bypasses duplicate detection)'
       def add(*paths)
         if paths.empty?
           puts 'Error: No paths provided'
-          puts 'Usage: ragdoll add PATH [PATH2] [PATH3]...'
+          puts 'Usage: ragdoll add PATH [PATH2] [PATH3]... [OPTIONS]'
           puts 'Examples:'
           puts '  ragdoll add file.pdf'
           puts '  ragdoll add ../docs'
           puts '  ragdoll add ../docs/**/*.md'
           puts '  ragdoll add file1.txt file2.pdf ../docs'
+          puts '  ragdoll add file.pdf --force-duplicate    # Force add even if duplicate'
+          puts '  ragdoll add ../docs --type=pdf            # Only process PDF files'
+          puts '  ragdoll add ../docs --skip-confirmation   # Skip prompts'
           exit 1
         end
 
@@ -338,12 +343,16 @@ module Ragdoll
 
         progressbar.finish
 
-        # Summary
+        # Summary with duplicate detection information
         success_count = all_results.count { |r| r && r[:status] == 'success' }
         error_count = all_results.count { |r| r && r[:status] == 'error' }
+        duplicate_count = all_results.count { |r| r && r[:status] == 'success' && r[:duplicate] }
+        new_count = success_count - duplicate_count
 
         puts "\nCompleted:"
-        puts "  Successfully added: #{success_count} files"
+        puts "  Successfully processed: #{success_count} files"
+        puts "    New documents: #{new_count}"
+        puts "    Duplicates #{options[:force_duplicate] ? 'forced' : 'detected'}: #{duplicate_count}" if duplicate_count > 0
         puts "  Errors: #{error_count} files"
 
         if error_count > 0
@@ -355,10 +364,33 @@ module Ragdoll
 
         return unless success_count > 0
 
-        puts "\nSuccessfully added files:"
-        all_results.select { |r| r && r[:status] == 'success' }.each do |result|
-          puts "  #{result[:file]} (ID: #{result[:document_id]})"
-          puts "    #{result[:message]}" if result[:message]
+        # Show new documents
+        new_documents = all_results.select { |r| r && r[:status] == 'success' && !r[:duplicate] }
+        if new_documents.any?
+          puts "\nNew documents added:"
+          new_documents.each do |result|
+            puts "  #{result[:file]} (ID: #{result[:document_id]})"
+            puts "    #{result[:message]}" if result[:message]
+          end
+        end
+
+        # Show duplicate information
+        duplicate_documents = all_results.select { |r| r && r[:status] == 'success' && r[:duplicate] }
+        if duplicate_documents.any?
+          if options[:force_duplicate]
+            puts "\nDuplicates forced to be added:"
+            duplicate_documents.each do |result|
+              puts "  #{result[:file]} (ID: #{result[:document_id]})"
+              puts "    #{result[:message]}" if result[:message]
+            end
+          else
+            puts "\nDuplicates detected (skipped):"
+            duplicate_documents.each do |result|
+              puts "  #{result[:file]} (existing ID: #{result[:document_id]})"
+              puts "    #{result[:message]}" if result[:message]
+            end
+            puts "\nTip: Use --force-duplicate (-f) to force adding duplicates"
+          end
         end
 
         puts "\nNote: Documents are being processed in the background."
@@ -412,12 +444,19 @@ module Ragdoll
 
       def process_single_file(client, path, options)
         begin
-          result = client.add_document(path)
+          # Pass force_duplicate parameter for duplicate detection
+          result = client.add_document(path, force_duplicate: options[:force_duplicate])
+          
+          # Determine if this was a duplicate detection
+          duplicate_detected = result[:duplicate] || (result[:message] && result[:message].include?('already exists'))
+          
           {
             file: path,
             document_id: result[:document_id],
             status: result[:success] ? 'success' : 'error',
-            message: result[:message]
+            message: result[:message],
+            duplicate: duplicate_detected,
+            forced: options[:force_duplicate]
           }
         rescue StandardError => e
           {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ragdoll-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.12
 platform: ruby
 authors:
 - Dewayne VanHoozer
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.10
+        version: 0.1.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.10
+        version: 0.1.12
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -177,8 +177,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Command-line interface for Ragdoll RAG system with semantic, full-text,
-  and hybrid search capabilities. Under development. Contributors welcome.
+description: Command-line interface for Ragdoll's unified text-based RAG system. Converts
+  all media types (images, audio, documents) to searchable text for powerful cross-modal
+  search capabilities. Under development. Contributors welcome.
 email:
 - dvanhoozer@gmail.com
 executables:
@@ -227,7 +228,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.1
+rubygems_version: 3.7.2
 specification_version: 4
-summary: Multi-Modal Retrieval Augmented Generation for the CLI
+summary: Unified Text-Based RAG CLI for Cross-Modal Search
 test_files: []