ragdoll-cli 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4794fa5ad365db7598b0cb87530391c6d8ca24c1ea431149e614e4997a4d5a6
-  data.tar.gz: 1a3fb4eb495a8ed99028b57925c6ddead351c4bc031e02b6b2c075c024c869aa
+  metadata.gz: 89c5200b031e287b125fb93d7025ea666ad3456bdc209fbc68a0dc70b673a410
+  data.tar.gz: 5bfa7dabb52f7b990b5935736ec1710a3b0f4cace386903bb9491bcf4f88720a
 SHA512:
-  metadata.gz: bd2009f1c320bc2cad1666b9f154f4e9c4899e9e703e89bded35cd3bc2a679d8f9a5726ebd3000546bab6c4a3fbe1bd79f2a98486ac85234fbdffde9a59479b6
-  data.tar.gz: dfa5ec3f7be52723c6850714ec7b3ebec1e9fd3cfd929fb970ab5bc1df2b77233924b2286f8ef3788a864e2d70ae817b7acc6ed361ac9278288d31d45087b40d
+  metadata.gz: f3eb42ababba85311e9d6bb3db1b5cf4f63f00e46a4a07ea8e9ec21841a9b4cdea44d36619b168a9298d08393d250917f66c14fa82c72b3866937833b3a44bae
+  data.tar.gz: 2a5085c3c38a9147f2bf26fb8a4ad2814bccdb05dfa3d5d76f5cb7ec21319d82d969758eece5b066f01b50100efa85ae9a9abfa20badca70d9144b5bd52c7906

data/README.md

@@ -42,9 +42,9 @@ This will install the `ragdoll` command-line tool.
    export OPENAI_API_KEY=your_api_key_here
    ```
 
-3. **Import documents:**
+3. **Add documents:**
    ```bash
-   ragdoll import "docs/*.pdf" --recursive
+   ragdoll add docs/*.pdf --recursive
    ```
 
 4. **Search for content:**
@@ -77,21 +77,57 @@ ragdoll config path
 ragdoll config database
 ```
 
-### Document Import
+### Document Management
 
 ```bash
-# Import files matching a pattern
-ragdoll import "documents/*.pdf"
+# Add a single document
+ragdoll add document.pdf
+
+# Add multiple documents and directories
+ragdoll add file1.pdf file2.txt ../docs
 
-# Import recursively from directory
-ragdoll import "docs/**/*" --recursive
+# Add files matching a pattern
+ragdoll add "documents/*.pdf"
+
+# Add recursively from directory (default: true)
+ragdoll add "docs/" --recursive
 
 # Filter by document type
-ragdoll import "files/*" --type pdf
+ragdoll add "files/*" --type pdf
+
+# Available types: pdf, docx, txt, md, html
+
+# Skip confirmation prompts
+ragdoll add docs/ --skip-confirmation
+
+# Force addition of duplicate documents
+ragdoll add document.pdf --force-duplicate
 
 # Available types: pdf, docx, txt, md, html
 ```
 
+#### Duplicate Detection
+
+Ragdoll automatically detects and prevents duplicate documents from being processed:
+
+```bash
+# Normal behavior - duplicates are detected and skipped
+ragdoll add document.pdf
+ragdoll add document.pdf  # Skipped (duplicate detected)
+
+# Force addition of duplicates when needed
+ragdoll add document.pdf --force-duplicate  # Creates new document despite duplicate
+
+# Batch processing safely handles mixed new/duplicate files
+ragdoll add docs/*.pdf  # Only processes new files, skips duplicates
+```
+
+**Duplicate Detection Features:**
+- **File-based detection**: Compares file location, modification time, and SHA256 hash
+- **Content-based detection**: Compares extracted text content and metadata
+- **Smart similarity**: Detects duplicates even with minor differences (5% tolerance)
+- **Performance optimized**: Uses database indexes for fast duplicate lookups
+
 ### Search
 
 ```bash
@@ -136,12 +172,9 @@ ragdoll search "backpropagation algorithm" --search-type fulltext
 ragdoll search "transformer architecture" --search-type hybrid --semantic-weight 0.7 --text-weight 0.3
 ```
 
-### Document Management
+### Document Operations
 
 ```bash
-# Add a single document
-ragdoll add <path>
-
 # List all documents
 ragdoll list
 
@@ -155,6 +188,10 @@ ragdoll list --format plain
 # Check document status
 ragdoll status <id>
 
+# Show detailed document information
+ragdoll show <id>
+ragdoll show <id> --format json
+
 # Update document metadata
 ragdoll update <id> --title "New Title"
 
@@ -164,8 +201,6 @@ ragdoll delete <id> --force  # Bypass confirmation
 
 # Show system statistics
 ragdoll stats
-ragdoll stats --format json
-ragdoll stats --format plain
 ```
 
 ### Retrieval Utilities

data/lib/ragdoll/cli/standalone_client.rb

@@ -6,7 +6,13 @@ module Ragdoll
       include DebugMe
 
       def add_document(path, **options)
-        Ragdoll.add_document(path: path, **options)
+        # Map force_duplicate option to force parameter for core library
+        core_options = options.dup
+        if core_options.key?(:force_duplicate)
+          core_options[:force] = core_options.delete(:force_duplicate)
+        end
+        
+        Ragdoll.add_document(path: path, **core_options)
       end
 
 

data/lib/ragdoll/cli/version.rb

@@ -3,6 +3,6 @@
 
 module Ragdoll
   module CLI
-    VERSION = "0.1.10"
+    VERSION = "0.1.11"
   end
 end

data/lib/ragdoll/cli.rb

@@ -285,17 +285,22 @@ module Ragdoll
                                 desc: 'Recursively process subdirectories (default: true)'
       method_option :type, type: :string, aliases: '-t',
                            desc: 'Filter by document type (pdf, docx, txt, md, html)'
-      method_option :force, type: :boolean, default: false, aliases: '-f',
+      method_option :skip_confirmation, type: :boolean, default: false, aliases: '-y',
                             desc: 'Skip confirmation prompts'
+      method_option :force_duplicate, type: :boolean, default: false, aliases: '-f',
+                            desc: 'Force addition of duplicate documents (bypasses duplicate detection)'
       def add(*paths)
         if paths.empty?
           puts 'Error: No paths provided'
-          puts 'Usage: ragdoll add PATH [PATH2] [PATH3]...'
+          puts 'Usage: ragdoll add PATH [PATH2] [PATH3]... [OPTIONS]'
           puts 'Examples:'
           puts '  ragdoll add file.pdf'
           puts '  ragdoll add ../docs'
           puts '  ragdoll add ../docs/**/*.md'
           puts '  ragdoll add file1.txt file2.pdf ../docs'
+          puts '  ragdoll add file.pdf --force-duplicate    # Force add even if duplicate'
+          puts '  ragdoll add ../docs --type=pdf            # Only process PDF files'
+          puts '  ragdoll add ../docs --skip-confirmation   # Skip prompts'
           exit 1
         end
 
@@ -338,12 +343,16 @@ module Ragdoll
 
         progressbar.finish
 
-        # Summary
+        # Summary with duplicate detection information
         success_count = all_results.count { |r| r && r[:status] == 'success' }
         error_count = all_results.count { |r| r && r[:status] == 'error' }
+        duplicate_count = all_results.count { |r| r && r[:status] == 'success' && r[:duplicate] }
+        new_count = success_count - duplicate_count
 
         puts "\nCompleted:"
-        puts "  Successfully added: #{success_count} files"
+        puts "  Successfully processed: #{success_count} files"
+        puts "    New documents: #{new_count}"
+        puts "    Duplicates #{options[:force_duplicate] ? 'forced' : 'detected'}: #{duplicate_count}" if duplicate_count > 0
         puts "  Errors: #{error_count} files"
 
         if error_count > 0
@@ -355,10 +364,33 @@ module Ragdoll
 
         return unless success_count > 0
 
-        puts "\nSuccessfully added files:"
-        all_results.select { |r| r && r[:status] == 'success' }.each do |result|
-          puts "  #{result[:file]} (ID: #{result[:document_id]})"
-          puts "    #{result[:message]}" if result[:message]
+        # Show new documents
+        new_documents = all_results.select { |r| r && r[:status] == 'success' && !r[:duplicate] }
+        if new_documents.any?
+          puts "\nNew documents added:"
+          new_documents.each do |result|
+            puts "  #{result[:file]} (ID: #{result[:document_id]})"
+            puts "    #{result[:message]}" if result[:message]
+          end
+        end
+
+        # Show duplicate information
+        duplicate_documents = all_results.select { |r| r && r[:status] == 'success' && r[:duplicate] }
+        if duplicate_documents.any?
+          if options[:force_duplicate]
+            puts "\nDuplicates forced to be added:"
+            duplicate_documents.each do |result|
+              puts "  #{result[:file]} (ID: #{result[:document_id]})"
+              puts "    #{result[:message]}" if result[:message]
+            end
+          else
+            puts "\nDuplicates detected (skipped):"
+            duplicate_documents.each do |result|
+              puts "  #{result[:file]} (existing ID: #{result[:document_id]})"
+              puts "    #{result[:message]}" if result[:message]
+            end
+            puts "\nTip: Use --force-duplicate (-f) to force adding duplicates"
+          end
         end
 
         puts "\nNote: Documents are being processed in the background."
@@ -412,12 +444,19 @@ module Ragdoll
 
       def process_single_file(client, path, options)
         begin
-          result = client.add_document(path)
+          # Pass force_duplicate parameter for duplicate detection
+          result = client.add_document(path, force_duplicate: options[:force_duplicate])
+          
+          # Determine if this was a duplicate detection
+          duplicate_detected = result[:duplicate] || (result[:message] && result[:message].include?('already exists'))
+          
           {
             file: path,
             document_id: result[:document_id],
             status: result[:success] ? 'success' : 'error',
-            message: result[:message]
+            message: result[:message],
+            duplicate: duplicate_detected,
+            forced: options[:force_duplicate]
           }
         rescue StandardError => e
           {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ragdoll-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.11
 platform: ruby
 authors:
 - Dewayne VanHoozer