ragdoll-cli 0.1.8 → 0.1.10

data/lib/ragdoll/cli.rb

@@ -14,6 +14,8 @@ require_relative 'cli/commands/search'
 require_relative 'cli/commands/config'
 require_relative 'cli/commands/delete'
 require_relative 'cli/commands/update'
+require_relative 'cli/commands/analytics'
+require_relative 'cli/commands/keywords'
 
 module Ragdoll
   module CLI
@@ -37,16 +39,32 @@ module Ragdoll
       desc 'search QUERY', 'Search for documents matching the query'
       method_option :limit, type: :numeric, default: 10, aliases: '-l',
                             desc: 'Maximum number of results to return'
+      method_option :threshold, type: :numeric,
+                               desc: 'Similarity threshold (0.0-1.0, lower = more results)'
       method_option :content_type, type: :string, aliases: '-c',
                                    desc: 'Filter by content type (text, image, audio)'
       method_option :classification, type: :string, aliases: '-C',
                                      desc: 'Filter by classification'
       method_option :keywords, type: :string, aliases: '-k',
-                               desc: 'Filter by keywords (comma-separated)'
+                               desc: 'Filter by keywords (comma-separated). Use ragdoll keywords for keyword-only search'
+      method_option :keywords_all, type: :boolean, default: false, aliases: '-K',
+                                   desc: 'Require ALL keywords to match (default: any keyword matches)'
       method_option :tags, type: :string, aliases: '-T',
                            desc: 'Filter by tags (comma-separated)'
       method_option :format, type: :string, default: 'table', aliases: '-f',
                              desc: 'Output format (table, json, plain)'
+      method_option :session_id, type: :string, aliases: '-s',
+                                 desc: 'Session ID for search tracking'
+      method_option :user_id, type: :string, aliases: '-u',
+                               desc: 'User ID for search tracking'
+      method_option :track_search, type: :boolean, default: true, aliases: '-t',
+                                   desc: 'Enable search tracking (default: true)'
+      method_option :search_type, type: :string, default: 'semantic', aliases: '-S',
+                                  desc: 'Search type: semantic, hybrid, fulltext (default: semantic)'
+      method_option :semantic_weight, type: :numeric, aliases: '-w',
+                                      desc: 'Weight for semantic search in hybrid mode (0.0-1.0, default: 0.7)'
+      method_option :text_weight, type: :numeric, aliases: '-W',
+                                 desc: 'Weight for text search in hybrid mode (0.0-1.0, default: 0.3)'
       def search(query)
         Search.new.call(query, options)
       end
@@ -54,6 +72,12 @@ module Ragdoll
       desc 'config SUBCOMMAND', 'Manage configuration'
       subcommand 'config', Config
 
+      desc 'analytics SUBCOMMAND', 'Search analytics and reporting'
+      subcommand 'analytics', Analytics
+
+      desc 'keywords SUBCOMMAND', 'Manage and search by document keywords'
+      subcommand 'keywords', Keywords
+
       desc 'stats', 'Show document and embedding statistics'
       def stats
         client = StandaloneClient.new
@@ -78,11 +102,36 @@ module Ragdoll
           end
         end
 
-        return unless stats[:content_types]
+        if stats[:content_types]
+          puts "\nContent Types:"
+          stats[:content_types].each do |type, count|
+            puts "  #{type}: #{count}"
+          end
+        end
 
-        puts "\nContent Types:"
-        stats[:content_types].each do |type, count|
-          puts "  #{type}: #{count}"
+        # Add search analytics if available
+        begin
+          search_analytics = client.search_analytics(days: 30)
+          if search_analytics && !search_analytics.empty?
+            puts "\nSearch Analytics (last 30 days):"
+            puts "  Total searches: #{search_analytics[:total_searches] || 0}"
+            puts "  Unique queries: #{search_analytics[:unique_queries] || 0}"
+            puts "  Avg results per search: #{search_analytics[:avg_results_per_search] || 0}"
+            puts "  Avg execution time: #{search_analytics[:avg_execution_time] || 0}ms"
+            
+            if search_analytics[:search_types]
+              puts "  Search types:"
+              search_analytics[:search_types].each do |type, count|
+                puts "    #{type}: #{count}"
+              end
+            end
+            
+            puts "  Searches with results: #{search_analytics[:searches_with_results] || 0}"
+            puts "  Avg click-through rate: #{search_analytics[:avg_click_through_rate] || 0}%"
+          end
+        rescue StandardError => e
+          # Search analytics not available - silently continue
+          puts "\nSearch analytics: Not available (#{e.message})"
         end
       end
 
@@ -120,12 +169,22 @@ module Ragdoll
             puts "  Status: #{document[:status]}"
             puts "  Embeddings Count: #{document[:embeddings_count]}"
             puts "  Content Length: #{document[:content_length]} characters"
+            
+            # Show keywords prominently
+            keywords = document[:keywords] || document['keywords'] || []
+            if keywords.any?
+              puts "  Keywords: #{keywords.join(', ')}"
+            else
+              puts "  Keywords: (none)"
+            end
+            
             puts "  Created: #{document[:created_at]}"
             puts "  Updated: #{document[:updated_at]}"
 
-            if document[:metadata]
+            if document[:metadata] && document[:metadata].any?
               puts "\nMetadata:"
               document[:metadata].each do |key, value|
+                next if key == 'keywords' # Already displayed above
                 puts "  #{key}: #{value}"
               end
             end
@@ -154,9 +213,25 @@ module Ragdoll
                             desc: 'Maximum number of documents to list'
       method_option :format, type: :string, default: 'table', aliases: '-f',
                              desc: 'Output format (table, json, plain)'
+      method_option :keywords, type: :string, aliases: '-k',
+                               desc: 'Filter by keywords (comma-separated)'
+      method_option :keywords_all, type: :boolean, default: false, aliases: '-K',
+                                   desc: 'Require ALL keywords to match (default: any keyword matches)'
       def list
         client = StandaloneClient.new
-        documents = client.list_documents(limit: options[:limit])
+        
+        # Handle keyword filtering if provided
+        if options[:keywords]
+          keywords_array = options[:keywords].split(',').map(&:strip)
+          search_method = options[:keywords_all] ? :search_by_keywords_all : :search_by_keywords
+          documents = client.public_send(search_method, keywords_array, limit: options[:limit])
+          
+          puts "Listing documents with keywords: #{keywords_array.join(', ')}"
+          puts "Mode: #{options[:keywords_all] ? 'ALL keywords (AND)' : 'ANY keywords (OR)'}"
+          puts
+        else
+          documents = client.list_documents(limit: options[:limit])
+        end
 
         # Get accurate embeddings count for all documents
         documents.each do |doc|
@@ -176,16 +251,30 @@ module Ragdoll
             puts "#{doc[:id]}: #{doc[:title] || 'Untitled'}"
           end
         else
-          # Table format
-          puts 'ID'.ljust(10) + 'Title'.ljust(40) + 'Status'.ljust(12) + 'Embeddings'
-          puts '-' * 80
-          documents.each do |doc|
-            id = (doc[:id] || doc['id'] || '')[0..9].ljust(10)
-            title = (doc[:title] || doc['title'] || 'Untitled')[0..39].ljust(40)
-            status = (doc[:status] || doc['status'] || 'unknown')[0..11].ljust(12)
-            embeddings = (doc[:embeddings_count] || doc['embeddings_count'] || 0).to_s
-
-            puts "#{id}#{title}#{status}#{embeddings}"
+          # Table format - show keywords if keyword filtering is being used
+          if options[:keywords]
+            puts 'ID'.ljust(10) + 'Title'.ljust(30) + 'Keywords'.ljust(35) + 'Status'.ljust(10) + 'Emb'
+            puts '-' * 90
+            documents.each do |doc|
+              id = (doc[:id] || doc['id'] || '')[0..9].ljust(10)
+              title = (doc[:title] || doc['title'] || 'Untitled')[0..29].ljust(30)
+              keywords = (doc[:keywords] || doc['keywords'] || []).join(', ')[0..34].ljust(35)
+              status = (doc[:status] || doc['status'] || 'unknown')[0..9].ljust(10)
+              embeddings = (doc[:embeddings_count] || doc['embeddings_count'] || 0).to_s
+
+              puts "#{id}#{title}#{keywords}#{status}#{embeddings}"
+            end
+          else
+            puts 'ID'.ljust(10) + 'Title'.ljust(40) + 'Status'.ljust(12) + 'Embeddings'
+            puts '-' * 80
+            documents.each do |doc|
+              id = (doc[:id] || doc['id'] || '')[0..9].ljust(10)
+              title = (doc[:title] || doc['title'] || 'Untitled')[0..39].ljust(40)
+              status = (doc[:status] || doc['status'] || 'unknown')[0..11].ljust(12)
+              embeddings = (doc[:embeddings_count] || doc['embeddings_count'] || 0).to_s
+
+              puts "#{id}#{title}#{status}#{embeddings}"
+            end
           end
         end
       end
@@ -354,27 +443,150 @@ module Ragdoll
       end
 
       desc 'context QUERY', 'Get context for RAG applications'
-      method_option :limit, type: :numeric, default: 5, aliases: '-l', desc: 'Maximum number of context chunks'
+      method_option :limit, type: :numeric, default: 10, aliases: '-l', desc: 'Maximum number of context chunks'
+      method_option :threshold, type: :numeric, desc: 'Similarity threshold (0.0-1.0, lower = more results)'
       def context(query)
         client = StandaloneClient.new
-        ctx = client.get_context(query, limit: options[:limit])
-        puts JSON.pretty_generate(ctx)
+        context_options = { limit: options[:limit] }
+        context_options[:threshold] = options[:threshold] if options[:threshold]
+        ctx = client.get_context(query, **context_options)
+        
+        # Check if no context was found and provide enhanced feedback
+        if ctx[:context_chunks].empty?
+          # Get the underlying search response for statistics
+          search_response = client.search(query, **context_options)
+          display_no_results_feedback(query, search_response, 'context')
+        else
+          puts JSON.pretty_generate(ctx)
+        end
       end
 
       desc 'enhance PROMPT', 'Enhance a prompt with context'
-      method_option :context_limit, type: :numeric, default: 5, aliases: '-l', desc: 'Number of context chunks to include'
+      method_option :limit, type: :numeric, default: 10, aliases: '-l', desc: 'Maximum number of context chunks to include'
+      method_option :threshold, type: :numeric, desc: 'Similarity threshold (0.0-1.0, lower = more results)'
       def enhance(prompt)
         client = StandaloneClient.new
-        enhanced = client.enhance_prompt(prompt, context_limit: options[:context_limit])
-        puts enhanced
+        enhance_options = { context_limit: options[:limit] }
+        enhance_options[:threshold] = options[:threshold] if options[:threshold]
+        enhanced = client.enhance_prompt(prompt, **enhance_options)
+        
+        # Check if no context was found and provide enhanced feedback
+        if enhanced[:context_count] == 0
+          # Get the underlying search response for statistics
+          search_response = client.search(prompt, limit: enhance_options[:context_limit], threshold: enhance_options[:threshold])
+          display_no_results_feedback(prompt, search_response, 'enhance')
+        else
+          puts enhanced[:enhanced_prompt]
+        end
       end
 
+      desc 'search-history', 'Show recent search history'
+      method_option :limit, type: :numeric, default: 20, aliases: '-l',
+                            desc: 'Number of searches to show (default: 20)'
+      method_option :user_id, type: :string, aliases: '-u',
+                              desc: 'Filter by user ID'
+      method_option :session_id, type: :string, aliases: '-s',
+                                  desc: 'Filter by session ID'
+      method_option :format, type: :string, default: 'table', aliases: '-f',
+                             desc: 'Output format (table, json, plain)'
+      def search_history
+        analytics = Analytics.new
+        analytics.options = options
+        analytics.history
+      end
+
+      desc 'search-stats', 'Show detailed search analytics'
+      method_option :days, type: :numeric, default: 30, aliases: '-d',
+                           desc: 'Number of days to analyze (default: 30)'
+      method_option :format, type: :string, default: 'table', aliases: '-f',
+                             desc: 'Output format (table, json)'
+      def search_stats
+        analytics = Analytics.new
+        analytics.options = options
+        analytics.overview
+      end
+
+      desc 'trending', 'Show trending search queries'
+      method_option :limit, type: :numeric, default: 10, aliases: '-l',
+                            desc: 'Number of queries to show (default: 10)'
+      method_option :days, type: :numeric, default: 7, aliases: '-d',
+                           desc: 'Time period in days (default: 7)'
+      method_option :format, type: :string, default: 'table', aliases: '-f',
+                             desc: 'Output format (table, json)'
+      def trending
+        analytics = Analytics.new
+        analytics.options = options
+        analytics.trending
+      end
+
+      desc 'cleanup-searches', 'Cleanup old search records'
+      method_option :days, type: :numeric, default: 30, aliases: '-d',
+                           desc: 'Remove searches older than N days (default: 30)'
+      method_option :dry_run, type: :boolean, default: true, aliases: '-n',
+                              desc: 'Show what would be deleted without actually deleting (default: true)'
+      method_option :force, type: :boolean, default: false, aliases: '-f',
+                            desc: 'Actually perform the cleanup (overrides dry_run)'
+      def cleanup_searches
+        analytics = Analytics.new
+        analytics.options = options
+        analytics.cleanup
+      end
 
       private
 
       def load_configuration
         ConfigurationLoader.new.load
       end
+
+      def display_no_results_feedback(query, search_response, command_type)
+        puts "No results found for '#{query}'"
+        puts
+        
+        # Get statistics for better feedback
+        statistics = search_response[:statistics] || search_response['statistics']
+        execution_time = search_response[:execution_time_ms] || search_response['execution_time_ms']
+        
+        if statistics
+          threshold = statistics[:threshold_used] || statistics['threshold_used']
+          highest = statistics[:highest_similarity] || statistics['highest_similarity']
+          lowest = statistics[:lowest_similarity] || statistics['lowest_similarity']
+          average = statistics[:average_similarity] || statistics['average_similarity']
+          above_threshold = statistics[:similarities_above_threshold] || statistics['similarities_above_threshold']
+          total_checked = statistics[:total_embeddings_checked] || statistics['total_embeddings_checked']
+          
+          puts "Search Analysis:"
+          puts "  • Similarity threshold: #{threshold&.round(3) || 'N/A'}"
+          puts "  • Embeddings analyzed: #{total_checked || 0}"
+          if highest && lowest && average
+            puts "  • Similarity range: #{lowest.round(3)} - #{highest.round(3)} (avg: #{average.round(3)})"
+          end
+          puts "  • Results above threshold: #{above_threshold || 0}"
+          puts "  • Search time: #{execution_time || 0}ms"
+          puts
+          
+          # Provide actionable suggestions
+          if highest && threshold
+            if highest < threshold
+              suggested_threshold = (highest * 0.9).round(3)
+              puts "💡 Suggestions:"
+              puts "  • Lower the similarity threshold (highest found: #{highest.round(3)})"
+              puts "  • Try: ragdoll #{command_type} '#{query}' --threshold=#{suggested_threshold}"
+              if highest < 0.3
+                puts "  • Your query might not match the document content well"
+                puts "  • Try different or more specific search terms"
+                puts "  • Try keyword-based search: ragdoll keywords search KEYWORD"
+                puts "  • List available keywords: ragdoll keywords list"
+              end
+            elsif above_threshold > 0
+              puts "💡 Note: Found #{above_threshold} similar content above threshold #{threshold}"
+              puts "  This suggests an issue with result processing."
+            end
+          end
+        else
+          puts "No similarity statistics available."
+          puts "💡 Try lowering the similarity threshold with --threshold=0.5"
+        end
+      end
     end
   end
 end

metadata

@@ -1,15 +1,184 @@
 --- !ruby/object:Gem::Specification
 name: ragdoll-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.10
 platform: ruby
 authors:
 - Dewayne VanHoozer
 bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies: []
-description: Under development.  Contributors welcome.
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ragdoll
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.10
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debug_me
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: undercover
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Command-line interface for Ragdoll RAG system with semantic, full-text,
+  and hybrid search capabilities. Under development. Contributors welcome.
 email:
 - dvanhoozer@gmail.com
 executables:
@@ -21,9 +190,11 @@ files:
 - Rakefile
 - bin/ragdoll
 - lib/ragdoll/cli.rb
+- lib/ragdoll/cli/commands/analytics.rb
 - lib/ragdoll/cli/commands/config.rb
 - lib/ragdoll/cli/commands/delete.rb
 - lib/ragdoll/cli/commands/health.rb
+- lib/ragdoll/cli/commands/keywords.rb
 - lib/ragdoll/cli/commands/list.rb
 - lib/ragdoll/cli/commands/search.rb
 - lib/ragdoll/cli/commands/stats.rb