ragnar-cli 0.1.0.pre.3 → 0.1.0.pre.4

data/lib/ragnar/cli.rb

@@ -1,27 +1,82 @@
+require_relative "cli_visualization"
+require_relative "config"
+require "thor/interactive"
+require "stringio"
+require "fileutils"
+
 module Ragnar
   class CLI < Thor
+    include CLIVisualization
+    include Thor::Interactive::Command
+
+    # Configure interactive mode
+    configure_interactive(
+      prompt: Config.instance.interactive_prompt,
+      allow_nested: false,
+      history_file: Config.instance.history_file,
+      default_handler: proc do |input, thor_instance|
+        puts "[DEBUG] Default handler called: #{input}" if ENV["DEBUG"]
+
+        begin
+          # IMPORTANT: Use direct method call, NOT invoke(), to avoid Thor's
+          # silent deduplication that prevents repeated calls to the same method
+          result = thor_instance.query(input.strip)
+          puts "[DEBUG] Default handler completed" if ENV["DEBUG"]
+          result
+        rescue => e
+          puts "[DEBUG] Default handler error: #{e.message}" if ENV["DEBUG"]
+          puts "[DEBUG] Backtrace: #{e.backtrace.first(3)}" if ENV["DEBUG"]
+          raise e
+        end
+      end
+    )
+
+    # Class variables for caching expensive resources in interactive mode
+    class_variable_set(:@@cached_database, nil)
+    class_variable_set(:@@cached_embedder, nil)
+    class_variable_set(:@@cached_llm_manager, nil)
+    class_variable_set(:@@cached_query_processor, nil)
+    class_variable_set(:@@cached_db_path, nil)
+
     desc "index PATH", "Index text files from PATH (file or directory)"
-    option :db_path, type: :string, default: Ragnar::DEFAULT_DB_PATH, desc: "Path to Lance database"
-    option :chunk_size, type: :numeric, default: Ragnar::DEFAULT_CHUNK_SIZE, desc: "Chunk size in tokens"
-    option :chunk_overlap, type: :numeric, default: Ragnar::DEFAULT_CHUNK_OVERLAP, desc: "Chunk overlap in tokens"
-    option :model, type: :string, default: Ragnar::DEFAULT_EMBEDDING_MODEL, desc: "Embedding model to use"
+    option :db_path, type: :string, desc: "Path to Lance database (default from config)"
+    option :chunk_size, type: :numeric, desc: "Chunk size in tokens (default from config)"
+    option :chunk_overlap, type: :numeric, desc: "Chunk overlap in tokens (default from config)"
+    option :model, type: :string, desc: "Embedding model to use (default from config)"
     def index(path)
-      unless File.exist?(path)
-        say "Error: Path does not exist: #{path}", :red
+      # Expand user paths (handle ~ in user input)
+      expanded_path = File.expand_path(path)
+      
+      unless File.exist?(expanded_path)
+        say "Error: Path does not exist: #{expanded_path}", :red
         exit 1
       end
 
       say "Indexing files from: #{path}", :green
 
+      # Debug options in interactive mode
+      puts "Debug - options: #{options.inspect}" if ENV['DEBUG']
+
+      # Get config instance
+      config = Config.instance
+      
+      # Clear database cache when indexing new content
+      db_path = options[:db_path] || config.database_path
+      if @@cached_db_path == db_path
+        @@cached_database = nil
+        @@cached_query_processor = nil
+      end
+
       indexer = Indexer.new(
-        db_path: options[:db_path],
-        chunk_size: options[:chunk_size],
-        chunk_overlap: options[:chunk_overlap],
-        embedding_model: options[:model]
+        db_path: db_path,
+        chunk_size: options[:chunk_size] || config.chunk_size,
+        chunk_overlap: options[:chunk_overlap] || config.chunk_overlap,
+        embedding_model: options[:model] || config.embedding_model,
+        show_progress: config.show_progress?
       )
 
       begin
-        stats = indexer.index_path(path)
+        stats = indexer.index_path(expanded_path)
         say "\nIndexing complete!", :green
         say "Files processed: #{stats[:files_processed]}"
         say "Chunks created: #{stats[:chunks_created]}"
@@ -33,31 +88,39 @@ module Ragnar
     end
 
     desc "train-umap", "Train UMAP model on existing embeddings"
-    option :db_path, type: :string, default: Ragnar::DEFAULT_DB_PATH, desc: "Path to Lance database"
+    option :db_path, type: :string, desc: "Path to Lance database (default from config)"
     option :n_components, type: :numeric, default: 50, desc: "Number of dimensions for reduction"
     option :n_neighbors, type: :numeric, default: 15, desc: "Number of neighbors for UMAP"
     option :min_dist, type: :numeric, default: 0.1, desc: "Minimum distance for UMAP"
-    option :model_path, type: :string, default: "umap_model.bin", desc: "Path to save UMAP model"
+    option :model_path, type: :string, desc: "Path to save UMAP model"
     def train_umap
       say "Training UMAP model on embeddings...", :green
 
+      config = Config.instance
+      # Use model_path from options if provided, otherwise use config models_dir
+      model_path = if options[:model_path]
+        options[:model_path]
+      else
+        File.join(config.models_dir, "umap_model.bin")
+      end
+      
       processor = UmapProcessor.new(
-        db_path: options[:db_path],
-        model_path: options[:model_path]
+        db_path: options[:db_path] || config.database_path,
+        model_path: model_path
       )
 
       begin
         stats = processor.train(
-          n_components: options[:n_components],
-          n_neighbors: options[:n_neighbors],
-          min_dist: options[:min_dist]
+          n_components: options[:n_components] || 50,
+          n_neighbors: options[:n_neighbors] || 15,
+          min_dist: options[:min_dist] || 0.1
         )
 
         say "\nUMAP training complete!", :green
         say "Embeddings processed: #{stats[:embeddings_count]}"
         say "Original dimensions: #{stats[:original_dims]}"
         say "Reduced dimensions: #{stats[:reduced_dims]}"
-        say "Model saved to: #{options[:model_path]}"
+        say "Model saved to: #{processor.model_path}"
       rescue => e
         say "Error during UMAP training: #{e.message}", :red
         exit 1
@@ -65,12 +128,19 @@ module Ragnar
     end
 
     desc "apply-umap", "Apply trained UMAP model to reduce embedding dimensions"
-    option :db_path, type: :string, default: Ragnar::DEFAULT_DB_PATH, desc: "Path to Lance database"
-    option :model_path, type: :string, default: "umap_model.bin", desc: "Path to UMAP model"
+    option :db_path, type: :string, desc: "Path to Lance database (default from config)"
+    option :model_path, type: :string, desc: "Path to UMAP model"
     option :batch_size, type: :numeric, default: 100, desc: "Batch size for processing"
     def apply_umap
-      unless File.exist?(options[:model_path])
-        say "Error: UMAP model not found at: #{options[:model_path]}", :red
+      config = Config.instance
+      model_path = if options[:model_path]
+        options[:model_path]
+      else
+        File.join(config.models_dir, "umap_model.bin")
+      end
+
+      unless File.exist?(model_path)
+        say "Error: UMAP model not found at: #{model_path}", :red
         say "Please run 'train-umap' first to create a model.", :yellow
         exit 1
       end
@@ -78,12 +148,12 @@ module Ragnar
       say "Applying UMAP model to embeddings...", :green
 
       processor = UmapProcessor.new(
-        db_path: options[:db_path],
-        model_path: options[:model_path]
+        db_path: options[:db_path] || config.database_path,
+        model_path: model_path
       )
 
       begin
-        stats = processor.apply(batch_size: options[:batch_size])
+        stats = processor.apply(batch_size: options[:batch_size] || 100)
 
         say "\nUMAP application complete!", :green
         say "Embeddings processed: #{stats[:processed]}"
@@ -96,18 +166,21 @@ module Ragnar
     end
 
     desc "topics", "Extract and display topics from indexed documents"
-    option :db_path, type: :string, default: Ragnar::DEFAULT_DB_PATH, desc: "Path to Lance database"
+    option :db_path, type: :string, desc: "Path to Lance database (default from config)"
     option :min_cluster_size, type: :numeric, default: 5, desc: "Minimum documents per topic"
     option :method, type: :string, default: "hybrid", desc: "Labeling method: fast, quality, or hybrid"
     option :export, type: :string, desc: "Export topics to file (json or html)"
     option :verbose, type: :boolean, default: false, aliases: "-v", desc: "Show detailed processing"
+    option :summarize, type: :boolean, default: false, aliases: "-s", desc: "Generate human-readable topic summaries using LLM"
+    option :llm_model, type: :string, default: "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", desc: "LLM model for summarization"
+    option :gguf_file, type: :string, default: "tinyllama-1.1b-chat-v1.0.q4_k_m.gguf", desc: "GGUF file name for LLM model"
     def topics
       require_relative 'topic_modeling'
 
       say "Extracting topics from indexed documents...", :green
 
-      # Load embeddings and documents from database
-      database = Database.new(options[:db_path])
+      # Load embeddings and documents from database - use cache in interactive mode
+      database = get_cached_database(options[:db_path] || Config.instance.database_path)
 
       begin
         # Get all documents with embeddings
@@ -130,7 +203,7 @@ module Ragnar
         # Check if we have reduced embeddings available
         first_doc = docs_with_embeddings.first
         has_reduced = first_doc[:reduced_embedding] && !first_doc[:reduced_embedding].empty?
-        
+
         if has_reduced
           embeddings = docs_with_embeddings.map { |d| d[:reduced_embedding] }
           say "Using reduced embeddings (#{embeddings.first.size} dimensions)", :yellow if options[:verbose]
@@ -142,7 +215,7 @@ module Ragnar
           # Let the engine handle dimensionality reduction if needed
           reduce_dims = true
         end
-        
+
         documents = docs_with_embeddings.map { |d| d[:chunk_text] }
         metadata = docs_with_embeddings.map { |d| { file_path: d[:file_path], chunk_index: d[:chunk_index] } }
 
@@ -164,12 +237,36 @@ module Ragnar
           metadata: metadata
         )
 
+        # Generate summaries if requested
+        if options[:summarize] && topics.any?
+          say "Generating topic summaries with LLM...", :yellow
+          begin
+            require 'red-candle'
+
+            # Initialize LLM for summarization once
+            say "Loading model: #{options[:llm_model]}", :cyan if options[:verbose]
+            llm = Candle::LLM.from_pretrained(options[:llm_model], gguf_file: options[:gguf_file])
+
+            # Add summaries to topics
+            topics.each_with_index do |topic, i|
+              say "  Summarizing topic #{i+1}/#{topics.length}...", :yellow if options[:verbose]
+              topic.instance_variable_set(:@summary, summarize_topic(topic, llm))
+            end
+
+            say "Topic summaries generated!", :green
+          rescue => e
+            say "Warning: Could not generate topic summaries: #{e.message}", :yellow
+            say "Proceeding without summaries...", :yellow
+          end
+        end
+
         # Display results
-        display_topics(topics)
+        display_topics(topics, show_summaries: options[:summarize])
 
         # Export if requested
         if options[:export]
-          export_topics(topics, options[:export])
+          # Pass embeddings and cluster IDs for visualization
+          export_topics(topics, options[:export], embeddings: embeddings, cluster_ids: engine.instance_variable_get(:@cluster_ids))
         end
 
       rescue => e
@@ -184,51 +281,80 @@ module Ragnar
     option :k, type: :numeric, default: 5, desc: "Number of results to return"
     option :show_scores, type: :boolean, default: false, desc: "Show similarity scores"
     def search(query_text)
-      database = Database.new(options[:database])
-      embedder = Embedder.new
-      
+      database = get_cached_database(options[:database] || Config.instance.database_path)
+      embedder = get_cached_embedder()
+
       # Generate embedding for query
       query_embedding = embedder.embed_text(query_text)
-      
+
       # Search for similar documents
       results = database.search_similar(query_embedding, k: options[:k])
-      
+
       if results.empty?
         say "No results found.", :yellow
         return
       end
-      
+
       say "Found #{results.length} results:\n", :green
-      
+
       results.each_with_index do |result, idx|
         say "#{idx + 1}. File: #{result[:file_path]}", :cyan
         say "   Chunk: #{result[:chunk_index]}"
-        
+
         if options[:show_scores]
           say "   Distance: #{result[:distance].round(4)}"
         end
-        
+
         # Show preview of content
         preview = result[:chunk_text][0..200].gsub(/\s+/, ' ')
         say "   Content: #{preview}..."
         say ""
       end
     end
-    
+
     desc "query QUESTION", "Query the RAG system"
-    option :db_path, type: :string, default: Ragnar::DEFAULT_DB_PATH, desc: "Path to Lance database"
+    option :db_path, type: :string, desc: "Path to Lance database (default from config)"
     option :top_k, type: :numeric, default: 3, desc: "Number of top documents to use"
     option :verbose, type: :boolean, default: false, aliases: "-v", desc: "Show detailed processing steps"
     option :json, type: :boolean, default: false, desc: "Output as JSON"
     def query(question)
-      processor = QueryProcessor.new(db_path: options[:db_path])
+      puts "Debug - Query called with: #{question.inspect}" if ENV['DEBUG']
+      puts "Debug - Options: #{options.inspect}" if ENV['DEBUG']
+
+      processor = get_cached_query_processor(options[:db_path] || Config.instance.database_path)
+      puts "Debug - Processor: #{processor.class}" if ENV['DEBUG']
 
       begin
-        result = processor.query(question, top_k: options[:top_k], verbose: options[:verbose])
+        config = Config.instance
+        result = processor.query(
+          question, 
+          top_k: options[:top_k] || config.query_top_k, 
+          verbose: options[:verbose] || false,
+          enable_rewriting: config.enable_query_rewriting?
+        )
+        puts "Debug - Result keys: #{result.keys}" if ENV['DEBUG']
 
         if options[:json]
           puts JSON.pretty_generate(result)
+        elsif interactive?
+          # Clean output for interactive mode - just answer, confidence, and sources
+          say "" # Add blank line before answer for spacing
+          say result[:answer]
+          
+          if result[:confidence]
+            say "\nConfidence: #{result[:confidence]}%", :magenta
+          end
+
+          if result[:sources] && !result[:sources].empty?
+            say "\nSources:", :blue
+            result[:sources].each_with_index do |source, idx|
+              say "  #{idx + 1}. #{source[:source_file]}" if source[:source_file]
+            end
+          end
+          
+          say "" # Add blank line for spacing
         else
+          # Full output for CLI mode
           say "\n" + "="*60, :green
           say "Query: #{result[:query]}", :cyan
 
@@ -250,7 +376,7 @@ module Ragnar
             end
           end
 
-          if options[:verbose] && result[:sub_queries]
+          if (options[:verbose] || false) && result[:sub_queries]
             say "\nSub-queries used:", :yellow
             result[:sub_queries].each { |sq| say "  - #{sq}" }
           end
@@ -259,15 +385,15 @@ module Ragnar
         end
       rescue => e
         say "Error processing query: #{e.message}", :red
-        say e.backtrace.first(5).join("\n") if options[:verbose]
+        puts "Debug - Full backtrace: #{e.backtrace.join("\n")}" if ENV['DEBUG']
         exit 1
       end
     end
 
     desc "stats", "Show database statistics"
-    option :db_path, type: :string, default: Ragnar::DEFAULT_DB_PATH, desc: "Path to Lance database"
+    option :db_path, type: :string, desc: "Path to Lance database (default from config)"
     def stats
-      db = Database.new(options[:db_path])
+      db = get_cached_database(options[:db_path] || Config.instance.database_path)
       stats = db.get_stats
 
       say "\nDatabase Statistics", :green
@@ -293,8 +419,325 @@ module Ragnar
       say "Ragnar v#{Ragnar::VERSION}"
     end
 
+    desc "config", "Show current configuration"
+    def config
+      config = Config.instance
+      
+      say "\nConfiguration Settings:", :cyan
+      say "-" * 40
+      
+      if config.config_exists?
+        say "Config file: #{config.config_file_path}", :green
+      else
+        say "Config file: None (using defaults)", :yellow
+      end
+      
+      say "\nPaths:", :cyan
+      say "  Database: #{config.database_path}"
+      say "  Models: #{config.models_dir}"
+      say "  History: #{config.history_file}"
+      
+      say "\nEmbeddings:", :cyan
+      say "  Model: #{config.embedding_model}"
+      say "  Chunk size: #{config.chunk_size}"
+      say "  Chunk overlap: #{config.chunk_overlap}"
+      
+      say "\nLLM:", :cyan
+      say "  Model: #{config.llm_model}"
+      say "  GGUF file: #{config.llm_gguf_file}"
+      
+      say "\nUMAP:", :cyan
+      say "  Reduced dimensions: #{config.get('umap.reduced_dimensions', Ragnar::DEFAULT_REDUCED_DIMENSIONS)}"
+      say "  N neighbors: #{config.get('umap.n_neighbors', 15)}"
+      say "  Min distance: #{config.get('umap.min_dist', 0.1)}"
+      
+      say "\nQuery:", :cyan
+      say "  Top K: #{config.query_top_k}"
+      say "  Query rewriting: #{config.enable_query_rewriting?}"
+    end
+    
+    desc "model", "Show current LLM model information"
+    def model
+      config = Config.instance
+      
+      say "\nLLM Model Configuration:", :cyan
+      say "-" * 40
+      
+      say "\nModel:", :green
+      say "  Repository: #{config.llm_model}"
+      say "  GGUF file: #{config.llm_gguf_file}"
+      
+      # Check if model files exist
+      model_path = File.join(config.models_dir, config.llm_gguf_file)
+      if File.exist?(model_path)
+        size_mb = (File.size(model_path) / 1024.0 / 1024.0).round(2)
+        say "\nModel file exists: #{model_path} (#{size_mb} MB)", :green
+      else
+        say "\nModel file not found: #{model_path}", :yellow
+        say "Run 'ragnar query' to download automatically", :yellow
+      end
+    end
+
+    desc "clear-cache", "Clear cached instances (useful in interactive mode)"
+    def clear_cache_command
+      clear_cache
+      say "Cache cleared. Next commands will create fresh instances.", :green
+    end
+
+    desc "reset", "Reset Ragnar data (database, models, cache)"
+    option :all, type: :boolean, default: false, aliases: "-a", desc: "Reset everything (database, models, cache)"
+    option :database, type: :boolean, default: false, aliases: "-d", desc: "Reset database only"
+    option :models, type: :boolean, default: false, aliases: "-m", desc: "Reset UMAP models only"
+    option :cache, type: :boolean, default: false, aliases: "-c", desc: "Clear cache only"
+    option :force, type: :boolean, default: false, aliases: "-f", desc: "Skip confirmation prompt"
+    def reset
+      # Determine what to reset
+      reset_all = options[:all]
+      reset_db = options[:database] || reset_all
+      reset_models = options[:models] || reset_all
+      reset_cache = options[:cache] || reset_all
+      
+      # If no specific options, default to all
+      if !reset_db && !reset_models && !reset_cache
+        reset_all = true
+        reset_db = reset_models = reset_cache = true
+      end
+      
+      # Build confirmation message
+      items_to_reset = []
+      items_to_reset << "database" if reset_db
+      items_to_reset << "UMAP models" if reset_models
+      items_to_reset << "cache" if reset_cache
+      
+      # Get paths that will be affected
+      config = Config.instance
+      db_path = options[:db_path] || config.database_path
+      model_path = File.join(config.models_dir, "umap_model.bin")
+      
+      # Show what will be deleted
+      say "\nWARNING: This will delete the following:", :red
+      say "-" * 40
+      
+      if reset_db
+        say "Database: #{db_path}", :cyan
+        if File.exist?(db_path)
+          stats = Database.new(db_path).get_stats rescue nil
+          if stats
+            say "  (#{stats[:total_documents]} documents, #{stats[:total_chunks]} chunks)", :white
+          end
+        else
+          say "  (does not exist)", :white
+        end
+      end
+      
+      if reset_models
+        say "UMAP models:", :cyan
+        model_files = [
+          model_path,
+          model_path.sub(/\.bin$/, '_metadata.json'),
+          model_path.sub(/\.bin$/, '_embeddings.json')  # Old format, if exists
+        ]
+        model_files.each do |file|
+          if File.exist?(file)
+            say "  #{file} (#{(File.size(file) / 1024.0).round(1)} KB)", :white
+          end
+        end
+        if model_files.none? { |f| File.exist?(f) }
+          say "  (no models found)", :white
+        end
+      end
+      
+      if reset_cache
+        cache_dir = File.expand_path("~/.cache/ragnar")
+        say "Cache directory: #{cache_dir}", :cyan
+        if Dir.exist?(cache_dir)
+          cache_size = Dir.glob(File.join(cache_dir, "**/*"))
+            .select { |f| File.file?(f) }
+            .sum { |f| File.size(f) } / 1024.0 / 1024.0
+          say "  (#{cache_size.round(1)} MB)", :white
+        else
+          say "  (does not exist)", :white
+        end
+      end
+      
+      say "-" * 40
+      
+      # Ask for confirmation unless --force
+      unless options[:force]
+        message = "\nAre you sure you want to reset #{items_to_reset.join(', ')}?"
+        
+        # Check if we're in interactive mode
+        if ENV['THOR_INTERACTIVE_SESSION'] == 'true'
+          # In interactive mode, use a simple prompt
+          say message, :yellow
+          response = ask("Type 'yes' to confirm, anything else to cancel:", :yellow)
+          confirmed = response.downcase == 'yes'
+        else
+          # In CLI mode, use Thor's yes? method
+          confirmed = yes?(message + " (y/N)", :yellow)
+        end
+        
+        unless confirmed
+          say "\nReset cancelled.", :cyan
+          return
+        end
+      end
+      
+      # Perform the reset
+      say "\nResetting...", :green
+      
+      if reset_db && File.exist?(db_path)
+        say "Removing database: #{db_path}"
+        FileUtils.rm_rf(db_path)
+        say "  ✓ Database removed", :green
+      end
+      
+      if reset_models
+        model_files = [
+          model_path,
+          model_path.sub(/\.bin$/, '_metadata.json'),
+          model_path.sub(/\.bin$/, '_embeddings.json')
+        ]
+        model_files.each do |file|
+          if File.exist?(file)
+            say "Removing model file: #{file}"
+            FileUtils.rm_f(file)
+            say "  ✓ Removed", :green
+          end
+        end
+      end
+      
+      if reset_cache
+        # Clear in-memory cache
+        clear_cache
+        
+        # Optionally clear cache directory (but preserve history)
+        cache_dir = File.expand_path("~/.cache/ragnar")
+        if Dir.exist?(cache_dir)
+          # Preserve history file
+          history_file = File.join(cache_dir, "history")
+          history_content = File.read(history_file) if File.exist?(history_file)
+          
+          # Remove cache directory contents except history
+          Dir.glob(File.join(cache_dir, "*")).each do |item|
+            next if File.basename(item) == "history"
+            if File.directory?(item)
+              FileUtils.rm_rf(item)
+            else
+              FileUtils.rm_f(item)
+            end
+            say "Removed cache item: #{File.basename(item)}", :green
+          end
+        end
+        say "  ✓ Cache cleared", :green
+      end
+      
+      say "\nReset complete!", :green
+      say "You can now start fresh with 'ragnar index <path>'", :cyan
+    end
+
+    desc "init-config", "Generate a configuration file with current defaults"
+    option :global, type: :boolean, default: false, aliases: "-g", desc: "Create global config in home directory"
+    option :force, type: :boolean, default: false, aliases: "-f", desc: "Overwrite existing config file"
+    def init_config
+      config = Config.instance
+      
+      if options[:global]
+        config_path = File.expand_path('~/.ragnar.yml')
+      else
+        config_path = File.join(Dir.pwd, '.ragnar.yml')
+      end
+      
+      if File.exist?(config_path) && !options[:force]
+        say "Config file already exists at: #{config_path}", :yellow
+        say "Use --force to overwrite, or choose a different location.", :yellow
+        return
+      end
+      
+      generated_path = config.generate_config_file(config_path)
+      say "Config file created at: #{generated_path}", :green
+      say "Edit this file to customize Ragnar's behavior.", :cyan
+      
+      if config.config_exists?
+        say "\nNote: Currently using config from: #{config.config_file_path}", :yellow
+      end
+    end
+
     private
 
+    # Cached instance helpers for interactive mode
+    def get_cached_database(db_path = nil)
+      # Use config default if no path provided
+      db_path ||= Config.instance.database_path
+      
+      # Cache database per path - clear cache if path changes
+      if @@cached_db_path != db_path
+        @@cached_database = nil
+        @@cached_db_path = db_path
+        @@cached_query_processor = nil  # Also clear dependent caches
+      end
+
+      @@cached_database ||= Database.new(db_path)
+    end
+
+    def get_cached_embedder(model_name = nil)
+      # Use config default if no model specified
+      model_name ||= Config.instance.embedding_model
+      @@cached_embedder ||= Embedder.new(model_name: model_name)
+    end
+
+    def get_cached_llm_manager
+      @@cached_llm_manager ||= LLMManager.instance
+    end
+
+    def get_cached_query_processor(db_path = nil)
+      # Use config default if no path provided
+      db_path ||= Config.instance.database_path
+      
+      # Cache query processor per database path
+      if @@cached_db_path != db_path || @@cached_query_processor.nil?
+        @@cached_query_processor = QueryProcessor.new(db_path: db_path)
+      end
+
+      @@cached_query_processor
+    end
+
+    def clear_cache
+      @@cached_database = nil
+      @@cached_embedder = nil
+      @@cached_llm_manager = nil
+      @@cached_query_processor = nil
+      @@cached_db_path = nil
+    end
+
+
+    def summarize_topic(topic, llm)
+      # Get representative documents for context
+      sample_docs = topic.representative_docs(k: 3)
+
+      # Simple, clear prompt for summarization
+      prompt = <<~PROMPT
+        Summarize what connects these documents in 1-2 sentences:
+
+        Key terms: #{topic.terms.first(5).join(', ')}
+
+        Documents:
+        #{sample_docs.map.with_index { |doc, i| "#{i+1}. #{doc}" }.join("\n")}
+
+        Summary:
+      PROMPT
+
+      begin
+        summary = llm.generate(prompt).strip
+        # Clean up common artifacts
+        summary = summary.lines.first&.strip || "Related documents"
+        summary = summary.gsub(/^(Summary:|Topic:|Documents:)/i, '').strip
+        summary.empty? ? "Documents about #{topic.terms.first(2).join(' and ')}" : summary
+      rescue => e
+        "Documents about #{topic.terms.first(2).join(' and ')}"
+      end
+    end
+
     def fetch_all_documents(database)
       # Temporary workaround to get all documents
       # In production, we'd add a proper method to Database class
@@ -321,9 +764,12 @@ module Ragnar
       []
     end
 
-    def display_topics(topics)
+    def display_topics(topics, show_summaries: false)
       say "\n" + "="*60, :green
       say "Topic Analysis Results", :cyan
+      if show_summaries
+        say "  (with LLM-generated summaries)", :yellow
+      end
       say "="*60, :green
 
       if topics.empty?
@@ -342,21 +788,21 @@ module Ragnar
         say "\n" + "─" * 40, :blue
         say "MAJOR TOPICS (≥20 docs)", :blue
         say "─" * 40, :blue
-        display_topic_group(large_topics, :cyan)
+        display_topic_group(large_topics, :cyan, show_summaries: show_summaries)
       end
 
       if medium_topics.any?
         say "\n" + "─" * 40, :yellow
         say "MEDIUM TOPICS (10-19 docs)", :yellow
         say "─" * 40, :yellow
-        display_topic_group(medium_topics, :yellow)
+        display_topic_group(medium_topics, :yellow, show_summaries: show_summaries)
       end
 
       if small_topics.any?
         say "\n" + "─" * 40, :white
         say "MINOR TOPICS (<10 docs)", :white
         say "─" * 40, :white
-        display_topic_group(small_topics, :white)
+        display_topic_group(small_topics, :white, show_summaries: show_summaries)
       end
 
       # Summary statistics
@@ -380,10 +826,18 @@ module Ragnar
       say "    Small (<10): #{small_topics.length} topics, #{small_topics.sum(&:size)} docs"
     end
 
-    def display_topic_group(topics, color)
+    def display_topic_group(topics, color, show_summaries: false)
       topics.sort_by { |t| -t.size }.each_with_index do |topic, idx|
         say "\n#{topic.label || 'Unlabeled'} (#{topic.size} docs)", color
 
+        # Show LLM summary if available
+        if show_summaries
+          summary = topic.instance_variable_get(:@summary)
+          if summary
+            say "  Summary: #{summary}", :green
+          end
+        end
+
         # Show coherence as a bar
         if topic.coherence > 0
           coherence_pct = (topic.coherence * 100).round(0)
@@ -395,8 +849,8 @@ module Ragnar
         # Compact term display
         say "  Terms: #{topic.terms.first(6).join(' • ')}" if topic.terms.any?
 
-        # Short sample
-        if topic.representative_docs(k: 1).any?
+        # Short sample (unless we showed a summary)
+        if !show_summaries && topic.representative_docs(k: 1).any?
           preview = topic.representative_docs(k: 1).first
           preview = preview[0..100] + "..." if preview.length > 100
           say "  \"#{preview}\"", :white
@@ -404,25 +858,34 @@ module Ragnar
       end
     end
 
-    def export_topics(topics, format)
+    def export_topics(topics, format, embeddings: nil, cluster_ids: nil)
       case format.downcase
       when 'json'
         export_topics_json(topics)
       when 'html'
-        export_topics_html(topics)
+        export_topics_html(topics, embeddings: embeddings, cluster_ids: cluster_ids)
       else
         say "Unknown export format: #{format}. Use 'json' or 'html'.", :red
       end
     end
 
     def export_topics_json(topics)
+      topics_data = topics.map do |topic|
+        topic_hash = topic.to_h
+        # Add summary if it exists
+        summary = topic.instance_variable_get(:@summary)
+        topic_hash[:summary] = summary if summary
+        topic_hash
+      end
+
       data = {
         generated_at: Time.now.iso8601,
-        topics: topics.map(&:to_h),
+        topics: topics_data,
         summary: {
           total_topics: topics.length,
           total_documents: topics.sum(&:size),
-          average_size: (topics.sum(&:size).to_f / topics.length).round(1)
+          average_size: (topics.sum(&:size).to_f / topics.length).round(1),
+          has_summaries: topics.any? { |t| t.instance_variable_get(:@summary) }
         }
       }
 
@@ -431,9 +894,9 @@ module Ragnar
       say "Topics exported to: #{filename}", :green
     end
 
-    def export_topics_html(topics)
+    def export_topics_html(topics, embeddings: nil, cluster_ids: nil)
       # Generate self-contained HTML with D3.js visualization
-      html = generate_topic_visualization_html(topics)
+      html = generate_topic_visualization_html(topics, embeddings: embeddings, cluster_ids: cluster_ids)
 
       filename = "topics_#{Time.now.strftime('%Y%m%d_%H%M%S')}.html"
       File.write(filename, html)
@@ -446,113 +909,5 @@ module Ragnar
       end
     end
 
-    def generate_topic_visualization_html(topics)
-      # Convert topics to JSON for D3.js
-      topics_json = topics.map do |topic|
-        {
-          id: topic.id,
-          label: topic.label || "Topic #{topic.id}",
-          size: topic.size,
-          terms: topic.terms.first(10),
-          coherence: topic.coherence,
-          samples: topic.representative_docs(k: 2).map { |d| d[0..200] }
-        }
-      end.to_json
-
-      # HTML template with embedded D3.js
-      <<~HTML
-        <!DOCTYPE html>
-        <html>
-        <head>
-          <meta charset="utf-8">
-          <title>Topic Visualization</title>
-          <script src="https://d3js.org/d3.v7.min.js"></script>
-          <style>
-            body { font-family: -apple-system, sans-serif; margin: 20px; }
-            #viz { width: 100%; height: 500px; border: 1px solid #ddd; }
-            .topic { cursor: pointer; }
-            .topic:hover { opacity: 0.8; }
-            #details { margin-top: 20px; padding: 15px; background: #f5f5f5; }
-            .term { display: inline-block; margin: 5px; padding: 5px 10px; background: #e0e0e0; border-radius: 3px; }
-          </style>
-        </head>
-        <body>
-          <h1>Topic Analysis Results</h1>
-          <div id="viz"></div>
-          <div id="details">Click on a topic to see details</div>
-
-          <script>
-            const data = #{topics_json};
-
-            // Create bubble chart
-            const width = document.getElementById('viz').clientWidth;
-            const height = 500;
-
-            const svg = d3.select("#viz")
-              .append("svg")
-              .attr("width", width)
-              .attr("height", height);
-
-            // Create scale for bubble sizes
-            const sizeScale = d3.scaleSqrt()
-              .domain([0, d3.max(data, d => d.size)])
-              .range([10, 50]);
-
-            // Create color scale
-            const colorScale = d3.scaleSequential(d3.interpolateViridis)
-              .domain([0, 1]);
-
-            // Create force simulation
-            const simulation = d3.forceSimulation(data)
-              .force("x", d3.forceX(width / 2).strength(0.05))
-              .force("y", d3.forceY(height / 2).strength(0.05))
-              .force("collide", d3.forceCollide(d => sizeScale(d.size) + 2));
-
-            // Create bubbles
-            const bubbles = svg.selectAll(".topic")
-              .data(data)
-              .enter().append("g")
-              .attr("class", "topic");
-
-            bubbles.append("circle")
-              .attr("r", d => sizeScale(d.size))
-              .attr("fill", d => colorScale(d.coherence))
-              .attr("stroke", "#fff")
-              .attr("stroke-width", 2);
-
-            bubbles.append("text")
-              .text(d => d.label)
-              .attr("text-anchor", "middle")
-              .attr("dy", ".3em")
-              .style("font-size", d => Math.min(sizeScale(d.size) / 3, 14) + "px");
-
-            // Add click handler
-            bubbles.on("click", function(event, d) {
-              showDetails(d);
-            });
-
-            // Update positions
-            simulation.on("tick", () => {
-              bubbles.attr("transform", d => `translate(${d.x},${d.y})`);
-            });
-
-            // Show topic details
-            function showDetails(topic) {
-              const details = document.getElementById('details');
-              details.innerHTML = `
-                <h2>${topic.label}</h2>
-                <p><strong>Documents:</strong> ${topic.size}</p>
-                <p><strong>Coherence:</strong> ${(topic.coherence * 100).toFixed(1)}%</p>
-                <p><strong>Top Terms:</strong></p>
-                <div>${topic.terms.map(t => `<span class="term">${t}</span>`).join('')}</div>
-                <p><strong>Sample Documents:</strong></p>
-                ${topic.samples.map(s => `<p style="font-size: 0.9em; color: #666;">"${s}..."</p>`).join('')}
-              `;
-            }
-          </script>
-        </body>
-        </html>
-      HTML
-    end
   end
-end
+end