ragnar-cli 0.1.0.pre.3 → 0.1.0.pre.5

data/lib/ragnar/cli_umap.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module Ragnar
+  class CLI < Thor
+    class Umap < Thor
+      desc "train", "Train UMAP model on existing embeddings"
+      option :db_path, type: :string, desc: "Path to Lance database (default from config)"
+      option :n_components, type: :numeric, default: 50, desc: "Number of dimensions for reduction"
+      option :n_neighbors, type: :numeric, default: 15, desc: "Number of neighbors for UMAP"
+      option :min_dist, type: :numeric, default: 0.1, desc: "Minimum distance for UMAP"
+      option :model_path, type: :string, desc: "Path to save UMAP model"
+      def train
+        say "Training UMAP model on embeddings...", :green
+
+        config = Config.instance
+        model_path = if options[:model_path]
+          options[:model_path]
+        else
+          File.join(config.models_dir, "umap_model.bin")
+        end
+
+        processor = UmapProcessor.new(
+          db_path: options[:db_path] || config.database_path,
+          model_path: model_path
+        )
+
+        begin
+          stats = processor.train(
+            n_components: options[:n_components] || 50,
+            n_neighbors: options[:n_neighbors] || 15,
+            min_dist: options[:min_dist] || 0.1
+          )
+
+          say "\nUMAP training complete!", :green
+          say "Embeddings processed: #{stats[:embeddings_count]}"
+          say "Original dimensions: #{stats[:original_dims]}"
+          say "Reduced dimensions: #{stats[:reduced_dims]}"
+          say "Model saved to: #{processor.model_path}"
+        rescue => e
+          say "Error during UMAP training: #{e.message}", :red
+          exit 1
+        end
+      end
+
+      desc "apply", "Apply trained UMAP model to reduce embedding dimensions"
+      option :db_path, type: :string, desc: "Path to Lance database (default from config)"
+      option :model_path, type: :string, desc: "Path to UMAP model"
+      option :batch_size, type: :numeric, default: 100, desc: "Batch size for processing"
+      def apply
+        config = Config.instance
+        model_path = if options[:model_path]
+          options[:model_path]
+        else
+          File.join(config.models_dir, "umap_model.bin")
+        end
+
+        unless File.exist?(model_path)
+          say "Error: UMAP model not found at: #{model_path}", :red
+          say "Please run 'ragnar umap train' first to create a model.", :yellow
+          exit 1
+        end
+
+        say "Applying UMAP model to embeddings...", :green
+
+        processor = UmapProcessor.new(
+          db_path: options[:db_path] || config.database_path,
+          model_path: model_path
+        )
+
+        begin
+          stats = processor.apply(batch_size: options[:batch_size] || 100)
+
+          say "\nUMAP application complete!", :green
+          say "Embeddings processed: #{stats[:processed]}"
+          say "Already processed: #{stats[:skipped]}"
+          say "Errors: #{stats[:errors]}" if stats[:errors] > 0
+        rescue => e
+          say "Error applying UMAP: #{e.message}", :red
+          exit 1
+        end
+      end
+    end
+  end
+end

data/lib/ragnar/cli_visualization.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Ragnar
+  module CLIVisualization
+    def generate_topic_visualization_html(topics, embeddings: nil, cluster_ids: nil)
+      # Convert topics to JSON for D3.js
+      topics_json = topics.map do |topic|
+        topic_data = {
+          id: topic.id,
+          label: topic.label || "Topic #{topic.id}",
+          size: topic.size,
+          terms: topic.terms.first(10),
+          coherence: topic.coherence,
+          samples: topic.representative_docs(k: 2).map { |d| d[0..200] }
+        }
+        
+        # Add summary if it exists
+        summary = topic.instance_variable_get(:@summary)
+        topic_data[:summary] = summary if summary
+        
+        topic_data
+      end.to_json
+
+      # HTML template with enhanced visualization
+      <<~HTML
+        <!DOCTYPE html>
+        <html>
+        <head>
+          <meta charset="utf-8">
+          <title>Topic Visualization</title>
+          <script src="https://d3js.org/d3.v7.min.js"></script>
+          <style>
+            body { 
+              font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; 
+              margin: 20px;
+              background: #f8f9fa;
+            }
+            .container {
+              max-width: 1400px;
+              margin: 0 auto;
+            }
+            h1 {
+              color: #2c3e50;
+              margin-bottom: 10px;
+            }
+            .viz-panel {
+              background: white;
+              border-radius: 8px;
+              box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+              padding: 15px;
+            }
+            #bubble-viz {
+              height: 500px;
+            }
+            .topic { cursor: pointer; }
+            .topic:hover { opacity: 0.8; }
+            #details { 
+              background: white;
+              padding: 20px;
+              border-radius: 8px;
+              box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+              margin-top: 20px;
+            }
+            .term { 
+              display: inline-block; 
+              margin: 5px; 
+              padding: 5px 10px; 
+              background: #e3f2fd; 
+              border-radius: 3px;
+              color: #1976d2;
+              font-size: 14px;
+            }
+          </style>
+        </head>
+        <body>
+          <div class="container">
+            <h1>Topic Analysis Results</h1>
+            
+            <div class="viz-panel">
+              <h3>Topic Overview</h3>
+              <div id="bubble-viz"></div>
+            </div>
+
+            <div id="details">Click on a topic to see details</div>
+          </div>
+
+          <script>
+            const topicsData = #{topics_json};
+
+            // Create bubble chart
+            function createBubbleChart() {
+              const width = document.getElementById('bubble-viz').clientWidth - 30;
+              const height = 470;
+
+              const svg = d3.select("#bubble-viz")
+                .append("svg")
+                .attr("width", width)
+                .attr("height", height);
+
+              // Create scale for bubble sizes
+              const sizeScale = d3.scaleSqrt()
+                .domain([0, d3.max(topicsData, d => d.size)])
+                .range([20, 60]);
+
+              // Create color scale
+              const colorScale = d3.scaleSequential(d3.interpolateTurbo)
+                .domain([0, 1]);
+
+              // Create force simulation
+              const simulation = d3.forceSimulation(topicsData)
+                .force("x", d3.forceX(width / 2).strength(0.05))
+                .force("y", d3.forceY(height / 2).strength(0.05))
+                .force("collide", d3.forceCollide(d => sizeScale(d.size) + 3));
+
+              // Create bubbles
+              const bubbles = svg.selectAll(".topic")
+                .data(topicsData)
+                .enter().append("g")
+                .attr("class", "topic");
+
+              bubbles.append("circle")
+                .attr("r", d => sizeScale(d.size))
+                .attr("fill", d => colorScale(d.coherence))
+                .attr("stroke", "#fff")
+                .attr("stroke-width", 2)
+                .style("filter", "drop-shadow(0px 2px 3px rgba(0,0,0,0.2))");
+
+              bubbles.append("text")
+                .text(d => d.label)
+                .attr("text-anchor", "middle")
+                .attr("dy", ".3em")
+                .style("font-size", d => Math.min(sizeScale(d.size) / 3, 14) + "px")
+                .style("fill", "white")
+                .style("font-weight", "500");
+
+              // Add click handler
+              bubbles.on("click", function(event, d) {
+                showDetails(d);
+              });
+
+              // Update positions
+              simulation.on("tick", () => {
+                bubbles.attr("transform", d => `translate(${d.x},${d.y})`);
+              });
+            }
+
+            // Show topic details
+            function showDetails(topic) {
+              const details = document.getElementById('details');
+              let summaryHtml = '';
+              if (topic.summary) {
+                summaryHtml = `
+                  <p><strong>Summary:</strong></p>
+                  <p style="font-size: 1.1em; color: #2c5234; padding: 15px; background: #e8f5e8; border-radius: 6px; border-left: 4px solid #4caf50; margin: 15px 0; line-height: 1.5;">${topic.summary}</p>
+                `;
+              }
+              
+              details.innerHTML = `
+                <h2>${topic.label}</h2>
+                <p><strong>Documents:</strong> ${topic.size}</p>
+                <p><strong>Coherence:</strong> ${(topic.coherence * 100).toFixed(1)}%</p>
+                ${summaryHtml}
+                <p><strong>Top Terms:</strong></p>
+                <div>${topic.terms.map(t => `<span class="term">${t}</span>`).join('')}</div>
+                <p><strong>Sample Documents:</strong></p>
+                ${topic.samples.map(s => `<p style="font-size: 0.9em; color: #666; padding: 10px; background: #f5f5f5; border-radius: 4px; margin: 10px 0;">"${s}..."</p>`).join('')}
+              `;
+            }
+
+            // Initialize visualizations
+            createBubbleChart();
+
+            // Show first topic details by default
+            if (topicsData.length > 0) {
+              showDetails(topicsData[0]);
+            }
+          </script>
+        </body>
+        </html>
+      HTML
+    end
+
+  end
+end

data/lib/ragnar/config.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'pathname' 
+require 'singleton'
+require 'fileutils'
+
+module Ragnar
+  class Config
+    include Singleton
+    
+    CONFIG_FILENAMES = ['.ragnar.yml', '.ragnarrc.yml', 'ragnar.yml'].freeze
+    
+    def initialize
+      @config = load_config
+      ensure_directories_exist
+    end
+    
+    # Main config access method
+    def get(key_path, default = nil)
+      keys = key_path.split('.')
+      value = keys.reduce(@config) { |config, key| config&.dig(key) }
+      
+      # Use default only if value is nil (not false)
+      result = value.nil? ? default : value
+      
+      # Expand paths that start with ~
+      if result.is_a?(String) && result.start_with?('~')
+        File.expand_path(result)
+      else
+        result
+      end
+    end
+    
+    # Common config accessors
+    def database_path
+      get('storage.database_path', default_database_path)
+    end
+    
+    def history_file
+      get('storage.history_file', default_history_file)
+    end
+    
+    def models_dir
+      get('storage.models_dir', default_models_dir)
+    end
+    
+    def embedding_model
+      get('embeddings.model', Ragnar::DEFAULT_EMBEDDING_MODEL)
+    end
+    
+    def chunk_size
+      get('embeddings.chunk_size', Ragnar::DEFAULT_CHUNK_SIZE)
+    end
+    
+    def chunk_overlap
+      get('embeddings.chunk_overlap', Ragnar::DEFAULT_CHUNK_OVERLAP)
+    end
+    
+    # LLM Profile support
+    # Profiles allow switching between LLM providers/models via --profile flag
+    # Backwards compatible: flat llm.provider/llm.default_model still work if no profiles defined
+
+    def set_active_profile(name)
+      name = name.to_s
+      profiles = llm_profiles
+      unless profiles.key?(name)
+        available = profiles.keys.join(', ')
+        raise ArgumentError, "Unknown profile '#{name}'. Available profiles: #{available}"
+      end
+      @active_profile = name
+    end
+
+    def llm_profile_name
+      @active_profile || get('llm.default_profile', nil) || llm_profiles.keys.first || 'default'
+    end
+
+    def llm_profiles
+      configured = get('llm.profiles', nil)
+      if configured.is_a?(Hash) && !configured.empty?
+        configured
+      else
+        # Backwards compat: synthesize a profile from flat keys
+        {
+          'default' => {
+            'provider' => get('llm.provider', 'red_candle'),
+            'model' => get('llm.default_model', 'MaziyarPanahi/Qwen3-4B-GGUF')
+          }
+        }
+      end
+    end
+
+    def llm_profile
+      llm_profiles[llm_profile_name] || llm_profiles.values.first
+    end
+
+    def available_profiles
+      llm_profiles.keys
+    end
+
+    # Create a new RubyLLM chat instance with the active profile's settings
+    def create_chat
+      api_key = llm_api_key
+      provider = llm_provider.to_sym
+
+      # Configure RubyLLM with the API key if present
+      if api_key
+        configure_provider_api_key(provider, api_key)
+      end
+
+      RubyLLM.chat(provider: provider, model: llm_model)
+    end
+
+    def llm_provider
+      llm_profile&.dig('provider') || get('llm.provider', 'red_candle')
+    end
+
+    def llm_model
+      llm_profile&.dig('model') || get('llm.default_model', 'MaziyarPanahi/Qwen3-4B-GGUF')
+    end
+
+    def llm_gguf_file
+      get('llm.default_gguf_file', "Qwen3-4B.Q4_K_M.gguf")
+    end
+
+    def llm_api_key
+      llm_profile&.dig('api_key') || get('llm.api_key', nil)
+    end
+    
+    def interactive_prompt
+      get('interactive.prompt', "ragnar> ")
+    end
+    
+    def quiet_mode?
+      get('interactive.quiet_mode', true)
+    end
+    
+    def show_progress?
+      get('output.show_progress', true)
+    end
+    
+    def query_top_k
+      get('query.top_k', 3)
+    end
+    
+    def enable_query_rewriting?
+      get('query.enable_query_rewriting', true)
+    end
+
+    def enable_reranking?
+      get('query.enable_reranking', true)
+    end
+
+    def reranker_model
+      get('query.reranker_model', 'BAAI/bge-reranker-base')
+    end
+    
+    # Config file management
+    def config_file_path
+      @config_file_path
+    end
+    
+    def config_exists?
+      !@config_file_path.nil?
+    end
+    
+    def reload!
+      @config = load_config
+      ensure_directories_exist
+    end
+    
+    # Generate a config file with current/default settings
+    def generate_config_file(path = nil)
+      path ||= File.expand_path('~/.ragnar.yml')
+      
+      config_content = {
+        'storage' => {
+          'database_path' => '~/.cache/ragnar/database',
+          'models_dir' => '~/.cache/ragnar/models', 
+          'history_file' => '~/.cache/ragnar/history'
+        },
+        'embeddings' => {
+          'model' => Ragnar::DEFAULT_EMBEDDING_MODEL,
+          'chunk_size' => Ragnar::DEFAULT_CHUNK_SIZE,
+          'chunk_overlap' => Ragnar::DEFAULT_CHUNK_OVERLAP
+        },
+        'umap' => {
+          'reduced_dimensions' => Ragnar::DEFAULT_REDUCED_DIMENSIONS,
+          'n_neighbors' => 15,
+          'min_dist' => 0.1,
+          'model_filename' => 'umap_model.bin'
+        },
+        'llm' => {
+          'default_profile' => 'red_candle',
+          'profiles' => {
+            'red_candle' => {
+              'provider' => 'red_candle',
+              'model' => 'MaziyarPanahi/Qwen3-4B-GGUF'
+            },
+            'opus' => {
+              'provider' => 'anthropic',
+              'model' => 'claude-opus-4-6'
+            },
+            'sonnet' => {
+              'provider' => 'anthropic',
+              'model' => 'claude-sonnet-4-6'
+            }
+          }
+        },
+        'query' => {
+          'top_k' => 3,
+          'enable_query_rewriting' => true,
+          'enable_reranking' => true,
+          'reranker_model' => 'BAAI/bge-reranker-base'
+        },
+        'interactive' => {
+          'prompt' => 'ragnar> ',
+          'quiet_mode' => true
+        },
+        'output' => {
+          'show_progress' => true
+        }
+      }
+      
+      # Ensure parent directory exists
+      FileUtils.mkdir_p(File.dirname(path))
+      
+      # Write config file with comments
+      File.write(path, generate_yaml_with_comments(config_content))
+      path
+    end
+    
+    private
+
+    def configure_provider_api_key(provider, api_key)
+      case provider
+      when :anthropic
+        RubyLLM.configure { |c| c.anthropic_api_key = api_key }
+      when :openai
+        RubyLLM.configure { |c| c.openai_api_key = api_key }
+      end
+    end
+
+    def load_config
+      @config_file_path = find_config_file
+      
+      if @config_file_path && File.exist?(@config_file_path)
+        YAML.load_file(@config_file_path) || {}
+      else
+        {}
+      end
+    rescue => e
+      warn "Warning: Error loading config file #{@config_file_path}: #{e.message}"
+      {}
+    end
+    
+    def find_config_file
+      # Search order: current directory → home directory
+      search_paths = [
+        Dir.pwd,
+        File.expand_path('~')
+      ]
+      
+      search_paths.each do |dir|
+        CONFIG_FILENAMES.each do |filename|
+          path = File.join(dir, filename)
+          return path if File.exist?(path)
+        end
+      end
+      
+      nil
+    end
+    
+    def ensure_directories_exist
+      directories = [
+        database_path,
+        models_dir,
+        File.dirname(history_file)
+      ]
+      
+      directories.each do |dir|
+        FileUtils.mkdir_p(dir) unless dir.nil?
+      end
+    end
+    
+    def default_database_path
+      xdg_cache_home = ENV['XDG_CACHE_HOME'] || File.expand_path('~/.cache')
+      File.join(xdg_cache_home, 'ragnar', 'database')
+    end
+    
+    def default_history_file
+      xdg_cache_home = ENV['XDG_CACHE_HOME'] || File.expand_path('~/.cache')
+      File.join(xdg_cache_home, 'ragnar', 'history')
+    end
+    
+    def default_models_dir
+      xdg_cache_home = ENV['XDG_CACHE_HOME'] || File.expand_path('~/.cache')
+      File.join(xdg_cache_home, 'ragnar', 'models')
+    end
+    
+    def generate_yaml_with_comments(config_hash)
+      yaml_content = YAML.dump(config_hash)
+      
+      # Add header comment
+      commented = <<~HEADER
+        # Ragnar Configuration File
+        # 
+        # This file configures default settings for Ragnar RAG system.
+        # Save as .ragnar.yml in your project directory or ~/.ragnar.yml for global settings.
+        # 
+        # Search order: ./.ragnar.yml → ~/.ragnar.yml → built-in defaults
+        #
+        # All paths support ~ expansion (e.g., ~/.cache/ragnar/database)
+
+      HEADER
+      
+      commented + yaml_content
+    end
+  end
+end