RubyGems - ragnar-cli - Versions diffs - 0.1.0.pre.2 → 0.1.0.pre.4 - Mend

ragnar-cli 0.1.0.pre.2 → 0.1.0.pre.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (20) hide show

checksums.yaml +4 -4
data/README.md +187 -36
data/lib/ragnar/cli.rb +543 -172
data/lib/ragnar/cli_visualization.rb +184 -0
data/lib/ragnar/config.rb +226 -0
data/lib/ragnar/database.rb +94 -8
data/lib/ragnar/llm_manager.rb +4 -1
data/lib/ragnar/query_processor.rb +38 -20
data/lib/ragnar/topic_modeling.rb +13 -10
data/lib/ragnar/umap_processor.rb +190 -73
data/lib/ragnar/umap_transform_service.rb +169 -88
data/lib/ragnar/version.rb +1 -1
metadata +43 -22
data/lib/ragnar/topic_modeling/engine.rb +0 -221
data/lib/ragnar/topic_modeling/labeling_strategies.rb +0 -300
data/lib/ragnar/topic_modeling/llm_adapter.rb +0 -131
data/lib/ragnar/topic_modeling/metrics.rb +0 -186
data/lib/ragnar/topic_modeling/term_extractor.rb +0 -170
data/lib/ragnar/topic_modeling/topic.rb +0 -117
data/lib/ragnar/topic_modeling/topic_labeler.rb +0 -61

data/lib/ragnar/cli_visualization.rb ADDED Viewed

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+module Ragnar
+  module CLIVisualization
+    def generate_topic_visualization_html(topics, embeddings: nil, cluster_ids: nil)
+      # Convert topics to JSON for D3.js
+      topics_json = topics.map do |topic|
+        topic_data = {
+          id: topic.id,
+          label: topic.label || "Topic #{topic.id}",
+          size: topic.size,
+          terms: topic.terms.first(10),
+          coherence: topic.coherence,
+          samples: topic.representative_docs(k: 2).map { |d| d[0..200] }
+        }
+        # Add summary if it exists
+        summary = topic.instance_variable_get(:@summary)
+        topic_data[:summary] = summary if summary
+        topic_data
+      end.to_json
+      # HTML template with enhanced visualization
+      <<~HTML
+        <!DOCTYPE html>
+        <html>
+        <head>
+          <meta charset="utf-8">
+          <title>Topic Visualization</title>
+          <script src="https://d3js.org/d3.v7.min.js"></script>
+          <style>
+            body {
+              font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+              margin: 20px;
+              background: #f8f9fa;
+            }
+            .container {
+              max-width: 1400px;
+              margin: 0 auto;
+            }
+            h1 {
+              color: #2c3e50;
+              margin-bottom: 10px;
+            }
+            .viz-panel {
+              background: white;
+              border-radius: 8px;
+              box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+              padding: 15px;
+            }
+            #bubble-viz {
+              height: 500px;
+            }
+            .topic { cursor: pointer; }
+            .topic:hover { opacity: 0.8; }
+            #details {
+              background: white;
+              padding: 20px;
+              border-radius: 8px;
+              box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+              margin-top: 20px;
+            }
+            .term {
+              display: inline-block;
+              margin: 5px;
+              padding: 5px 10px;
+              background: #e3f2fd;
+              border-radius: 3px;
+              color: #1976d2;
+              font-size: 14px;
+            }
+          </style>
+        </head>
+        <body>
+          <div class="container">
+            <h1>Topic Analysis Results</h1>
+            <div class="viz-panel">
+              <h3>Topic Overview</h3>
+              <div id="bubble-viz"></div>
+            </div>
+            <div id="details">Click on a topic to see details</div>
+          </div>
+          <script>
+            const topicsData = #{topics_json};
+            // Create bubble chart
+            function createBubbleChart() {
+              const width = document.getElementById('bubble-viz').clientWidth - 30;
+              const height = 470;
+              const svg = d3.select("#bubble-viz")
+                .append("svg")
+                .attr("width", width)
+                .attr("height", height);
+              // Create scale for bubble sizes
+              const sizeScale = d3.scaleSqrt()
+                .domain([0, d3.max(topicsData, d => d.size)])
+                .range([20, 60]);
+              // Create color scale
+              const colorScale = d3.scaleSequential(d3.interpolateTurbo)
+                .domain([0, 1]);
+              // Create force simulation
+              const simulation = d3.forceSimulation(topicsData)
+                .force("x", d3.forceX(width / 2).strength(0.05))
+                .force("y", d3.forceY(height / 2).strength(0.05))
+                .force("collide", d3.forceCollide(d => sizeScale(d.size) + 3));
+              // Create bubbles
+              const bubbles = svg.selectAll(".topic")
+                .data(topicsData)
+                .enter().append("g")
+                .attr("class", "topic");
+              bubbles.append("circle")
+                .attr("r", d => sizeScale(d.size))
+                .attr("fill", d => colorScale(d.coherence))
+                .attr("stroke", "#fff")
+                .attr("stroke-width", 2)
+                .style("filter", "drop-shadow(0px 2px 3px rgba(0,0,0,0.2))");
+              bubbles.append("text")
+                .text(d => d.label)
+                .attr("text-anchor", "middle")
+                .attr("dy", ".3em")
+                .style("font-size", d => Math.min(sizeScale(d.size) / 3, 14) + "px")
+                .style("fill", "white")
+                .style("font-weight", "500");
+              // Add click handler
+              bubbles.on("click", function(event, d) {
+                showDetails(d);
+              });
+              // Update positions
+              simulation.on("tick", () => {
+                bubbles.attr("transform", d => `translate(${d.x},${d.y})`);
+              });
+            }
+            // Show topic details
+            function showDetails(topic) {
+              const details = document.getElementById('details');
+              let summaryHtml = '';
+              if (topic.summary) {
+                summaryHtml = `
+                  <p><strong>Summary:</strong></p>
+                  <p style="font-size: 1.1em; color: #2c5234; padding: 15px; background: #e8f5e8; border-radius: 6px; border-left: 4px solid #4caf50; margin: 15px 0; line-height: 1.5;">${topic.summary}</p>
+                `;
+              }
+              details.innerHTML = `
+                <h2>${topic.label}</h2>
+                <p><strong>Documents:</strong> ${topic.size}</p>
+                <p><strong>Coherence:</strong> ${(topic.coherence * 100).toFixed(1)}%</p>
+                ${summaryHtml}
+                <p><strong>Top Terms:</strong></p>
+                <div>${topic.terms.map(t => `<span class="term">${t}</span>`).join('')}</div>
+                <p><strong>Sample Documents:</strong></p>
+                ${topic.samples.map(s => `<p style="font-size: 0.9em; color: #666; padding: 10px; background: #f5f5f5; border-radius: 4px; margin: 10px 0;">"${s}..."</p>`).join('')}
+              `;
+            }
+            // Initialize visualizations
+            createBubbleChart();
+            // Show first topic details by default
+            if (topicsData.length > 0) {
+              showDetails(topicsData[0]);
+            }
+          </script>
+        </body>
+        </html>
+      HTML
+    end
+  end
+end

data/lib/ragnar/config.rb ADDED Viewed

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+require 'yaml'
+require 'pathname'
+require 'singleton'
+require 'fileutils'
+module Ragnar
+  class Config
+    include Singleton
+    CONFIG_FILENAMES = ['.ragnar.yml', '.ragnarrc.yml', 'ragnar.yml'].freeze
+    def initialize
+      @config = load_config
+      ensure_directories_exist
+    end
+    # Main config access method
+    def get(key_path, default = nil)
+      keys = key_path.split('.')
+      value = keys.reduce(@config) { |config, key| config&.dig(key) }
+      # Use default only if value is nil (not false)
+      result = value.nil? ? default : value
+      # Expand paths that start with ~
+      if result.is_a?(String) && result.start_with?('~')
+        File.expand_path(result)
+      else
+        result
+      end
+    end
+    # Common config accessors
+    def database_path
+      get('storage.database_path', default_database_path)
+    end
+    def history_file
+      get('storage.history_file', default_history_file)
+    end
+    def models_dir
+      get('storage.models_dir', default_models_dir)
+    end
+    def embedding_model
+      get('embeddings.model', Ragnar::DEFAULT_EMBEDDING_MODEL)
+    end
+    def chunk_size
+      get('embeddings.chunk_size', Ragnar::DEFAULT_CHUNK_SIZE)
+    end
+    def chunk_overlap
+      get('embeddings.chunk_overlap', Ragnar::DEFAULT_CHUNK_OVERLAP)
+    end
+    def llm_model
+      get('llm.default_model', "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF")
+    end
+    def llm_gguf_file
+      get('llm.default_gguf_file', "tinyllama-1.1b-chat-v1.0.q4_k_m.gguf")
+    end
+    def interactive_prompt
+      get('interactive.prompt', "ragnar> ")
+    end
+    def quiet_mode?
+      get('interactive.quiet_mode', true)
+    end
+    def show_progress?
+      get('output.show_progress', true)
+    end
+    def query_top_k
+      get('query.top_k', 3)
+    end
+    def enable_query_rewriting?
+      get('query.enable_query_rewriting', true)
+    end
+    # Config file management
+    def config_file_path
+      @config_file_path
+    end
+    def config_exists?
+      !@config_file_path.nil?
+    end
+    def reload!
+      @config = load_config
+      ensure_directories_exist
+    end
+    # Generate a config file with current/default settings
+    def generate_config_file(path = nil)
+      path ||= File.expand_path('~/.ragnar.yml')
+      config_content = {
+        'storage' => {
+          'database_path' => '~/.cache/ragnar/database',
+          'models_dir' => '~/.cache/ragnar/models',
+          'history_file' => '~/.cache/ragnar/history'
+        },
+        'embeddings' => {
+          'model' => Ragnar::DEFAULT_EMBEDDING_MODEL,
+          'chunk_size' => Ragnar::DEFAULT_CHUNK_SIZE,
+          'chunk_overlap' => Ragnar::DEFAULT_CHUNK_OVERLAP
+        },
+        'umap' => {
+          'reduced_dimensions' => Ragnar::DEFAULT_REDUCED_DIMENSIONS,
+          'n_neighbors' => 15,
+          'min_dist' => 0.1,
+          'model_filename' => 'umap_model.bin'
+        },
+        'llm' => {
+          'default_model' => 'TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF',
+          'default_gguf_file' => 'tinyllama-1.1b-chat-v1.0.q4_k_m.gguf'
+        },
+        'query' => {
+          'top_k' => 3,
+          'enable_query_rewriting' => true
+        },
+        'interactive' => {
+          'prompt' => 'ragnar> ',
+          'quiet_mode' => true
+        },
+        'output' => {
+          'show_progress' => true
+        }
+      }
+      # Ensure parent directory exists
+      FileUtils.mkdir_p(File.dirname(path))
+      # Write config file with comments
+      File.write(path, generate_yaml_with_comments(config_content))
+      path
+    end
+    private
+    def load_config
+      @config_file_path = find_config_file
+      if @config_file_path && File.exist?(@config_file_path)
+        YAML.load_file(@config_file_path) || {}
+      else
+        {}
+      end
+    rescue => e
+      warn "Warning: Error loading config file #{@config_file_path}: #{e.message}"
+      {}
+    end
+    def find_config_file
+      # Search order: current directory → home directory
+      search_paths = [
+        Dir.pwd,
+        File.expand_path('~')
+      ]
+      search_paths.each do |dir|
+        CONFIG_FILENAMES.each do |filename|
+          path = File.join(dir, filename)
+          return path if File.exist?(path)
+        end
+      end
+      nil
+    end
+    def ensure_directories_exist
+      directories = [
+        database_path,
+        models_dir,
+        File.dirname(history_file)
+      ]
+      directories.each do |dir|
+        FileUtils.mkdir_p(dir) unless dir.nil?
+      end
+    end
+    def default_database_path
+      xdg_cache_home = ENV['XDG_CACHE_HOME'] || File.expand_path('~/.cache')
+      File.join(xdg_cache_home, 'ragnar', 'database')
+    end
+    def default_history_file
+      xdg_cache_home = ENV['XDG_CACHE_HOME'] || File.expand_path('~/.cache')
+      File.join(xdg_cache_home, 'ragnar', 'history')
+    end
+    def default_models_dir
+      xdg_cache_home = ENV['XDG_CACHE_HOME'] || File.expand_path('~/.cache')
+      File.join(xdg_cache_home, 'ragnar', 'models')
+    end
+    def generate_yaml_with_comments(config_hash)
+      yaml_content = YAML.dump(config_hash)
+      # Add header comment
+      commented = <<~HEADER
+        # Ragnar Configuration File
+        #
+        # This file configures default settings for Ragnar RAG system.
+        # Save as .ragnar.yml in your project directory or ~/.ragnar.yml for global settings.
+        #
+        # Search order: ./.ragnar.yml → ~/.ragnar.yml → built-in defaults
+        #
+        # All paths support ~ expansion (e.g., ~/.cache/ragnar/database)
+      HEADER
+      commented + yaml_content
+    end
+  end
+end

data/lib/ragnar/database.rb CHANGED Viewed

@@ -5,6 +5,7 @@ module Ragnar
     def initialize(db_path, table_name: "documents")
       @db_path = db_path
       @table_name = table_name
+      @dataset_cache = nil  # Cache to prevent file descriptor leaks
       ensure_database_exists
     end
@@ -34,16 +35,23 @@ module Ragnar
         metadata: :string
       }
+      # Clear cache before modifying dataset
+      clear_dataset_cache
       # Use the new open_or_create method from Lancelot
       # This automatically handles both creating new and opening existing datasets
       dataset = Lancelot::Dataset.open_or_create(@db_path, schema: schema)
       dataset.add_documents(data)
+      # Clear cache after modification to ensure fresh data on next read
+      clear_dataset_cache
     end
     def get_embeddings(limit: nil, offset: 0)
       return [] unless dataset_exists?
-      dataset = Lancelot::Dataset.open(@db_path)
+      dataset = cached_dataset
+      return [] unless dataset
       # Get all documents or a subset
       docs = if limit && offset > 0
@@ -67,7 +75,8 @@ module Ragnar
     def update_reduced_embeddings(updates)
       return if updates.empty?
-      dataset = Lancelot::Dataset.open(@db_path)
+      dataset = cached_dataset
+      return unless dataset
       # Get all existing documents and safely extract their data
       all_docs = dataset.to_a.map do |doc|
@@ -113,17 +122,24 @@ module Ragnar
         metadata: :string
       }
+      # Clear cache before recreating dataset
+      clear_dataset_cache
       # Remove old dataset and create new one with updated data
       FileUtils.rm_rf(@db_path)
       # Use open_or_create which will create since we just deleted the path
       dataset = Lancelot::Dataset.open_or_create(@db_path, schema: schema)
       dataset.add_documents(updated_docs)
+      # Clear cache after modification
+      clear_dataset_cache
     end
     def search_similar(embedding, k: 10, use_reduced: false)
       return [] unless dataset_exists?
-      dataset = Lancelot::Dataset.open(@db_path)
+      dataset = cached_dataset
+      return [] unless dataset
       embedding_field = use_reduced ? :reduced_embedding : :embedding
@@ -149,7 +165,9 @@ module Ragnar
     def count
       return 0 unless dataset_exists?
-      dataset = Lancelot::Dataset.open(@db_path)
+      dataset = cached_dataset
+      return 0 unless dataset
       dataset.to_a.size
     end
@@ -166,7 +184,18 @@ module Ragnar
         }
       end
-      dataset = Lancelot::Dataset.open(@db_path)
+      dataset = cached_dataset
+      unless dataset
+        return {
+          document_count: 0,
+          total_documents: 0,
+          unique_files: 0,
+          total_chunks: 0,
+          with_embeddings: 0,
+          with_reduced_embeddings: 0,
+          total_size_mb: 0.0
+        }
+      end
       # Get all documents
       all_docs = dataset.to_a
@@ -214,7 +243,9 @@ module Ragnar
     def get_all_documents_with_embeddings(limit: nil)
       return [] unless dataset_exists?
-      dataset = Lancelot::Dataset.open(@db_path)
+      dataset = cached_dataset
+      return [] unless dataset
       all_docs = limit ? dataset.first(limit) : dataset.to_a
       all_docs.select { |doc| doc[:embedding] && !doc[:embedding].empty? }
@@ -223,7 +254,8 @@ module Ragnar
     def full_text_search(query, limit: 10)
       return [] unless dataset_exists?
-      dataset = Lancelot::Dataset.open(@db_path)
+      dataset = cached_dataset
+      return [] unless dataset
       # Use Lancelot's full-text search
       results = dataset.full_text_search(
@@ -243,11 +275,49 @@ module Ragnar
       end
     end
+    # Get the total number of documents in the database
+    def document_count
+      count
+    end
+    # Get documents by their IDs
+    # @param ids [Array<String>] Document IDs to fetch
+    # @return [Array<Hash>] Documents with their embeddings
+    def get_documents_by_ids(ids)
+      return [] if ids.empty? || !dataset_exists?
+      dataset = cached_dataset
+      return [] unless dataset
+      # Create ID lookup set for efficiency
+      id_set = ids.to_set
+      # Filter documents by IDs
+      dataset.to_a.select { |doc| id_set.include?(doc[:id]) }.map do |doc|
+        {
+          id: doc[:id],
+          chunk_text: doc[:chunk_text],
+          file_path: doc[:file_path],
+          chunk_index: doc[:chunk_index],
+          embedding: doc[:embedding],
+          reduced_embedding: doc[:reduced_embedding],
+          metadata: JSON.parse(doc[:metadata] || "{}")
+        }
+      end
+    end
     def dataset_exists?
       return false unless File.exist?(@db_path)
+      # Try to use cached dataset if available
+      if @dataset_cache
+        return true
+      end
+      # Otherwise check if we can open it
       begin
-        Lancelot::Dataset.open(@db_path)
+        # Don't cache here, just check existence
+        dataset = Lancelot::Dataset.open(@db_path)
         true
       rescue
         false
@@ -263,5 +333,21 @@ module Ragnar
     def table_exists?
       dataset_exists?
     end
+    # Cached dataset accessor to prevent file descriptor leaks
+    def cached_dataset
+      return nil unless File.exist?(@db_path)
+      @dataset_cache ||= begin
+        Lancelot::Dataset.open(@db_path)
+      rescue => e
+        nil
+      end
+    end
+    # Clear the cached dataset (e.g., after modifications)
+    def clear_dataset_cache
+      @dataset_cache = nil
+    end
   end
 end

data/lib/ragnar/llm_manager.rb CHANGED Viewed

@@ -18,7 +18,10 @@ module Ragnar
       @mutex.synchronize do
         @llms[cache_key] ||= begin
-          puts "Loading LLM: #{model_id}..." unless @llms.key?(cache_key)
+          # Only show loading message if not in interactive mode or if verbose
+          show_loading = ENV['DEBUG'] # Only show in debug mode for now
+          puts "Loading LLM: #{model_id}..." if show_loading && !@llms.key?(cache_key)
           if gguf_file
             Candle::LLM.from_pretrained(model_id, gguf_file: gguf_file)
           else

data/lib/ragnar/query_processor.rb CHANGED Viewed

@@ -16,29 +16,47 @@ module Ragnar
       @reranker = nil # Will initialize when needed
     end
-    def query(user_query, top_k: 3, verbose: false)
+    def query(user_query, top_k: 3, verbose: false, enable_rewriting: true)
       puts "Processing query: #{user_query}" if verbose
-      # Step 1: Rewrite and analyze the query
-      puts "\n#{'-'*60}" if verbose
-      puts "STEP 1: Query Analysis & Rewriting" if verbose
-      puts "-"*60 if verbose
-      rewritten = @rewriter.rewrite(user_query)
-      if verbose
-        puts "\nOriginal Query: #{user_query}"
-        puts "\nRewritten Query Analysis:"
-        puts "  Clarified Intent: #{rewritten['clarified_intent']}"
-        puts "  Query Type: #{rewritten['query_type']}"
-        puts "  Context Needed: #{rewritten['context_needed']}"
-        puts "\nGenerated Sub-queries (#{rewritten['sub_queries'].length}):"
-        rewritten['sub_queries'].each_with_index do |sq, idx|
-          puts "  #{idx + 1}. #{sq}"
+      # Step 1: Rewrite and analyze the query (if enabled)
+      if enable_rewriting
+        puts "\n#{'-'*60}" if verbose
+        puts "STEP 1: Query Analysis & Rewriting" if verbose
+        puts "-"*60 if verbose
+        rewritten = @rewriter.rewrite(user_query)
+        if verbose
+          puts "\nOriginal Query: #{user_query}"
+          puts "\nRewritten Query Analysis:"
+          puts "  Clarified Intent: #{rewritten['clarified_intent']}"
+          puts "  Query Type: #{rewritten['query_type']}"
+          puts "  Context Needed: #{rewritten['context_needed']}"
+          puts "\nGenerated Sub-queries (#{rewritten['sub_queries'].length}):"
+          rewritten['sub_queries'].each_with_index do |sq, idx|
+            puts "  #{idx + 1}. #{sq}"
+          end
+          if rewritten['key_terms'] && !rewritten['key_terms'].empty?
+            puts "\nKey Terms Identified:"
+            puts "  #{rewritten['key_terms'].join(', ')}"
+          end
         end
-        if rewritten['key_terms'] && !rewritten['key_terms'].empty?
-          puts "\nKey Terms Identified:"
-          puts "  #{rewritten['key_terms'].join(', ')}"
+      else
+        # Skip rewriting - use original query directly
+        rewritten = {
+          'clarified_intent' => user_query,
+          'query_type' => 'direct',
+          'context_needed' => 'general',
+          'sub_queries' => [user_query],
+          'key_terms' => []
+        }
+        if verbose
+          puts "\n#{'-'*60}"
+          puts "STEP 1: Query Analysis (Rewriting Disabled)"
+          puts "-"*60
+          puts "\nUsing original query directly"
         end
       end