ragnar-cli 0.1.0.pre.1 → 0.1.0.pre.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0b9db7d48838621cadf2a90bff6fc4afca333ecd7fdc2364666196fce437474
-  data.tar.gz: '0468b4cdb2893fb80b7b52ad9c9dc4857bb6a90e0105c0e4356ba0612a5bcfae'
+  metadata.gz: 6b9a7fdbf0345f1c111f8028f8b881d8014a55226cfe3d02f6a76fd6cd9b213c
+  data.tar.gz: 2341e27f16b442c0631876303e0da5141438559ef6685e2c15514cd18416d99c
 SHA512:
-  metadata.gz: c398c02f5019e86476a59ebe64639ec42b009c4699d3704a39492950d00e3be0e252c0cdd84101b24121f359f12c8652c3129643376fc4b84b03c9ab99843f13
-  data.tar.gz: c295cfa1ec329d954d7d86e026b29536d83194ed47ed3a28c96527cfc3c8ae7c8d652a09d4b4e771599e1671a16523bcffa79abb01feddb5592edb48561b19ff
+  metadata.gz: a87f39a5dfd246732be4e24b19aba8b49a7a735f78d825d0241f04b0b776fc8b23c15f0b3488416dedfba37d9027ff6442f38ab2aad43bb79395e0c769247275
+  data.tar.gz: 00d7533c2e16b57da59786a840f1b653cfe472c63168a6e003abb31a79b3f6e0f056fdabe32f4b4566ea682d229e3d0710167358ab7c78fd18c497690d9a3675

data/README.md

@@ -1,4 +1,4 @@
-# Ragnar
+<img src="/docs/assets/ragnar-wide.png" alt="ragnar" height="80px">
 
 A complete Ruby implementation of Retrieval-Augmented Generation (RAG) pipeline using native Ruby ML/NLP gems.
 
@@ -419,11 +419,11 @@ MIT License - see LICENSE file for details
 ## Acknowledgments
 
 This project integrates several excellent Ruby gems:
-- [red-candle](https://github.com/red-candle) - Ruby ML/LLM toolkit
-- [lancelot](https://github.com/lancelot) - Lance database bindings
+- [red-candle](https://github.com/assaydepot/red-candle) - Ruby ML/LLM toolkit
+- [lancelot](https://github.com/cpetersen/lancelot) - Lance database bindings
 - [clusterkit](https://github.com/cpetersen/clusterkit) - UMAP and clustering implementation
 - [parsekit](https://github.com/cpetersen/parsekit) - Content extraction
-- [baran](https://github.com/baran) - Text splitting utilities
+- [baran](https://github.com/moeki0/baran) - Text splitting utilities
 
 ## Roadmap
 
@@ -436,4 +436,4 @@ This project integrates several excellent Ruby gems:
 - [ ] Performance benchmarking suite
 - [ ] Support for multiple embedding models simultaneously
 - [ ] Query result caching
-- [ ] Automatic index optimization
+- [ ] Automatic index optimization

data/lib/ragnar/cli.rb

@@ -127,7 +127,22 @@ module Ragnar
           exit 1
         end
 
-        embeddings = docs_with_embeddings.map { |d| d[:embedding] }
+        # Check if we have reduced embeddings available
+        first_doc = docs_with_embeddings.first
+        has_reduced = first_doc[:reduced_embedding] && !first_doc[:reduced_embedding].empty?
+        
+        if has_reduced
+          embeddings = docs_with_embeddings.map { |d| d[:reduced_embedding] }
+          say "Using reduced embeddings (#{embeddings.first.size} dimensions)", :yellow if options[:verbose]
+          # Already reduced, so don't reduce again in the engine
+          reduce_dims = false
+        else
+          embeddings = docs_with_embeddings.map { |d| d[:embedding] }
+          say "Using original embeddings (#{embeddings.first.size} dimensions)", :yellow if options[:verbose]
+          # Let the engine handle dimensionality reduction if needed
+          reduce_dims = true
+        end
+        
         documents = docs_with_embeddings.map { |d| d[:chunk_text] }
         metadata = docs_with_embeddings.map { |d| { file_path: d[:file_path], chunk_index: d[:chunk_index] } }
 
@@ -137,7 +152,8 @@ module Ragnar
         engine = Ragnar::TopicModeling::Engine.new(
           min_cluster_size: options[:min_cluster_size],
           labeling_method: options[:method].to_sym,
-          verbose: options[:verbose]
+          verbose: options[:verbose],
+          reduce_dimensions: reduce_dims
         )
 
         # Extract topics

data/lib/ragnar/topic_modeling/engine.rb

@@ -128,19 +128,99 @@ module Ragnar
       def reduce_dimensions(embeddings)
         require 'clusterkit'
         
-        umap = ClusterKit::Dimensionality::UMAP.new(
-          n_components: @n_components,
-          n_neighbors: 15,
-          random_seed: 42  # For reproducibility
-        )
-        
-        # Convert to format UMAP expects
-        umap.fit_transform(embeddings)
+        # Validate embeddings before UMAP
+        valid_embeddings, invalid_indices = validate_embeddings_for_umap(embeddings)
+        
+        if valid_embeddings.empty?
+          raise "No valid embeddings for dimensionality reduction.\n\n" \
+                "All embeddings contain invalid values (NaN, Infinity, or non-numeric).\n" \
+                "Try running without dimensionality reduction:\n" \
+                "  ragnar topics --reduce-dimensions false"
+        end
+        
+        if invalid_indices.any? && @verbose
+          puts "  ⚠️  Warning: #{invalid_indices.size} embeddings with invalid values removed"
+        end
+        
+        begin
+          # Adjust parameters based on data size
+          n_samples = valid_embeddings.size
+          n_components = [@n_components, n_samples - 1, 50].min
+          n_neighbors = [15, n_samples - 1].min
+          
+          if @verbose && n_components != @n_components
+            puts "  Adjusted n_components to #{n_components} (was #{@n_components}) for #{n_samples} samples"
+          end
+          
+          umap = ClusterKit::Dimensionality::UMAP.new(
+            n_components: n_components,
+            n_neighbors: n_neighbors,
+            random_seed: 42  # For reproducibility
+          )
+          
+          # Convert to format UMAP expects
+          reduced = umap.fit_transform(valid_embeddings)
+          
+          # If we had to remove invalid embeddings, reconstruct the full array
+          if invalid_indices.any?
+            full_reduced = []
+            valid_idx = 0
+            embeddings.size.times do |i|
+              if invalid_indices.include?(i)
+                # Use zeros for invalid embeddings (they'll be outliers anyway)
+                full_reduced << Array.new(n_components, 0.0)
+              else
+                full_reduced << reduced[valid_idx]
+                valid_idx += 1
+              end
+            end
+            full_reduced
+          else
+            reduced
+          end
+        rescue => e
+          if e.message.include?("index out of bounds")
+            error_msg = "\n❌ Dimensionality reduction failed\n\n"
+            error_msg += "The UMAP algorithm encountered an error with your data.\n\n"
+            error_msg += "This typically happens with:\n"
+            error_msg += "  • Embeddings containing invalid values\n"
+            error_msg += "  • Too few samples (#{valid_embeddings.size} valid embeddings)\n"
+            error_msg += "  • Incompatible parameters\n\n"
+            error_msg += "Solutions:\n"
+            error_msg += "  1. Run without dimensionality reduction:\n"
+            error_msg += "     ragnar topics --reduce-dimensions false\n\n"
+            error_msg += "  2. Use fewer dimensions:\n"
+            error_msg += "     ragnar topics --n-components 2\n\n"
+            error_msg += "  3. Re-index your documents:\n"
+            error_msg += "     ragnar index <path> --force\n"
+            raise error_msg
+          else
+            raise
+          end
+        end
       rescue LoadError
         puts "Warning: Dimensionality reduction requires ClusterKit. Using original embeddings." if @verbose
         embeddings
       end
       
+      private
+      
+      def validate_embeddings_for_umap(embeddings)
+        valid = []
+        invalid_indices = []
+        
+        embeddings.each_with_index do |embedding, idx|
+          if embedding.is_a?(Array) && 
+             embedding.all? { |v| v.is_a?(Numeric) && v.finite? }
+            valid << embedding
+          else
+            invalid_indices << idx
+          end
+        end
+        
+        [valid, invalid_indices]
+      end
+      
       def build_topics(cluster_ids)
         @cluster_ids = cluster_ids
         

data/lib/ragnar/umap_processor.rb

@@ -28,6 +28,67 @@ module Ragnar
       
       puts "Found #{embeddings.size} embeddings"
       
+      # Validate embeddings
+      embedding_dims = embeddings.map(&:size).uniq
+      if embedding_dims.size > 1
+        puts "  ⚠️  Warning: Inconsistent embedding dimensions found: #{embedding_dims.inspect}"
+        puts "     This may cause errors during UMAP training."
+        # Filter to only embeddings with the most common dimension
+        most_common_dim = embedding_dims.max_by { |dim| embeddings.count { |e| e.size == dim } }
+        embeddings = embeddings.select { |e| e.size == most_common_dim }
+        puts "     Using only embeddings with #{most_common_dim} dimensions (#{embeddings.size} embeddings)"
+      end
+      
+      # Check for nil or invalid values
+      invalid_count = 0
+      nan_count = 0
+      inf_count = 0
+      
+      valid_embeddings = embeddings.select do |embedding|
+        if !embedding.is_a?(Array)
+          invalid_count += 1
+          false
+        elsif embedding.any? { |v| !v.is_a?(Numeric) }
+          invalid_count += 1
+          false
+        elsif embedding.any?(&:nan?)
+          nan_count += 1
+          false
+        elsif embedding.any? { |v| !v.finite? }
+          inf_count += 1
+          false
+        else
+          true
+        end
+      end
+      
+      if valid_embeddings.size < embeddings.size
+        puts "\n  ⚠️  Data quality issues detected:"
+        puts "     • Invalid embeddings: #{invalid_count}" if invalid_count > 0
+        puts "     • Embeddings with NaN: #{nan_count}" if nan_count > 0
+        puts "     • Embeddings with Infinity: #{inf_count}" if inf_count > 0
+        puts "     • Total removed: #{embeddings.size - valid_embeddings.size}"
+        puts "     • Remaining valid: #{valid_embeddings.size}"
+        
+        embeddings = valid_embeddings
+      end
+      
+      if embeddings.empty?
+        raise "No valid embeddings found after validation.\n\n" \
+              "All embeddings contain invalid values (NaN, Infinity, or non-numeric).\n" \
+              "This suggests a problem with the embedding model or indexing process.\n\n" \
+              "Please try:\n" \
+              "  1. Re-indexing your documents: ragnar index <path> --force\n" \
+              "  2. Using a different embedding model\n" \
+              "  3. Checking your document content for unusual characters"
+      end
+      
+      if embeddings.size < 10
+        raise "Too few valid embeddings (#{embeddings.size}) for UMAP training.\n\n" \
+              "UMAP requires at least 10 samples to work effectively.\n" \
+              "Please index more documents or check for data quality issues."
+      end
+      
       # Adjust parameters based on the number of samples
       # UMAP requires n_neighbors < n_samples
       # Also, n_components should be less than n_samples for stability
@@ -55,41 +116,68 @@ module Ragnar
       embedding_matrix = embeddings
       original_dims = embeddings.first.size
       
+      # Ensure n_components is reasonable
+      if n_components >= original_dims
+        puts "  ⚠️  Warning: n_components (#{n_components}) >= original dimensions (#{original_dims})"
+        n_components = [original_dims / 2, 50].min
+        puts "     Reducing n_components to #{n_components}"
+      end
+      
+      # For very high dimensional data, be more conservative
+      if original_dims > 500 && n_components > 50
+        puts "  ⚠️  Note: High dimensional data (#{original_dims}D) being reduced to #{n_components}D"
+        puts "     Consider using n_components <= 50 for stability"
+      end
+      
       puts "\nTraining UMAP model..."
       puts "  Original dimensions: #{original_dims}"
       puts "  Target dimensions: #{n_components}"
       puts "  Neighbors: #{n_neighbors}"
       puts "  Min distance: #{min_dist}"
       
-      # Use the simple ClusterKit.umap method
-      progressbar = TTY::ProgressBar.new(
-        "Training UMAP [:bar] :percent",
-        total: 100,
-        bar_format: :block,
-        width: 30
-      )
-      
-      # Start progress in background (ClusterKit doesn't provide callbacks)
-      progress_thread = Thread.new do
-        100.times do
-          sleep(0.05)
-          progressbar.advance
-          break if @training_complete
+      # Perform the actual training using the class-based API
+      puts "  Training UMAP model (this may take a moment)..."
+      
+      begin
+        @umap_instance = ClusterKit::Dimensionality::UMAP.new(
+          n_components: n_components,
+          n_neighbors: n_neighbors
+        )
+        
+        @reduced_embeddings = @umap_instance.fit_transform(embedding_matrix)
+        
+        puts "  ✓ UMAP training complete"
+      rescue => e
+        # Provide helpful error message without exposing internal stack trace
+        error_msg = "\n❌ UMAP training failed\n\n"
+        
+        if e.message.include?("index out of bounds")
+          error_msg += "The UMAP algorithm encountered an index out of bounds error.\n\n"
+          error_msg += "This typically happens when:\n"
+          error_msg += "  • The embedding data contains invalid values (NaN, Infinity)\n"
+          error_msg += "  • The parameters are incompatible with your data\n"
+          error_msg += "  • There are duplicate or corrupted embeddings\n\n"
+          error_msg += "Suggested solutions:\n"
+          error_msg += "  1. Try with more conservative parameters:\n"
+          error_msg += "     ragnar train-umap --n-components 10 --n-neighbors 5\n\n"
+          error_msg += "  2. Re-index your documents to regenerate embeddings:\n"
+          error_msg += "     ragnar index <path> --force\n\n"
+          error_msg += "  3. Check your embedding model configuration\n\n"
+          error_msg += "Current parameters:\n"
+          error_msg += "  • n_components: #{n_components}\n"
+          error_msg += "  • n_neighbors: #{n_neighbors}\n"
+          error_msg += "  • embeddings: #{embeddings.size} samples\n"
+          error_msg += "  • dimensions: #{original_dims}\n"
+        else
+          error_msg += "Error: #{e.message}\n\n"
+          error_msg += "This may be due to incompatible parameters or data issues.\n"
+          error_msg += "Try using more conservative parameters:\n"
+          error_msg += "  ragnar train-umap --n-components 10 --n-neighbors 5\n"
         end
+        
+        raise RuntimeError, error_msg
       end
       
-      # Perform the actual training using the class-based API
-      @umap_instance = ClusterKit::Dimensionality::UMAP.new(
-        n_components: n_components,
-        n_neighbors: n_neighbors
-      )
-      
-      @reduced_embeddings = @umap_instance.fit_transform(embedding_matrix)
-      
-      @training_complete = true
-      progress_thread.join
-      progressbar.finish
-      
       # Store the parameters for saving
       @model_params = {
         n_components: n_components,

data/lib/ragnar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ragnar
-  VERSION = "0.1.0.pre.1"
+  VERSION = "0.1.0.pre.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ragnar-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.1
+  version: 0.1.0.pre.3
 platform: ruby
 authors:
 - Chris Petersen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-08-22 00:00:00.000000000 Z
+date: 2025-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: lancelot
   requirement: !ruby/object:Gem::Requirement