ragnar-cli 0.1.0.pre.1

data/lib/ragnar/umap_transform_service.rb

@@ -0,0 +1,124 @@
+require 'clusterkit'
+
+module Ragnar
+  class UmapTransformService
+    include Singleton
+    
+    def initialize
+      @umap_model = nil
+      @model_path = "umap_model.bin"
+    end
+    
+    # Transform a query embedding to reduced space using saved UMAP model
+    def transform_query(query_embedding, model_path = nil)
+      # Use the real UMAP model's transform capability
+      model_path ||= @model_path
+      
+      # Load the model if not already loaded
+      load_model(model_path) unless @umap_model
+      
+      # Transform the query embedding using the trained UMAP model
+      # The transform method expects a 2D array (even for a single embedding)
+      result = @umap_model.transform([query_embedding])
+      
+      # Return the first (and only) transformed embedding
+      result.first
+    rescue => e
+      # Fall back to k-NN approximation if model loading fails
+      puts "Warning: Could not use UMAP model for transform: #{e.message}"
+      puts "Falling back to k-NN approximation..."
+      knn_approximate_transform(query_embedding)
+    end
+    
+    # Check if we can do transforms
+    def model_available?(model_path = nil)
+      model_path ||= @model_path
+      
+      # First check if the actual UMAP model file exists
+      if File.exist?(model_path)
+        return true
+      end
+      
+      # Fallback: check if the database has reduced embeddings for k-NN approximation
+      database = Database.new("./rag_database")
+      stats = database.get_stats
+      stats[:with_reduced_embeddings] > 0
+    end
+    
+    private
+    
+    def load_model(model_path)
+      unless File.exist?(model_path)
+        raise "UMAP model not found at #{model_path}. Please train a model first."
+      end
+      
+      @umap_model = ClusterKit::Dimensionality::UMAP.load_model(model_path)
+      puts "UMAP model loaded for query transformation"
+    end
+    
+    def knn_approximate_transform(query_embedding)
+      # Fallback k-NN approximation method
+      # Get database stats to know dimensions
+      database = Database.new("./rag_database")
+      stats = database.get_stats
+      
+      # If we don't have reduced embeddings, we can't transform
+      if stats[:with_reduced_embeddings] == 0
+        raise "No reduced embeddings available in database"
+      end
+      
+      # Get all documents with their embeddings
+      all_docs = database.get_embeddings
+      
+      # Find k nearest neighbors in full embedding space
+      k = 5
+      neighbors = []
+      
+      all_docs.each_with_index do |doc, idx|
+        next unless doc[:embedding] && doc[:reduced_embedding]
+        
+        distance = euclidean_distance(query_embedding, doc[:embedding])
+        neighbors << { idx: idx, distance: distance, reduced: doc[:reduced_embedding] }
+      end
+      
+      # Sort by distance and take k nearest
+      neighbors.sort_by! { |n| n[:distance] }
+      k_nearest = neighbors.first(k)
+      
+      # Average the reduced embeddings of k nearest neighbors
+      # This is a simple approximation of the transform
+      if k_nearest.empty?
+        raise "No neighbors found for transform"
+      end
+      
+      reduced_dims = k_nearest.first[:reduced].size
+      averaged = Array.new(reduced_dims, 0.0)
+      
+      # Weighted average based on inverse distance
+      total_weight = 0.0
+      k_nearest.each do |neighbor|
+        # Use inverse distance as weight (closer = higher weight)
+        weight = 1.0 / (neighbor[:distance] + 0.001) # Add small epsilon to avoid division by zero
+        total_weight += weight
+        
+        neighbor[:reduced].each_with_index do |val, idx|
+          averaged[idx] += val * weight
+        end
+      end
+      
+      # Normalize by total weight
+      averaged.map { |val| val / total_weight }
+    end
+    
+    def euclidean_distance(vec1, vec2)
+      return Float::INFINITY if vec1.size != vec2.size
+      
+      sum = 0.0
+      vec1.each_with_index do |val, idx|
+        diff = val - vec2[idx]
+        sum += diff * diff
+      end
+      Math.sqrt(sum)
+    end
+  end
+end

data/lib/ragnar/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Ragnar
+  VERSION = "0.1.0.pre.1"
+end

data/lib/ragnar.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "thor"
+require "red-candle"
+require "lancelot"
+require "clusterkit"
+require "baran"
+require "tty-progressbar"
+require "securerandom"
+require "json"
+require "fileutils"
+require "singleton"
+
+module Ragnar
+  class Error < StandardError; end
+
+  DEFAULT_DB_PATH = "ragnar_database"
+  DEFAULT_CHUNK_SIZE = 512
+  DEFAULT_CHUNK_OVERLAP = 50
+  DEFAULT_EMBEDDING_MODEL = "jinaai/jina-embeddings-v2-base-en"
+  DEFAULT_REDUCED_DIMENSIONS = 64  # Reduce embeddings from 768D to 64D for faster search
+end
+
+require_relative "ragnar/version"
+require_relative "ragnar/database"
+require_relative "ragnar/chunker"
+require_relative "ragnar/embedder"
+require_relative "ragnar/indexer"
+require_relative "ragnar/umap_processor"
+require_relative "ragnar/llm_manager"
+require_relative "ragnar/context_repacker"
+require_relative "ragnar/query_rewriter"
+require_relative "ragnar/umap_transform_service"
+require_relative "ragnar/query_processor"
+require_relative "ragnar/topic_modeling"
+require_relative "ragnar/cli"

data/lib/ragnar_cli.rb

@@ -0,0 +1,2 @@
+# frozen_string_literal: true
+require 'ragnar'

metadata

@@ -0,0 +1,234 @@
+--- !ruby/object:Gem::Specification
+name: ragnar-cli
+version: !ruby/object:Gem::Version
+  version: 0.1.0.pre.1
+platform: ruby
+authors:
+- Chris Petersen
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2025-08-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: red-candle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: lancelot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+- !ruby/object:Gem::Dependency
+  name: clusterkit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0.pre.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0.pre.2
+- !ruby/object:Gem::Dependency
+  name: baran
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: parsekit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0.pre.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0.pre.1
+- !ruby/object:Gem::Dependency
+  name: tty-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+description: Ragnar is a high-performance RAG system that leverages Rust libraries
+  through Ruby bindings for embeddings, vector search, and topic modeling. It provides
+  a complete CLI for indexing documents and querying with LLMs.
+email:
+- chris@example.com
+executables:
+- ragnar
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- exe/ragnar
+- lib/ragnar.rb
+- lib/ragnar/chunker.rb
+- lib/ragnar/cli.rb
+- lib/ragnar/context_repacker.rb
+- lib/ragnar/database.rb
+- lib/ragnar/embedder.rb
+- lib/ragnar/indexer.rb
+- lib/ragnar/llm_manager.rb
+- lib/ragnar/query_processor.rb
+- lib/ragnar/query_rewriter.rb
+- lib/ragnar/topic_modeling.rb
+- lib/ragnar/topic_modeling/engine.rb
+- lib/ragnar/topic_modeling/labeling_strategies.rb
+- lib/ragnar/topic_modeling/llm_adapter.rb
+- lib/ragnar/topic_modeling/metrics.rb
+- lib/ragnar/topic_modeling/term_extractor.rb
+- lib/ragnar/topic_modeling/topic.rb
+- lib/ragnar/topic_modeling/topic_labeler.rb
+- lib/ragnar/umap_processor.rb
+- lib/ragnar/umap_transform_service.rb
+- lib/ragnar/version.rb
+- lib/ragnar_cli.rb
+homepage: https://github.com/cpetersen/ragnar
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/cpetersen/ragnar
+  source_code_uri: https://github.com/cpetersen/ragnar
+  changelog_uri: https://github.com/cpetersen/ragnar/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.3
+signing_key:
+specification_version: 4
+summary: A Ruby + Rust powered RAG (Retrieval-Augmented Generation) system
+test_files: []