ragnar-cli 0.1.0.pre.1

data/lib/ragnar/topic_modeling/term_extractor.rb

@@ -0,0 +1,170 @@
+require 'set'
+
+module Ragnar
+  module TopicModeling
+    class TermExtractor
+      # Common English stop words to filter out
+      STOP_WORDS = Set.new(%w[
+        the be to of and a in that have i it for not on with he as you do at
+        this but his by from they we say her she or an will my one all would
+        there their what so up out if about who get which go me when make can
+        like time no just him know take people into year your good some could
+        them see other than then now look only come its over think also back
+        after use two how our work first well way even new want because any
+        these give day most us is was are been has had were said did get may
+      ])
+      
+      def initialize(stop_words: STOP_WORDS, min_word_length: 3, max_word_length: 20)
+        @stop_words = stop_words
+        @min_word_length = min_word_length
+        @max_word_length = max_word_length
+      end
+      
+      # Extract distinctive terms using c-TF-IDF
+      def extract_distinctive_terms(topic_docs:, all_docs:, top_n: 20)
+        # Tokenize and count terms in topic
+        topic_terms = count_terms(topic_docs)
+        
+        # Tokenize and count document frequency across all docs
+        doc_frequencies = compute_document_frequencies(all_docs)
+        
+        # Compute c-TF-IDF scores
+        scores = {}
+        total_docs = all_docs.length.to_f
+        
+        topic_terms.each do |term, tf|
+          # c-TF-IDF formula: tf * log(N / df)
+          df = doc_frequencies[term] || 1
+          idf = Math.log(total_docs / df)
+          scores[term] = tf * idf
+        end
+        
+        # Return top scoring terms
+        scores.sort_by { |_, score| -score }
+               .first(top_n)
+               .map(&:first)
+      end
+      
+      # Standard TF-IDF implementation
+      def extract_tfidf_terms(documents:, top_n: 20)
+        # Document frequency
+        doc_frequencies = compute_document_frequencies(documents)
+        total_docs = documents.length.to_f
+        
+        # Compute TF-IDF for each document
+        all_scores = []
+        
+        documents.each do |doc|
+          terms = count_terms([doc])
+          doc_length = terms.values.sum.to_f
+          
+          scores = {}
+          terms.each do |term, count|
+            tf = count / doc_length  # Normalized term frequency
+            df = doc_frequencies[term] || 1
+            idf = Math.log(total_docs / df)
+            scores[term] = tf * idf
+          end
+          
+          all_scores << scores
+        end
+        
+        # Aggregate scores across all documents
+        aggregated = {}
+        all_scores.each do |doc_scores|
+          doc_scores.each do |term, score|
+            aggregated[term] ||= 0
+            aggregated[term] += score
+          end
+        end
+        
+        # Return top terms
+        aggregated.sort_by { |_, score| -score }
+                  .first(top_n)
+                  .map(&:first)
+      end
+      
+      # Simple term frequency extraction
+      def extract_frequent_terms(documents:, top_n: 20)
+        terms = count_terms(documents)
+        terms.sort_by { |_, count| -count }
+             .first(top_n)
+             .map(&:first)
+      end
+      
+      private
+      
+      def tokenize(text)
+        # Simple tokenization - can be improved with proper NLP tokenizer
+        text.downcase
+            .split(/\W+/)
+            .select { |word| valid_word?(word) }
+      end
+      
+      def valid_word?(word)
+        word.length >= @min_word_length &&
+        word.length <= @max_word_length &&
+        !@stop_words.include?(word) &&
+        !word.match?(/^\d+$/)  # Not pure numbers
+      end
+      
+      def count_terms(documents)
+        terms = Hash.new(0)
+        
+        documents.each do |doc|
+          tokenize(doc).each do |word|
+            terms[word] += 1
+          end
+        end
+        
+        terms
+      end
+      
+      def compute_document_frequencies(documents)
+        doc_frequencies = Hash.new(0)
+        
+        documents.each do |doc|
+          # Use set to count each term once per document
+          unique_terms = Set.new(tokenize(doc))
+          unique_terms.each do |term|
+            doc_frequencies[term] += 1
+          end
+        end
+        
+        doc_frequencies
+      end
+      
+      # N-gram extraction for phrases
+      def extract_ngrams(text, n: 2)
+        words = tokenize(text)
+        ngrams = []
+        
+        (0..words.length - n).each do |i|
+          ngram = words[i, n].join(" ")
+          ngrams << ngram
+        end
+        
+        ngrams
+      end
+      
+      # Extract both unigrams and bigrams
+      def extract_mixed_terms(documents:, top_n: 20)
+        all_terms = Hash.new(0)
+        
+        documents.each do |doc|
+          # Unigrams
+          tokenize(doc).each { |word| all_terms[word] += 1 }
+          
+          # Bigrams
+          extract_ngrams(doc, n: 2).each { |bigram| all_terms[bigram] += 1 }
+        end
+        
+        # Filter and return top terms
+        all_terms.select { |term, count| count > 1 }  # Appears more than once
+                 .sort_by { |_, count| -count }
+                 .first(top_n)
+                 .map(&:first)
+      end
+    end
+  end
+end

data/lib/ragnar/topic_modeling/topic.rb

@@ -0,0 +1,117 @@
+module Ragnar
+  module TopicModeling
+    class Topic
+      attr_reader :id, :document_indices, :documents, :embeddings, :metadata
+      attr_accessor :terms, :label
+      
+      def initialize(id:, document_indices:, documents:, embeddings:, metadata: nil)
+        @id = id
+        @document_indices = document_indices
+        @documents = documents
+        @embeddings = embeddings
+        @metadata = metadata || []
+        @terms = []
+        @label = nil
+      end
+      
+      def size
+        @documents.length
+      end
+      
+      def centroid
+        @centroid ||= compute_centroid
+      end
+      
+      def representative_docs(k: 3)
+        return @documents if @documents.length <= k
+        
+        # Find documents closest to centroid
+        distances = @embeddings.map do |embedding|
+          distance_to_centroid(embedding)
+        end
+        
+        # Get indices of k smallest distances
+        top_indices = distances.each_with_index.sort_by(&:first).first(k).map(&:last)
+        top_indices.map { |i| @documents[i] }
+      end
+      
+      def coherence
+        @coherence ||= Metrics.compute_coherence(@terms, @documents)
+      end
+      
+      def distinctiveness(other_topics)
+        @distinctiveness ||= Metrics.compute_distinctiveness(self, other_topics)
+      end
+      
+      def set_terms(terms)
+        @terms = terms
+        @centroid = nil  # Reset centroid cache
+      end
+      
+      def set_label(label)
+        @label = label
+      end
+      
+      def summary
+        {
+          id: @id,
+          label: @label || "Topic #{@id}",
+          size: size,
+          terms: @terms.first(10),
+          coherence: coherence.round(3),
+          representative_docs: representative_docs(k: 2).map { |d| d[0..100] + "..." }
+        }
+      end
+      
+      def to_h
+        {
+          id: @id,
+          label: @label,
+          document_indices: @document_indices,
+          terms: @terms,
+          centroid: centroid,
+          size: size,
+          coherence: coherence
+        }
+      end
+      
+      def self.from_h(hash)
+        topic = new(
+          id: hash[:id],
+          document_indices: hash[:document_indices],
+          documents: [],  # Would need to be reconstructed
+          embeddings: [],  # Would need to be reconstructed
+          metadata: []
+        )
+        topic.set_label(hash[:label])
+        topic.set_terms(hash[:terms])
+        topic
+      end
+      
+      private
+      
+      def compute_centroid
+        return [] if @embeddings.empty?
+        
+        # Compute mean of all embeddings
+        dim = @embeddings.first.length
+        centroid = Array.new(dim, 0.0)
+        
+        @embeddings.each do |embedding|
+          embedding.each_with_index do |val, idx|
+            centroid[idx] += val
+          end
+        end
+        
+        centroid.map { |val| val / @embeddings.length }
+      end
+      
+      def distance_to_centroid(embedding)
+        # Euclidean distance
+        Math.sqrt(
+          embedding.zip(centroid).map { |a, b| (a - b) ** 2 }.sum
+        )
+      end
+    end
+  end
+end

data/lib/ragnar/topic_modeling/topic_labeler.rb

@@ -0,0 +1,61 @@
+require_relative 'labeling_strategies'
+
+module Ragnar
+  module TopicModeling
+    class TopicLabeler
+      attr_reader :strategy
+      
+      def initialize(method: :hybrid, llm_client: nil)
+        @method = method
+        @llm_client = llm_client
+        @strategy = LabelingStrategies.create(method, llm_client: llm_client)
+      end
+      
+      # Generate a human-readable label for a topic
+      # Returns a hash with label, description, and metadata
+      def generate_label(topic: nil, terms:, documents: [], method: nil)
+        # Allow method override per call
+        if method && method != @method
+          strategy = LabelingStrategies.create(method, llm_client: @llm_client)
+        else
+          strategy = @strategy
+        end
+        
+        # Generate label using selected strategy
+        result = strategy.generate_label(
+          topic: topic,
+          terms: terms,
+          documents: documents
+        )
+        
+        # Ensure we always return a consistent structure
+        normalize_result(result)
+      end
+      
+      # Convenience method for simple label string
+      def generate_simple_label(terms:, documents: [], method: nil)
+        result = generate_label(terms: terms, documents: documents, method: method)
+        result[:label]
+      end
+      
+      # Change strategy at runtime
+      def set_strategy(method)
+        @method = method
+        @strategy = LabelingStrategies.create(method, llm_client: @llm_client)
+      end
+      
+      private
+      
+      def normalize_result(result)
+        {
+          label: result[:label] || "Unknown Topic",
+          description: result[:description] || nil,
+          method: result[:method] || @method,
+          confidence: result[:confidence] || 0.5,
+          themes: result[:themes] || [],
+          metadata: result.reject { |k, _| [:label, :description, :method, :confidence, :themes].include?(k) }
+        }
+      end
+    end
+  end
+end

data/lib/ragnar/topic_modeling.rb

@@ -0,0 +1,24 @@
+# Main entry point for topic modeling functionality
+# Designed for future extraction into a separate gem
+
+require_relative 'topic_modeling/topic'
+require_relative 'topic_modeling/term_extractor'
+require_relative 'topic_modeling/metrics'
+require_relative 'topic_modeling/topic_labeler'
+require_relative 'topic_modeling/engine'
+
+module Ragnar
+  module TopicModeling
+    
+    # Convenience method to create a new topic modeling engine
+    def self.new(**options)
+      Engine.new(**options)
+    end
+    
+    # Extract topics from embeddings and documents (simple interface)
+    def self.extract(embeddings:, documents:, **options)
+      engine = Engine.new(**options)
+      engine.fit(embeddings: embeddings, documents: documents)
+    end
+  end
+end

data/lib/ragnar/umap_processor.rb

@@ -0,0 +1,228 @@
+require 'json'
+
+module Ragnar
+  class UmapProcessor
+    attr_reader :database, :model_path
+    
+    def initialize(db_path: Ragnar::DEFAULT_DB_PATH, model_path: "umap_model.bin")
+      @database = Database.new(db_path)
+      @model_path = model_path
+      @umap_model = nil
+    end
+    
+    def train(n_components: Ragnar::DEFAULT_REDUCED_DIMENSIONS, n_neighbors: 15, min_dist: 0.1)
+      puts "Loading embeddings from database..."
+      
+      # Get all embeddings
+      docs = @database.get_embeddings
+      
+      if docs.empty?
+        raise "No embeddings found in database. Please index some documents first."
+      end
+      
+      embeddings = docs.map { |d| d[:embedding] }.compact
+      
+      if embeddings.empty?
+        raise "No valid embeddings found in database."
+      end
+      
+      puts "Found #{embeddings.size} embeddings"
+      
+      # Adjust parameters based on the number of samples
+      # UMAP requires n_neighbors < n_samples
+      # Also, n_components should be less than n_samples for stability
+      n_samples = embeddings.size
+      
+      if n_neighbors >= n_samples
+        n_neighbors = [3, (n_samples - 1) / 2].max.to_i
+        puts "  Adjusted n_neighbors to #{n_neighbors} (was #{15}, but only have #{n_samples} samples)"
+      end
+      
+      if n_components >= n_samples
+        n_components = [2, n_samples - 1].min
+        puts "  Adjusted n_components to #{n_components} (was #{50}, but only have #{n_samples} samples)"
+      end
+      
+      # Warn if we have very few samples
+      if n_samples < 100
+        puts "\n  ⚠️  Warning: UMAP works best with at least 100 samples."
+        puts "     You currently have #{n_samples} samples."
+        puts "     Consider indexing more documents for better results."
+      end
+      
+      # Convert to matrix format for ClusterKit
+      # ClusterKit expects a 2D array or Numo::NArray
+      embedding_matrix = embeddings
+      original_dims = embeddings.first.size
+      
+      puts "\nTraining UMAP model..."
+      puts "  Original dimensions: #{original_dims}"
+      puts "  Target dimensions: #{n_components}"
+      puts "  Neighbors: #{n_neighbors}"
+      puts "  Min distance: #{min_dist}"
+      
+      # Use the simple ClusterKit.umap method
+      progressbar = TTY::ProgressBar.new(
+        "Training UMAP [:bar] :percent",
+        total: 100,
+        bar_format: :block,
+        width: 30
+      )
+      
+      # Start progress in background (ClusterKit doesn't provide callbacks)
+      progress_thread = Thread.new do
+        100.times do
+          sleep(0.05)
+          progressbar.advance
+          break if @training_complete
+        end
+      end
+      
+      # Perform the actual training using the class-based API
+      @umap_instance = ClusterKit::Dimensionality::UMAP.new(
+        n_components: n_components,
+        n_neighbors: n_neighbors
+      )
+      
+      @reduced_embeddings = @umap_instance.fit_transform(embedding_matrix)
+      
+      @training_complete = true
+      progress_thread.join
+      progressbar.finish
+      
+      # Store the parameters for saving
+      @model_params = {
+        n_components: n_components,
+        n_neighbors: n_neighbors,
+        min_dist: min_dist
+      }
+      
+      # Save the model
+      save_model
+      
+      {
+        embeddings_count: embeddings.size,
+        original_dims: original_dims,
+        reduced_dims: n_components
+      }
+    end
+    
+    def apply(batch_size: 100)
+      # Load the trained UMAP model (reduced embeddings)
+      reduced_embeddings = load_model
+      
+      puts "Applying saved UMAP embeddings to database..."
+      
+      # Get all embeddings from database
+      all_docs = @database.get_embeddings
+      
+      if all_docs.empty?
+        puts "No embeddings found in database."
+        return {
+          processed: 0,
+          skipped: 0,
+          errors: 0
+        }
+      end
+      
+      puts "Found #{all_docs.size} documents in database"
+      puts "Loaded #{reduced_embeddings.size} reduced embeddings from model"
+      
+      if all_docs.size != reduced_embeddings.size
+        puts "⚠️  Warning: Mismatch between database documents (#{all_docs.size}) and model embeddings (#{reduced_embeddings.size})"
+        puts "   This suggests the model was trained on a different dataset."
+        puts "   Please retrain the UMAP model after indexing all your documents."
+        return {
+          processed: 0,
+          skipped: 0,
+          errors: 1
+        }
+      end
+      
+      # Prepare updates - match document IDs to reduced embeddings
+      updates = all_docs.each_with_index.map do |doc, idx|
+        {
+          id: doc[:id],
+          reduced_embedding: reduced_embeddings[idx]
+        }
+      end
+      
+      puts "Updating database with reduced embeddings..."
+      @database.update_reduced_embeddings(updates)
+      
+      {
+        processed: updates.size,
+        skipped: 0,
+        errors: 0
+      }
+    end
+    
+    private
+    
+    def process_batch(docs)
+      # Extract embeddings
+      embeddings = docs.map { |d| d[:embedding] }
+      
+      # Transform using UMAP
+      # The transform method returns a 2D array where each row is a reduced embedding
+      reduced = @umap_model.transform(embeddings)
+      
+      # Prepare updates
+      updates = docs.each_with_index.map do |doc, idx|
+        {
+          id: doc[:id],
+          reduced_embedding: reduced[idx]
+        }
+      end
+      
+      # Update database
+      @database.update_reduced_embeddings(updates)
+    end
+    
+    def save_model
+      return unless @umap_instance && @reduced_embeddings
+      
+      # Save the trained UMAP model for transforming new queries
+      @umap_instance.save_model(@model_path)
+      puts "UMAP model saved to: #{@model_path}"
+      
+      # Also cache the reduced embeddings separately for the apply method
+      embeddings_path = @model_path.sub(/\.bin$/, '_embeddings.json')
+      ClusterKit::Dimensionality::UMAP.save_data(@reduced_embeddings, embeddings_path)
+      puts "Reduced embeddings cached to: #{embeddings_path}"
+    end
+    
+    def load_model
+      return @reduced_embeddings if @reduced_embeddings
+      
+      # For the apply method, we need the pre-computed embeddings
+      embeddings_path = @model_path.sub(/\.bin$/, '_embeddings.json')
+      unless File.exist?(embeddings_path)
+        raise "Cached embeddings not found at #{embeddings_path}. Please train a model first."
+      end
+      
+      @reduced_embeddings = ClusterKit::Dimensionality::UMAP.load_data(embeddings_path)
+      puts "Cached embeddings loaded from: #{embeddings_path}"
+      @reduced_embeddings
+    end
+    
+    def load_umap_model
+      # Load the actual UMAP model for transforming new data
+      unless File.exist?(@model_path)
+        raise "UMAP model not found at #{@model_path}. Please train a model first."
+      end
+      
+      @umap_instance ||= ClusterKit::Dimensionality::UMAP.load_model(@model_path)
+      puts "UMAP model loaded from: #{@model_path}"
+      @umap_instance
+    end
+    
+    def self.optimal_dimensions(original_dims, target_ratio: 0.1)
+      # Suggest optimal number of dimensions for reduction
+      # Common heuristic: reduce to 10% of original dimensions
+      # but keep at least 50 dimensions for good quality
+      suggested = (original_dims * target_ratio).to_i
+      [suggested, 50].max
+    end
+  end
+end