ragnar-cli 0.1.0.pre.2 → 0.1.0.pre.4

data/lib/ragnar/topic_modeling/llm_adapter.rb

@@ -1,131 +0,0 @@
-# Adapter to allow different LLM backends (red-candle, remote APIs, etc.)
-module Ragnar
-  module TopicModeling
-    class LLMAdapter
-      # Factory method to create appropriate LLM client
-      def self.create(type: :auto, **options)
-        case type
-        when :red_candle
-          RedCandleAdapter.new(**options)
-        when :openai
-          # Future: OpenAIAdapter.new(**options)
-          raise NotImplementedError, "OpenAI adapter not yet implemented"
-        when :anthropic
-          # Future: AnthropicAdapter.new(**options)
-          raise NotImplementedError, "Anthropic adapter not yet implemented"
-        when :auto
-          # Try red-candle first, then fall back to others
-          begin
-            RedCandleAdapter.new(**options)
-          rescue LoadError
-            nil  # No LLM available
-          end
-        else
-          raise ArgumentError, "Unknown LLM type: #{type}"
-        end
-      end
-    end
-    
-    # Adapter for red-candle (local LLMs)
-    class RedCandleAdapter
-      def initialize(model: nil, **options)
-        require 'candle'
-        
-        @model = model || default_model
-        @options = options
-        @llm = load_or_create_llm
-      end
-      
-      def generate(prompt:, max_tokens: 100, temperature: 0.3, response_format: nil)
-        # Red-candle specific generation
-        response = @llm.generate(
-          prompt,
-          max_length: max_tokens,
-          temperature: temperature,
-          do_sample: temperature > 0
-        )
-        
-        # Handle JSON response format if requested
-        if response_format && response_format[:type] == "json_object"
-          ensure_json_response(response)
-        else
-          response
-        end
-      end
-      
-      def available?
-        true
-      end
-      
-      private
-      
-      def default_model
-        # Use a small, fast model by default for topic labeling
-        "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
-      end
-      
-      def load_or_create_llm
-        # Check if already loaded in ruby-rag
-        if defined?(Ragnar::LLMManager)
-          begin
-            return Ragnar::LLMManager.instance.get_llm(@model)
-          rescue
-            # Fall through to create new
-          end
-        end
-        
-        # Create new LLM instance
-        Candle::Model.new(
-          model_id: @model,
-          model_type: :llama,
-          quantized: true
-        )
-      end
-      
-      def ensure_json_response(response)
-        # Try to extract JSON from response
-        begin
-          # Look for JSON-like content
-          json_match = response.match(/\{.*\}/m)
-          if json_match
-            JSON.parse(json_match[0])
-            json_match[0]  # Return the JSON string if valid
-          else
-            # Generate a basic JSON response
-            generate_fallback_json(response)
-          end
-        rescue JSON::ParserError
-          generate_fallback_json(response)
-        end
-      end
-      
-      def generate_fallback_json(text)
-        # Create a simple JSON from text response
-        label = text.lines.first&.strip || "Unknown"
-        {
-          label: label,
-          description: text,
-          confidence: 0.5
-        }.to_json
-      end
-    end
-    
-    # Future adapter for remote LLMs
-    class RemoteAdapter
-      def initialize(api_key:, endpoint:, **options)
-        @api_key = api_key
-        @endpoint = endpoint
-        @options = options
-      end
-      
-      def generate(prompt:, max_tokens: 100, temperature: 0.3, response_format: nil)
-        # Make API call
-        raise NotImplementedError, "Remote LLM adapter coming soon"
-      end
-      
-      def available?
-        !@api_key.nil?
-      end
-    end
-  end
-end

data/lib/ragnar/topic_modeling/metrics.rb

@@ -1,186 +0,0 @@
-module Ragnar
-  module TopicModeling
-    module Metrics
-      extend self
-      
-      # Compute UMass Coherence for topic quality
-      # Higher coherence = more interpretable topic
-      def compute_coherence(terms, documents, top_n: 10)
-        return 0.0 if terms.empty? || documents.empty?
-        
-        # Use top N terms
-        eval_terms = terms.first(top_n)
-        return 0.0 if eval_terms.length < 2
-        
-        # Create document term matrix for co-occurrence
-        doc_term_counts = count_cooccurrences(eval_terms, documents)
-        
-        # Compute UMass coherence
-        coherence_sum = 0.0
-        pairs_count = 0
-        
-        eval_terms.each_with_index do |term_i, i|
-          eval_terms.each_with_index do |term_j, j|
-            next unless j < i  # Only upper triangle
-            
-            # P(term_i, term_j) = co-occurrence count
-            cooccur = doc_term_counts["#{term_i},#{term_j}"] || 0
-            # P(term_j) = document frequency
-            doc_freq_j = doc_term_counts[term_j] || 0
-            
-            if cooccur > 0 && doc_freq_j > 0
-              # UMass: log((cooccur + 1) / doc_freq_j)
-              coherence_sum += Math.log((cooccur + 1.0) / doc_freq_j)
-              pairs_count += 1
-            end
-          end
-        end
-        
-        return 0.0 if pairs_count == 0
-        
-        # Normalize by number of pairs
-        coherence = coherence_sum / pairs_count
-        
-        # Transform to 0-1 range (coherence is typically negative)
-        # More negative = less coherent, so we reverse and bound
-        normalized = 1.0 / (1.0 + Math.exp(-coherence))
-        normalized
-      end
-      
-      # Compute how distinct a topic is from others
-      def compute_distinctiveness(topic, other_topics)
-        return 1.0 if other_topics.empty?
-        
-        topic_terms = Set.new(topic.terms.first(20))
-        
-        # Compare with other topics
-        overlaps = other_topics.map do |other|
-          next if other.id == topic.id
-          
-          other_terms = Set.new(other.terms.first(20))
-          overlap = (topic_terms & other_terms).size.to_f
-          
-          # Jaccard similarity
-          union_size = (topic_terms | other_terms).size
-          union_size > 0 ? overlap / union_size : 0
-        end.compact
-        
-        return 1.0 if overlaps.empty?
-        
-        # Distinctiveness = 1 - average overlap
-        1.0 - (overlaps.sum / overlaps.length)
-      end
-      
-      # Compute diversity across all topics
-      def compute_diversity(topics)
-        return 0.0 if topics.length < 2
-        
-        # Collect all term sets
-        term_sets = topics.map { |t| Set.new(t.terms.first(20)) }
-        
-        # Compute pairwise Jaccard distances
-        distances = []
-        term_sets.each_with_index do |set_i, i|
-          term_sets.each_with_index do |set_j, j|
-            next unless j > i  # Only upper triangle
-            
-            intersection = (set_i & set_j).size.to_f
-            union = (set_i | set_j).size.to_f
-            
-            # Jaccard distance = 1 - Jaccard similarity
-            distance = union > 0 ? 1.0 - (intersection / union) : 1.0
-            distances << distance
-          end
-        end
-        
-        # Average distance = diversity
-        distances.sum / distances.length
-      end
-      
-      # Compute coverage (what fraction of docs are in topics vs outliers)
-      def compute_coverage(topics, total_documents)
-        return 0.0 if total_documents == 0
-        
-        docs_in_topics = topics.sum(&:size)
-        docs_in_topics.to_f / total_documents
-      end
-      
-      # Silhouette score for cluster quality
-      def compute_silhouette_score(topic, all_topics, embeddings)
-        return 0.0 if topic.embeddings.empty?
-        
-        silhouettes = []
-        
-        topic.embeddings.each_with_index do |embedding, idx|
-          # a(i) = average distance to other points in same cluster
-          if topic.embeddings.length > 1
-            a_i = topic.embeddings.each_with_index
-                      .reject { |_, j| j == idx }
-                      .map { |other, _| euclidean_distance(embedding, other) }
-                      .sum.to_f / (topic.embeddings.length - 1)
-          else
-            a_i = 0.0
-          end
-          
-          # b(i) = minimum average distance to points in other clusters
-          b_values = all_topics.reject { |t| t.id == topic.id }.map do |other_topic|
-            next if other_topic.embeddings.empty?
-            
-            avg_dist = other_topic.embeddings
-                                 .map { |other| euclidean_distance(embedding, other) }
-                                 .sum.to_f / other_topic.embeddings.length
-            avg_dist
-          end.compact
-          
-          b_i = b_values.min || a_i
-          
-          # Silhouette coefficient
-          if a_i == 0 && b_i == 0
-            s_i = 0
-          else
-            s_i = (b_i - a_i) / [a_i, b_i].max
-          end
-          
-          silhouettes << s_i
-        end
-        
-        # Average silhouette score for topic
-        silhouettes.sum / silhouettes.length
-      end
-      
-      private
-      
-      def count_cooccurrences(terms, documents)
-        counts = Hash.new(0)
-        
-        documents.each do |doc|
-          doc_lower = doc.downcase
-          
-          # Count individual term occurrences
-          terms.each do |term|
-            counts[term] += 1 if doc_lower.include?(term.downcase)
-          end
-          
-          # Count co-occurrences
-          terms.each_with_index do |term_i, i|
-            terms.each_with_index do |term_j, j|
-              next unless j < i
-              
-              if doc_lower.include?(term_i.downcase) && doc_lower.include?(term_j.downcase)
-                counts["#{term_i},#{term_j}"] += 1
-              end
-            end
-          end
-        end
-        
-        counts
-      end
-      
-      def euclidean_distance(vec1, vec2)
-        Math.sqrt(
-          vec1.zip(vec2).map { |a, b| (a - b) ** 2 }.sum
-        )
-      end
-    end
-  end
-end

data/lib/ragnar/topic_modeling/term_extractor.rb

@@ -1,170 +0,0 @@
-require 'set'
-
-module Ragnar
-  module TopicModeling
-    class TermExtractor
-      # Common English stop words to filter out
-      STOP_WORDS = Set.new(%w[
-        the be to of and a in that have i it for not on with he as you do at
-        this but his by from they we say her she or an will my one all would
-        there their what so up out if about who get which go me when make can
-        like time no just him know take people into year your good some could
-        them see other than then now look only come its over think also back
-        after use two how our work first well way even new want because any
-        these give day most us is was are been has had were said did get may
-      ])
-      
-      def initialize(stop_words: STOP_WORDS, min_word_length: 3, max_word_length: 20)
-        @stop_words = stop_words
-        @min_word_length = min_word_length
-        @max_word_length = max_word_length
-      end
-      
-      # Extract distinctive terms using c-TF-IDF
-      def extract_distinctive_terms(topic_docs:, all_docs:, top_n: 20)
-        # Tokenize and count terms in topic
-        topic_terms = count_terms(topic_docs)
-        
-        # Tokenize and count document frequency across all docs
-        doc_frequencies = compute_document_frequencies(all_docs)
-        
-        # Compute c-TF-IDF scores
-        scores = {}
-        total_docs = all_docs.length.to_f
-        
-        topic_terms.each do |term, tf|
-          # c-TF-IDF formula: tf * log(N / df)
-          df = doc_frequencies[term] || 1
-          idf = Math.log(total_docs / df)
-          scores[term] = tf * idf
-        end
-        
-        # Return top scoring terms
-        scores.sort_by { |_, score| -score }
-               .first(top_n)
-               .map(&:first)
-      end
-      
-      # Standard TF-IDF implementation
-      def extract_tfidf_terms(documents:, top_n: 20)
-        # Document frequency
-        doc_frequencies = compute_document_frequencies(documents)
-        total_docs = documents.length.to_f
-        
-        # Compute TF-IDF for each document
-        all_scores = []
-        
-        documents.each do |doc|
-          terms = count_terms([doc])
-          doc_length = terms.values.sum.to_f
-          
-          scores = {}
-          terms.each do |term, count|
-            tf = count / doc_length  # Normalized term frequency
-            df = doc_frequencies[term] || 1
-            idf = Math.log(total_docs / df)
-            scores[term] = tf * idf
-          end
-          
-          all_scores << scores
-        end
-        
-        # Aggregate scores across all documents
-        aggregated = {}
-        all_scores.each do |doc_scores|
-          doc_scores.each do |term, score|
-            aggregated[term] ||= 0
-            aggregated[term] += score
-          end
-        end
-        
-        # Return top terms
-        aggregated.sort_by { |_, score| -score }
-                  .first(top_n)
-                  .map(&:first)
-      end
-      
-      # Simple term frequency extraction
-      def extract_frequent_terms(documents:, top_n: 20)
-        terms = count_terms(documents)
-        terms.sort_by { |_, count| -count }
-             .first(top_n)
-             .map(&:first)
-      end
-      
-      private
-      
-      def tokenize(text)
-        # Simple tokenization - can be improved with proper NLP tokenizer
-        text.downcase
-            .split(/\W+/)
-            .select { |word| valid_word?(word) }
-      end
-      
-      def valid_word?(word)
-        word.length >= @min_word_length &&
-        word.length <= @max_word_length &&
-        !@stop_words.include?(word) &&
-        !word.match?(/^\d+$/)  # Not pure numbers
-      end
-      
-      def count_terms(documents)
-        terms = Hash.new(0)
-        
-        documents.each do |doc|
-          tokenize(doc).each do |word|
-            terms[word] += 1
-          end
-        end
-        
-        terms
-      end
-      
-      def compute_document_frequencies(documents)
-        doc_frequencies = Hash.new(0)
-        
-        documents.each do |doc|
-          # Use set to count each term once per document
-          unique_terms = Set.new(tokenize(doc))
-          unique_terms.each do |term|
-            doc_frequencies[term] += 1
-          end
-        end
-        
-        doc_frequencies
-      end
-      
-      # N-gram extraction for phrases
-      def extract_ngrams(text, n: 2)
-        words = tokenize(text)
-        ngrams = []
-        
-        (0..words.length - n).each do |i|
-          ngram = words[i, n].join(" ")
-          ngrams << ngram
-        end
-        
-        ngrams
-      end
-      
-      # Extract both unigrams and bigrams
-      def extract_mixed_terms(documents:, top_n: 20)
-        all_terms = Hash.new(0)
-        
-        documents.each do |doc|
-          # Unigrams
-          tokenize(doc).each { |word| all_terms[word] += 1 }
-          
-          # Bigrams
-          extract_ngrams(doc, n: 2).each { |bigram| all_terms[bigram] += 1 }
-        end
-        
-        # Filter and return top terms
-        all_terms.select { |term, count| count > 1 }  # Appears more than once
-                 .sort_by { |_, count| -count }
-                 .first(top_n)
-                 .map(&:first)
-      end
-    end
-  end
-end

data/lib/ragnar/topic_modeling/topic.rb

@@ -1,117 +0,0 @@
-module Ragnar
-  module TopicModeling
-    class Topic
-      attr_reader :id, :document_indices, :documents, :embeddings, :metadata
-      attr_accessor :terms, :label
-      
-      def initialize(id:, document_indices:, documents:, embeddings:, metadata: nil)
-        @id = id
-        @document_indices = document_indices
-        @documents = documents
-        @embeddings = embeddings
-        @metadata = metadata || []
-        @terms = []
-        @label = nil
-      end
-      
-      def size
-        @documents.length
-      end
-      
-      def centroid
-        @centroid ||= compute_centroid
-      end
-      
-      def representative_docs(k: 3)
-        return @documents if @documents.length <= k
-        
-        # Find documents closest to centroid
-        distances = @embeddings.map do |embedding|
-          distance_to_centroid(embedding)
-        end
-        
-        # Get indices of k smallest distances
-        top_indices = distances.each_with_index.sort_by(&:first).first(k).map(&:last)
-        top_indices.map { |i| @documents[i] }
-      end
-      
-      def coherence
-        @coherence ||= Metrics.compute_coherence(@terms, @documents)
-      end
-      
-      def distinctiveness(other_topics)
-        @distinctiveness ||= Metrics.compute_distinctiveness(self, other_topics)
-      end
-      
-      def set_terms(terms)
-        @terms = terms
-        @centroid = nil  # Reset centroid cache
-      end
-      
-      def set_label(label)
-        @label = label
-      end
-      
-      def summary
-        {
-          id: @id,
-          label: @label || "Topic #{@id}",
-          size: size,
-          terms: @terms.first(10),
-          coherence: coherence.round(3),
-          representative_docs: representative_docs(k: 2).map { |d| d[0..100] + "..." }
-        }
-      end
-      
-      def to_h
-        {
-          id: @id,
-          label: @label,
-          document_indices: @document_indices,
-          terms: @terms,
-          centroid: centroid,
-          size: size,
-          coherence: coherence
-        }
-      end
-      
-      def self.from_h(hash)
-        topic = new(
-          id: hash[:id],
-          document_indices: hash[:document_indices],
-          documents: [],  # Would need to be reconstructed
-          embeddings: [],  # Would need to be reconstructed
-          metadata: []
-        )
-        topic.set_label(hash[:label])
-        topic.set_terms(hash[:terms])
-        topic
-      end
-      
-      private
-      
-      def compute_centroid
-        return [] if @embeddings.empty?
-        
-        # Compute mean of all embeddings
-        dim = @embeddings.first.length
-        centroid = Array.new(dim, 0.0)
-        
-        @embeddings.each do |embedding|
-          embedding.each_with_index do |val, idx|
-            centroid[idx] += val
-          end
-        end
-        
-        centroid.map { |val| val / @embeddings.length }
-      end
-      
-      def distance_to_centroid(embedding)
-        # Euclidean distance
-        Math.sqrt(
-          embedding.zip(centroid).map { |a, b| (a - b) ** 2 }.sum
-        )
-      end
-    end
-  end
-end

data/lib/ragnar/topic_modeling/topic_labeler.rb

@@ -1,61 +0,0 @@
-require_relative 'labeling_strategies'
-
-module Ragnar
-  module TopicModeling
-    class TopicLabeler
-      attr_reader :strategy
-      
-      def initialize(method: :hybrid, llm_client: nil)
-        @method = method
-        @llm_client = llm_client
-        @strategy = LabelingStrategies.create(method, llm_client: llm_client)
-      end
-      
-      # Generate a human-readable label for a topic
-      # Returns a hash with label, description, and metadata
-      def generate_label(topic: nil, terms:, documents: [], method: nil)
-        # Allow method override per call
-        if method && method != @method
-          strategy = LabelingStrategies.create(method, llm_client: @llm_client)
-        else
-          strategy = @strategy
-        end
-        
-        # Generate label using selected strategy
-        result = strategy.generate_label(
-          topic: topic,
-          terms: terms,
-          documents: documents
-        )
-        
-        # Ensure we always return a consistent structure
-        normalize_result(result)
-      end
-      
-      # Convenience method for simple label string
-      def generate_simple_label(terms:, documents: [], method: nil)
-        result = generate_label(terms: terms, documents: documents, method: method)
-        result[:label]
-      end
-      
-      # Change strategy at runtime
-      def set_strategy(method)
-        @method = method
-        @strategy = LabelingStrategies.create(method, llm_client: @llm_client)
-      end
-      
-      private
-      
-      def normalize_result(result)
-        {
-          label: result[:label] || "Unknown Topic",
-          description: result[:description] || nil,
-          method: result[:method] || @method,
-          confidence: result[:confidence] || 0.5,
-          themes: result[:themes] || [],
-          metadata: result.reject { |k, _| [:label, :description, :method, :confidence, :themes].include?(k) }
-        }
-      end
-    end
-  end
-end