ragnar-cli 0.1.0.pre.3 → 0.1.0.pre.5

data/lib/ragnar/topic_modeling/term_extractor.rb

@@ -1,170 +0,0 @@
-require 'set'
-
-module Ragnar
-  module TopicModeling
-    class TermExtractor
-      # Common English stop words to filter out
-      STOP_WORDS = Set.new(%w[
-        the be to of and a in that have i it for not on with he as you do at
-        this but his by from they we say her she or an will my one all would
-        there their what so up out if about who get which go me when make can
-        like time no just him know take people into year your good some could
-        them see other than then now look only come its over think also back
-        after use two how our work first well way even new want because any
-        these give day most us is was are been has had were said did get may
-      ])
-      
-      def initialize(stop_words: STOP_WORDS, min_word_length: 3, max_word_length: 20)
-        @stop_words = stop_words
-        @min_word_length = min_word_length
-        @max_word_length = max_word_length
-      end
-      
-      # Extract distinctive terms using c-TF-IDF
-      def extract_distinctive_terms(topic_docs:, all_docs:, top_n: 20)
-        # Tokenize and count terms in topic
-        topic_terms = count_terms(topic_docs)
-        
-        # Tokenize and count document frequency across all docs
-        doc_frequencies = compute_document_frequencies(all_docs)
-        
-        # Compute c-TF-IDF scores
-        scores = {}
-        total_docs = all_docs.length.to_f
-        
-        topic_terms.each do |term, tf|
-          # c-TF-IDF formula: tf * log(N / df)
-          df = doc_frequencies[term] || 1
-          idf = Math.log(total_docs / df)
-          scores[term] = tf * idf
-        end
-        
-        # Return top scoring terms
-        scores.sort_by { |_, score| -score }
-               .first(top_n)
-               .map(&:first)
-      end
-      
-      # Standard TF-IDF implementation
-      def extract_tfidf_terms(documents:, top_n: 20)
-        # Document frequency
-        doc_frequencies = compute_document_frequencies(documents)
-        total_docs = documents.length.to_f
-        
-        # Compute TF-IDF for each document
-        all_scores = []
-        
-        documents.each do |doc|
-          terms = count_terms([doc])
-          doc_length = terms.values.sum.to_f
-          
-          scores = {}
-          terms.each do |term, count|
-            tf = count / doc_length  # Normalized term frequency
-            df = doc_frequencies[term] || 1
-            idf = Math.log(total_docs / df)
-            scores[term] = tf * idf
-          end
-          
-          all_scores << scores
-        end
-        
-        # Aggregate scores across all documents
-        aggregated = {}
-        all_scores.each do |doc_scores|
-          doc_scores.each do |term, score|
-            aggregated[term] ||= 0
-            aggregated[term] += score
-          end
-        end
-        
-        # Return top terms
-        aggregated.sort_by { |_, score| -score }
-                  .first(top_n)
-                  .map(&:first)
-      end
-      
-      # Simple term frequency extraction
-      def extract_frequent_terms(documents:, top_n: 20)
-        terms = count_terms(documents)
-        terms.sort_by { |_, count| -count }
-             .first(top_n)
-             .map(&:first)
-      end
-      
-      private
-      
-      def tokenize(text)
-        # Simple tokenization - can be improved with proper NLP tokenizer
-        text.downcase
-            .split(/\W+/)
-            .select { |word| valid_word?(word) }
-      end
-      
-      def valid_word?(word)
-        word.length >= @min_word_length &&
-        word.length <= @max_word_length &&
-        !@stop_words.include?(word) &&
-        !word.match?(/^\d+$/)  # Not pure numbers
-      end
-      
-      def count_terms(documents)
-        terms = Hash.new(0)
-        
-        documents.each do |doc|
-          tokenize(doc).each do |word|
-            terms[word] += 1
-          end
-        end
-        
-        terms
-      end
-      
-      def compute_document_frequencies(documents)
-        doc_frequencies = Hash.new(0)
-        
-        documents.each do |doc|
-          # Use set to count each term once per document
-          unique_terms = Set.new(tokenize(doc))
-          unique_terms.each do |term|
-            doc_frequencies[term] += 1
-          end
-        end
-        
-        doc_frequencies
-      end
-      
-      # N-gram extraction for phrases
-      def extract_ngrams(text, n: 2)
-        words = tokenize(text)
-        ngrams = []
-        
-        (0..words.length - n).each do |i|
-          ngram = words[i, n].join(" ")
-          ngrams << ngram
-        end
-        
-        ngrams
-      end
-      
-      # Extract both unigrams and bigrams
-      def extract_mixed_terms(documents:, top_n: 20)
-        all_terms = Hash.new(0)
-        
-        documents.each do |doc|
-          # Unigrams
-          tokenize(doc).each { |word| all_terms[word] += 1 }
-          
-          # Bigrams
-          extract_ngrams(doc, n: 2).each { |bigram| all_terms[bigram] += 1 }
-        end
-        
-        # Filter and return top terms
-        all_terms.select { |term, count| count > 1 }  # Appears more than once
-                 .sort_by { |_, count| -count }
-                 .first(top_n)
-                 .map(&:first)
-      end
-    end
-  end
-end

data/lib/ragnar/topic_modeling/topic.rb

@@ -1,117 +0,0 @@
-module Ragnar
-  module TopicModeling
-    class Topic
-      attr_reader :id, :document_indices, :documents, :embeddings, :metadata
-      attr_accessor :terms, :label
-      
-      def initialize(id:, document_indices:, documents:, embeddings:, metadata: nil)
-        @id = id
-        @document_indices = document_indices
-        @documents = documents
-        @embeddings = embeddings
-        @metadata = metadata || []
-        @terms = []
-        @label = nil
-      end
-      
-      def size
-        @documents.length
-      end
-      
-      def centroid
-        @centroid ||= compute_centroid
-      end
-      
-      def representative_docs(k: 3)
-        return @documents if @documents.length <= k
-        
-        # Find documents closest to centroid
-        distances = @embeddings.map do |embedding|
-          distance_to_centroid(embedding)
-        end
-        
-        # Get indices of k smallest distances
-        top_indices = distances.each_with_index.sort_by(&:first).first(k).map(&:last)
-        top_indices.map { |i| @documents[i] }
-      end
-      
-      def coherence
-        @coherence ||= Metrics.compute_coherence(@terms, @documents)
-      end
-      
-      def distinctiveness(other_topics)
-        @distinctiveness ||= Metrics.compute_distinctiveness(self, other_topics)
-      end
-      
-      def set_terms(terms)
-        @terms = terms
-        @centroid = nil  # Reset centroid cache
-      end
-      
-      def set_label(label)
-        @label = label
-      end
-      
-      def summary
-        {
-          id: @id,
-          label: @label || "Topic #{@id}",
-          size: size,
-          terms: @terms.first(10),
-          coherence: coherence.round(3),
-          representative_docs: representative_docs(k: 2).map { |d| d[0..100] + "..." }
-        }
-      end
-      
-      def to_h
-        {
-          id: @id,
-          label: @label,
-          document_indices: @document_indices,
-          terms: @terms,
-          centroid: centroid,
-          size: size,
-          coherence: coherence
-        }
-      end
-      
-      def self.from_h(hash)
-        topic = new(
-          id: hash[:id],
-          document_indices: hash[:document_indices],
-          documents: [],  # Would need to be reconstructed
-          embeddings: [],  # Would need to be reconstructed
-          metadata: []
-        )
-        topic.set_label(hash[:label])
-        topic.set_terms(hash[:terms])
-        topic
-      end
-      
-      private
-      
-      def compute_centroid
-        return [] if @embeddings.empty?
-        
-        # Compute mean of all embeddings
-        dim = @embeddings.first.length
-        centroid = Array.new(dim, 0.0)
-        
-        @embeddings.each do |embedding|
-          embedding.each_with_index do |val, idx|
-            centroid[idx] += val
-          end
-        end
-        
-        centroid.map { |val| val / @embeddings.length }
-      end
-      
-      def distance_to_centroid(embedding)
-        # Euclidean distance
-        Math.sqrt(
-          embedding.zip(centroid).map { |a, b| (a - b) ** 2 }.sum
-        )
-      end
-    end
-  end
-end

data/lib/ragnar/topic_modeling/topic_labeler.rb

@@ -1,61 +0,0 @@
-require_relative 'labeling_strategies'
-
-module Ragnar
-  module TopicModeling
-    class TopicLabeler
-      attr_reader :strategy
-      
-      def initialize(method: :hybrid, llm_client: nil)
-        @method = method
-        @llm_client = llm_client
-        @strategy = LabelingStrategies.create(method, llm_client: llm_client)
-      end
-      
-      # Generate a human-readable label for a topic
-      # Returns a hash with label, description, and metadata
-      def generate_label(topic: nil, terms:, documents: [], method: nil)
-        # Allow method override per call
-        if method && method != @method
-          strategy = LabelingStrategies.create(method, llm_client: @llm_client)
-        else
-          strategy = @strategy
-        end
-        
-        # Generate label using selected strategy
-        result = strategy.generate_label(
-          topic: topic,
-          terms: terms,
-          documents: documents
-        )
-        
-        # Ensure we always return a consistent structure
-        normalize_result(result)
-      end
-      
-      # Convenience method for simple label string
-      def generate_simple_label(terms:, documents: [], method: nil)
-        result = generate_label(terms: terms, documents: documents, method: method)
-        result[:label]
-      end
-      
-      # Change strategy at runtime
-      def set_strategy(method)
-        @method = method
-        @strategy = LabelingStrategies.create(method, llm_client: @llm_client)
-      end
-      
-      private
-      
-      def normalize_result(result)
-        {
-          label: result[:label] || "Unknown Topic",
-          description: result[:description] || nil,
-          method: result[:method] || @method,
-          confidence: result[:confidence] || 0.5,
-          themes: result[:themes] || [],
-          metadata: result.reject { |k, _| [:label, :description, :method, :confidence, :themes].include?(k) }
-        }
-      end
-    end
-  end
-end