lda-ruby 0.3.9 → 0.4.0

data/lib/lda-ruby/backends/rust.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module Lda
+  module Backends
+    class Rust < Base
+      SETTINGS = %i[max_iter convergence em_max_iter em_convergence num_topics init_alpha est_alpha verbose].freeze
+
+      def self.available?
+        return false unless defined?(::Lda::RUST_EXTENSION_LOADED) && ::Lda::RUST_EXTENSION_LOADED
+        return false unless defined?(::Lda::RustBackend)
+
+        if ::Lda::RustBackend.respond_to?(:available?)
+          ::Lda::RustBackend.available?
+        else
+          true
+        end
+      rescue StandardError
+        false
+      end
+
+      SETTINGS.each do |setting_name|
+        define_method(setting_name) do
+          @fallback.public_send(setting_name)
+        end
+
+        define_method("#{setting_name}=") do |value|
+          @fallback.public_send("#{setting_name}=", value)
+        end
+      end
+
+      def initialize(random_seed: nil)
+        super(random_seed: random_seed)
+        raise LoadError, "Rust backend is unavailable for this environment" unless self.class.available?
+
+        @fallback = PureRuby.new(random_seed: random_seed)
+        @fallback.topic_weights_kernel = method(:rust_topic_weights_for_word)
+        @fallback.topic_term_accumulator_kernel = method(:rust_accumulate_topic_term_counts)
+        @fallback.document_inference_kernel = method(:rust_infer_document)
+        @fallback.corpus_iteration_kernel = method(:rust_infer_corpus_iteration)
+        @fallback.topic_term_finalizer_kernel = method(:rust_finalize_topic_term_counts)
+        @fallback.gamma_shift_kernel = method(:rust_average_gamma_shift)
+        @fallback.topic_document_probability_kernel = method(:rust_topic_document_probability)
+        @fallback.topic_term_seed_kernel = method(:rust_seeded_topic_term_probabilities)
+        @fallback.trusted_kernel_outputs = true
+      end
+
+      def name
+        "rust"
+      end
+
+      def corpus=(corpus)
+        @corpus = corpus
+        @fallback.corpus = corpus
+        true
+      end
+
+      def fast_load_corpus_from_file(filename)
+        loaded = @fallback.fast_load_corpus_from_file(filename)
+        @corpus = @fallback.corpus
+        loaded
+      end
+
+      def load_settings(settings_file)
+        loaded = @fallback.load_settings(settings_file)
+        @corpus = @fallback.corpus
+        loaded
+      end
+
+      def set_config(init_alpha, num_topics, max_iter, convergence, em_max_iter, em_convergence, est_alpha)
+        @fallback.set_config(init_alpha, num_topics, max_iter, convergence, em_max_iter, em_convergence, est_alpha)
+      end
+
+      def em(start)
+        rust_before_em(start)
+        @fallback.em(start)
+      end
+
+      def beta
+        @fallback.beta
+      end
+
+      def gamma
+        @fallback.gamma
+      end
+
+      def compute_phi
+        @fallback.compute_phi
+      end
+
+      def model
+        @fallback.model
+      end
+
+      private
+
+      def rust_before_em(start)
+        return unless defined?(::Lda::RustBackend)
+        return unless ::Lda::RustBackend.respond_to?(:before_em)
+
+        ::Lda::RustBackend.before_em(start.to_s, @corpus&.num_docs.to_i, @corpus&.num_terms.to_i)
+      rescue StandardError
+        nil
+      end
+
+      def rust_topic_weights_for_word(beta_probabilities, gamma, word_index, min_probability)
+        return nil unless defined?(::Lda::RustBackend)
+        return nil unless ::Lda::RustBackend.respond_to?(:topic_weights_for_word)
+
+        ::Lda::RustBackend.topic_weights_for_word(
+          beta_probabilities,
+          gamma,
+          Integer(word_index),
+          Float(min_probability)
+        )
+      rescue StandardError
+        nil
+      end
+
+      def rust_accumulate_topic_term_counts(topic_term_counts, phi_d, words, counts)
+        return nil unless defined?(::Lda::RustBackend)
+        return nil unless ::Lda::RustBackend.respond_to?(:accumulate_topic_term_counts)
+
+        ::Lda::RustBackend.accumulate_topic_term_counts(
+          topic_term_counts,
+          phi_d,
+          words,
+          counts
+        )
+      rescue StandardError
+        nil
+      end
+
+      def rust_infer_document(beta_probabilities, gamma_initial, words, counts, max_iter, convergence, min_probability, init_alpha)
+        return nil unless defined?(::Lda::RustBackend)
+        return nil unless ::Lda::RustBackend.respond_to?(:infer_document)
+
+        output = ::Lda::RustBackend.infer_document(
+          beta_probabilities,
+          gamma_initial,
+          words,
+          counts,
+          Integer(max_iter),
+          Float(convergence),
+          Float(min_probability),
+          Float(init_alpha)
+        )
+
+        return nil unless output.is_a?(Array)
+        return nil if output.empty?
+
+        gamma = output.first
+        phi_rows = output[1..] || []
+        [gamma, phi_rows]
+      rescue StandardError
+        nil
+      end
+
+      def rust_infer_corpus_iteration(beta_probabilities, document_words, document_counts, max_iter, convergence, min_probability, init_alpha)
+        return nil unless defined?(::Lda::RustBackend)
+        return nil unless ::Lda::RustBackend.respond_to?(:infer_corpus_iteration)
+
+        ::Lda::RustBackend.infer_corpus_iteration(
+          beta_probabilities,
+          document_words,
+          document_counts,
+          Integer(max_iter),
+          Float(convergence),
+          Float(min_probability),
+          Float(init_alpha)
+        )
+      rescue StandardError
+        nil
+      end
+
+      def rust_finalize_topic_term_counts(topic_term_counts, min_probability)
+        return nil unless defined?(::Lda::RustBackend)
+        return nil unless ::Lda::RustBackend.respond_to?(:normalize_topic_term_counts)
+
+        ::Lda::RustBackend.normalize_topic_term_counts(
+          topic_term_counts,
+          Float(min_probability)
+        )
+      rescue StandardError
+        nil
+      end
+
+      def rust_average_gamma_shift(previous_gamma, current_gamma)
+        return nil unless defined?(::Lda::RustBackend)
+        return nil unless ::Lda::RustBackend.respond_to?(:average_gamma_shift)
+
+        ::Lda::RustBackend.average_gamma_shift(previous_gamma, current_gamma)
+      rescue StandardError
+        nil
+      end
+
+      def rust_topic_document_probability(phi_matrix, document_counts, num_topics, min_probability)
+        return nil unless defined?(::Lda::RustBackend)
+        return nil unless ::Lda::RustBackend.respond_to?(:topic_document_probability)
+
+        ::Lda::RustBackend.topic_document_probability(
+          phi_matrix,
+          document_counts,
+          Integer(num_topics),
+          Float(min_probability)
+        )
+      rescue StandardError
+        nil
+      end
+
+      def rust_seeded_topic_term_probabilities(document_words, document_counts, topics, terms, min_probability)
+        return nil unless defined?(::Lda::RustBackend)
+        return nil unless ::Lda::RustBackend.respond_to?(:seeded_topic_term_probabilities)
+
+        ::Lda::RustBackend.seeded_topic_term_probabilities(
+          document_words,
+          document_counts,
+          Integer(topics),
+          Integer(terms),
+          Float(min_probability)
+        )
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/lda-ruby/backends.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "lda-ruby/backends/base"
+require "lda-ruby/backends/rust"
+require "lda-ruby/backends/native"
+require "lda-ruby/backends/pure_ruby"
+
+module Lda
+  module Backends
+    class << self
+      def build(host:, requested: nil, random_seed: nil)
+        mode = normalize_mode(requested)
+
+        case mode
+        when :auto
+          if Rust.available?
+            Rust.new(random_seed: random_seed)
+          elsif Native.available?(host)
+            Native.new(host, random_seed: random_seed)
+          else
+            PureRuby.new(random_seed: random_seed)
+          end
+        when :rust
+          raise LoadError, "Rust backend is unavailable for this environment" unless Rust.available?
+
+          Rust.new(random_seed: random_seed)
+        when :native
+          raise LoadError, "Native backend is unavailable for this environment" unless Native.available?(host)
+
+          Native.new(host, random_seed: random_seed)
+        when :pure
+          PureRuby.new(random_seed: random_seed)
+        else
+          raise ArgumentError, "Unknown backend mode: #{requested.inspect}"
+        end
+      end
+
+      private
+
+      def normalize_mode(requested)
+        raw_mode = requested || ENV.fetch("LDA_RUBY_BACKEND", "auto")
+
+        case raw_mode.to_s.strip.downcase
+        when "", "auto"
+          :auto
+        when "native", "c"
+          :native
+        when "rust", "rust_native"
+          :rust
+        when "pure", "ruby", "pure_ruby"
+          :pure
+        else
+          raw_mode
+        end
+      end
+    end
+  end
+end

data/lib/lda-ruby/corpus/corpus.rb

@@ -1,24 +1,26 @@
-require 'set'
+require "set"
+require "yaml"
 
 module Lda
   class Corpus
     attr_reader :documents, :num_docs, :num_terms, :vocabulary, :stopwords
 
     def initialize(stop_word_list = nil)
-      @documents = Array.new
+      @documents = []
       @all_terms = Set.new
       @num_terms = @num_docs = 0
       @vocabulary = Vocabulary.new
-      if stop_word_list.nil?
-        @stopwords = YAML.load_file(File.join(File.dirname(__FILE__), '..', 'config', 'stopwords.yml'))
-      else
-        @stopwords = YAML.load_file(stop_word_list)
-      end
-      @stopwords.map! { |w| w.strip }
+      @stopwords =  if stop_word_list.nil?
+                      File.join(File.dirname(__FILE__), '..', 'config', 'stopwords.yml')
+                    else
+                      stop_word_list
+                    end
+      @stopwords = YAML.load_file(@stopwords)
+      @stopwords.map!(&:strip)
     end
-    
+
     def add_document(doc)
-      raise 'Parameter +doc+ must be of type Document' unless doc.kind_of?(Document)
+      raise 'Parameter +doc+ must be of type Document' unless doc.is_a?(Document)
 
       @documents << doc
 
@@ -29,11 +31,11 @@ module Lda
       update_vocabulary(doc)
       nil
     end
-	
-	def remove_word(word)
-		@vocabulary.words.delete word
-	end
-	
+
+    def remove_word(word)
+      @vocabulary.words.delete word
+    end
+
     protected
 
     def update_vocabulary(doc)

data/lib/lda-ruby/corpus/data_corpus.rb

@@ -12,11 +12,11 @@ module Lda
     protected
 
     def load_from_file
-      txt = File.open(@filename, 'r') { |f| f.read }
+      txt = File.open(@filename, 'r', &:read)
       lines = txt.split(/[\r\n]+/)
       lines.each do |line|
         add_document(DataDocument.new(self, line))
       end
     end
   end
-end
+end

data/lib/lda-ruby/corpus/directory_corpus.rb

@@ -15,11 +15,11 @@ module Lda
     protected
 
     def load_from_directory
-      dir_glob = File.join(@path, (@extension ? "*.#{@extension}" : "*"))
+      dir_glob = File.join(@path, (@extension ? "*.#{@extension}" : '*'))
 
       Dir.glob(dir_glob).each do |filename|
         add_document(TextDocument.build_from_file(self, filename))
       end
     end
   end
-end
+end

data/lib/lda-ruby/corpus/text_corpus.rb

@@ -6,7 +6,7 @@ module Lda
     def initialize(input_data)
       super()
 
-      docs = if input_data.is_a?(String) && File.exists?(input_data)
+      docs = if input_data.is_a?(String) && File.exist?(input_data)
         # yaml file containing an array of strings representing each document
         YAML.load_file(input_data)
       elsif input_data.is_a?(Array)
@@ -16,7 +16,7 @@ module Lda
         # a single string representing one document
         [input_data]
       else
-        raise "Unknown input type: please pass in a valid filename or an array of strings."
+        raise 'Unknown input type: please pass in a valid filename or an array of strings.'
       end
 
       docs.each do |doc|

data/lib/lda-ruby/document/document.rb

@@ -1,4 +1,3 @@
-# coding: utf-8
 require 'yaml'
 
 module Lda
@@ -8,9 +7,9 @@ module Lda
     def initialize(corpus)
       @corpus = corpus
 
-      @words  = Array.new
-      @counts = Array.new
-      @tokens = Array.new
+      @words  = []
+      @counts = []
+      @tokens = []
       @length = 0
       @total  = 0
     end
@@ -23,7 +22,7 @@ module Lda
       @length = @words.size
     end
 
-    def has_text?
+    def text?
       false
     end
 
@@ -32,7 +31,8 @@ module Lda
     end
 
     def tokenize(text)
-      clean_text = text.gsub(/[^a-zäöüß'-]+/i, ' ').gsub(/\s+/, ' ').downcase  # remove everything but letters and ' and leave only single spaces
+      # remove everything but letters and ' and leave only single spaces
+      clean_text = text.gsub(/[^a-zäöüß'-]+/i, ' ').gsub(/\s+/, ' ').downcase
       @tokens = handle(clean_text.split(' '))
       nil
     end

data/lib/lda-ruby/document/text_document.rb

@@ -11,14 +11,15 @@ module Lda
       build_from_tokens
     end
 
-    def has_text?
+    def text?
       true
     end
 
     def self.build_from_file(corpus, filename)
-      @filename = filename.dup.freeze
-      text = File.open(@filename, 'r') { |f| f.read }
-      self.new(corpus, text)
+      text = File.read(filename)
+      document = new(corpus, text)
+      document.instance_variable_set(:@filename, filename.dup.freeze)
+      document
     end
 
     protected

data/lib/lda-ruby/rust_build_policy.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Lda
+  module RustBuildPolicy
+    ENV_KEY = "LDA_RUBY_RUST_BUILD"
+    AUTO = "auto"
+    ALWAYS = "always"
+    NEVER = "never"
+    VALID_VALUES = [AUTO, ALWAYS, NEVER].freeze
+
+    module_function
+
+    def resolve(raw_value = ENV[ENV_KEY])
+      value = raw_value.to_s.strip.downcase
+      return AUTO if value.empty?
+      return value if VALID_VALUES.include?(value)
+
+      AUTO
+    end
+  end
+end

data/lib/lda-ruby/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Lda
+  VERSION = "0.4.0"
+end