lda-ruby 0.3.9 → 0.4.0

data/lib/lda-ruby.rb

@@ -1,29 +1,125 @@
-$LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
-
-require 'lda-ruby/lda'
-require 'lda-ruby/document/document'
-require 'lda-ruby/document/data_document'
-require 'lda-ruby/document/text_document'
-require 'lda-ruby/corpus/corpus'
-require 'lda-ruby/corpus/data_corpus'
-require 'lda-ruby/corpus/text_corpus'
-require 'lda-ruby/corpus/directory_corpus'
-require 'lda-ruby/vocabulary'
+# frozen_string_literal: true
+
+require "lda-ruby/version"
+require "rbconfig"
+
+rust_extension_loaded = false
+rust_dlext = RbConfig::CONFIG.fetch("DLEXT")
+
+[
+  "lda_ruby_rust",
+  "../ext/lda-ruby-rust/target/release/lda_ruby_rust",
+  "../ext/lda-ruby-rust/target/release/lda_ruby_rust.#{rust_dlext}",
+  "../ext/lda-ruby-rust/target/debug/lda_ruby_rust",
+  "../ext/lda-ruby-rust/target/debug/lda_ruby_rust.#{rust_dlext}"
+].each do |rust_extension_candidate|
+  begin
+    if rust_extension_candidate.start_with?("../")
+      require_relative rust_extension_candidate
+    else
+      require rust_extension_candidate
+    end
+
+    rust_extension_loaded = true
+    break
+  rescue LoadError
+    next
+  end
+end
+
+native_extension_loaded = false
+
+begin
+  require "lda-ruby/lda"
+  native_extension_loaded = true
+rescue LoadError
+  begin
+    require_relative "../ext/lda-ruby/lda"
+    native_extension_loaded = true
+  rescue LoadError
+    native_extension_loaded = false
+  end
+end
+
+LDA_RUBY_NATIVE_EXTENSION_LOADED = native_extension_loaded unless defined?(LDA_RUBY_NATIVE_EXTENSION_LOADED)
+LDA_RUBY_RUST_EXTENSION_LOADED = rust_extension_loaded unless defined?(LDA_RUBY_RUST_EXTENSION_LOADED)
+
+require "lda-ruby/document/document"
+require "lda-ruby/document/data_document"
+require "lda-ruby/document/text_document"
+require "lda-ruby/corpus/corpus"
+require "lda-ruby/corpus/data_corpus"
+require "lda-ruby/corpus/text_corpus"
+require "lda-ruby/corpus/directory_corpus"
+require "lda-ruby/vocabulary"
+require "lda-ruby/backends"
 
 module Lda
+  RUST_EXTENSION_LOADED = LDA_RUBY_RUST_EXTENSION_LOADED unless const_defined?(:RUST_EXTENSION_LOADED)
+  NATIVE_EXTENSION_LOADED = LDA_RUBY_NATIVE_EXTENSION_LOADED unless const_defined?(:NATIVE_EXTENSION_LOADED)
+
   class Lda
-    attr_reader :vocab, :corpus
+    NATIVE_ALIAS_MAP = {
+      fast_load_corpus_from_file: :__native_fast_load_corpus_from_file,
+      "corpus=": :__native_set_corpus,
+      em: :__native_em,
+      load_settings: :__native_load_settings,
+      set_config: :__native_set_config,
+      max_iter: :__native_max_iter,
+      "max_iter=": :__native_set_max_iter,
+      convergence: :__native_convergence,
+      "convergence=": :__native_set_convergence,
+      em_max_iter: :__native_em_max_iter,
+      "em_max_iter=": :__native_set_em_max_iter,
+      em_convergence: :__native_em_convergence,
+      "em_convergence=": :__native_set_em_convergence,
+      init_alpha: :__native_init_alpha,
+      "init_alpha=": :__native_set_init_alpha,
+      est_alpha: :__native_est_alpha,
+      "est_alpha=": :__native_set_est_alpha,
+      num_topics: :__native_num_topics,
+      "num_topics=": :__native_set_num_topics,
+      verbose: :__native_verbose,
+      "verbose=": :__native_set_verbose,
+      beta: :__native_beta,
+      gamma: :__native_gamma,
+      compute_phi: :__native_compute_phi,
+      model: :__native_model
+    }.freeze
+
+    NATIVE_ALIAS_MAP.each do |native_name, alias_name|
+      next unless method_defined?(native_name)
+
+      alias_method alias_name, native_name
+      private alias_name
+    end
+
+    attr_reader :vocab, :corpus, :backend
+
+    def initialize(corpus, backend: nil, random_seed: nil)
+      @backend = Backends.build(host: self, requested: backend, random_seed: random_seed)
 
-    def initialize(corpus)
       load_default_settings
 
       @vocab = nil
       self.corpus = corpus
-      @vocab = corpus.vocabulary.to_a if corpus.vocabulary
+      @vocab = corpus.vocabulary.to_a if corpus.respond_to?(:vocabulary) && corpus.vocabulary
 
       @phi = nil
     end
 
+    def backend_name
+      @backend.name
+    end
+
+    def native_backend?
+      backend_name == "native"
+    end
+
+    def rust_backend?
+      backend_name == "rust"
+    end
+
     def load_default_settings
       self.max_iter = 20
       self.convergence = 1e-6
@@ -36,25 +132,138 @@ module Lda
       [20, 1e-6, 100, 1e-4, 20, 0.3, 1]
     end
 
-    def load_corpus(filename)
-      @corpus = Corpus.new
-      @corpus.load_from_file(filename)
+    def set_config(init_alpha, num_topics, max_iter, convergence, em_max_iter, em_convergence = self.em_convergence, est_alpha = self.est_alpha)
+      @backend.set_config(
+        Float(init_alpha),
+        Integer(num_topics),
+        Integer(max_iter),
+        Float(convergence),
+        Integer(em_max_iter),
+        Float(em_convergence),
+        Integer(est_alpha)
+      )
+    end
+
+    def max_iter
+      @backend.max_iter
+    end
+
+    def max_iter=(value)
+      @backend.max_iter = Integer(value)
+    end
+
+    def convergence
+      @backend.convergence
+    end
+
+    def convergence=(value)
+      @backend.convergence = Float(value)
+    end
+
+    def em_max_iter
+      @backend.em_max_iter
+    end
+
+    def em_max_iter=(value)
+      @backend.em_max_iter = Integer(value)
+    end
+
+    def em_convergence
+      @backend.em_convergence
+    end
+
+    def em_convergence=(value)
+      @backend.em_convergence = Float(value)
+    end
+
+    def num_topics
+      @backend.num_topics
+    end
+
+    def num_topics=(value)
+      @backend.num_topics = Integer(value)
+    end
 
+    def init_alpha
+      @backend.init_alpha
+    end
+
+    def init_alpha=(value)
+      @backend.init_alpha = Float(value)
+    end
+
+    def est_alpha
+      @backend.est_alpha
+    end
+
+    def est_alpha=(value)
+      @backend.est_alpha = Integer(value)
+    end
+
+    def verbose
+      @backend.verbose
+    end
+
+    def verbose=(value)
+      @backend.verbose = !!value
+    end
+
+    def corpus=(corpus)
+      @corpus = corpus
+      @backend.corpus = corpus
       true
     end
 
+    def load_corpus(filename)
+      fast_load_corpus_from_file(filename)
+    end
+
+    def fast_load_corpus_from_file(filename)
+      loaded = @backend.fast_load_corpus_from_file(filename)
+
+      if @backend.corpus
+        @corpus = @backend.corpus
+        @vocab = @corpus.vocabulary.to_a if @corpus.respond_to?(:vocabulary) && @corpus.vocabulary
+      elsif @corpus.nil?
+        @corpus = DataCorpus.new(filename)
+      end
+
+      !!loaded
+    end
+
+    def load_settings(settings_file)
+      @backend.load_settings(settings_file)
+    end
+
     def load_vocabulary(vocab)
       if vocab.is_a?(Array)
-        @vocab = Marshal::load(Marshal::dump(vocab))      # deep clone array
+        @vocab = Marshal.load(Marshal.dump(vocab)) # deep clone array
       elsif vocab.is_a?(Vocabulary)
         @vocab = vocab.to_a
       else
-        @vocab = File.open(vocab, 'r') { |f| f.read.split(/\s+/) }
+        @vocab = File.read(vocab).split(/\s+/)
       end
 
       true
     end
 
+    def em(start = "random")
+      @phi = nil
+      @backend.em(start.to_s)
+    end
+
+    def beta
+      @backend.beta
+    end
+
+    def gamma
+      @backend.gamma
+    end
+
+    def model
+      @backend.model
+    end
+
     #
     # Visualization method for printing out the top +words_per_topic+ words
     # for each topic.
@@ -62,14 +271,18 @@ module Lda
     # See also +top_words+.
     #
     def print_topics(words_per_topic = 10)
-      raise 'No vocabulary loaded.' unless @vocab
+      raise "No vocabulary loaded." unless @vocab
 
-      self.beta.each_with_index do |topic, topic_num|
-        # Sort the topic array and return the sorted indices of the best scores
-        indices = (topic.zip((0...@vocab.size).to_a).sort { |i, j| i[0] <=> j[0] }.map { |i, j| j }.reverse)[0...words_per_topic]
+      beta.each_with_index do |topic, topic_num|
+        indices = topic
+          .each_with_index
+          .sort_by { |score, _index| score }
+          .reverse
+          .first(words_per_topic)
+          .map { |_score, index| index }
 
         puts "Topic #{topic_num}"
-        puts "\t#{indices.map {|i| @vocab[i]}.join("\n\t")}"
+        puts "\t#{indices.map { |i| @vocab[i] }.join("\n\t")}"
         puts ""
       end
 
@@ -87,21 +300,24 @@ module Lda
     # See also +print_topics+.
     #
     def top_word_indices(words_per_topic = 10)
-      raise 'No vocabulary loaded.' unless @vocab
+      raise "No vocabulary loaded." unless @vocab
 
-      # find the highest scoring words per topic
-      topics = Hash.new
-      indices = (0...@vocab.size).to_a
+      topics = {}
 
-      self.beta.each_with_index do |topic, topic_num|
-        topics[topic_num] = (topic.zip((0...@vocab.size).to_a).sort { |i, j| i[0] <=> j[0] }.map { |i, j| j }.reverse)[0...words_per_topic]
+      beta.each_with_index do |topic, topic_num|
+        topics[topic_num] = topic
+          .each_with_index
+          .sort_by { |score, _index| score }
+          .reverse
+          .first(words_per_topic)
+          .map { |_score, index| index }
       end
 
       topics
     end
 
     def top_words(words_per_topic = 10)
-      output = Hash.new
+      output = {}
 
       topics = top_word_indices(words_per_topic)
       topics.each_pair do |topic_num, words|
@@ -118,49 +334,78 @@ module Lda
     # after the first call, so if it needs to be recomputed, set the +recompute+
     # value to true.
     #
-    def phi(recompute=false)
-      if @phi.nil? || recompute
-        @phi = self.compute_phi
-      end
+    def phi(recompute = false)
+      @phi = compute_phi if @phi.nil? || recompute
 
       @phi
     end
 
+    def compute_phi
+      @backend.compute_phi
+    end
+
     #
     # Compute the average log probability for each topic for each document in the corpus.
     # This method returns a matrix:  num_docs x num_topics with the average log probability
     # for the topic in the document.
     #
     def compute_topic_document_probability
-      outp = Array.new
+      phi_matrix = phi
+      document_counts = @corpus.documents.map(&:counts)
+
+      backend_output = @backend.topic_document_probability(phi_matrix, document_counts)
+      if valid_topic_document_probability_output?(backend_output, document_counts.size, num_topics)
+        return backend_output
+      end
+
+      outp = []
 
       @corpus.documents.each_with_index do |doc, idx|
-        tops = [0.0] * self.num_topics
-        ttl  = doc.counts.inject(0.0) {|sum, i| sum + i}
-        self.phi[idx].each_with_index do |word_dist, word_idx|
+        tops = [0.0] * num_topics
+        ttl = doc.counts.inject(0.0) { |sum, i| sum + i }
+
+        phi_matrix[idx].each_with_index do |word_dist, word_idx|
           word_dist.each_with_index do |top_prob, top_idx|
-            tops[top_idx] += Math.log(top_prob) * doc.counts[word_idx]
+            tops[top_idx] += Math.log([top_prob, 1e-300].max) * doc.counts[word_idx]
           end
         end
-        tops = tops.map {|i| i / ttl}
+
+        tops = tops.map { |i| i / ttl }
         outp << tops
       end
 
       outp
     end
 
+    def valid_topic_document_probability_output?(output, expected_docs, expected_topics)
+      return false unless output.is_a?(Array)
+      return false unless output.size == expected_docs
+
+      output.each do |row|
+        return false unless row.is_a?(Array)
+        return false unless row.size == expected_topics
+        row.each do |value|
+          return false unless value.is_a?(Numeric)
+          return false unless value.finite?
+        end
+      end
+
+      true
+    end
+
     #
     # String representation displaying current settings.
     #
     def to_s
       outp = ["LDA Settings:"]
-      outp << "    Initial alpha: %0.6f" % self.init_alpha
-      outp << "      # of topics: %d" % self.num_topics
-      outp << "   Max iterations: %d" % self.max_iter
-      outp << "      Convergence: %0.6f" % self.convergence
-      outp << "EM max iterations: %d" % self.em_max_iter
-      outp << "   EM convergence: %0.6f" % self.em_convergence
-      outp << "   Estimate alpha: %d" % self.est_alpha
+      outp << format("    Initial alpha: %0.6f", init_alpha)
+      outp << format("      # of topics: %d", num_topics)
+      outp << format("   Max iterations: %d", max_iter)
+      outp << format("      Convergence: %0.6f", convergence)
+      outp << format("EM max iterations: %d", em_max_iter)
+      outp << format("   EM convergence: %0.6f", em_convergence)
+      outp << format("   Estimate alpha: %d", est_alpha)
+      outp << format("         Backend: %s", backend_name)
 
       outp.join("\n")
     end

data/test/backend_compatibility_test.rb

@@ -0,0 +1,146 @@
+require_relative "test_helper"
+
+class BackendCompatibilityTest < Test::Unit::TestCase
+  FIXTURE_DOCUMENTS = [
+    "apple banana apple banana fruit sweet fruit",
+    "truck wheel truck road engine metal road",
+    "ruby code gem ruby class module test",
+    "banana fruit apple orchard fresh sweet",
+    "engine road truck wheel fuel highway",
+    "module ruby class object gem code"
+  ].freeze
+
+  def setup
+    @corpus = Lda::TextCorpus.new(FIXTURE_DOCUMENTS)
+  end
+
+  def test_pure_backend_seeded_fixture
+    lda = build_and_train(:pure)
+
+    assert_equal "pure_ruby", lda.backend_name
+    assert_backend_output_valid(lda)
+  end
+
+  def test_native_backend_seeded_fixture
+    return unless Lda::NATIVE_EXTENSION_LOADED
+
+    lda = build_and_train(:native)
+
+    assert_equal "native", lda.backend_name
+    assert_backend_output_valid(lda)
+  end
+
+  def test_native_and_pure_backend_agree_on_shapes
+    return unless Lda::NATIVE_EXTENSION_LOADED
+
+    native = build_and_train(:native)
+    pure = build_and_train(:pure)
+
+    assert_equal native.model[0], pure.model[0]
+    assert_equal native.model[1], pure.model[1]
+    assert_equal native.beta.size, pure.beta.size
+    assert_equal native.gamma.size, pure.gamma.size
+    assert_equal native.phi.size, pure.phi.size
+  end
+
+  def test_rust_backend_seeded_fixture
+    return unless Lda::RUST_EXTENSION_LOADED
+
+    rust = build_and_train(:rust)
+
+    assert_equal "rust", rust.backend_name
+    assert_backend_output_valid(rust)
+  end
+
+  def test_rust_and_pure_backend_numeric_parity
+    return unless Lda::RUST_EXTENSION_LOADED
+
+    pure = build_and_train(:pure)
+    rust = build_and_train(:rust)
+
+    assert_nested_close(pure.gamma, rust.gamma, 1e-9)
+    assert_nested_close(pure.beta, rust.beta, 1e-9)
+    assert_nested_close(pure.phi, rust.phi, 1e-9)
+    assert_nested_close(
+      exponentiate_nested(pure.compute_topic_document_probability),
+      exponentiate_nested(rust.compute_topic_document_probability),
+      1e-6
+    )
+  end
+
+  private
+
+  def build_and_train(backend)
+    lda = Lda::Lda.new(@corpus, backend: backend, random_seed: 1234)
+    lda.verbose = false
+    lda.num_topics = 3
+    lda.max_iter = 25
+    lda.em_max_iter = 40
+    lda.convergence = 1e-5
+    lda.em_convergence = 1e-4
+    lda.em("seeded")
+    lda
+  end
+
+  def assert_backend_output_valid(lda)
+    assert_equal 3, lda.model[0]
+    assert lda.model[1] > 0
+
+    assert_equal @corpus.num_docs, lda.gamma.size
+    lda.gamma.each do |topic_weights|
+      assert_equal 3, topic_weights.size
+      topic_weights.each do |weight|
+        assert weight.is_a?(Numeric)
+        assert weight.finite?
+        assert weight.positive?
+      end
+    end
+
+    assert_equal 3, lda.beta.size
+    lda.beta.each do |topic_log_probs|
+      assert topic_log_probs.size > 0
+      probabilities = topic_log_probs.map { |log_prob| Math.exp(log_prob) }
+      assert_in_delta 1.0, probabilities.sum, 1e-3
+    end
+
+    phi = lda.phi
+    assert_equal @corpus.num_docs, phi.size
+    phi.each_with_index do |doc_phi, doc_index|
+      assert_equal @corpus.documents[doc_index].length, doc_phi.size
+      doc_phi.each do |word_topic_distribution|
+        assert_equal 3, word_topic_distribution.size
+        assert_in_delta 1.0, word_topic_distribution.sum, 1e-3
+      end
+    end
+
+    probabilities = lda.compute_topic_document_probability
+    assert_equal @corpus.num_docs, probabilities.size
+    probabilities.each do |row|
+      assert_equal 3, row.size
+      row.each { |value| assert value.finite? }
+    end
+
+    top_words = lda.top_words(4)
+    assert_equal 3, top_words.size
+    top_words.each_value { |words| assert_equal 4, words.size }
+  end
+
+  def assert_nested_close(left, right, tolerance)
+    assert_equal left.class, right.class
+
+    if left.is_a?(Array)
+      assert_equal left.size, right.size
+      left.each_with_index do |left_item, index|
+        assert_nested_close(left_item, right[index], tolerance)
+      end
+    else
+      assert_in_delta left.to_f, right.to_f, tolerance
+    end
+  end
+
+  def exponentiate_nested(value)
+    return Math.exp(value.to_f) unless value.is_a?(Array)
+
+    value.map { |item| exponentiate_nested(item) }
+  end
+end

data/test/backends_selection_test.rb

@@ -0,0 +1,100 @@
+require_relative "test_helper"
+
+class BackendsSelectionTest < Test::Unit::TestCase
+  RUST_ALIAS = :__test_original_rust_available__
+  NATIVE_ALIAS = :__test_original_native_available__
+
+  setup do
+    @host = Object.new
+    @rust_singleton = Lda::Backends::Rust.singleton_class
+    @native_singleton = Lda::Backends::Native.singleton_class
+
+    @rust_singleton.send(:alias_method, RUST_ALIAS, :available?)
+    @native_singleton.send(:alias_method, NATIVE_ALIAS, :available?)
+    @previous_env_backend = ENV["LDA_RUBY_BACKEND"]
+  end
+
+  teardown do
+    restore_availability_stubs
+    ENV["LDA_RUBY_BACKEND"] = @previous_env_backend
+  end
+
+  should "prefer rust over native in auto mode when both are available" do
+    stub_rust_available(true)
+    stub_native_available(true)
+
+    backend = Lda::Backends.build(host: @host, requested: :auto)
+    assert_instance_of Lda::Backends::Rust, backend
+  end
+
+  should "fall back to native in auto mode when rust is unavailable" do
+    stub_rust_available(false)
+    stub_native_available(true)
+
+    backend = Lda::Backends.build(host: @host, requested: :auto)
+    assert_instance_of Lda::Backends::Native, backend
+  end
+
+  should "fall back to pure in auto mode when rust and native are unavailable" do
+    stub_rust_available(false)
+    stub_native_available(false)
+
+    backend = Lda::Backends.build(host: @host, requested: :auto)
+    assert_instance_of Lda::Backends::PureRuby, backend
+  end
+
+  should "respect LDA_RUBY_BACKEND env override when requested mode is nil" do
+    stub_rust_available(true)
+    stub_native_available(true)
+    ENV["LDA_RUBY_BACKEND"] = "pure_ruby"
+
+    backend = Lda::Backends.build(host: @host, requested: nil)
+    assert_instance_of Lda::Backends::PureRuby, backend
+  end
+
+  should "raise for unknown backend mode" do
+    stub_rust_available(false)
+    stub_native_available(false)
+
+    error = assert_raise(ArgumentError) do
+      Lda::Backends.build(host: @host, requested: :unknown_backend)
+    end
+
+    assert_match(/Unknown backend mode/i, error.message)
+  end
+
+  private
+
+  def stub_rust_available(value)
+    silence_redefinition_warnings do
+      @rust_singleton.send(:define_method, :available?) do
+        value
+      end
+    end
+  end
+
+  def stub_native_available(value)
+    silence_redefinition_warnings do
+      @native_singleton.send(:define_method, :available?) do |_host|
+        value
+      end
+    end
+  end
+
+  def restore_availability_stubs
+    silence_redefinition_warnings do
+      @rust_singleton.send(:alias_method, :available?, RUST_ALIAS)
+      @native_singleton.send(:alias_method, :available?, NATIVE_ALIAS)
+    end
+    @rust_singleton.send(:remove_method, RUST_ALIAS)
+    @native_singleton.send(:remove_method, NATIVE_ALIAS)
+  end
+
+  def silence_redefinition_warnings
+    previous_verbose = $VERBOSE
+    $VERBOSE = nil
+    yield
+  ensure
+    $VERBOSE = previous_verbose
+  end
+end

data/test/gemspec_test.rb

@@ -0,0 +1,27 @@
+require_relative "test_helper"
+
+class GemspecTest < Test::Unit::TestCase
+  def test_gemspec_excludes_local_rust_build_artifacts
+    spec = Gem::Specification.load(File.expand_path("../lda-ruby.gemspec", __dir__))
+    assert_not_nil spec
+
+    rust_target_files = spec.files.grep(%r{\Aext/lda-ruby-rust/target/})
+    assert_equal [], rust_target_files
+    assert(!spec.files.include?("ext/lda-ruby-rust/Cargo.lock"))
+    assert(!spec.files.include?("ext/lda-ruby-rust/Makefile"))
+  end
+
+  def test_gemspec_declares_rust_extconf
+    spec = Gem::Specification.load(File.expand_path("../lda-ruby.gemspec", __dir__))
+    assert_not_nil spec
+
+    assert(spec.extensions.include?("ext/lda-ruby-rust/extconf.rb"))
+  end
+
+  def test_gemspec_includes_release_runbook
+    spec = Gem::Specification.load(File.expand_path("../lda-ruby.gemspec", __dir__))
+    assert_not_nil spec
+
+    assert(spec.files.include?("docs/release-runbook.md"))
+  end
+end