lda-ruby 0.3.9 → 0.5.0

data/test/backend_compatibility_test.rb

@@ -0,0 +1,146 @@
+require_relative "test_helper"
+
+class BackendCompatibilityTest < Test::Unit::TestCase
+  FIXTURE_DOCUMENTS = [
+    "apple banana apple banana fruit sweet fruit",
+    "truck wheel truck road engine metal road",
+    "ruby code gem ruby class module test",
+    "banana fruit apple orchard fresh sweet",
+    "engine road truck wheel fuel highway",
+    "module ruby class object gem code"
+  ].freeze
+
+  def setup
+    @corpus = Lda::TextCorpus.new(FIXTURE_DOCUMENTS)
+  end
+
+  def test_pure_backend_seeded_fixture
+    lda = build_and_train(:pure)
+
+    assert_equal "pure_ruby", lda.backend_name
+    assert_backend_output_valid(lda)
+  end
+
+  def test_native_backend_seeded_fixture
+    return unless Lda::NATIVE_EXTENSION_LOADED
+
+    lda = build_and_train(:native)
+
+    assert_equal "native", lda.backend_name
+    assert_backend_output_valid(lda)
+  end
+
+  def test_native_and_pure_backend_agree_on_shapes
+    return unless Lda::NATIVE_EXTENSION_LOADED
+
+    native = build_and_train(:native)
+    pure = build_and_train(:pure)
+
+    assert_equal native.model[0], pure.model[0]
+    assert_equal native.model[1], pure.model[1]
+    assert_equal native.beta.size, pure.beta.size
+    assert_equal native.gamma.size, pure.gamma.size
+    assert_equal native.phi.size, pure.phi.size
+  end
+
+  def test_rust_backend_seeded_fixture
+    return unless Lda::RUST_EXTENSION_LOADED
+
+    rust = build_and_train(:rust)
+
+    assert_equal "rust", rust.backend_name
+    assert_backend_output_valid(rust)
+  end
+
+  def test_rust_and_pure_backend_numeric_parity
+    return unless Lda::RUST_EXTENSION_LOADED
+
+    pure = build_and_train(:pure)
+    rust = build_and_train(:rust)
+
+    assert_nested_close(pure.gamma, rust.gamma, 1e-9)
+    assert_nested_close(pure.beta, rust.beta, 1e-9)
+    assert_nested_close(pure.phi, rust.phi, 1e-9)
+    assert_nested_close(
+      exponentiate_nested(pure.compute_topic_document_probability),
+      exponentiate_nested(rust.compute_topic_document_probability),
+      1e-6
+    )
+  end
+
+  private
+
+  def build_and_train(backend)
+    lda = Lda::Lda.new(@corpus, backend: backend, random_seed: 1234)
+    lda.verbose = false
+    lda.num_topics = 3
+    lda.max_iter = 25
+    lda.em_max_iter = 40
+    lda.convergence = 1e-5
+    lda.em_convergence = 1e-4
+    lda.em("seeded")
+    lda
+  end
+
+  def assert_backend_output_valid(lda)
+    assert_equal 3, lda.model[0]
+    assert lda.model[1] > 0
+
+    assert_equal @corpus.num_docs, lda.gamma.size
+    lda.gamma.each do |topic_weights|
+      assert_equal 3, topic_weights.size
+      topic_weights.each do |weight|
+        assert weight.is_a?(Numeric)
+        assert weight.finite?
+        assert weight.positive?
+      end
+    end
+
+    assert_equal 3, lda.beta.size
+    lda.beta.each do |topic_log_probs|
+      assert topic_log_probs.size > 0
+      probabilities = topic_log_probs.map { |log_prob| Math.exp(log_prob) }
+      assert_in_delta 1.0, probabilities.sum, 1e-3
+    end
+
+    phi = lda.phi
+    assert_equal @corpus.num_docs, phi.size
+    phi.each_with_index do |doc_phi, doc_index|
+      assert_equal @corpus.documents[doc_index].length, doc_phi.size
+      doc_phi.each do |word_topic_distribution|
+        assert_equal 3, word_topic_distribution.size
+        assert_in_delta 1.0, word_topic_distribution.sum, 1e-3
+      end
+    end
+
+    probabilities = lda.compute_topic_document_probability
+    assert_equal @corpus.num_docs, probabilities.size
+    probabilities.each do |row|
+      assert_equal 3, row.size
+      row.each { |value| assert value.finite? }
+    end
+
+    top_words = lda.top_words(4)
+    assert_equal 3, top_words.size
+    top_words.each_value { |words| assert_equal 4, words.size }
+  end
+
+  def assert_nested_close(left, right, tolerance)
+    assert_equal left.class, right.class
+
+    if left.is_a?(Array)
+      assert_equal left.size, right.size
+      left.each_with_index do |left_item, index|
+        assert_nested_close(left_item, right[index], tolerance)
+      end
+    else
+      assert_in_delta left.to_f, right.to_f, tolerance
+    end
+  end
+
+  def exponentiate_nested(value)
+    return Math.exp(value.to_f) unless value.is_a?(Array)
+
+    value.map { |item| exponentiate_nested(item) }
+  end
+end

data/test/backends_selection_test.rb

@@ -0,0 +1,100 @@
+require_relative "test_helper"
+
+class BackendsSelectionTest < Test::Unit::TestCase
+  RUST_ALIAS = :__test_original_rust_available__
+  NATIVE_ALIAS = :__test_original_native_available__
+
+  setup do
+    @host = Object.new
+    @rust_singleton = Lda::Backends::Rust.singleton_class
+    @native_singleton = Lda::Backends::Native.singleton_class
+
+    @rust_singleton.send(:alias_method, RUST_ALIAS, :available?)
+    @native_singleton.send(:alias_method, NATIVE_ALIAS, :available?)
+    @previous_env_backend = ENV["LDA_RUBY_BACKEND"]
+  end
+
+  teardown do
+    restore_availability_stubs
+    ENV["LDA_RUBY_BACKEND"] = @previous_env_backend
+  end
+
+  should "prefer rust over native in auto mode when both are available" do
+    stub_rust_available(true)
+    stub_native_available(true)
+
+    backend = Lda::Backends.build(host: @host, requested: :auto)
+    assert_instance_of Lda::Backends::Rust, backend
+  end
+
+  should "fall back to native in auto mode when rust is unavailable" do
+    stub_rust_available(false)
+    stub_native_available(true)
+
+    backend = Lda::Backends.build(host: @host, requested: :auto)
+    assert_instance_of Lda::Backends::Native, backend
+  end
+
+  should "fall back to pure in auto mode when rust and native are unavailable" do
+    stub_rust_available(false)
+    stub_native_available(false)
+
+    backend = Lda::Backends.build(host: @host, requested: :auto)
+    assert_instance_of Lda::Backends::PureRuby, backend
+  end
+
+  should "respect LDA_RUBY_BACKEND env override when requested mode is nil" do
+    stub_rust_available(true)
+    stub_native_available(true)
+    ENV["LDA_RUBY_BACKEND"] = "pure_ruby"
+
+    backend = Lda::Backends.build(host: @host, requested: nil)
+    assert_instance_of Lda::Backends::PureRuby, backend
+  end
+
+  should "raise for unknown backend mode" do
+    stub_rust_available(false)
+    stub_native_available(false)
+
+    error = assert_raise(ArgumentError) do
+      Lda::Backends.build(host: @host, requested: :unknown_backend)
+    end
+
+    assert_match(/Unknown backend mode/i, error.message)
+  end
+
+  private
+
+  def stub_rust_available(value)
+    silence_redefinition_warnings do
+      @rust_singleton.send(:define_method, :available?) do
+        value
+      end
+    end
+  end
+
+  def stub_native_available(value)
+    silence_redefinition_warnings do
+      @native_singleton.send(:define_method, :available?) do |_host|
+        value
+      end
+    end
+  end
+
+  def restore_availability_stubs
+    silence_redefinition_warnings do
+      @rust_singleton.send(:alias_method, :available?, RUST_ALIAS)
+      @native_singleton.send(:alias_method, :available?, NATIVE_ALIAS)
+    end
+    @rust_singleton.send(:remove_method, RUST_ALIAS)
+    @native_singleton.send(:remove_method, NATIVE_ALIAS)
+  end
+
+  def silence_redefinition_warnings
+    previous_verbose = $VERBOSE
+    $VERBOSE = nil
+    yield
+  ensure
+    $VERBOSE = previous_verbose
+  end
+end

data/test/benchmark_scripts_test.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "test_helper"
+require "open3"
+
+class BenchmarkScriptsTest < Test::Unit::TestCase
+  def setup
+    @repo_root = File.expand_path("..", __dir__)
+    @check_rust_benchmark = File.join(@repo_root, "bin", "check-rust-benchmark")
+  end
+
+  def test_check_rust_benchmark_help
+    stdout, stderr, status = Open3.capture3(@check_rust_benchmark, "--help", chdir: @repo_root)
+    assert(status.success?, stderr)
+    assert_match(/Usage: \.\/bin\/check-rust-benchmark/, stdout)
+  end
+
+  def test_check_rust_benchmark_rejects_unknown_argument
+    _stdout, stderr, status = Open3.capture3(@check_rust_benchmark, "--unknown", chdir: @repo_root)
+    assert(!status.success?, "expected check-rust-benchmark to fail for unknown args")
+    assert_match(/Unknown argument/, stderr)
+  end
+end

data/test/gemspec_test.rb

@@ -0,0 +1,27 @@
+require_relative "test_helper"
+
+class GemspecTest < Test::Unit::TestCase
+  def test_gemspec_excludes_local_rust_build_artifacts
+    spec = Gem::Specification.load(File.expand_path("../lda-ruby.gemspec", __dir__))
+    assert_not_nil spec
+
+    rust_target_files = spec.files.grep(%r{\Aext/lda-ruby-rust/target/})
+    assert_equal [], rust_target_files
+    assert(!spec.files.include?("ext/lda-ruby-rust/Cargo.lock"))
+    assert(!spec.files.include?("ext/lda-ruby-rust/Makefile"))
+  end
+
+  def test_gemspec_declares_rust_extconf
+    spec = Gem::Specification.load(File.expand_path("../lda-ruby.gemspec", __dir__))
+    assert_not_nil spec
+
+    assert(spec.extensions.include?("ext/lda-ruby-rust/extconf.rb"))
+  end
+
+  def test_gemspec_includes_release_runbook
+    spec = Gem::Specification.load(File.expand_path("../lda-ruby.gemspec", __dir__))
+    assert_not_nil spec
+
+    assert(spec.files.include?("docs/release-runbook.md"))
+  end
+end

data/test/lda_ruby_test.rb

@@ -1,11 +1,4 @@
-require 'rubygems'
-require 'test/unit'
-require 'shoulda'
-require 'yaml'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'lda-ruby'
+require_relative "test_helper"
 
 class LdaRubyTest < Test::Unit::TestCase
   context "A Document instance" do
@@ -19,7 +12,7 @@ class LdaRubyTest < Test::Unit::TestCase
       end
 
       should "not have text" do
-        assert !@document.has_text?
+        assert !@document.text?
       end
 
       should "be empty" do
@@ -51,7 +44,7 @@ class LdaRubyTest < Test::Unit::TestCase
       end
 
       should "not have text" do
-        assert !@document.has_text?
+        assert !@document.text?
       end
 
       should "have word count equal to what was added" do
@@ -78,7 +71,7 @@ class LdaRubyTest < Test::Unit::TestCase
       end
 
       should "have text" do
-        assert @document.has_text?
+        assert @document.text?
       end
 
       should "have word count equal to what was added" do
@@ -240,6 +233,33 @@ class LdaRubyTest < Test::Unit::TestCase
       assert !@lda.est_alpha.nil?
     end
 
+    should "expose the selected backend name" do
+      assert(["native", "pure_ruby", "rust"].include?(@lda.backend_name))
+    end
+
+    should "raise when rust backend is requested but extension is unavailable" do
+      if Lda::RUST_EXTENSION_LOADED
+        assert true
+      else
+        assert_raise(LoadError) { Lda::Lda.new(@corpus, backend: :rust) }
+      end
+    end
+
+    should "run with rust backend when extension is available" do
+      if Lda::RUST_EXTENSION_LOADED
+        rust_lda = Lda::Lda.new(@corpus, backend: :rust, random_seed: 1234)
+        rust_lda.verbose = false
+        rust_lda.num_topics = 4
+        rust_lda.em("seeded")
+
+        assert_equal "rust", rust_lda.backend_name
+        assert_equal @corpus.num_docs, rust_lda.gamma.size
+        assert_equal @corpus.num_docs, rust_lda.phi.size
+      else
+        assert true
+      end
+    end
+
     context "after running em" do
       setup do
         @lda.verbose = false
@@ -277,5 +297,23 @@ class LdaRubyTest < Test::Unit::TestCase
         end
       end
     end
+
+    context "using the pure-ruby backend" do
+      setup do
+        @lda = Lda::Lda.new(@corpus, backend: :pure, random_seed: 1234)
+        @lda.verbose = false
+        @lda.num_topics = 6
+        @lda.max_iter = 20
+        @lda.em_max_iter = 30
+        @lda.em('random')
+      end
+
+      should "run em and generate model matrices" do
+        assert_equal "pure_ruby", @lda.backend_name
+        assert_equal @lda.num_topics, @lda.beta.size
+        assert_equal @corpus.num_docs, @lda.gamma.size
+        assert_equal @corpus.num_docs, @lda.phi.size
+      end
+    end
   end
 end

data/test/packaged_gem_smoke_test.rb

@@ -0,0 +1,33 @@
+require_relative "test_helper"
+require "tmpdir"
+require "fileutils"
+require_relative "../bin/packaged-gem-smoke"
+
+class PackagedGemSmokeTest < Test::Unit::TestCase
+  def test_gem_path_under_prefix_handles_symlinked_prefixes
+    Dir.mktmpdir("packaged-smoke") do |tmpdir|
+      real_root = File.join(tmpdir, "real")
+      link_root = File.join(tmpdir, "link")
+      gem_dir = File.join(real_root, "gems", "lda-ruby-0.4.0")
+
+      FileUtils.mkdir_p(gem_dir)
+      File.symlink(real_root, link_root)
+
+      assert(
+        Lda::PackagedGemSmoke.gem_path_under_prefix?(gem_dir, link_root),
+        "expected symlinked prefix to match real gem path"
+      )
+      assert(
+        Lda::PackagedGemSmoke.gem_path_under_prefix?(File.join(link_root, "gems", "lda-ruby-0.4.0"), real_root),
+        "expected real prefix to match symlinked gem path"
+      )
+    end
+  end
+
+  def test_gem_path_under_prefix_rejects_neighbor_prefixes
+    assert(
+      !Lda::PackagedGemSmoke.gem_path_under_prefix?("/tmp/gemhome-other/gems/lda-ruby-0.4.0", "/tmp/gemhome"),
+      "neighbor prefixes should not match"
+    )
+  end
+end

data/test/pure_ruby_orchestration_test.rb

@@ -0,0 +1,109 @@
+require_relative "test_helper"
+
+class PureRubyOrchestrationTest < Test::Unit::TestCase
+  FIXTURE_DOCUMENTS = [
+    "apple banana apple banana fruit sweet fruit",
+    "truck wheel truck road engine metal road",
+    "ruby code gem ruby class module test",
+    "banana fruit apple orchard fresh sweet",
+    "engine road truck wheel fuel highway",
+    "module ruby class object gem code"
+  ].freeze
+
+  def test_rust_em_input_includes_expected_fields
+    backend = build_backend
+
+    em_input = backend.rust_em_input("seeded")
+
+    assert_equal 3, em_input[:topics]
+    assert_equal em_input[:document_words].size, em_input[:document_counts].size
+    assert_equal em_input[:document_words].map(&:length), em_input[:document_lengths]
+    assert_equal em_input[:document_counts].map { |counts| counts.sum.to_f }, em_input[:document_totals]
+    assert_equal 3, em_input[:initial_beta_probabilities].size
+    assert em_input[:terms] > 0
+  end
+
+  def test_em_from_input_matches_seeded_em_output
+    direct = build_backend
+    from_input = build_backend
+
+    direct.em("seeded")
+    em_input = from_input.rust_em_input("seeded")
+    from_input.em_from_input(em_input)
+
+    assert_nested_close(direct.gamma, from_input.gamma, 1e-9)
+    assert_nested_close(direct.beta, from_input.beta, 1e-9)
+    assert_nested_close(direct.compute_phi, from_input.compute_phi, 1e-9)
+  end
+
+  def test_rust_initial_beta_probabilities_matches_rust_em_input_for_random_start
+    from_helper = build_backend
+    from_input = build_backend
+
+    document_words = from_helper.corpus.documents.map { |document| document.words.map(&:to_i) }
+    document_counts = from_helper.corpus.documents.map { |document| document.counts.map(&:to_f) }
+    terms = from_helper.corpus.documents.flat_map(&:words).max + 1
+
+    helper_beta = from_helper.rust_initial_beta_probabilities(
+      "random",
+      document_words,
+      document_counts,
+      from_helper.num_topics,
+      terms
+    )
+    em_input = from_input.rust_em_input("random")
+
+    assert_nested_close(helper_beta, em_input[:initial_beta_probabilities], 1e-12)
+  end
+
+  def test_apply_em_state_sets_outputs
+    backend = build_backend
+
+    docs = backend.corpus.documents
+    topics = 3
+    terms = 5
+
+    beta_probabilities = Array.new(topics) { Array.new(terms, 1.0 / terms) }
+    beta_log = beta_probabilities.map { |row| row.map { |probability| Math.log(probability) } }
+    gamma = Array.new(docs.size) { Array.new(topics, 1.0) }
+    phi = docs.map { |document| Array.new(document.length) { Array.new(topics, 1.0 / topics) } }
+
+    backend.apply_em_state(
+      beta_probabilities: beta_probabilities,
+      beta_log: beta_log,
+      gamma: gamma,
+      phi: phi
+    )
+
+    assert_equal beta_log, backend.beta
+    assert_equal gamma, backend.gamma
+    assert_equal phi, backend.compute_phi
+  end
+
+  private
+
+  def build_backend
+    backend = Lda::Backends::PureRuby.new(random_seed: 1234)
+    backend.corpus = Lda::TextCorpus.new(FIXTURE_DOCUMENTS)
+    backend.verbose = false
+    backend.num_topics = 3
+    backend.max_iter = 25
+    backend.em_max_iter = 40
+    backend.convergence = 1e-5
+    backend.em_convergence = 1e-4
+    backend
+  end
+
+  def assert_nested_close(left, right, tolerance)
+    assert_equal left.class, right.class
+
+    if left.is_a?(Array)
+      assert_equal left.size, right.size
+      left.each_with_index do |left_item, index|
+        assert_nested_close(left_item, right[index], tolerance)
+      end
+    else
+      assert_in_delta left.to_f, right.to_f, tolerance
+    end
+  end
+end

data/test/release_scripts_test.rb

@@ -0,0 +1,93 @@
+require_relative "test_helper"
+require "open3"
+
+class ReleaseScriptsTest < Test::Unit::TestCase
+  def setup
+    @repo_root = File.expand_path("..", __dir__)
+    @check_version_sync = File.join(@repo_root, "bin", "check-version-sync")
+    @release_prepare = File.join(@repo_root, "bin", "release-prepare")
+    @verify_rubygems_api_key = File.join(@repo_root, "bin", "verify-rubygems-api-key")
+    @verify_release_publish = File.join(@repo_root, "bin", "verify-release-publish")
+  end
+
+  def test_check_version_sync_passes_for_repository_versions
+    stdout, stderr, status = Open3.capture3(@check_version_sync, chdir: @repo_root)
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_match(/Version sync OK:/, stdout)
+  end
+
+  def test_check_version_sync_fails_for_mismatched_tag
+    _stdout, stderr, status = Open3.capture3(@check_version_sync, "--tag", "v9.9.9", chdir: @repo_root)
+    assert(!status.success?, "expected check-version-sync to fail for mismatched tag")
+    assert_match(/does not match expected tag/, stderr)
+  end
+
+  def test_check_version_sync_print_tag_matches_library_version
+    stdout, stderr, status = Open3.capture3(@check_version_sync, "--print-tag", chdir: @repo_root)
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_equal("v#{Lda::VERSION}", stdout.strip)
+  end
+
+  def test_release_prepare_dry_run_does_not_change_files
+    version_rb_path = File.join(@repo_root, "lib", "lda-ruby", "version.rb")
+    version_yml_path = File.join(@repo_root, "VERSION.yml")
+    changelog_path = File.join(@repo_root, "CHANGELOG.md")
+
+    baseline = {
+      version_rb_path => File.read(version_rb_path),
+      version_yml_path => File.read(version_yml_path),
+      changelog_path => File.read(changelog_path)
+    }
+
+    stdout, stderr, status = Open3.capture3(
+      @release_prepare,
+      "9.9.9",
+      "--allow-dirty",
+      "--dry-run",
+      chdir: @repo_root
+    )
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_match(/Dry run: would update/, stdout)
+
+    baseline.each do |path, original|
+      assert_equal(original, File.read(path), "#{path} changed during dry-run")
+    end
+  end
+
+  def test_verify_rubygems_api_key_help
+    stdout, stderr, status = Open3.capture3(@verify_rubygems_api_key, "--help", chdir: @repo_root)
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_match(/Usage: \.\/bin\/verify-rubygems-api-key/, stdout)
+  end
+
+  def test_verify_rubygems_api_key_rejects_unknown_argument
+    _stdout, stderr, status = Open3.capture3(@verify_rubygems_api_key, "--unknown-flag", chdir: @repo_root)
+    assert(!status.success?, "expected verify-rubygems-api-key to fail for unknown arguments")
+    assert_match(/unknown argument/, stderr)
+  end
+
+  def test_verify_release_publish_help
+    stdout, stderr, status = Open3.capture3(@verify_release_publish, "--help", chdir: @repo_root)
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_match(/Usage: \.\/bin\/verify-release-publish/, stdout)
+  end
+
+  def test_verify_release_publish_rejects_unknown_argument
+    _stdout, stderr, status = Open3.capture3(@verify_release_publish, "--unknown-flag", chdir: @repo_root)
+    assert(!status.success?, "expected verify-release-publish to fail for unknown arguments")
+    assert_match(/unknown argument/, stderr)
+  end
+
+  def test_verify_release_publish_rejects_invalid_tag_format
+    _stdout, stderr, status = Open3.capture3(
+      @verify_release_publish,
+      "--tag",
+      "0.4.0",
+      "--skip-rubygems",
+      "--skip-github",
+      chdir: @repo_root
+    )
+    assert(!status.success?, "expected verify-release-publish to fail for invalid tag format")
+    assert_match(/tag must be in format vX\.Y\.Z/, stderr)
+  end
+end

data/test/rust_build_policy_test.rb

@@ -0,0 +1,23 @@
+require_relative "test_helper"
+require "lda-ruby/rust_build_policy"
+
+class RustBuildPolicyTest < Test::Unit::TestCase
+  def test_default_policy_is_auto
+    assert_equal "auto", Lda::RustBuildPolicy.resolve(nil)
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("")
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("  ")
+  end
+
+  def test_resolves_valid_values_case_insensitively
+    assert_equal "always", Lda::RustBuildPolicy.resolve("always")
+    assert_equal "always", Lda::RustBuildPolicy.resolve("ALWAYS")
+    assert_equal "never", Lda::RustBuildPolicy.resolve("never")
+    assert_equal "never", Lda::RustBuildPolicy.resolve("  NeVeR  ")
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("AUTO")
+  end
+
+  def test_invalid_policy_falls_back_to_auto
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("sometimes")
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("true")
+  end
+end