lda-ruby 0.5.0-x64-mingw-ucrt

data/test/lda_ruby_test.rb

@@ -0,0 +1,319 @@
+require_relative "test_helper"
+
+class LdaRubyTest < Test::Unit::TestCase
+  context "A Document instance" do
+    setup do
+      @corpus = Lda::Corpus.new
+    end
+
+    context "A typical Document" do
+      setup do
+        @document = Lda::Document.new(@corpus)
+      end
+
+      should "not have text" do
+        assert !@document.text?
+      end
+
+      should "be empty" do
+        assert_equal @document.total, 0
+        assert_equal @document.length, 0
+      end
+
+      context "after adding words" do
+        setup do
+          @document.words << 1 << 2 << 3 << 4 << 5
+          @document.counts << 2 << 1 << 1 << 1 << 3
+          @document.recompute
+        end
+
+        should "have word count equal to what was added" do
+          assert_equal @document.length, 5
+        end
+
+        should "have total words equal to the sum of the counts" do
+          assert_equal @document.total, 8
+        end
+      end
+    end
+
+    context "A typical DataDocument" do
+      setup do
+        @data = '5 1:2 2:1 3:1 4:1 5:3'
+        @document = Lda::DataDocument.new(@corpus, @data)
+      end
+
+      should "not have text" do
+        assert !@document.text?
+      end
+
+      should "have word count equal to what was added" do
+        assert_equal @document.length, 5
+      end
+
+      should "have total words equal to the sum of the counts" do
+        assert_equal @document.total, 8
+      end
+
+      should "have words equal to the order they were entered" do
+        assert_equal @document.words, [1, 2, 3, 4, 5]
+      end
+
+      should "have counts equal to the order they were entered" do
+        assert_equal @document.counts, [2, 1, 1, 1, 3]
+      end
+    end
+
+    context "A typical TextDocument" do
+      setup do
+        @text = 'stop words stop stop masterful stoppage buffalo buffalo buffalo'
+        @document = Lda::TextDocument.new(@corpus, @text)
+      end
+
+      should "have text" do
+        assert @document.text?
+      end
+
+      should "have word count equal to what was added" do
+        assert_equal @document.length, 5
+      end
+
+      should "have total words equal to the sum of the counts" do
+        assert_equal @document.total, @text.split(/ /).size
+      end
+
+      should "have tokens in the order they were entered" do
+        assert_equal @document.tokens, @text.split(/ /)
+      end
+    end
+  end
+
+  context "A Corpus instance" do
+    context "A typical Lda::Corpus instance" do
+      setup do
+        @corpus = Lda::Corpus.new
+        @document1 = Lda::TextDocument.new(@corpus, 'This is the document that never ends.  Oh wait yeah it does.')
+        @document2 = Lda::TextDocument.new(@corpus, 'A second document that is just as lame as the first.')
+      end
+
+      should "be able to add new documents" do
+        assert @corpus.respond_to?(:add_document)
+        @corpus.add_document(@document1)
+        assert_equal @corpus.documents.size, 1
+      end
+
+      should "update vocabulary with words in the document" do
+        @corpus.add_document(@document2)
+        assert_equal @corpus.vocabulary.words.member?('lame'), true
+      end
+    end
+
+    context "An Lda::DataCorpus instance loaded from a file" do
+      setup do
+        @filename = File.join(File.dirname(__FILE__), 'data', 'docs.dat')
+        @filetext = File.open(@filename, 'r') { |f| f.read }
+        @corpus = Lda::DataCorpus.new(@filename)
+      end
+
+      should "contain the number of documents equivalent to the number of lines in the file" do
+        assert_equal @corpus.num_docs, @filetext.split(/\n/).size
+      end
+
+      should "not load any words into the vocabulary since none were given" do
+        assert_equal @corpus.vocabulary.words.size, 0
+      end
+    end
+
+    context "An Lda::TextCorpus instance loaded from a file" do
+      setup do
+        @filename = File.join(File.dirname(__FILE__), 'data', 'wiki-test-docs.yml')
+        @filedocs = YAML::load_file(@filename)
+        @corpus = Lda::TextCorpus.new(@filename)
+      end
+
+      should "contain the number of documents equivalent to the number of lines in the file" do
+        assert_equal @corpus.num_docs, @filedocs.size
+      end
+
+      should "update the vocabulary with the words that were loaded" do
+        assert @corpus.vocabulary.words.size > 0
+      end
+    end
+
+    context "An Lda::DirectoryCorpus instance loaded from a directory" do
+      setup do
+        @path = File.join(File.dirname(__FILE__), 'data', 'tmp')
+        @extension = 'txt'
+        Dir.mkdir(@path)
+        @original_filename = File.join(File.dirname(__FILE__), 'data', 'wiki-test-docs.yml')
+        @filedocs = YAML::load_file(@original_filename)
+        @filedocs.each_with_index do |doc, idx|
+          File.open(File.join(@path, "doc_#{idx + 1}.txt"), 'w') { |f| f.write(doc) }
+        end
+
+        @corpus = Lda::DirectoryCorpus.new(@path, @extension)
+      end
+
+      should "load a document for every file in the directory" do
+        assert_equal @corpus.num_docs, @filedocs.size
+      end
+
+      should "update the vocabulary with the words that were loaded" do
+        assert @corpus.vocabulary.words.size > 0
+      end
+
+      teardown do
+        Dir.glob(File.join(@path, "*.#{@extension}")).each { |f| File.unlink(f) }
+        Dir.rmdir(@path)
+      end
+    end
+  end
+
+  context "A Vocabulary instance" do
+    setup do
+      @vocab = Lda::Vocabulary.new
+      @words = ['word1', 'word2', 'word3', 'word4', 'word5', 'word6']
+      @filename1 = File.join(File.dirname(__FILE__), 'data', 'tmp_file.txt')
+      File.open(@filename1, 'w') do |f|
+        @words.each { |w| f.write("#{w}\n") }
+      end
+      @filename2 = File.join(File.dirname(__FILE__), 'data', 'tmp_file.yml')
+      File.open(@filename2, 'w') { |f| YAML::dump(@words, f) }
+    end
+
+    should "load a file containing a list of words, one per line" do
+      assert @vocab.num_words == 0
+      @vocab.load_file(@filename1)
+      assert @vocab.words.size > 0
+    end
+
+    should "load a yaml file containing a list of words" do
+      assert @vocab.num_words == 0
+      @vocab.load_yaml(@filename2)
+      assert @vocab.num_words > 0
+    end
+
+    should "return indexes for words in the order they were loaded" do
+      @vocab.load_yaml(@filename2)
+      @words.each_with_index do |word, idx|
+        assert_equal @vocab.check_word(word), idx + 1
+      end
+    end
+
+    teardown do
+      File.unlink(@filename1)
+      File.unlink(@filename2)
+    end
+  end
+
+  context "An Lda::Lda instance" do
+    setup do
+      @filename = File.join(File.dirname(__FILE__), 'data', 'wiki-test-docs.yml')
+      @filedocs = YAML::load_file(@filename)
+      @corpus = Lda::TextCorpus.new(@filename)
+
+      @lda = Lda::Lda.new(@corpus)
+    end
+
+    should "have loaded the vocabulary from the corpus" do
+      assert !@lda.vocab.nil?
+    end
+
+    should "have loaded the same number of words in the vocabulary as are in the original" do
+      assert_equal @lda.vocab.size, @corpus.vocabulary.num_words
+    end
+
+    should "have default values for the main settings" do
+      assert !@lda.max_iter.nil?
+      assert !@lda.convergence.nil?
+      assert !@lda.em_max_iter.nil?
+      assert !@lda.em_convergence.nil?
+      assert !@lda.num_topics.nil?
+      assert !@lda.init_alpha.nil?
+      assert !@lda.est_alpha.nil?
+    end
+
+    should "expose the selected backend name" do
+      assert(["native", "pure_ruby", "rust"].include?(@lda.backend_name))
+    end
+
+    should "raise when rust backend is requested but extension is unavailable" do
+      if Lda::RUST_EXTENSION_LOADED
+        assert true
+      else
+        assert_raise(LoadError) { Lda::Lda.new(@corpus, backend: :rust) }
+      end
+    end
+
+    should "run with rust backend when extension is available" do
+      if Lda::RUST_EXTENSION_LOADED
+        rust_lda = Lda::Lda.new(@corpus, backend: :rust, random_seed: 1234)
+        rust_lda.verbose = false
+        rust_lda.num_topics = 4
+        rust_lda.em("seeded")
+
+        assert_equal "rust", rust_lda.backend_name
+        assert_equal @corpus.num_docs, rust_lda.gamma.size
+        assert_equal @corpus.num_docs, rust_lda.phi.size
+      else
+        assert true
+      end
+    end
+
+    context "after running em" do
+      setup do
+        @lda.verbose = false
+        @lda.num_topics = 8
+        @lda.em('random')
+      end
+
+      should "phi should be defined" do
+        assert !@lda.phi.nil?
+      end
+
+      should "return the top 10 list of words for each topic" do
+        topics = @lda.top_words(10)
+        assert topics.is_a?(Hash)
+        assert_equal topics.size, @lda.num_topics
+
+        topics.each_pair do |topic, top_n_words|
+          assert_equal top_n_words.size, 10
+        end
+      end
+
+      context "after computing topic-document probabilities" do
+        setup do
+          @topic_doc_probs = @lda.compute_topic_document_probability
+        end
+
+        should "have a row for each document" do
+          assert_equal @topic_doc_probs.size, @corpus.num_docs
+        end
+
+        should "have columns for each topic" do
+          @topic_doc_probs.each do |doc|
+            assert_equal doc.size, @lda.num_topics
+          end
+        end
+      end
+    end
+
+    context "using the pure-ruby backend" do
+      setup do
+        @lda = Lda::Lda.new(@corpus, backend: :pure, random_seed: 1234)
+        @lda.verbose = false
+        @lda.num_topics = 6
+        @lda.max_iter = 20
+        @lda.em_max_iter = 30
+        @lda.em('random')
+      end
+
+      should "run em and generate model matrices" do
+        assert_equal "pure_ruby", @lda.backend_name
+        assert_equal @lda.num_topics, @lda.beta.size
+        assert_equal @corpus.num_docs, @lda.gamma.size
+        assert_equal @corpus.num_docs, @lda.phi.size
+      end
+    end
+  end
+end

data/test/packaged_gem_smoke_test.rb

@@ -0,0 +1,33 @@
+require_relative "test_helper"
+require "tmpdir"
+require "fileutils"
+require_relative "../bin/packaged-gem-smoke"
+
+class PackagedGemSmokeTest < Test::Unit::TestCase
+  def test_gem_path_under_prefix_handles_symlinked_prefixes
+    Dir.mktmpdir("packaged-smoke") do |tmpdir|
+      real_root = File.join(tmpdir, "real")
+      link_root = File.join(tmpdir, "link")
+      gem_dir = File.join(real_root, "gems", "lda-ruby-0.4.0")
+
+      FileUtils.mkdir_p(gem_dir)
+      File.symlink(real_root, link_root)
+
+      assert(
+        Lda::PackagedGemSmoke.gem_path_under_prefix?(gem_dir, link_root),
+        "expected symlinked prefix to match real gem path"
+      )
+      assert(
+        Lda::PackagedGemSmoke.gem_path_under_prefix?(File.join(link_root, "gems", "lda-ruby-0.4.0"), real_root),
+        "expected real prefix to match symlinked gem path"
+      )
+    end
+  end
+
+  def test_gem_path_under_prefix_rejects_neighbor_prefixes
+    assert(
+      !Lda::PackagedGemSmoke.gem_path_under_prefix?("/tmp/gemhome-other/gems/lda-ruby-0.4.0", "/tmp/gemhome"),
+      "neighbor prefixes should not match"
+    )
+  end
+end

data/test/pure_ruby_orchestration_test.rb

@@ -0,0 +1,109 @@
+require_relative "test_helper"
+
+class PureRubyOrchestrationTest < Test::Unit::TestCase
+  FIXTURE_DOCUMENTS = [
+    "apple banana apple banana fruit sweet fruit",
+    "truck wheel truck road engine metal road",
+    "ruby code gem ruby class module test",
+    "banana fruit apple orchard fresh sweet",
+    "engine road truck wheel fuel highway",
+    "module ruby class object gem code"
+  ].freeze
+
+  def test_rust_em_input_includes_expected_fields
+    backend = build_backend
+
+    em_input = backend.rust_em_input("seeded")
+
+    assert_equal 3, em_input[:topics]
+    assert_equal em_input[:document_words].size, em_input[:document_counts].size
+    assert_equal em_input[:document_words].map(&:length), em_input[:document_lengths]
+    assert_equal em_input[:document_counts].map { |counts| counts.sum.to_f }, em_input[:document_totals]
+    assert_equal 3, em_input[:initial_beta_probabilities].size
+    assert em_input[:terms] > 0
+  end
+
+  def test_em_from_input_matches_seeded_em_output
+    direct = build_backend
+    from_input = build_backend
+
+    direct.em("seeded")
+    em_input = from_input.rust_em_input("seeded")
+    from_input.em_from_input(em_input)
+
+    assert_nested_close(direct.gamma, from_input.gamma, 1e-9)
+    assert_nested_close(direct.beta, from_input.beta, 1e-9)
+    assert_nested_close(direct.compute_phi, from_input.compute_phi, 1e-9)
+  end
+
+  def test_rust_initial_beta_probabilities_matches_rust_em_input_for_random_start
+    from_helper = build_backend
+    from_input = build_backend
+
+    document_words = from_helper.corpus.documents.map { |document| document.words.map(&:to_i) }
+    document_counts = from_helper.corpus.documents.map { |document| document.counts.map(&:to_f) }
+    terms = from_helper.corpus.documents.flat_map(&:words).max + 1
+
+    helper_beta = from_helper.rust_initial_beta_probabilities(
+      "random",
+      document_words,
+      document_counts,
+      from_helper.num_topics,
+      terms
+    )
+    em_input = from_input.rust_em_input("random")
+
+    assert_nested_close(helper_beta, em_input[:initial_beta_probabilities], 1e-12)
+  end
+
+  def test_apply_em_state_sets_outputs
+    backend = build_backend
+
+    docs = backend.corpus.documents
+    topics = 3
+    terms = 5
+
+    beta_probabilities = Array.new(topics) { Array.new(terms, 1.0 / terms) }
+    beta_log = beta_probabilities.map { |row| row.map { |probability| Math.log(probability) } }
+    gamma = Array.new(docs.size) { Array.new(topics, 1.0) }
+    phi = docs.map { |document| Array.new(document.length) { Array.new(topics, 1.0 / topics) } }
+
+    backend.apply_em_state(
+      beta_probabilities: beta_probabilities,
+      beta_log: beta_log,
+      gamma: gamma,
+      phi: phi
+    )
+
+    assert_equal beta_log, backend.beta
+    assert_equal gamma, backend.gamma
+    assert_equal phi, backend.compute_phi
+  end
+
+  private
+
+  def build_backend
+    backend = Lda::Backends::PureRuby.new(random_seed: 1234)
+    backend.corpus = Lda::TextCorpus.new(FIXTURE_DOCUMENTS)
+    backend.verbose = false
+    backend.num_topics = 3
+    backend.max_iter = 25
+    backend.em_max_iter = 40
+    backend.convergence = 1e-5
+    backend.em_convergence = 1e-4
+    backend
+  end
+
+  def assert_nested_close(left, right, tolerance)
+    assert_equal left.class, right.class
+
+    if left.is_a?(Array)
+      assert_equal left.size, right.size
+      left.each_with_index do |left_item, index|
+        assert_nested_close(left_item, right[index], tolerance)
+      end
+    else
+      assert_in_delta left.to_f, right.to_f, tolerance
+    end
+  end
+end

data/test/release_scripts_test.rb

@@ -0,0 +1,93 @@
+require_relative "test_helper"
+require "open3"
+
+class ReleaseScriptsTest < Test::Unit::TestCase
+  def setup
+    @repo_root = File.expand_path("..", __dir__)
+    @check_version_sync = File.join(@repo_root, "bin", "check-version-sync")
+    @release_prepare = File.join(@repo_root, "bin", "release-prepare")
+    @verify_rubygems_api_key = File.join(@repo_root, "bin", "verify-rubygems-api-key")
+    @verify_release_publish = File.join(@repo_root, "bin", "verify-release-publish")
+  end
+
+  def test_check_version_sync_passes_for_repository_versions
+    stdout, stderr, status = Open3.capture3(@check_version_sync, chdir: @repo_root)
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_match(/Version sync OK:/, stdout)
+  end
+
+  def test_check_version_sync_fails_for_mismatched_tag
+    _stdout, stderr, status = Open3.capture3(@check_version_sync, "--tag", "v9.9.9", chdir: @repo_root)
+    assert(!status.success?, "expected check-version-sync to fail for mismatched tag")
+    assert_match(/does not match expected tag/, stderr)
+  end
+
+  def test_check_version_sync_print_tag_matches_library_version
+    stdout, stderr, status = Open3.capture3(@check_version_sync, "--print-tag", chdir: @repo_root)
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_equal("v#{Lda::VERSION}", stdout.strip)
+  end
+
+  def test_release_prepare_dry_run_does_not_change_files
+    version_rb_path = File.join(@repo_root, "lib", "lda-ruby", "version.rb")
+    version_yml_path = File.join(@repo_root, "VERSION.yml")
+    changelog_path = File.join(@repo_root, "CHANGELOG.md")
+
+    baseline = {
+      version_rb_path => File.read(version_rb_path),
+      version_yml_path => File.read(version_yml_path),
+      changelog_path => File.read(changelog_path)
+    }
+
+    stdout, stderr, status = Open3.capture3(
+      @release_prepare,
+      "9.9.9",
+      "--allow-dirty",
+      "--dry-run",
+      chdir: @repo_root
+    )
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_match(/Dry run: would update/, stdout)
+
+    baseline.each do |path, original|
+      assert_equal(original, File.read(path), "#{path} changed during dry-run")
+    end
+  end
+
+  def test_verify_rubygems_api_key_help
+    stdout, stderr, status = Open3.capture3(@verify_rubygems_api_key, "--help", chdir: @repo_root)
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_match(/Usage: \.\/bin\/verify-rubygems-api-key/, stdout)
+  end
+
+  def test_verify_rubygems_api_key_rejects_unknown_argument
+    _stdout, stderr, status = Open3.capture3(@verify_rubygems_api_key, "--unknown-flag", chdir: @repo_root)
+    assert(!status.success?, "expected verify-rubygems-api-key to fail for unknown arguments")
+    assert_match(/unknown argument/, stderr)
+  end
+
+  def test_verify_release_publish_help
+    stdout, stderr, status = Open3.capture3(@verify_release_publish, "--help", chdir: @repo_root)
+    assert(status.success?, "stdout=#{stdout}\nstderr=#{stderr}")
+    assert_match(/Usage: \.\/bin\/verify-release-publish/, stdout)
+  end
+
+  def test_verify_release_publish_rejects_unknown_argument
+    _stdout, stderr, status = Open3.capture3(@verify_release_publish, "--unknown-flag", chdir: @repo_root)
+    assert(!status.success?, "expected verify-release-publish to fail for unknown arguments")
+    assert_match(/unknown argument/, stderr)
+  end
+
+  def test_verify_release_publish_rejects_invalid_tag_format
+    _stdout, stderr, status = Open3.capture3(
+      @verify_release_publish,
+      "--tag",
+      "0.4.0",
+      "--skip-rubygems",
+      "--skip-github",
+      chdir: @repo_root
+    )
+    assert(!status.success?, "expected verify-release-publish to fail for invalid tag format")
+    assert_match(/tag must be in format vX\.Y\.Z/, stderr)
+  end
+end

data/test/rust_build_policy_test.rb

@@ -0,0 +1,23 @@
+require_relative "test_helper"
+require "lda-ruby/rust_build_policy"
+
+class RustBuildPolicyTest < Test::Unit::TestCase
+  def test_default_policy_is_auto
+    assert_equal "auto", Lda::RustBuildPolicy.resolve(nil)
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("")
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("  ")
+  end
+
+  def test_resolves_valid_values_case_insensitively
+    assert_equal "always", Lda::RustBuildPolicy.resolve("always")
+    assert_equal "always", Lda::RustBuildPolicy.resolve("ALWAYS")
+    assert_equal "never", Lda::RustBuildPolicy.resolve("never")
+    assert_equal "never", Lda::RustBuildPolicy.resolve("  NeVeR  ")
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("AUTO")
+  end
+
+  def test_invalid_policy_falls_back_to_auto
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("sometimes")
+    assert_equal "auto", Lda::RustBuildPolicy.resolve("true")
+  end
+end