tomoto 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a27c5c0ff4a71b0e0b084773adf7a2a0ede39152b210973787c12e98637cb7d3
+  data.tar.gz: c70cabfa9e8685e86edae56c4b50c52a8bb6baf4d3f7684c3e28d7345e460551
+SHA512:
+  metadata.gz: d23a02abb149799facf1b557004ebc2d749131d37eb33e8f70e8aa109f117cc79a3db847326206fb08d5114b983959fa782082c9fe239e3573afe363d81f5066
+  data.tar.gz: 6ca2548b92c30adea217437dfad8a5e0ef802c2789a21cd5d40d88514607889b854afa6a2b7e1ef06f7ecabaa1226655c0dfc29aaa5439993861c2895542ae98

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2020-10-09)
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2019
+Copyright (c) 2020 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,123 @@
+# Tomoto
+
+[Tomoto](https://github.com/bab2min/tomotopy) - high performance topic modeling - for Ruby
+
+## Installation
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'tomoto'
+```
+
+It can take around 10 minutes to compile the extension.
+
+## Getting Started
+
+Train a model
+
+```ruby
+model = Tomoto::LDA.new(k: 3)
+model.add_doc("text from document one")
+model.add_doc("text from document two")
+model.add_doc("text from document three")
+model.train(100)
+```
+
+Get topic words
+
+```ruby
+model.topic_words
+```
+
+Save the model to a file
+
+```ruby
+model.save("model.bin")
+```
+
+Load the model from a file
+
+```ruby
+model = Tomoto::LDA.load("model.bin")
+```
+
+Get the number of words for each topic
+
+```ruby
+model.count_by_topics
+```
+
+Get the vocab
+
+```ruby
+model.vocabs
+```
+
+Get the log likelihood per word
+
+```ruby
+model.ll_per_word
+```
+
+## Models
+
+Supports:
+
+- Latent Dirichlet Allocation (`LDA`)
+- Hierarchical Dirichlet Process (`HDP`)
+- Correlated Topic Model (`CT`)
+
+## Parameters
+
+```ruby
+Tomoto::LDA.new(
+  tw: :one, # or :idf, :pmi
+  min_cf: 0,
+  min_df: 0,
+  rm_top: 0,
+  k: 1,
+  alpha: 0.1,
+  eta: 0.01,
+  seed: nil
+)
+```
+
+## Tokenization
+
+Documents are tokenized by whitespace by default, or you can perform your own tokenization.
+
+```ruby
+model.add_doc(["tokens", "from", "document", "one"])
+```
+
+## Performance
+
+Tomoto uses AVX2, AVX, or SSE2 instructions to increase performance on machines that support it. Check what it’s using with:
+
+```ruby
+Tomoto.isa
+```
+
+## History
+
+View the [changelog](https://github.com/ankane/tomoto/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/tomoto/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/tomoto/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone --recursive https://github.com/ankane/tomoto.git
+cd tomoto
+bundle install
+bundle exec rake compile
+bundle exec rake test
+```

data/ext/tomoto/ext.cpp

@@ -0,0 +1,245 @@
+// tomoto
+#include <CT.h>
+#include <HDP.h>
+#include <LDA.h>
+
+// rice
+#include <rice/Array.hpp>
+#include <rice/Hash.hpp>
+#include <rice/Module.hpp>
+
+using Rice::Array;
+using Rice::Class;
+using Rice::Hash;
+using Rice::Module;
+using Rice::Object;
+using Rice::define_class_under;
+using Rice::define_module;
+
+template<>
+Object to_ruby<std::vector<float>>(std::vector<float> const & x)
+{
+  Array res;
+  for (auto const& v : x) {
+    res.push(v);
+  }
+  return res;
+}
+
+extern "C"
+void Init_ext()
+{
+  Module rb_mTomoto = define_module("Tomoto")
+    .define_singleton_method(
+      "isa",
+      *[]() {
+        #ifdef __AVX2__
+          return "avx2";
+        #elif defined(__AVX__)
+          return "avx";
+        #elif defined(__SSE2__) || defined(__x86_64__) || defined(_WIN64)
+          return "sse2";
+        #else
+          return "none";
+        #endif
+      });
+
+  Class rb_cLDA = define_class_under<tomoto::ILDAModel>(rb_mTomoto, "LDA")
+    .define_singleton_method(
+      "_new",
+      *[](size_t tw, size_t k, float alpha, float eta, int seed) {
+        if (seed < 0) {
+          seed = std::random_device{}();
+        }
+        return tomoto::ILDAModel::create((tomoto::TermWeight)tw, k, alpha, eta, seed);
+      })
+    .define_method(
+      "_add_doc",
+      *[](tomoto::ILDAModel& self, Array rb_words) {
+        std::vector<std::string> words;
+        words.reserve(rb_words.size());
+        for (auto const& v : rb_words) {
+          words.push_back(from_ruby<std::string>(v));
+        }
+        self.addDoc(words);
+      })
+    .define_method(
+      "alpha",
+      *[](tomoto::ILDAModel& self) {
+        return self.getAlpha();
+      })
+    .define_method(
+      "burn_in",
+      *[](tomoto::ILDAModel& self) {
+        return self.getBurnInIteration();
+      })
+    .define_method(
+      "burn_in=",
+      *[](tomoto::ILDAModel& self, size_t iteration) {
+        self.setBurnInIteration(iteration);
+        return iteration;
+      })
+    .define_method(
+      "_count_by_topics",
+      *[](tomoto::ILDAModel& self) {
+        Array res;
+        for (auto const& v : self.getCountByTopic()) {
+          res.push(v);
+        }
+        return res;
+      })
+    .define_method(
+      "eta",
+      *[](tomoto::ILDAModel& self) {
+        return self.getEta();
+      })
+    .define_method(
+      "k",
+      *[](tomoto::ILDAModel& self) {
+        return self.getK();
+      })
+    .define_method(
+      "_load",
+      *[](tomoto::ILDAModel& self, const char* filename) {
+        std::ifstream str{ filename, std::ios_base::binary };
+        if (!str) throw std::runtime_error{ std::string("cannot open file '") + filename + std::string("'") };
+        std::vector<uint8_t> extra_data;
+        self.loadModel(str, &extra_data);
+      })
+    .define_method(
+      "ll_per_word",
+      *[](tomoto::ILDAModel& self) {
+        return self.getLLPerWord();
+      })
+    .define_method(
+      "num_words",
+      *[](tomoto::ILDAModel& self) {
+        return self.getN();
+      })
+    .define_method(
+      "num_vocabs",
+      *[](tomoto::ILDAModel& self) {
+        return self.getV();
+      })
+    .define_method(
+      "_prepare",
+      *[](tomoto::ILDAModel& self, size_t minCnt, size_t minDf, size_t rmTop) {
+        self.prepare(true, minCnt, minDf, rmTop);
+      })
+    .define_method(
+      "_removed_top_words",
+      *[](tomoto::ILDAModel& self, size_t rmTop) {
+        Array res;
+        auto dict = self.getVocabDict();
+        size_t size = dict.size();
+        for (size_t i = rmTop; i > 0; i--) {
+          res.push(dict.toWord(size - i));
+        }
+        return res;
+      })
+    .define_method(
+      "_save",
+      *[](tomoto::ILDAModel& self, const char* filename, bool full) {
+        std::ofstream str{ filename, std::ios_base::binary };
+        std::vector<uint8_t> extra_data;
+        self.saveModel(str, full, &extra_data);
+      })
+    .define_method(
+      "_topic_words",
+      *[](tomoto::ILDAModel& self, size_t topicId, size_t topN) {
+        Hash res;
+        for (auto const& v : self.getWordsByTopicSorted(topicId, topN)) {
+          res[v.first] = v.second;
+        }
+        return res;
+      })
+    .define_method(
+      "_train",
+      *[](tomoto::ILDAModel& self, size_t iteration, size_t workers) {
+        size_t ps = 0;
+        self.train(iteration, workers, (tomoto::ParallelScheme)ps);
+      })
+    .define_method(
+      "vocabs",
+      *[](tomoto::ILDAModel& self) {
+        auto dict = self.getVocabDict();
+        Array res;
+        auto utf8 = Class(rb_cEncoding).call("const_get", "UTF_8");
+        for (size_t i = 0; i < dict.size(); i++) {
+          res.push(to_ruby<std::string>(dict.toWord(i)).call("force_encoding", utf8));
+        }
+        return res;
+      });
+
+  Class rb_cCT = define_class_under<tomoto::ICTModel, tomoto::ILDAModel>(rb_mTomoto, "CT")
+    .define_singleton_method(
+      "_new",
+      *[](size_t tw, size_t k, float alpha, float eta, int seed) {
+        if (seed < 0) {
+          seed = std::random_device{}();
+        }
+        return tomoto::ICTModel::create((tomoto::TermWeight)tw, k, alpha, eta, seed);
+      })
+    .define_method(
+      "num_beta_sample",
+      *[](tomoto::ICTModel& self) {
+        return self.getNumBetaSample();
+      })
+    .define_method(
+      "num_beta_sample=",
+      *[](tomoto::ICTModel& self, size_t numSample) {
+        self.setNumBetaSample(numSample);
+        return numSample;
+      })
+    .define_method(
+      "num_tmn_sample",
+      *[](tomoto::ICTModel& self) {
+        return self.getNumTMNSample();
+      })
+    .define_method(
+      "num_tmn_sample=",
+      *[](tomoto::ICTModel& self, size_t numSample) {
+        self.setNumTMNSample(numSample);
+        return numSample;
+      })
+    .define_method(
+      "prior_cov",
+      *[](tomoto::ICTModel& self) {
+        return self.getPriorCov();
+      })
+    .define_method(
+      "prior_mean",
+      *[](tomoto::ICTModel& self) {
+        return self.getPriorMean();
+      });
+
+  Class rb_cHDP = define_class_under<tomoto::IHDPModel, tomoto::ILDAModel>(rb_mTomoto, "HDP")
+    .define_singleton_method(
+      "_new",
+      *[](size_t tw, size_t k, float alpha, float eta, float gamma, int seed) {
+        if (seed < 0) {
+          seed = std::random_device{}();
+        }
+        return tomoto::IHDPModel::create((tomoto::TermWeight)tw, k, alpha, eta, gamma, seed);
+      })
+    .define_method(
+      "gamma",
+      *[](tomoto::IHDPModel& self) {
+        return self.getGamma();
+      })
+    .define_method(
+      "live_k",
+      *[](tomoto::IHDPModel& self) {
+        return self.getLiveK();
+      })
+    .define_method(
+      "live_topic?",
+      *[](tomoto::IHDPModel& self, size_t tid) {
+        return self.isLiveTopic(tid);
+      })
+    .define_method(
+      "num_tables",
+      *[](tomoto::IHDPModel& self) {
+        return self.getTotalTables();
+      });
+}

data/ext/tomoto/extconf.rb

@@ -0,0 +1,28 @@
+require "mkmf-rice"
+
+abort "Missing stdc++" unless have_library("stdc++")
+
+$CXXFLAGS += " -std=c++11 -DEIGEN_MPL2_ONLY"
+
+$CXXFLAGS += " -march=native"
+
+apple_clang = RbConfig::CONFIG["CC_VERSION_MESSAGE"] =~ /apple clang/i
+
+if apple_clang
+  # silence rice warnings
+  $CXXFLAGS += " -Wno-deprecated-declarations"
+end
+
+# silence tomoto warnings
+$CXXFLAGS += " -Wno-unused-variable -Wno-switch"
+
+ext = File.expand_path(".", __dir__)
+tomoto = File.expand_path("../../vendor/tomotopy/src/TopicModel", __dir__)
+eigen = File.expand_path("../../vendor/eigen", __dir__)
+eigen_rand = File.expand_path("../../vendor/EigenRand", __dir__)
+
+$srcs = Dir["{#{ext},#{tomoto}}/*.cpp"]
+$INCFLAGS += " -I#{tomoto} -I#{eigen} -I#{eigen_rand}"
+$VPATH << tomoto
+
+create_makefile("tomoto/ext")

data/lib/tomoto.rb

@@ -0,0 +1,12 @@
+# ext
+require "tomoto/ext"
+
+# modules
+require "tomoto/ct"
+require "tomoto/hdp"
+require "tomoto/lda"
+require "tomoto/version"
+
+module Tomoto
+  TERM_WEIGHT = [:one, :idf, :pmi]
+end