tomoto 0.2.2 → 0.2.3

data/ext/tomoto/plda.cpp

@@ -1,14 +1,14 @@
 #include <PLDA.h>
 
-#include <rice/Module.hpp>
+#include <rice/rice.hpp>
 
 #include "utils.h"
 
 void init_plda(Rice::Module& m) {
   Rice::define_class_under<tomoto::IPLDAModel, tomoto::ILLDAModel>(m, "PLDA")
-    .define_singleton_method(
+    .define_singleton_function(
       "_new",
-      *[](size_t tw, size_t latent_topics, tomoto::Float alpha, tomoto::Float eta, size_t seed) {
+      [](size_t tw, size_t latent_topics, tomoto::Float alpha, tomoto::Float eta, size_t seed) {
         tomoto::PLDAArgs args;
         args.numLatentTopics = latent_topics;
         args.alpha = {alpha};
@@ -17,17 +17,17 @@ void init_plda(Rice::Module& m) {
           args.seed = seed;
         }
         return tomoto::IPLDAModel::create((tomoto::TermWeight)tw, args);
-      })
+      }, Rice::Return().takeOwnership())
     .define_method(
       "_add_doc",
-      *[](tomoto::IPLDAModel& self, std::vector<std::string> words, std::vector<std::string> labels) {
+      [](tomoto::IPLDAModel& self, std::vector<std::string> words, std::vector<std::string> labels) {
         auto doc = buildDoc(words);
         doc.misc["labels"] = labels;
         return self.addDoc(doc);
       })
     .define_method(
       "latent_topics",
-      *[](tomoto::IPLDAModel& self) {
+      [](tomoto::IPLDAModel& self) {
         return self.getNumLatentTopics();
       });
 }

data/ext/tomoto/slda.cpp

@@ -1,18 +1,18 @@
 #include <SLDA.h>
 
-#include <rice/Module.hpp>
+#include <rice/rice.hpp>
 
 #include "utils.h"
 
 void init_slda(Rice::Module& m) {
   Rice::define_class_under<tomoto::ISLDAModel, tomoto::ILDAModel>(m, "SLDA")
-    .define_singleton_method(
+    .define_singleton_function(
       "_new",
-      *[](size_t tw, size_t k, Array rb_vars, tomoto::Float alpha, tomoto::Float eta, std::vector<tomoto::Float> mu, std::vector<tomoto::Float> nu_sq, std::vector<tomoto::Float> glm_param, size_t seed) {
+      [](size_t tw, size_t k, Array rb_vars, tomoto::Float alpha, tomoto::Float eta, std::vector<tomoto::Float> mu, std::vector<tomoto::Float> nu_sq, std::vector<tomoto::Float> glm_param, size_t seed) {
         std::vector<tomoto::ISLDAModel::GLM> vars;
         vars.reserve(rb_vars.size());
         for (auto const& v : rb_vars) {
-          vars.push_back((tomoto::ISLDAModel::GLM) from_ruby<int>(v));
+          vars.push_back((tomoto::ISLDAModel::GLM) Rice::detail::From_Ruby<int>().convert(v.value()));
         }
         tomoto::SLDAArgs args;
         args.k = k;
@@ -26,22 +26,22 @@ void init_slda(Rice::Module& m) {
           args.seed = seed;
         }
         return tomoto::ISLDAModel::create((tomoto::TermWeight)tw, args);
-      })
+      }, Rice::Return().takeOwnership())
     .define_method(
       "_add_doc",
-      *[](tomoto::ISLDAModel& self, std::vector<std::string> words, std::vector<tomoto::Float> y) {
+      [](tomoto::ISLDAModel& self, std::vector<std::string> words, std::vector<tomoto::Float> y) {
         auto doc = buildDoc(words);
         doc.misc["y"] = y;
         return self.addDoc(doc);
       })
     .define_method(
       "f",
-      *[](tomoto::ISLDAModel& self) {
+      [](tomoto::ISLDAModel& self) {
         return self.getF();
       })
     .define_method(
       "_var_type",
-      *[](tomoto::ISLDAModel& self, size_t var_id) {
+      [](tomoto::ISLDAModel& self, size_t var_id) {
         if (var_id >= self.getF()) throw std::runtime_error{ "'var_id' must be < 'f'" };
         return self.getTypeOfVar(var_id) == tomoto::ISLDAModel::GLM::linear ? "l" : "b";
       });

data/ext/tomoto/utils.h

@@ -1,80 +1,26 @@
 #pragma once
 
-#include <rice/Array.hpp>
+#include <rice/rice.hpp>
+#include <rice/stl.hpp>
 
 using Rice::Array;
 using Rice::Object;
 
-template<>
-inline
-Object to_ruby<std::vector<tomoto::Float>>(std::vector<tomoto::Float> const & x)
+namespace Rice::detail
 {
-  Array res;
-  for (auto const& v : x) {
-    res.push(v);
-  }
-  return res;
-}
-
-template<>
-inline
-Object to_ruby<std::vector<uint32_t>>(std::vector<uint32_t> const & x)
-{
-  Array res;
-  for (auto const& v : x) {
-    res.push(v);
-  }
-  return res;
-}
-
-template<>
-inline
-Object to_ruby<std::vector<uint64_t>>(std::vector<uint64_t> const & x)
-{
-  Array res;
-  for (auto const& v : x) {
-    res.push(v);
-  }
-  return res;
-}
-
-template<>
-inline
-std::vector<std::string> from_ruby<std::vector<std::string>>(Object x)
-{
-  Array a = Array(x);
-  std::vector<std::string> res;
-  res.reserve(a.size());
-  for (auto const& v : a) {
-    res.push_back(from_ruby<std::string>(v));
-  }
-  return res;
-}
-
-template<>
-inline
-std::vector<tomoto::Float> from_ruby<std::vector<tomoto::Float>>(Object x)
-{
-  Array a = Array(x);
-  std::vector<tomoto::Float> res;
-  res.reserve(a.size());
-  for (auto const& v : a) {
-    res.push_back(from_ruby<tomoto::Float>(v));
-  }
-  return res;
-}
-
-template<>
-inline
-std::vector<uint64_t> from_ruby<std::vector<uint64_t>>(Object x)
-{
-  Array a = Array(x);
-  std::vector<uint64_t> res;
-  res.reserve(a.size());
-  for (auto const& v : a) {
-    res.push_back(from_ruby<uint64_t>(v));
-  }
-  return res;
+  template<typename T>
+  class To_Ruby<std::vector<T>>
+  {
+  public:
+    VALUE convert(std::vector<T> const & x)
+    {
+      auto a = rb_ary_new2(x.size());
+      for (const auto& v : x) {
+        detail::protect(rb_ary_push, a, To_Ruby<T>().convert(v));
+      }
+      return a;
+    }
+  };
 }
 
 inline tomoto::RawDoc buildDoc(std::vector<std::string>& words) {

data/lib/tomoto/version.rb

@@ -1,3 +1,3 @@
 module Tomoto
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/vendor/tomotopy/README.kr.rst

@@ -198,6 +198,57 @@ add_doc은 `tomotopy.LDAModel.train`을 시작하기 전까지만 사용할 수
 infer 메소드는 `tomotopy.Document` 인스턴스 하나를 추론하거나 `tomotopy.Document` 인스턴스의 `list`를 추론하는데 사용할 수 있습니다. 
 자세한 것은 `tomotopy.LDAModel.infer`을 참조하길 바랍니다.
 
+Corpus와 transform
+--------------------
+`tomotopy`의 모든 토픽 모델들은 각자 별도의 내부적인 문헌 타입을 가지고 있습니다.
+그리고 이 문헌 타입들에 맞는 문헌들은 각 모델의 `add_doc` 메소드를 통해 생성될 수 있습니다.
+하지만 이 때문에 동일한 목록의 문헌들을 서로 다른 토픽 모델에 입력해야 하는 경우
+매 모델에 각 문헌을 추가할때마다 `add_doc`을 호출해야하기 때문에 비효율이 발생합니다.
+따라서 `tomotopy`에서는 여러 문헌을 묶어서 관리해주는 `tomotopy.utils.Corpus` 클래스를 제공합니다.
+토픽 모델 객체를 생성할때 `tomotopy.utils.Corpus`를 `__init__` 메소드의 `corpus` 인자로 넘겨줌으로써 
+어떤 모델에든 쉽게 문헌들을 삽입할 수 있게 해줍니다.
+`tomotopy.utils.Corpus`를 토픽 모델에 삽입하면 corpus 객체가 가지고 있는 문헌들 전부가 모델에 자동으로 삽입됩니다.
+
+그런데 일부 토픽 모델의 경우 문헌을 생성하기 위해 서로 다른 데이터를 요구합니다.
+예를 들어 `tomotopy.DMRModel`는 `metadata`라는 `str` 타입의 데이터를 요구하고, 
+`tomotopy.PLDAModel`는 `labels`라는 `List[str]` 타입의 데이터를 요구합니다. 
+그러나 `tomotopy.utils.Corpus`는 토픽 모델에 종속되지 않은 독립적인 문헌 데이터를 보관하기 때문에,
+corpus가 가지고 있는 문헌 데이터가 실제 토픽 모델이 요구하는 데이터와 일치하지 않을 가능성이 있습니다.
+이 경우 `transform`라는 인자를 통해 corpus 내의 데이터를 변형시켜 토픽 모델이 요구하는 실제 데이터와 일치시킬 수 있습니다.
+자세한 내용은 아래의 코드를 확인해주세요:
+
+::
+
+    from tomotopy import DMRModel
+    from tomotopy.utils import Corpus
+
+    corpus = Corpus()
+    corpus.add_doc("a b c d e".split(), a_data=1)
+    corpus.add_doc("e f g h i".split(), a_data=2)
+    corpus.add_doc("i j k l m".split(), a_data=3)
+
+    model = DMRModel(k=10)
+    model.add_corpus(corpus) 
+    # `corpus`에 있던 `a_data`는 사라지고
+    # `DMRModel`이 요구하는 `metadata`에는 기본값인 빈 문자열이 채워집니다.
+
+    assert model.docs[0].metadata == ''
+    assert model.docs[1].metadata == ''
+    assert model.docs[2].metadata == ''
+
+    def transform_a_data_to_metadata(misc: dict):
+        return {'metadata': str(misc['a_data'])}
+    # 이 함수는 `a_data`를 `metadata`로 변환합니다.
+
+    model = DMRModel(k=10)
+    model.add_corpus(corpus, transform=transform_a_data_to_metadata)
+    # 이제 `model`에는 기본값이 아닌 `metadata`가 입력됩니다. 이들은 `transform`에 의해 `a_data`로부터 생성됩니다.
+
+    assert model.docs[0].metadata == '1'
+    assert model.docs[1].metadata == '2'
+    assert model.docs[2].metadata == '3'
+
+
 병렬 샘플링 알고리즘
 ----------------------------
 `tomotopy`는 0.5.0버전부터 병렬 알고리즘을 고를 수 있는 선택지를 제공합니다.
@@ -254,6 +305,12 @@ tomotopy의 Python3 예제 코드는 https://github.com/bab2min/tomotopy/blob/ma
 
 역사
 -------
+* 0.12.1 (2021-06-20)
+    * `tomotopy.LDAModel.set_word_prior()`가 크래시를 발생시키던 문제를 해결했습니다.
+    * 이제 `tomotopy.LDAModel.perplexity`와 `tomotopy.LDAModel.ll_per_word`가 TermWeight가 ONE이 아닌 경우에도 정확한 값을 반환합니다.
+    * 용어가중치가 적용된 빈도수를 반환하는 `tomotopy.LDAModel.used_vocab_weighted_freq`가 추가되었습니다.
+    * 이제 `tomotopy.LDAModel.summary()`가 단어의 엔트로피뿐만 아니라, 용어 가중치가 적용된 단어의 엔트로피도 함께 보여줍니다.
+
 * 0.12.0 (2021-04-26)
     * 이제 `tomotopy.DMRModel`와 `tomotopy.GDMRModel`가 다중 메타데이터를 지원합니다. (https://github.com/bab2min/tomotopy/blob/main/examples/dmr_multi_label.py 참조)
     * `tomotopy.GDMRModel`의 성능이 개선되었습니다.

data/vendor/tomotopy/README.rst

@@ -202,6 +202,55 @@ Inference for unseen document should be performed using `tomotopy.LDAModel.infer
 The `infer` method can infer only one instance of `tomotopy.Document` or a `list` of instances of `tomotopy.Document`. 
 See more at `tomotopy.LDAModel.infer`.
 
+Corpus and transform
+--------------------
+Every topic model in `tomotopy` has its own internal document type.
+A document can be created and added into suitable for each model through each model's `add_doc` method. 
+However, trying to add the same list of documents to different models becomes quite inconvenient, 
+because `add_doc` should be called for the same list of documents to each different model.
+Thus, `tomotopy` provides `tomotopy.utils.Corpus` class that holds a list of documents. 
+`tomotopy.utils.Corpus` can be inserted into any model by passing as argument `corpus` to `__init__` or `add_corpus` method of each model. 
+So, inserting `tomotopy.utils.Corpus` just has the same effect to inserting documents the corpus holds.
+
+Some topic models requires different data for its documents. 
+For example, `tomotopy.DMRModel` requires argument `metadata` in `str` type, 
+but `tomotopy.PLDAModel` requires argument `labels` in `List[str]` type. 
+Since `tomotopy.utils.Corpus` holds an independent set of documents rather than being tied to a specific topic model, 
+data types required by a topic model may be inconsistent when a corpus is added into that topic model. 
+In this case, miscellaneous data can be transformed to be fitted target topic model using argument `transform`. 
+See more details in the following code:
+
+::
+
+    from tomotopy import DMRModel
+    from tomotopy.utils import Corpus
+
+    corpus = Corpus()
+    corpus.add_doc("a b c d e".split(), a_data=1)
+    corpus.add_doc("e f g h i".split(), a_data=2)
+    corpus.add_doc("i j k l m".split(), a_data=3)
+
+    model = DMRModel(k=10)
+    model.add_corpus(corpus) 
+    # You lose `a_data` field in `corpus`, 
+    # and `metadata` that `DMRModel` requires is filled with the default value, empty str.
+
+    assert model.docs[0].metadata == ''
+    assert model.docs[1].metadata == ''
+    assert model.docs[2].metadata == ''
+
+    def transform_a_data_to_metadata(misc: dict):
+        return {'metadata': str(misc['a_data'])}
+    # this function transforms `a_data` to `metadata`
+
+    model = DMRModel(k=10)
+    model.add_corpus(corpus, transform=transform_a_data_to_metadata)
+    # Now docs in `model` has non-default `metadata`, that generated from `a_data` field.
+
+    assert model.docs[0].metadata == '1'
+    assert model.docs[1].metadata == '2'
+    assert model.docs[2].metadata == '3'
+
 Parallel Sampling Algorithms
 ----------------------------
 Since version 0.5.0, `tomotopy` allows you to choose a parallelism algorithm. 
@@ -260,6 +309,12 @@ meaning you can use it for any reasonable purpose and remain in complete ownersh
 
 History
 -------
+* 0.12.1 (2021-06-20)
+    * An issue where `tomotopy.LDAModel.set_word_prior()` causes a crash has been fixed.
+    * Now `tomotopy.LDAModel.perplexity` and `tomotopy.LDAModel.ll_per_word` return the accurate value when `TermWeight` is not `ONE`.
+    * `tomotopy.LDAModel.used_vocab_weighted_freq` was added, which returns term-weighted frequencies of words.
+    * Now `tomotopy.LDAModel.summary()` shows not only the entropy of words, but also the entropy of term-weighted words.
+
 * 0.12.0 (2021-04-26)
     * Now `tomotopy.DMRModel` and `tomotopy.GDMRModel` support multiple values of metadata (see https://github.com/bab2min/tomotopy/blob/main/examples/dmr_multi_label.py )
     * The performance of `tomotopy.GDMRModel` was improved.

data/vendor/tomotopy/src/Labeling/Phraser.hpp

@@ -316,7 +316,7 @@ namespace tomoto
 				}
 			}
 
-			float totN = std::accumulate(vocabFreqs.begin(), vocabFreqs.end(), (size_t)0);
+			float totN = (float)std::accumulate(vocabFreqs.begin(), vocabFreqs.end(), (size_t)0);
 			const float logTotN = std::log(totN);
 
 			// calculating PMIs
@@ -489,7 +489,7 @@ namespace tomoto
 
 				float rbe = branchingEntropy(trieNodes[0].getNext(bigram.first)->getNext(bigram.second), candMinCnt);
 				float lbe = branchingEntropy(trieNodesBw[0].getNext(bigram.second)->getNext(bigram.first), candMinCnt);
-				float nbe = std::sqrt(rbe * lbe) / std::log(p.second);
+				float nbe = std::sqrt(rbe * lbe) / (float)std::log(p.second);
 				if (nbe < minNBE) continue;
 				candidates.emplace_back(npmi * nbe, bigram.first, bigram.second);
 				candidates.back().cf = p.second;
@@ -512,7 +512,7 @@ namespace tomoto
 
 					float rbe = branchingEntropy(node, candMinCnt);
 					float lbe = branchingEntropy(trieNodesBw[0].findNode(rkeys.rbegin(), rkeys.rend()), candMinCnt);
-					float nbe = std::sqrt(rbe * lbe) / std::log(node->val);
+					float nbe = std::sqrt(rbe * lbe) / (float)std::log(node->val);
 					if (nbe < minNBE) return;
 					candidates.emplace_back(npmi * nbe, rkeys);
 					candidates.back().cf = node->val;

data/vendor/tomotopy/src/TopicModel/CTModel.hpp

@@ -33,7 +33,10 @@ namespace tomoto
 		friend typename BaseClass::BaseClass;
 		using WeightType = typename BaseClass::WeightType;
 
-		static constexpr char TMID[] = "CTM\0";
+		static constexpr auto tmid()
+		{
+			return serializer::to_key("CTM\0");
+		}
 
 		uint64_t numBetaSample = 10;
 		uint64_t numTMNSample = 5;
@@ -247,7 +250,7 @@ namespace tomoto
 			this->optimInterval = 2;
 		}
 
-		std::vector<Float> getTopicsByDoc(const _DocType& doc, bool normalize) const
+		std::vector<Float> _getTopicsByDoc(const _DocType& doc, bool normalize) const
 		{
 			std::vector<Float> ret(this->K);
 			Eigen::Map<Eigen::Array<Float, -1, 1>> m{ ret.data(), this->K };

data/vendor/tomotopy/src/TopicModel/DMRModel.hpp

@@ -47,7 +47,10 @@ namespace tomoto
 		friend typename BaseClass::BaseClass;
 		using WeightType = typename BaseClass::WeightType;
 
-		static constexpr char TMID[] = "DMR\0";
+		static constexpr auto tmid()
+		{
+			return serializer::to_key("DMR\0");
+		}
 
 		Matrix lambda;
 		mutable std::unordered_map<std::pair<uint64_t, Vector>, size_t, MdHash> mdHashMap;
@@ -449,7 +452,7 @@ namespace tomoto
 			optimRepeat = _optimRepeat;
 		}
 
-		std::vector<Float> getTopicsByDoc(const _DocType& doc, bool normalize) const
+		std::vector<Float> _getTopicsByDoc(const _DocType& doc, bool normalize) const
 		{
 			std::vector<Float> ret(this->K);
 			auto alphaDoc = getCachedAlpha(doc);

data/vendor/tomotopy/src/TopicModel/DTModel.hpp

@@ -41,7 +41,10 @@ namespace tomoto
 		friend typename BaseClass::BaseClass;
 		using WeightType = typename BaseClass::WeightType;
 
-		static constexpr char TMID[] = "DTM\0";
+		static constexpr auto tmid()
+		{
+			return serializer::to_key("DTM\0");
+		}
 
 		uint64_t T;
 		Float shapeA = 0.03f, shapeB = 0.1f, shapeC = 0.55f;
@@ -54,7 +57,7 @@ namespace tomoto
 		std::vector<sample::AliasMethod<>> wordAliasTables; // Dim: (Word * Time)
 
 		template<int _inc>
-		inline void addWordTo(_ModelState& ld, _DocType& doc, uint32_t pid, Vid vid, Tid tid) const
+		inline void addWordTo(_ModelState& ld, _DocType& doc, size_t pid, Vid vid, Tid tid) const
 		{
 			assert(tid < this->K);
 			assert(vid < this->realV);

data/vendor/tomotopy/src/TopicModel/HDPModel.hpp

@@ -168,7 +168,7 @@ namespace tomoto
 		}
 
 		template<int _inc> 
-		inline void addWordTo(_ModelState& ld, _DocType& doc, uint32_t pid, Vid vid, size_t tableId, Tid tid) const
+		inline void addWordTo(_ModelState& ld, _DocType& doc, size_t pid, Vid vid, size_t tableId, Tid tid) const
 		{
 			addOnlyWordTo<_inc>(ld, doc, pid, vid, tid);
 			constexpr bool _dec = _inc < 0 && _tw != TermWeight::one;
@@ -490,7 +490,7 @@ namespace tomoto
 			THROW_ERROR_WITH_INFO(exc::Unimplemented, "HDPModel doesn't provide setWordPrior function.");
 		}
 
-		std::vector<Float> getTopicsByDoc(const _DocType& doc, bool normalize) const
+		std::vector<Float> _getTopicsByDoc(const _DocType& doc, bool normalize) const
 		{
 			std::vector<Float> ret(this->K);
 			Eigen::Map<Eigen::Array<Float, -1, 1>> m{ ret.data(), this->K };
@@ -522,7 +522,7 @@ namespace tomoto
 			for (size_t i = 0; i < cntIdx.size(); ++i)
 			{
 				if (i && cntIdx[i].first / sum <= topicThreshold) break;
-				newK[cntIdx[i].second] = i;
+				newK[cntIdx[i].second] = (Tid)i;
 				liveK++;
 			}
 
@@ -558,7 +558,7 @@ namespace tomoto
 						lda->docs[i].Zs[j] = non_topic_id;
 						continue;
 					}
-					size_t newTopic = newK[this->docs[i].numTopicByTable[this->docs[i].Zs[j]].topic];
+					Tid newTopic = newK[this->docs[i].numTopicByTable[this->docs[i].Zs[j]].topic];
 					while (newTopic == (Tid)-1) newTopic = newK[randomTopic(rng)];
 					lda->docs[i].Zs[j] = newTopic;
 				}