tomoto 0.3.3 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: daa9c454c4cf09e120dbbe17305d225be58ac5937c463886e87ea1b3c3b5d466
-  data.tar.gz: f6c0c353a0efcc6026964e9125f1156b50e0d119506ecab2812522f7b716042d
+  metadata.gz: 86215ec57ae6cf6e36531ee2896e2b81d591f61909eb5454ef70b69c5db0a39d
+  data.tar.gz: 3f31adcb38a1793caaaedc516f99c9ffce4b82ff0c93f2a169b85377e116433b
 SHA512:
-  metadata.gz: 874f531a75a62d2291793ded080f380f8103682c2ae2b087dd31a014533443d5f35a7ea4e634aabd246fab1564ece35679c60c0b6ffbb6a627d57048e32bf790
-  data.tar.gz: e91bf3c618394f34f208fe4945729db6719a2cab1a8e7192e646b232d7e38274f2087808d145dccddb94b3cc632b9dddce82f942f051a4bf522ec5ec9d2c43b3
+  metadata.gz: db0a4bd9831cecae6711e150ecc2c5d23b87ada83d418f784474cf2f260627e52e40d871d9af974c6a79790e8d0d060ac08fbb69775e1d2de316085421ef76af
+  data.tar.gz: '07779f29aa9bdb4b71d9a0acdfe26c84d041312ad769c54539d6377b0ab01327dba045d97b51179a50498b438d137b567a3708bd3a2ab902c16adec37ad3a779'

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.4.1 (2024-09-04)
+
+- Updated tomoto to 0.13.0
+
+## 0.4.0 (2023-12-28)
+
+- Added support for Ruby 3.3
+- Added precompiled gem for Linux ARM
+- Updated tomoto to 0.12.7
+- Dropped support for Ruby < 3
+
 ## 0.3.3 (2023-02-01)
 
 - Added `topic_label_dict` method to `LLDA`

data/README.md

@@ -2,7 +2,7 @@
 
 :tomato: [tomoto](https://github.com/bab2min/tomotopy) - high performance topic modeling - for Ruby
 
-[![Build Status](https://github.com/ankane/tomoto-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/tomoto-ruby/actions)
+[![Build Status](https://github.com/ankane/tomoto-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/tomoto-ruby/actions)
 
 ## Installation
 

data/ext/tomoto/extconf.rb

@@ -27,16 +27,18 @@ else
 end
 
 # silence tomoto warnings
-$CXXFLAGS += " -Wno-unused-variable -Wno-switch"
+$CXXFLAGS += " -Wno-unused-variable -Wno-switch -Wno-unqualified-std-cast-call"
 
 ext = File.expand_path(".", __dir__)
 tomoto = File.expand_path("../../vendor/tomotopy/src/TopicModel", __dir__)
+tomoto_utils = File.expand_path("../../vendor/tomotopy/src/Utils", __dir__)
 eigen = File.expand_path("../../vendor/eigen", __dir__)
 eigen_rand = File.expand_path("../../vendor/EigenRand", __dir__)
 variant = File.expand_path("../../vendor/variant/include", __dir__)
 
-$srcs = Dir["{#{ext},#{tomoto}}/*.cpp"]
+$srcs = Dir["{#{ext},#{tomoto},#{tomoto_utils}}/*.cpp"]
 $INCFLAGS += " -I#{tomoto} -I#{eigen} -I#{eigen_rand} -I#{variant}"
 $VPATH << tomoto
+$VPATH << tomoto_utils
 
 create_makefile("tomoto/tomoto")

data/lib/tomoto/version.rb

@@ -1,3 +1,3 @@
 module Tomoto
-  VERSION = "0.3.3"
+  VERSION = "0.4.1"
 end

data/lib/tomoto.rb

@@ -6,20 +6,20 @@ rescue LoadError
 end
 
 # modules
-require "tomoto/ct"
-require "tomoto/dmr"
-require "tomoto/dt"
-require "tomoto/gdmr"
-require "tomoto/hdp"
-require "tomoto/hlda"
-require "tomoto/hpa"
-require "tomoto/lda"
-require "tomoto/llda"
-require "tomoto/mglda"
-require "tomoto/pa"
-require "tomoto/plda"
-require "tomoto/slda"
-require "tomoto/version"
+require_relative "tomoto/ct"
+require_relative "tomoto/dmr"
+require_relative "tomoto/dt"
+require_relative "tomoto/gdmr"
+require_relative "tomoto/hdp"
+require_relative "tomoto/hlda"
+require_relative "tomoto/hpa"
+require_relative "tomoto/lda"
+require_relative "tomoto/llda"
+require_relative "tomoto/mglda"
+require_relative "tomoto/pa"
+require_relative "tomoto/plda"
+require_relative "tomoto/slda"
+require_relative "tomoto/version"
 
 module Tomoto
   PARALLEL_SCHEME = [:default, :none, :copy_merge, :partition]

data/vendor/tomotopy/README.kr.rst

@@ -7,7 +7,7 @@ tomotopy
 .. image:: https://zenodo.org/badge/186155463.svg
    :target: https://zenodo.org/badge/latestdoi/186155463
 
-🎌
+🌐
 `English`_,
 **한국어**.
 
@@ -305,6 +305,32 @@ tomotopy의 Python3 예제 코드는 https://github.com/bab2min/tomotopy/blob/ma
 
 역사
 -------
+* 0.13.0 (2024-08-05)
+    * 신규 기능
+        * 토픽 모델 뷰어인 `tomotopy.viewer.open_viewer()`의 주요 기능이 완성되었습니다.
+        * `tomotopy.LDAModel.get_hash()`가 추가되었습니다. 모델의 128bit 해시를 구해줍니다.
+        * `ngram_list` 인자가 `tomotopy.utils.SimpleTokenizer`에 추가되었습니다.
+    * Bug fixes
+        * `Corpus.concat_ngrams` 호출 후에 `spans`이 비일관적인 버그가 수정되었습니다.
+        * `tomotopy.LDAModel.load()`와 `tomotopy.LDAModel.save()`의 병목을 최적화하여 속도를 10배 이상 개선했습니다.
+
+* 0.12.7 (2023-12-19)
+    * 신규 기능
+        * 토픽 모델 뷰어인 `tomotopy.viewer.open_viewer()`가 추가되었습니다.
+        * `tomotopy.utils.Corpus.process()`의 속도를 개선했습니다.
+    * Bug fixes
+        * `Document.span`이 이제 바이트 단위가 아니라 문자 단위로 범위를 제대로 반환합니다.
+
+* 0.12.6 (2023-12-11)
+    * 신규 기능
+        * `tomotopy.LDAModel.train`과 `tomotopy.LDAModel.set_word_prior`에 몇가지 편의 기능을 추가했습니다.
+        * `LDAModel.train`가 이제 학습 진행상황을 모니터링할 수 있는 `callback`, `callback_interval`, `show_progres` 인자를 지원합니다.
+        * `LDAModel.set_word_prior`가 이제 `prior` 인자로 `Dict[int, float]` 타입도 받을 수 있게 되었습니다.
+
+* 0.12.5 (2023-08-03)
+    * 신규 기능
+        * Linux ARM64 아키텍처에 대한 지원을 추가했습니다.
+
 * 0.12.4 (2023-01-22)
     * New features
         * macOS ARM64 아키텍처에 대한 지원을 추가했습니다.

data/vendor/tomotopy/README.rst

@@ -7,7 +7,7 @@ tomotopy
 .. image:: https://zenodo.org/badge/186155463.svg
    :target: https://zenodo.org/badge/latestdoi/186155463
 
-🎌
+🌐
 **English**,
 `한국어`_.
 
@@ -309,6 +309,32 @@ meaning you can use it for any reasonable purpose and remain in complete ownersh
 
 History
 -------
+* 0.13.0 (2024-08-05)
+    * New features
+        * Major features of Topic Model Viewer `tomotopy.viewer.open_viewer()` are ready now.
+        * `tomotopy.LDAModel.get_hash()` is added. You can get 128bit hash value of the model.
+        * Add an argument `ngram_list` to `tomotopy.utils.SimpleTokenizer`.
+    * Bug fixes
+        * Fixed inconsistent `spans` bug after `Corpus.concat_ngrams` is called.
+        * Optimized the bottleneck of `tomotopy.LDAModel.load()` and `tomotopy.LDAModel.save()` and improved its speed more than 10 times.
+
+* 0.12.7 (2023-12-19)
+    * New features
+        * Added Topic Model Viewer `tomotopy.viewer.open_viewer()`
+        * Optimized the performance of `tomotopy.utils.Corpus.process()`
+    * Bug fixes
+        * `Document.span` now returns the ranges in character unit, not in byte unit.
+
+* 0.12.6 (2023-12-11)
+    * New features
+        * Added some convenience features to `tomotopy.LDAModel.train` and `tomotopy.LDAModel.set_word_prior`.
+        * `LDAModel.train` now has new arguments `callback`, `callback_interval` and `show_progres` to monitor the training progress.
+        * `LDAModel.set_word_prior` now can accept `Dict[int, float]` type as its argument `prior`.
+
+* 0.12.5 (2023-08-03)
+    * New features
+        * Added support for Linux ARM64 architecture.
+
 * 0.12.4 (2023-01-22)
     * New features
         * Added support for macOS ARM64 architecture.

data/vendor/tomotopy/src/TopicModel/CT.h

@@ -11,8 +11,8 @@ namespace tomoto
 		Matrix beta; // Dim: (K, betaSample)
 		Vector smBeta; // Dim: K
 		
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, smBeta);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, smBeta);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct CTArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/CTModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentCTM, BaseDocument, 0, smBeta);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentCTM, BaseDocument, 1, 0x00010001, smBeta);
+
+	TMT_INSTANTIATE_DOC(DocumentCTM);
+
 	ICTModel* ICTModel::create(TermWeight _weight, const CTArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, CTModel, args);

data/vendor/tomotopy/src/TopicModel/CTModel.hpp

@@ -243,6 +243,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, numBetaSample, numTMNSample, topicPrior);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, numBetaSample, numTMNSample, topicPrior);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, numBetaSample, numTMNSample, topicPrior);
 
 		CTModel(const CTArgs& args)
 			: BaseClass(args)

data/vendor/tomotopy/src/TopicModel/DMR.h

@@ -18,8 +18,8 @@ namespace tomoto
 
 		RawDoc::MiscType makeMisc(const ITopicModel* tm) const override;
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, metadata);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, metadata, multiMetadata);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct DMRArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/DMRModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentDMR, BaseDocument, 0, metadata);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentDMR, BaseDocument, 1, 0x00010001, metadata, multiMetadata);
+
+	TMT_INSTANTIATE_DOC(DocumentDMR);
+
 	IDMRModel* IDMRModel::create(TermWeight _weight, const DMRArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, DMRModel, args);

data/vendor/tomotopy/src/TopicModel/DMRModel.hpp

@@ -364,6 +364,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, sigma, alphaEps, metadataDict, lambda);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, sigma, alphaEps, metadataDict, lambda, multiMetadataDict);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, sigma, alphaEps, metadataDict, lambda, multiMetadataDict);
 
 		DMRModel(const DMRArgs& args)
 			: BaseClass(args), sigma(args.sigma), alphaEps(args.alphaEps)

data/vendor/tomotopy/src/TopicModel/DT.h

@@ -21,8 +21,8 @@ namespace tomoto
 			return ret;
 		}
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, timepoint);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, timepoint);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct DTArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/DTModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentDTM, BaseDocument, 0, timepoint);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentDTM, BaseDocument, 1, 0x00010001, timepoint);
+
+	TMT_INSTANTIATE_DOC(DocumentDTM);
+
 	IDTModel* IDTModel::create(TermWeight _weight, const DTArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, DTModel, args);

data/vendor/tomotopy/src/TopicModel/DTModel.hpp

@@ -22,6 +22,7 @@ namespace tomoto
 		Eigen::Matrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic * Time, Vocabs)
 		//ShareableMatrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic * Time, Vocabs)
 		DEFINE_SERIALIZER(numByTopic, numByTopicWord);
+		DEFINE_HASHER(numByTopic, numByTopicWord);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -365,6 +366,7 @@ namespace tomoto
 		{
 			double ll = 0;
 			const size_t V = this->realV;
+			if (V == 0) return 0;
 			for (Tid t = 0; t < T; ++t)
 			{
 				// topic-word distribution
@@ -495,6 +497,8 @@ namespace tomoto
 			T, shapeA, shapeB, shapeC, alphaVar, etaVar, phiVar, alphas, etaByDoc, phi);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, 
 			T, shapeA, shapeB, shapeC, alphaVar, etaVar, phiVar, alphas, etaByDoc, phi);
+		DEFINE_HASHER_AFTER_BASE(BaseClass,
+			T, shapeA, shapeB, shapeC, alphaVar, etaVar, phiVar, alphas, etaByDoc, phi);
 
 		GETTER(T, size_t, T);
 		GETTER(NumDocsByT, std::vector<uint32_t>, numDocsByTime);

data/vendor/tomotopy/src/TopicModel/GDMR.h

@@ -17,8 +17,8 @@ namespace tomoto
 			return ret;
 		}
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, metadataOrg);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, metadataOrg, metadataNormalized);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct GDMRArgs : public DMRArgs

data/vendor/tomotopy/src/TopicModel/GDMRModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentGDMR, BaseDocument, 0, metadataOrg);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentGDMR, BaseDocument, 1, 0x00010001, metadataOrg, metadataNormalized);
+
+	TMT_INSTANTIATE_DOC(DocumentGDMR);
+
     IGDMRModel* IGDMRModel::create(TermWeight _weight, const GDMRArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, GDMRModel, args);

data/vendor/tomotopy/src/TopicModel/GDMRModel.hpp

@@ -412,6 +412,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, sigma0, degreeByF, mdCoefs, mdIntercepts);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, sigma0, orderDecay, degreeByF, mdCoefs, mdIntercepts, mdMax);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, sigma0, orderDecay, degreeByF, mdCoefs, mdIntercepts, mdMax);
 
 		GDMRModel(const GDMRArgs& args)
 			: BaseClass(args), sigma0(args.sigma0), orderDecay(args.orderDecay), degreeByF(args.degrees)

data/vendor/tomotopy/src/TopicModel/HDP.h

@@ -39,8 +39,8 @@ namespace tomoto
 		};
 		std::vector<TableTopicInfo> numTopicByTable;
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, numTopicByTable);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, numTopicByTable);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 
 		size_t getNumTable() const
 		{

data/vendor/tomotopy/src/TopicModel/HDPModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentHDP, BaseDocument, 0, numTopicByTable);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentHDP, BaseDocument, 1, 0x00010001, numTopicByTable);
+
+	TMT_INSTANTIATE_DOC(DocumentHDP);
+
     IHDPModel* IHDPModel::create(TermWeight _weight, const HDPArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, HDPModel, args);

data/vendor/tomotopy/src/TopicModel/HDPModel.hpp

@@ -19,6 +19,7 @@ namespace tomoto
 		size_t totalTable = 0;
 
 		DEFINE_SERIALIZER_AFTER_BASE(ModelStateLDA<_tw>, numTableByTopic, totalTable);
+		DEFINE_HASHER_AFTER_BASE(ModelStateLDA<_tw>, numTableByTopic, totalTable);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -457,6 +458,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, gamma);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, gamma);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, gamma);
 
 		HDPModel(const HDPArgs& args)
 			: BaseClass(args), gamma(args.gamma)

data/vendor/tomotopy/src/TopicModel/HLDA.h

@@ -16,8 +16,8 @@ namespace tomoto
 
 		template<typename _TopicModel> void update(WeightType* ptr, const _TopicModel& mdl);
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, path);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, path);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct HLDAArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/HLDAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentHLDA, BaseDocument, 0, path);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentHLDA, BaseDocument, 1, 0x00010001, path);
+
+	TMT_INSTANTIATE_DOC(DocumentHLDA);
+
 	IHLDAModel* IHLDAModel::create(TermWeight _weight, const HLDAArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, HLDAModel, args);

data/vendor/tomotopy/src/TopicModel/HLDAModel.hpp

@@ -18,6 +18,7 @@ namespace tomoto
 			int32_t parent = 0, sibling = 0, child = 0;
 
 			DEFINE_SERIALIZER(numCustomers, level, parent, sibling, child);
+			DEFINE_HASHER(numCustomers, level, parent, sibling, child);
 
 			NCRPNode* getParent() const
 			{
@@ -118,6 +119,7 @@ namespace tomoto
 			Vector nodeWLikelihoods; //
 
 			DEFINE_SERIALIZER(nodes, levelBlocks);
+			DEFINE_HASHER(nodes, levelBlocks);
 
 			template<bool _makeNewPath = true>
 			void calcNodeLikelihood(Float gamma, size_t levelDepth)
@@ -317,6 +319,12 @@ namespace tomoto
 			ModelStateLDA<_tw>::serializerWrite(ostr);
 			nt->serializerWrite(ostr);
 		}
+
+		uint64_t computeHash(uint64_t seed) const
+		{
+			seed = ModelStateLDA<_tw>::computeHash(seed);
+			return nt->computeHash(seed);
+		}
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -596,6 +604,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, gamma);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, gamma);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, gamma);
 
 		HLDAModel(const HLDAArgs& args)
 			: BaseClass(args), gamma(args.gamma)

data/vendor/tomotopy/src/TopicModel/HPA.h

@@ -12,8 +12,8 @@ namespace tomoto
 
 		template<typename _TopicModel> void update(WeightType* ptr, const _TopicModel& mdl);
 
-		DEFINE_SERIALIZER_BASE_WITH_VERSION(BaseDocument, 0);
-		DEFINE_SERIALIZER_BASE_WITH_VERSION(BaseDocument, 1);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct HPAArgs : public PAArgs

data/vendor/tomotopy/src/TopicModel/HPAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_BASE_WITH_VERSION(DocumentHPA, BaseDocument, 0);
+	DEFINE_OUT_SERIALIZER_BASE_WITH_VERSION(DocumentHPA, BaseDocument, 1);
+
+	TMT_INSTANTIATE_DOC(DocumentHPA);
+
     IHPAModel* IHPAModel::create(TermWeight _weight, bool _exclusive, const HPAArgs& args, bool scalarRng)
 	{
 		if (_exclusive)

data/vendor/tomotopy/src/TopicModel/HPAModel.hpp

@@ -21,6 +21,7 @@ namespace tomoto
 		Eigen::Matrix<WeightType, -1, -1> numByTopic1_2;
 
 		DEFINE_SERIALIZER_AFTER_BASE(ModelStateLDA<_tw>, numByTopicWord, numByTopic, numByTopic1_2);
+		DEFINE_HASHER_AFTER_BASE(ModelStateLDA<_tw>, numByTopicWord, numByTopic, numByTopic1_2);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -439,6 +440,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, K2, subAlphas, subAlphaSum);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, K2, subAlphas, subAlphaSum);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, K2, subAlphas, subAlphaSum);
 
 		HPAModel(const HPAArgs& args)
 			: BaseClass(args, false), K2(args.k2)

data/vendor/tomotopy/src/TopicModel/LDA.h

@@ -93,6 +93,12 @@ namespace tomoto
 			if (!ostr.write((const char*)this->data(), sizeof(_Scalar) * this->size()))
 				throw std::ios_base::failure(std::string("writing type '") + typeid(_Scalar).name() + std::string("' is failed"));
 		}
+
+		uint64_t computeHash(uint64_t seed) const
+		{
+			seed = serializer::computeHashMany(seed, (uint32_t)this->rows(), (uint32_t)this->cols());
+			return serializer::computeFastHash(this->data(), sizeof(_Scalar) * this->size(), seed);
+		}
 	};
 
 	template<typename _Base, TermWeight _tw>
@@ -139,8 +145,8 @@ namespace tomoto
 		tvector<Float> wordWeights;
 		ShareableMatrix<WeightType, -1, 1> numByTopic;
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentBase, 0, Zs, wordWeights);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentBase, 1, 0x00010001, Zs, wordWeights);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 
 		template<typename _TopicModel> void update(WeightType* ptr, const _TopicModel& mdl);
 		

data/vendor/tomotopy/src/TopicModel/LDAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+    DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentLDA, DocumentBase, 0, Zs, wordWeights);
+    DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentLDA, DocumentBase, 1, 0x00010001, Zs, wordWeights);
+
+    TMT_INSTANTIATE_DOC(DocumentLDA);
+
     ILDAModel* ILDAModel::create(TermWeight _weight, const LDAArgs& args, bool scalarRng)
     {
         TMT_SWITCH_TW(_weight, scalarRng, LDAModel, args);

data/vendor/tomotopy/src/TopicModel/LDAModel.hpp

@@ -47,6 +47,10 @@ Term Weighting Scheme is based on following paper:
 		return nullptr; } while(0)
 #endif
 
+#define TMT_INSTANTIATE_DOC(CLS) template struct CLS<TermWeight::one>; \
+	template struct CLS<TermWeight::idf>; \
+	template struct CLS<TermWeight::pmi>;
+
 #define GETTER(name, type, field) type get##name() const override { return field; }
 
 namespace tomoto
@@ -61,6 +65,7 @@ namespace tomoto
 		//Eigen::Matrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic, Vocabs)
 		ShareableMatrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic, Vocabs)
 		DEFINE_SERIALIZER(numByTopic, numByTopicWord);
+		DEFINE_HASHER(numByTopic, numByTopicWord);
 	};
 
 	namespace flags
@@ -954,6 +959,8 @@ namespace tomoto
 		DEFINE_TAGGED_SERIALIZER_WITH_VERSION(1, 0x00010001, vocabWeights, alpha, alphas, eta, K, etaByWord,
 			burnIn, optimInterval);
 
+		DEFINE_HASHER(vocabWeights, alpha, alphas, eta, K, /*etaByWord,*/ burnIn, optimInterval);
+
 		LDAModel(const LDAArgs& args, bool checkAlpha = true)
 			: BaseClass(args.seed), K(args.k), alpha(args.alpha[0]), eta(args.eta)
 		{
@@ -1066,6 +1073,7 @@ namespace tomoto
 		void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0, bool updateStopwords = true) override
 		{
 			if (initDocs && updateStopwords) this->removeStopwords(minWordCnt, minWordDf, removeTopN);
+			static_cast<DerivedClass*>(this)->updateWordFormCnts();
 			static_cast<DerivedClass*>(this)->updateWeakArray();
 			static_cast<DerivedClass*>(this)->initGlobalState(initDocs);
 			static_cast<DerivedClass*>(this)->prepareWordPriors();

data/vendor/tomotopy/src/TopicModel/LLDA.h

@@ -11,8 +11,8 @@ namespace tomoto
 		using WeightType = typename DocumentLDA<_tw>::WeightType;
 		Eigen::Matrix<int8_t, -1, 1> labelMask;
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, labelMask);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, labelMask);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	class ILLDAModel : public ILDAModel

data/vendor/tomotopy/src/TopicModel/LLDAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentLLDA, BaseDocument, 0, labelMask);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentLLDA, BaseDocument, 1, 0x00010001, labelMask);
+
+	TMT_INSTANTIATE_DOC(DocumentLLDA);
+
 	ILLDAModel* ILLDAModel::create(TermWeight _weight, const LDAArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, LLDAModel, args);

data/vendor/tomotopy/src/TopicModel/LLDAModel.hpp

@@ -107,6 +107,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, topicLabelDict);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, topicLabelDict);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, topicLabelDict);
 
 		LLDAModel(const LDAArgs& args)
 			: BaseClass(args)

data/vendor/tomotopy/src/TopicModel/MGLDA.h

@@ -22,8 +22,8 @@ namespace tomoto
 		Eigen::Matrix<WeightType, -1, 1> numByWin; // number of words in the window (len = S + T - 1)
 		Eigen::Matrix<WeightType, -1, -1> numByWinTopicL; // number of words in the loc. topic in the window (len = KL * (S + T - 1))
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, sents, Vs, numGl, numBySentWin, numByWinL, numByWin, numByWinTopicL);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, sents, Vs, numGl, numBySentWin, numByWinL, numByWin, numByWinTopicL);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 
 		template<typename _TopicModel> void update(WeightType* ptr, const _TopicModel& mdl);
 	};

data/vendor/tomotopy/src/TopicModel/MGLDAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentMGLDA, BaseDocument, 0, sents, Vs, numGl, numBySentWin, numByWinL, numByWin, numByWinTopicL);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentMGLDA, BaseDocument, 1, 0x00010001, sents, Vs, numGl, numBySentWin, numByWinL, numByWin, numByWinTopicL);
+
+	TMT_INSTANTIATE_DOC(DocumentMGLDA);
+
     IMGLDAModel* IMGLDAModel::create(TermWeight _weight, const MGLDAArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, MGLDAModel, args);

data/vendor/tomotopy/src/TopicModel/MGLDAModel.hpp

@@ -370,6 +370,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, alphaL, alphaM, alphaML, etaL, gamma, KL, T);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, alphaL, alphaM, alphaML, etaL, gamma, KL, T);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, alphaL, alphaM, alphaML, etaL, gamma, KL, T);
 
 		MGLDAModel(const MGLDAArgs& args)
 			: BaseClass(args), KL(args.kL), T(args.t),
@@ -516,9 +517,14 @@ namespace tomoto
 			return std::make_unique<_DocType>(as_mutable(this)->template _makeFromRawDoc<true>(rawDoc));
 		}
 
+		size_t getNumTopicsForPrior() const override
+		{
+			return this->K + KL;
+		}
+
 		void setWordPrior(const std::string& word, const std::vector<Float>& priors) override
 		{
-			if (priors.size() != this->K + KL) THROW_ERROR_WITH_INFO(exc::InvalidArgument, "priors.size() must be equal to K.");
+			if (priors.size() != this->K + KL) THROW_ERROR_WITH_INFO(exc::InvalidArgument, "priors.size() must be equal to K + KL.");
 			for (auto p : priors)
 			{
 				if (p < 0) THROW_ERROR_WITH_INFO(exc::InvalidArgument, "priors must not be less than 0.");