tomoto 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7d16410002670991fd881e13f64195db9de29dfa5c383da2287d44c9053b500
-  data.tar.gz: 290254c48ed1c3ce1ff51e2bbe07a46ed02d05dab6bcc095f38cdbf499883561
+  metadata.gz: 86215ec57ae6cf6e36531ee2896e2b81d591f61909eb5454ef70b69c5db0a39d
+  data.tar.gz: 3f31adcb38a1793caaaedc516f99c9ffce4b82ff0c93f2a169b85377e116433b
 SHA512:
-  metadata.gz: 817a074c0f9969ded7592d70a2b3096ca91142470552e019bc95668b45b658d24010160bce53b356810b09f25288fd9fb9c070841b3587ea23e6099f528f94b0
-  data.tar.gz: 94764d26429358b30766a36ef899b6856b61c848ed575cf62911dae9a352344c9736cd86be859601642cde479ba810b607dd41730db920d0fa9170b71dc9fdf2
+  metadata.gz: db0a4bd9831cecae6711e150ecc2c5d23b87ada83d418f784474cf2f260627e52e40d871d9af974c6a79790e8d0d060ac08fbb69775e1d2de316085421ef76af
+  data.tar.gz: '07779f29aa9bdb4b71d9a0acdfe26c84d041312ad769c54539d6377b0ab01327dba045d97b51179a50498b438d137b567a3708bd3a2ab902c16adec37ad3a779'

data/CHANGELOG.md

@@ -1,5 +1,10 @@
+## 0.4.1 (2024-09-04)
+
+- Updated tomoto to 0.13.0
+
 ## 0.4.0 (2023-12-28)
 
+- Added support for Ruby 3.3
 - Added precompiled gem for Linux ARM
 - Updated tomoto to 0.12.7
 - Dropped support for Ruby < 3

data/README.md

@@ -2,7 +2,7 @@
 
 :tomato: [tomoto](https://github.com/bab2min/tomotopy) - high performance topic modeling - for Ruby
 
-[![Build Status](https://github.com/ankane/tomoto-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/tomoto-ruby/actions)
+[![Build Status](https://github.com/ankane/tomoto-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/tomoto-ruby/actions)
 
 ## Installation
 

data/ext/tomoto/extconf.rb

@@ -27,16 +27,18 @@ else
 end
 
 # silence tomoto warnings
-$CXXFLAGS += " -Wno-unused-variable -Wno-switch"
+$CXXFLAGS += " -Wno-unused-variable -Wno-switch -Wno-unqualified-std-cast-call"
 
 ext = File.expand_path(".", __dir__)
 tomoto = File.expand_path("../../vendor/tomotopy/src/TopicModel", __dir__)
+tomoto_utils = File.expand_path("../../vendor/tomotopy/src/Utils", __dir__)
 eigen = File.expand_path("../../vendor/eigen", __dir__)
 eigen_rand = File.expand_path("../../vendor/EigenRand", __dir__)
 variant = File.expand_path("../../vendor/variant/include", __dir__)
 
-$srcs = Dir["{#{ext},#{tomoto}}/*.cpp"]
+$srcs = Dir["{#{ext},#{tomoto},#{tomoto_utils}}/*.cpp"]
 $INCFLAGS += " -I#{tomoto} -I#{eigen} -I#{eigen_rand} -I#{variant}"
 $VPATH << tomoto
+$VPATH << tomoto_utils
 
 create_makefile("tomoto/tomoto")

data/lib/tomoto/version.rb

@@ -1,3 +1,3 @@
 module Tomoto
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/vendor/tomotopy/README.kr.rst

@@ -7,7 +7,7 @@ tomotopy
 .. image:: https://zenodo.org/badge/186155463.svg
    :target: https://zenodo.org/badge/latestdoi/186155463
 
-🎌
+🌐
 `English`_,
 **한국어**.
 
@@ -305,6 +305,15 @@ tomotopy의 Python3 예제 코드는 https://github.com/bab2min/tomotopy/blob/ma
 
 역사
 -------
+* 0.13.0 (2024-08-05)
+    * 신규 기능
+        * 토픽 모델 뷰어인 `tomotopy.viewer.open_viewer()`의 주요 기능이 완성되었습니다.
+        * `tomotopy.LDAModel.get_hash()`가 추가되었습니다. 모델의 128bit 해시를 구해줍니다.
+        * `ngram_list` 인자가 `tomotopy.utils.SimpleTokenizer`에 추가되었습니다.
+    * Bug fixes
+        * `Corpus.concat_ngrams` 호출 후에 `spans`이 비일관적인 버그가 수정되었습니다.
+        * `tomotopy.LDAModel.load()`와 `tomotopy.LDAModel.save()`의 병목을 최적화하여 속도를 10배 이상 개선했습니다.
+
 * 0.12.7 (2023-12-19)
     * 신규 기능
         * 토픽 모델 뷰어인 `tomotopy.viewer.open_viewer()`가 추가되었습니다.

data/vendor/tomotopy/README.rst

@@ -7,7 +7,7 @@ tomotopy
 .. image:: https://zenodo.org/badge/186155463.svg
    :target: https://zenodo.org/badge/latestdoi/186155463
 
-🎌
+🌐
 **English**,
 `한국어`_.
 
@@ -309,6 +309,15 @@ meaning you can use it for any reasonable purpose and remain in complete ownersh
 
 History
 -------
+* 0.13.0 (2024-08-05)
+    * New features
+        * Major features of Topic Model Viewer `tomotopy.viewer.open_viewer()` are ready now.
+        * `tomotopy.LDAModel.get_hash()` is added. You can get 128bit hash value of the model.
+        * Add an argument `ngram_list` to `tomotopy.utils.SimpleTokenizer`.
+    * Bug fixes
+        * Fixed inconsistent `spans` bug after `Corpus.concat_ngrams` is called.
+        * Optimized the bottleneck of `tomotopy.LDAModel.load()` and `tomotopy.LDAModel.save()` and improved its speed more than 10 times.
+
 * 0.12.7 (2023-12-19)
     * New features
         * Added Topic Model Viewer `tomotopy.viewer.open_viewer()`

data/vendor/tomotopy/src/TopicModel/CT.h

@@ -11,8 +11,8 @@ namespace tomoto
 		Matrix beta; // Dim: (K, betaSample)
 		Vector smBeta; // Dim: K
 		
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, smBeta);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, smBeta);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct CTArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/CTModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentCTM, BaseDocument, 0, smBeta);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentCTM, BaseDocument, 1, 0x00010001, smBeta);
+
+	TMT_INSTANTIATE_DOC(DocumentCTM);
+
 	ICTModel* ICTModel::create(TermWeight _weight, const CTArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, CTModel, args);

data/vendor/tomotopy/src/TopicModel/CTModel.hpp

@@ -243,6 +243,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, numBetaSample, numTMNSample, topicPrior);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, numBetaSample, numTMNSample, topicPrior);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, numBetaSample, numTMNSample, topicPrior);
 
 		CTModel(const CTArgs& args)
 			: BaseClass(args)

data/vendor/tomotopy/src/TopicModel/DMR.h

@@ -18,8 +18,8 @@ namespace tomoto
 
 		RawDoc::MiscType makeMisc(const ITopicModel* tm) const override;
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, metadata);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, metadata, multiMetadata);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct DMRArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/DMRModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentDMR, BaseDocument, 0, metadata);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentDMR, BaseDocument, 1, 0x00010001, metadata, multiMetadata);
+
+	TMT_INSTANTIATE_DOC(DocumentDMR);
+
 	IDMRModel* IDMRModel::create(TermWeight _weight, const DMRArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, DMRModel, args);

data/vendor/tomotopy/src/TopicModel/DMRModel.hpp

@@ -364,6 +364,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, sigma, alphaEps, metadataDict, lambda);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, sigma, alphaEps, metadataDict, lambda, multiMetadataDict);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, sigma, alphaEps, metadataDict, lambda, multiMetadataDict);
 
 		DMRModel(const DMRArgs& args)
 			: BaseClass(args), sigma(args.sigma), alphaEps(args.alphaEps)

data/vendor/tomotopy/src/TopicModel/DT.h

@@ -21,8 +21,8 @@ namespace tomoto
 			return ret;
 		}
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, timepoint);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, timepoint);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct DTArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/DTModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentDTM, BaseDocument, 0, timepoint);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentDTM, BaseDocument, 1, 0x00010001, timepoint);
+
+	TMT_INSTANTIATE_DOC(DocumentDTM);
+
 	IDTModel* IDTModel::create(TermWeight _weight, const DTArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, DTModel, args);

data/vendor/tomotopy/src/TopicModel/DTModel.hpp

@@ -22,6 +22,7 @@ namespace tomoto
 		Eigen::Matrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic * Time, Vocabs)
 		//ShareableMatrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic * Time, Vocabs)
 		DEFINE_SERIALIZER(numByTopic, numByTopicWord);
+		DEFINE_HASHER(numByTopic, numByTopicWord);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -496,6 +497,8 @@ namespace tomoto
 			T, shapeA, shapeB, shapeC, alphaVar, etaVar, phiVar, alphas, etaByDoc, phi);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, 
 			T, shapeA, shapeB, shapeC, alphaVar, etaVar, phiVar, alphas, etaByDoc, phi);
+		DEFINE_HASHER_AFTER_BASE(BaseClass,
+			T, shapeA, shapeB, shapeC, alphaVar, etaVar, phiVar, alphas, etaByDoc, phi);
 
 		GETTER(T, size_t, T);
 		GETTER(NumDocsByT, std::vector<uint32_t>, numDocsByTime);

data/vendor/tomotopy/src/TopicModel/GDMR.h

@@ -17,8 +17,8 @@ namespace tomoto
 			return ret;
 		}
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, metadataOrg);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, metadataOrg, metadataNormalized);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct GDMRArgs : public DMRArgs

data/vendor/tomotopy/src/TopicModel/GDMRModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentGDMR, BaseDocument, 0, metadataOrg);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentGDMR, BaseDocument, 1, 0x00010001, metadataOrg, metadataNormalized);
+
+	TMT_INSTANTIATE_DOC(DocumentGDMR);
+
     IGDMRModel* IGDMRModel::create(TermWeight _weight, const GDMRArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, GDMRModel, args);

data/vendor/tomotopy/src/TopicModel/GDMRModel.hpp

@@ -412,6 +412,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, sigma0, degreeByF, mdCoefs, mdIntercepts);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, sigma0, orderDecay, degreeByF, mdCoefs, mdIntercepts, mdMax);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, sigma0, orderDecay, degreeByF, mdCoefs, mdIntercepts, mdMax);
 
 		GDMRModel(const GDMRArgs& args)
 			: BaseClass(args), sigma0(args.sigma0), orderDecay(args.orderDecay), degreeByF(args.degrees)

data/vendor/tomotopy/src/TopicModel/HDP.h

@@ -39,8 +39,8 @@ namespace tomoto
 		};
 		std::vector<TableTopicInfo> numTopicByTable;
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, numTopicByTable);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, numTopicByTable);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 
 		size_t getNumTable() const
 		{

data/vendor/tomotopy/src/TopicModel/HDPModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentHDP, BaseDocument, 0, numTopicByTable);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentHDP, BaseDocument, 1, 0x00010001, numTopicByTable);
+
+	TMT_INSTANTIATE_DOC(DocumentHDP);
+
     IHDPModel* IHDPModel::create(TermWeight _weight, const HDPArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, HDPModel, args);

data/vendor/tomotopy/src/TopicModel/HDPModel.hpp

@@ -19,6 +19,7 @@ namespace tomoto
 		size_t totalTable = 0;
 
 		DEFINE_SERIALIZER_AFTER_BASE(ModelStateLDA<_tw>, numTableByTopic, totalTable);
+		DEFINE_HASHER_AFTER_BASE(ModelStateLDA<_tw>, numTableByTopic, totalTable);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -457,6 +458,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, gamma);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, gamma);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, gamma);
 
 		HDPModel(const HDPArgs& args)
 			: BaseClass(args), gamma(args.gamma)

data/vendor/tomotopy/src/TopicModel/HLDA.h

@@ -16,8 +16,8 @@ namespace tomoto
 
 		template<typename _TopicModel> void update(WeightType* ptr, const _TopicModel& mdl);
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, path);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, path);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct HLDAArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/HLDAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentHLDA, BaseDocument, 0, path);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentHLDA, BaseDocument, 1, 0x00010001, path);
+
+	TMT_INSTANTIATE_DOC(DocumentHLDA);
+
 	IHLDAModel* IHLDAModel::create(TermWeight _weight, const HLDAArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, HLDAModel, args);

data/vendor/tomotopy/src/TopicModel/HLDAModel.hpp

@@ -18,6 +18,7 @@ namespace tomoto
 			int32_t parent = 0, sibling = 0, child = 0;
 
 			DEFINE_SERIALIZER(numCustomers, level, parent, sibling, child);
+			DEFINE_HASHER(numCustomers, level, parent, sibling, child);
 
 			NCRPNode* getParent() const
 			{
@@ -118,6 +119,7 @@ namespace tomoto
 			Vector nodeWLikelihoods; //
 
 			DEFINE_SERIALIZER(nodes, levelBlocks);
+			DEFINE_HASHER(nodes, levelBlocks);
 
 			template<bool _makeNewPath = true>
 			void calcNodeLikelihood(Float gamma, size_t levelDepth)
@@ -317,6 +319,12 @@ namespace tomoto
 			ModelStateLDA<_tw>::serializerWrite(ostr);
 			nt->serializerWrite(ostr);
 		}
+
+		uint64_t computeHash(uint64_t seed) const
+		{
+			seed = ModelStateLDA<_tw>::computeHash(seed);
+			return nt->computeHash(seed);
+		}
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -596,6 +604,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, gamma);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, gamma);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, gamma);
 
 		HLDAModel(const HLDAArgs& args)
 			: BaseClass(args), gamma(args.gamma)

data/vendor/tomotopy/src/TopicModel/HPA.h

@@ -12,8 +12,8 @@ namespace tomoto
 
 		template<typename _TopicModel> void update(WeightType* ptr, const _TopicModel& mdl);
 
-		DEFINE_SERIALIZER_BASE_WITH_VERSION(BaseDocument, 0);
-		DEFINE_SERIALIZER_BASE_WITH_VERSION(BaseDocument, 1);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct HPAArgs : public PAArgs

data/vendor/tomotopy/src/TopicModel/HPAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_BASE_WITH_VERSION(DocumentHPA, BaseDocument, 0);
+	DEFINE_OUT_SERIALIZER_BASE_WITH_VERSION(DocumentHPA, BaseDocument, 1);
+
+	TMT_INSTANTIATE_DOC(DocumentHPA);
+
     IHPAModel* IHPAModel::create(TermWeight _weight, bool _exclusive, const HPAArgs& args, bool scalarRng)
 	{
 		if (_exclusive)

data/vendor/tomotopy/src/TopicModel/HPAModel.hpp

@@ -21,6 +21,7 @@ namespace tomoto
 		Eigen::Matrix<WeightType, -1, -1> numByTopic1_2;
 
 		DEFINE_SERIALIZER_AFTER_BASE(ModelStateLDA<_tw>, numByTopicWord, numByTopic, numByTopic1_2);
+		DEFINE_HASHER_AFTER_BASE(ModelStateLDA<_tw>, numByTopicWord, numByTopic, numByTopic1_2);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -439,6 +440,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, K2, subAlphas, subAlphaSum);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, K2, subAlphas, subAlphaSum);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, K2, subAlphas, subAlphaSum);
 
 		HPAModel(const HPAArgs& args)
 			: BaseClass(args, false), K2(args.k2)

data/vendor/tomotopy/src/TopicModel/LDA.h

@@ -93,6 +93,12 @@ namespace tomoto
 			if (!ostr.write((const char*)this->data(), sizeof(_Scalar) * this->size()))
 				throw std::ios_base::failure(std::string("writing type '") + typeid(_Scalar).name() + std::string("' is failed"));
 		}
+
+		uint64_t computeHash(uint64_t seed) const
+		{
+			seed = serializer::computeHashMany(seed, (uint32_t)this->rows(), (uint32_t)this->cols());
+			return serializer::computeFastHash(this->data(), sizeof(_Scalar) * this->size(), seed);
+		}
 	};
 
 	template<typename _Base, TermWeight _tw>
@@ -139,8 +145,8 @@ namespace tomoto
 		tvector<Float> wordWeights;
 		ShareableMatrix<WeightType, -1, 1> numByTopic;
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentBase, 0, Zs, wordWeights);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentBase, 1, 0x00010001, Zs, wordWeights);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 
 		template<typename _TopicModel> void update(WeightType* ptr, const _TopicModel& mdl);
 		

data/vendor/tomotopy/src/TopicModel/LDAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+    DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentLDA, DocumentBase, 0, Zs, wordWeights);
+    DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentLDA, DocumentBase, 1, 0x00010001, Zs, wordWeights);
+
+    TMT_INSTANTIATE_DOC(DocumentLDA);
+
     ILDAModel* ILDAModel::create(TermWeight _weight, const LDAArgs& args, bool scalarRng)
     {
         TMT_SWITCH_TW(_weight, scalarRng, LDAModel, args);

data/vendor/tomotopy/src/TopicModel/LDAModel.hpp

@@ -47,6 +47,10 @@ Term Weighting Scheme is based on following paper:
 		return nullptr; } while(0)
 #endif
 
+#define TMT_INSTANTIATE_DOC(CLS) template struct CLS<TermWeight::one>; \
+	template struct CLS<TermWeight::idf>; \
+	template struct CLS<TermWeight::pmi>;
+
 #define GETTER(name, type, field) type get##name() const override { return field; }
 
 namespace tomoto
@@ -61,6 +65,7 @@ namespace tomoto
 		//Eigen::Matrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic, Vocabs)
 		ShareableMatrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic, Vocabs)
 		DEFINE_SERIALIZER(numByTopic, numByTopicWord);
+		DEFINE_HASHER(numByTopic, numByTopicWord);
 	};
 
 	namespace flags
@@ -954,6 +959,8 @@ namespace tomoto
 		DEFINE_TAGGED_SERIALIZER_WITH_VERSION(1, 0x00010001, vocabWeights, alpha, alphas, eta, K, etaByWord,
 			burnIn, optimInterval);
 
+		DEFINE_HASHER(vocabWeights, alpha, alphas, eta, K, /*etaByWord,*/ burnIn, optimInterval);
+
 		LDAModel(const LDAArgs& args, bool checkAlpha = true)
 			: BaseClass(args.seed), K(args.k), alpha(args.alpha[0]), eta(args.eta)
 		{
@@ -1066,6 +1073,7 @@ namespace tomoto
 		void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0, bool updateStopwords = true) override
 		{
 			if (initDocs && updateStopwords) this->removeStopwords(minWordCnt, minWordDf, removeTopN);
+			static_cast<DerivedClass*>(this)->updateWordFormCnts();
 			static_cast<DerivedClass*>(this)->updateWeakArray();
 			static_cast<DerivedClass*>(this)->initGlobalState(initDocs);
 			static_cast<DerivedClass*>(this)->prepareWordPriors();

data/vendor/tomotopy/src/TopicModel/LLDA.h

@@ -11,8 +11,8 @@ namespace tomoto
 		using WeightType = typename DocumentLDA<_tw>::WeightType;
 		Eigen::Matrix<int8_t, -1, 1> labelMask;
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, labelMask);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, labelMask);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	class ILLDAModel : public ILDAModel

data/vendor/tomotopy/src/TopicModel/LLDAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentLLDA, BaseDocument, 0, labelMask);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentLLDA, BaseDocument, 1, 0x00010001, labelMask);
+
+	TMT_INSTANTIATE_DOC(DocumentLLDA);
+
 	ILLDAModel* ILLDAModel::create(TermWeight _weight, const LDAArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, LLDAModel, args);

data/vendor/tomotopy/src/TopicModel/LLDAModel.hpp

@@ -107,6 +107,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, topicLabelDict);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, topicLabelDict);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, topicLabelDict);
 
 		LLDAModel(const LDAArgs& args)
 			: BaseClass(args)

data/vendor/tomotopy/src/TopicModel/MGLDA.h

@@ -22,8 +22,8 @@ namespace tomoto
 		Eigen::Matrix<WeightType, -1, 1> numByWin; // number of words in the window (len = S + T - 1)
 		Eigen::Matrix<WeightType, -1, -1> numByWinTopicL; // number of words in the loc. topic in the window (len = KL * (S + T - 1))
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, sents, Vs, numGl, numBySentWin, numByWinL, numByWin, numByWinTopicL);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, sents, Vs, numGl, numBySentWin, numByWinL, numByWin, numByWinTopicL);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 
 		template<typename _TopicModel> void update(WeightType* ptr, const _TopicModel& mdl);
 	};

data/vendor/tomotopy/src/TopicModel/MGLDAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentMGLDA, BaseDocument, 0, sents, Vs, numGl, numBySentWin, numByWinL, numByWin, numByWinTopicL);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentMGLDA, BaseDocument, 1, 0x00010001, sents, Vs, numGl, numBySentWin, numByWinL, numByWin, numByWinTopicL);
+
+	TMT_INSTANTIATE_DOC(DocumentMGLDA);
+
     IMGLDAModel* IMGLDAModel::create(TermWeight _weight, const MGLDAArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, MGLDAModel, args);

data/vendor/tomotopy/src/TopicModel/MGLDAModel.hpp

@@ -370,6 +370,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, alphaL, alphaM, alphaML, etaL, gamma, KL, T);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, alphaL, alphaM, alphaML, etaL, gamma, KL, T);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, alphaL, alphaM, alphaML, etaL, gamma, KL, T);
 
 		MGLDAModel(const MGLDAArgs& args)
 			: BaseClass(args), KL(args.kL), T(args.t),

data/vendor/tomotopy/src/TopicModel/PA.h

@@ -15,8 +15,8 @@ namespace tomoto
 
 		template<typename _TopicModel> void update(WeightType* ptr, const _TopicModel& mdl);
 
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, Z2s);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, Z2s);
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct PAArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/PAModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentPA, BaseDocument, 0, Z2s);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentPA, BaseDocument, 1, 0x00010001, Z2s);
+
+	TMT_INSTANTIATE_DOC(DocumentPA);
+
 	IPAModel* IPAModel::create(TermWeight _weight, const PAArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, PAModel, args);

data/vendor/tomotopy/src/TopicModel/PAModel.hpp

@@ -19,6 +19,7 @@ namespace tomoto
 		Vector subTmp;
 
 		DEFINE_SERIALIZER_AFTER_BASE(ModelStateLDA<_tw>, numByTopic1_2, numByTopic2);
+		DEFINE_HASHER_AFTER_BASE(ModelStateLDA<_tw>, numByTopic1_2, numByTopic2);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -364,6 +365,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, K2, subAlphas, subAlphaSum);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, K2, subAlphas, subAlphaSum);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, K2, subAlphas, subAlphaSum);
 
 		PAModel(const PAArgs& args)
 			: BaseClass(args), K2(args.k2)

data/vendor/tomotopy/src/TopicModel/PLDAModel.hpp

@@ -111,6 +111,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, topicLabelDict, numLatentTopics, numTopicsPerLabel);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, topicLabelDict, numLatentTopics, numTopicsPerLabel);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, topicLabelDict, numLatentTopics, numTopicsPerLabel);
 
 		PLDAModel(const PLDAArgs& args)
 			: BaseClass(args.setK(1)),

data/vendor/tomotopy/src/TopicModel/PT.h

@@ -11,9 +11,9 @@ namespace tomoto
 		using WeightType = typename DocumentLDA<_tw>::WeightType;
 		
 		uint64_t pseudoDoc = 0;
-
-		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, pseudoDoc);
-		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, pseudoDoc);
+		
+		DECLARE_SERIALIZER_WITH_VERSION(0);
+		DECLARE_SERIALIZER_WITH_VERSION(1);
 	};
 
 	struct PTArgs : public LDAArgs

data/vendor/tomotopy/src/TopicModel/PTModel.cpp

@@ -2,6 +2,11 @@
 
 namespace tomoto
 {
+	DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentPT, BaseDocument, 0, pseudoDoc);
+	DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(DocumentPT, BaseDocument, 1, 0x00010001, pseudoDoc);
+
+	TMT_INSTANTIATE_DOC(DocumentPT);
+
 	IPTModel* IPTModel::create(TermWeight _weight, const PTArgs& args, bool scalarRng)
 	{
 		TMT_SWITCH_TW(_weight, scalarRng, PTModel, args);

data/vendor/tomotopy/src/TopicModel/PTModel.hpp

@@ -266,6 +266,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, numPDocs, lambda);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, numPDocs, lambda);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, numPDocs, lambda);
 
 		GETTER(P, size_t, numPDocs);