tomoto 0.1.0

data/vendor/tomotopy/src/TopicModel/CT.h

@@ -0,0 +1,37 @@
+#pragma once
+#include "LDA.h"
+
+namespace tomoto
+{
+    template<TermWeight _tw>
+	struct DocumentCTM : public DocumentLDA<_tw>
+	{
+		using BaseDocument = DocumentLDA<_tw>;
+		using DocumentLDA<_tw>::DocumentLDA;
+		Eigen::Matrix<Float, -1, -1> beta; // Dim: (K, betaSample)
+		Eigen::Matrix<Float, -1, 1> smBeta; // Dim: K
+		
+		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, smBeta);
+		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, smBeta);
+	};
+
+	class ICTModel : public ILDAModel
+	{
+	public:
+		using DefaultDocType = DocumentCTM<TermWeight::one>;
+		static ICTModel* create(TermWeight _weight, size_t _K = 1,
+			Float smoothingAlpha = 0.1,  Float _eta = 0.01,
+			size_t seed = std::random_device{}(),
+			bool scalarRng = false);
+
+		virtual void setNumBetaSample(size_t numSample) = 0;
+		virtual size_t getNumBetaSample() const = 0;
+		virtual void setNumTMNSample(size_t numSample) = 0;
+		virtual size_t getNumTMNSample() const = 0;
+		virtual void setNumDocBetaSample(size_t numSample) = 0;
+		virtual size_t getNumDocBetaSample() const = 0;
+		virtual std::vector<Float> getPriorMean() const = 0;
+		virtual std::vector<Float> getPriorCov() const = 0;
+		virtual std::vector<Float> getCorrelationTopic(Tid k) const = 0;
+	};
+}

data/vendor/tomotopy/src/TopicModel/CTModel.cpp

@@ -0,0 +1,13 @@
+#include "CTModel.hpp"
+
+namespace tomoto
+{
+	/*template class CTModel<TermWeight::one>;
+	template class CTModel<TermWeight::idf>;
+	template class CTModel<TermWeight::pmi>;*/
+
+	ICTModel* ICTModel::create(TermWeight _weight, size_t _K, Float smoothingAlpha, Float _eta, size_t seed, bool scalarRng)
+	{
+		TMT_SWITCH_TW(_weight, scalarRng, CTModel, _K, smoothingAlpha, _eta, seed);
+	}
+}

data/vendor/tomotopy/src/TopicModel/CTModel.hpp

@@ -0,0 +1,293 @@
+#pragma once
+#include "LDAModel.hpp"
+#include "../Utils/MultiNormalDistribution.hpp"
+#include "../Utils/TruncMultiNormal.hpp"
+#include "CT.h"
+/*
+Implementation of CTM using Gibbs sampling by bab2min
+* Blei, D., & Lafferty, J. (2006). Correlated topic models. Advances in neural information processing systems, 18, 147.
+* Mimno, D., Wallach, H., & McCallum, A. (2008, December). Gibbs sampling for logistic normal topic models with graph-based priors. In NIPS Workshop on Analyzing Graphs (Vol. 61).
+*/
+
+namespace tomoto
+{
+	template<TermWeight _tw>
+	struct ModelStateCTM : public ModelStateLDA<_tw>
+	{
+	};
+
+	template<TermWeight _tw, typename _RandGen,
+		size_t _Flags = flags::partitioned_multisampling,
+		typename _Interface = ICTModel,
+		typename _Derived = void,
+		typename _DocType = DocumentCTM<_tw>,
+		typename _ModelState = ModelStateCTM<_tw>>
+	class CTModel : public LDAModel<_tw, _RandGen, _Flags, _Interface,
+		typename std::conditional<std::is_same<_Derived, void>::value, CTModel<_tw, _RandGen, _Flags>, _Derived>::type,
+		_DocType, _ModelState>
+	{
+	protected:
+		using DerivedClass = typename std::conditional<std::is_same<_Derived, void>::value, CTModel<_tw, _RandGen>, _Derived>::type;
+		using BaseClass = LDAModel<_tw, _RandGen, _Flags, _Interface, DerivedClass, _DocType, _ModelState>;
+		friend BaseClass;
+		friend typename BaseClass::BaseClass;
+		using WeightType = typename BaseClass::WeightType;
+
+		static constexpr char TMID[] = "CTM\0";
+
+		uint64_t numBetaSample = 10;
+		uint64_t numTMNSample = 5;
+		uint64_t numDocBetaSample = -1;
+		math::MultiNormalDistribution<Float> topicPrior;
+		
+		template<bool _asymEta>
+		Float* getZLikelihoods(_ModelState& ld, const _DocType& doc, size_t docId, size_t vid) const
+		{
+			const size_t V = this->realV;
+			assert(vid < V);
+			auto etaHelper = this->template getEtaHelper<_asymEta>();
+			auto& zLikelihood = ld.zLikelihood;
+			zLikelihood = doc.smBeta.array()
+				* (ld.numByTopicWord.col(vid).array().template cast<Float>() + etaHelper.getEta(vid))
+				/ (ld.numByTopic.array().template cast<Float>() + etaHelper.getEtaSum());
+			sample::prefixSum(zLikelihood.data(), this->K);
+			return &zLikelihood[0];
+		}
+
+		void updateBeta(_DocType& doc, _RandGen& rg) const
+		{
+			Eigen::Matrix<Float, -1, 1> pbeta, lowerBound, upperBound;
+			constexpr Float epsilon = 1e-8;
+			constexpr size_t burnIn = 3;
+
+			pbeta = lowerBound = upperBound = Eigen::Matrix<Float, -1, 1>::Zero(this->K);
+			for (size_t i = 0; i < numBetaSample + burnIn; ++i)
+			{
+				if (i == 0) pbeta = Eigen::Matrix<Float, -1, 1>::Ones(this->K);
+				else pbeta = doc.beta.col(i % numBetaSample).array().exp();
+				Float betaESum = pbeta.sum() + 1;
+				pbeta /= betaESum;
+				for (size_t k = 0; k < this->K; ++k)
+				{
+					Float N_k = doc.numByTopic[k] + this->alpha;
+					Float N_nk = doc.getSumWordWeight() + this->alpha * (this->K + 1) - N_k;
+					Float u1 = rg.uniform_real(), u2 = rg.uniform_real();
+					Float max_uk = epsilon + pow(u1, (Float)1 / N_k)  * (pbeta[k] - epsilon);
+					Float min_unk = (1 - pow(u2, (Float)1 / N_nk))
+						* (1 - pbeta[k]) + pbeta[k];
+
+					Float c = betaESum * (1 - pbeta[k]);
+					lowerBound[k] = log(c * max_uk / (1 - max_uk));
+					upperBound[k] = log(c * min_unk / (1 - min_unk));
+					if (lowerBound[k] > upperBound[k])
+					{
+						THROW_ERROR_WITH_INFO(exception::TrainingError,
+							text::format("Bound Error: LB(%f) > UB(%f)\n"
+								"max_uk: %f, min_unk: %f, c: %f", lowerBound[k], upperBound[k], max_uk, min_unk, c));
+					}
+				}
+
+				try
+				{
+					math::sampleFromTruncatedMultiNormal(doc.beta.col((i + 1) % numBetaSample),
+						topicPrior, lowerBound, upperBound, rg, numTMNSample);
+
+					if (!std::isfinite(doc.beta.col((i + 1) % numBetaSample)[0])) 
+						THROW_ERROR_WITH_INFO(exception::TrainingError,
+							text::format("doc.beta.col(%d) is %f", (i + 1) % numBetaSample, 
+							doc.beta.col((i + 1) % numBetaSample)[0]));
+				}
+				catch (const std::runtime_error& e)
+				{
+					std::cerr << e.what() << std::endl;
+					THROW_ERROR_WITH_INFO(exception::TrainingError, e.what());
+				}
+			}
+
+			// update softmax-applied beta coefficient
+			doc.smBeta.head(this->K) = doc.beta.block(0, 0, this->K, std::min(numBetaSample, numDocBetaSample)).rowwise().mean();
+			doc.smBeta = doc.smBeta.array().exp();
+			doc.smBeta /= doc.smBeta.array().sum();
+		}
+
+		template<ParallelScheme _ps, bool _infer, typename _ExtraDocData>
+		void sampleDocument(_DocType& doc, const _ExtraDocData& edd, size_t docId, _ModelState& ld, _RandGen& rgs, size_t iterationCnt, size_t partitionId = 0) const
+		{
+			BaseClass::template sampleDocument<_ps, _infer>(doc, edd, docId, ld, rgs, iterationCnt, partitionId);
+			/*if (iterationCnt >= this->burnIn && this->optimInterval && (iterationCnt + 1) % this->optimInterval == 0)
+			{
+				updateBeta(doc, rgs);
+			}*/
+		}
+
+		template<typename _DocIter>
+		void sampleGlobalLevel(ThreadPool* pool, _ModelState* localData, _RandGen* rgs, _DocIter first, _DocIter last) const
+		{
+			if (this->globalStep < this->burnIn || !this->optimInterval || (this->globalStep + 1) % this->optimInterval != 0) return;
+
+			if (pool)
+			{
+				std::vector<std::future<void>> res;
+				const size_t chStride = pool->getNumWorkers() * 8;
+				size_t dist = std::distance(first, last);
+				for (size_t ch = 0; ch < chStride; ++ch)
+				{
+					auto b = first, e = first;
+					std::advance(b, dist * ch / chStride);
+					std::advance(e, dist * (ch + 1) / chStride);
+					res.emplace_back(pool->enqueue([&, ch, chStride](size_t threadId, _DocIter b, _DocIter e)
+					{
+						for (auto doc = b; doc != e; ++doc)
+						{
+							updateBeta(*doc, rgs[threadId]);
+						}
+					}, b, e));
+				}
+				for (auto& r : res) r.get();
+			}
+			else
+			{
+				for (auto doc = first; doc != last; ++doc)
+				{
+					updateBeta(*doc, rgs[0]);
+				}
+			}
+		}
+
+		int restoreFromTrainingError(const exception::TrainingError& e, ThreadPool& pool, _ModelState* localData, _RandGen* rgs)
+		{
+			std::cerr << "Failed to sample! Reset prior and retry!" << std::endl;
+			const size_t chStride = std::min(pool.getNumWorkers() * 8, this->docs.size());
+			topicPrior = math::MultiNormalDistribution<Float>{ this->K };
+			std::vector<std::future<void>> res;
+			for (size_t ch = 0; ch < chStride; ++ch)
+			{
+				res.emplace_back(pool.enqueue([&, this](size_t threadId, size_t ch)
+				{
+					for (size_t i = ch; i < this->docs.size(); i += chStride)
+					{
+						this->docs[i].beta.setZero();
+						updateBeta(this->docs[i], rgs[threadId]);
+					}
+				}, ch));
+			}
+			for (auto& r : res) r.get();
+			return 0;
+		}
+
+		void optimizeParameters(ThreadPool& pool, _ModelState* localData, _RandGen* rgs)
+		{
+			std::vector<std::future<void>> res;
+			topicPrior = math::MultiNormalDistribution<Float>::estimate([this](size_t i)
+			{
+				return this->docs[i / numBetaSample].beta.col(i % numBetaSample);
+			}, this->docs.size() * numBetaSample);
+			if (!std::isfinite(topicPrior.mean[0]))
+				THROW_ERROR_WITH_INFO(exception::TrainingError, 
+					text::format("topicPrior.mean is %f", topicPrior.mean[0]));
+		}
+
+		template<typename _DocIter>
+		double getLLDocs(_DocIter _first, _DocIter _last) const
+		{
+			const auto K = this->K;
+			const auto alpha = this->alpha;
+
+			double ll = 0;
+			for (; _first != _last; ++_first)
+			{
+				auto& doc = *_first;
+				Eigen::Matrix<Float, -1, 1> pbeta = doc.smBeta.array().log();
+				Float last = pbeta[K - 1];
+				for (Tid k = 0; k < K; ++k)
+				{
+					ll += pbeta[k] * (doc.numByTopic[k] + alpha) - math::lgammaT(doc.numByTopic[k] + alpha + 1);
+				}
+				pbeta.array() -= last;
+				ll += topicPrior.getLL(pbeta.head(this->K));
+				ll += math::lgammaT(doc.getSumWordWeight() + alpha * K + 1);
+			}
+			return ll;
+		}
+
+		void prepareDoc(_DocType& doc, size_t docId, size_t wordSize) const
+		{
+			BaseClass::prepareDoc(doc, docId, wordSize);
+			doc.beta = Eigen::Matrix<Float, -1, -1>::Zero(this->K, numBetaSample);
+			doc.smBeta = Eigen::Matrix<Float, -1, 1>::Constant(this->K, (Float)1 / this->K);
+		}
+
+		void updateDocs()
+		{
+			BaseClass::updateDocs();
+			for (auto& doc : this->docs)
+			{
+				doc.beta = Eigen::Matrix<Float, -1, -1>::Zero(this->K, numBetaSample);
+			}
+		}
+
+		void initGlobalState(bool initDocs)
+		{
+			BaseClass::initGlobalState(initDocs);
+			if (initDocs)
+			{
+				topicPrior = math::MultiNormalDistribution<Float>{ this->K };
+			}
+		}
+
+	public:
+		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, numBetaSample, numTMNSample, topicPrior);
+		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, numBetaSample, numTMNSample, topicPrior);
+
+		CTModel(size_t _K = 1, Float smoothingAlpha = 0.1, Float _eta = 0.01, size_t _rg = std::random_device{}())
+			: BaseClass(_K, smoothingAlpha, _eta, _rg)
+		{
+			this->optimInterval = 2;
+		}
+
+		std::vector<Float> getTopicsByDoc(const _DocType& doc) const
+		{
+			std::vector<Float> ret(this->K);
+			Eigen::Map<Eigen::Matrix<Float, -1, 1>>{ret.data(), this->K}.array() =
+				doc.numByTopic.array().template cast<Float>() / doc.getSumWordWeight();
+			return ret;
+		}
+
+		std::vector<Float> getPriorMean() const override
+		{
+			return { topicPrior.mean.data(), topicPrior.mean.data() + topicPrior.mean.size() };
+		}
+
+		std::vector<Float> getPriorCov() const override
+		{
+			return { topicPrior.cov.data(), topicPrior.cov.data() + topicPrior.cov.size() };
+		}
+
+		std::vector<Float> getCorrelationTopic(Tid k) const override
+		{
+			Eigen::Matrix<Float, -1, 1> ret = topicPrior.cov.col(k).array() / (topicPrior.cov.diagonal().array() * topicPrior.cov(k, k)).sqrt();
+			return { ret.data(), ret.data() + ret.size() };
+		}
+
+		GETTER(NumBetaSample, size_t, numBetaSample);
+
+		void setNumBetaSample(size_t _numSample) override
+		{
+			numBetaSample = _numSample;
+		}
+
+		GETTER(NumDocBetaSample, size_t, numDocBetaSample);
+
+		void setNumDocBetaSample(size_t _numSample) override
+		{
+			numDocBetaSample = _numSample;
+		}
+
+		GETTER(NumTMNSample, size_t, numTMNSample);
+
+		void setNumTMNSample(size_t _numSample) override
+		{
+			numTMNSample = _numSample;
+		}
+	};
+}

data/vendor/tomotopy/src/TopicModel/DMR.h

@@ -0,0 +1,51 @@
+#pragma once
+#include "LDA.h"
+
+namespace tomoto
+{
+	template<TermWeight _tw>
+	struct DocumentDMR : public DocumentLDA<_tw>
+	{
+		using BaseDocument = DocumentLDA<_tw>;
+		using DocumentLDA<_tw>::DocumentLDA;
+		size_t metadata = 0;
+
+		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, metadata);
+		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, metadata);
+	};
+
+    class IDMRModel : public ILDAModel
+	{
+	public:
+		using DefaultDocType = DocumentDMR<TermWeight::one>;
+		static IDMRModel* create(TermWeight _weight, size_t _K = 1,
+			Float defaultAlpha = 1.0, Float _sigma = 1.0, Float _eta = 0.01, Float _alphaEps = 1e-10,
+			size_t seed = std::random_device{}(),
+			bool scalarRng = false);
+
+		virtual size_t addDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) = 0;
+		virtual std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) const = 0;
+		
+		virtual size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
+			const std::vector<std::string>& metadata) = 0;
+		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
+			const std::vector<std::string>& metadata) const = 0;
+
+		virtual size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
+			const std::vector<std::string>& metadata) = 0;
+		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
+			const std::vector<std::string>& metadata) const = 0;
+
+		virtual void setAlphaEps(Float _alphaEps) = 0;
+		virtual Float getAlphaEps() const = 0;
+		virtual void setOptimRepeat(size_t repeat) = 0;
+		virtual size_t getOptimRepeat() const = 0;
+		virtual size_t getF() const = 0;
+		virtual Float getSigma() const = 0;
+		virtual const Dictionary& getMetadataDict() const = 0;
+		virtual std::vector<Float> getLambdaByMetadata(size_t metadataId) const = 0;
+		virtual std::vector<Float> getLambdaByTopic(Tid tid) const = 0;
+	};
+}

data/vendor/tomotopy/src/TopicModel/DMRModel.cpp

@@ -0,0 +1,13 @@
+#include "DMRModel.hpp"
+
+namespace tomoto
+{
+	/*template class DMRModel<TermWeight::one>;
+	template class DMRModel<TermWeight::idf>;
+	template class DMRModel<TermWeight::pmi>;*/
+
+	IDMRModel* IDMRModel::create(TermWeight _weight, size_t _K, Float _defaultAlpha, Float _sigma, Float _eta, Float _alphaEps, size_t seed, bool scalarRng)
+	{
+		TMT_SWITCH_TW(_weight, scalarRng, DMRModel, _K, _defaultAlpha, _sigma, _eta, _alphaEps, seed);
+	}
+}

data/vendor/tomotopy/src/TopicModel/DMRModel.hpp

@@ -0,0 +1,374 @@
+#pragma once
+#include "LDAModel.hpp"
+#include "../Utils/LBFGS.h"
+#include "../Utils/text.hpp"
+#include "DMR.h"
+/*
+Implementation of DMR using Gibbs sampling by bab2min
+* Mimno, D., & McCallum, A. (2012). Topic models conditioned on arbitrary features with dirichlet-multinomial regression. arXiv preprint arXiv:1206.3278.
+*/
+
+namespace tomoto
+{
+	template<TermWeight _tw>
+	struct ModelStateDMR : public ModelStateLDA<_tw>
+	{
+		Eigen::Matrix<Float, -1, 1> tmpK;
+	};
+
+	template<TermWeight _tw, typename _RandGen, 
+		size_t _Flags = flags::partitioned_multisampling,
+		typename _Interface = IDMRModel,
+		typename _Derived = void,
+		typename _DocType = DocumentDMR<_tw>,
+		typename _ModelState = ModelStateDMR<_tw>>
+	class DMRModel : public LDAModel<_tw, _RandGen, _Flags, _Interface,
+		typename std::conditional<std::is_same<_Derived, void>::value, DMRModel<_tw, _RandGen, _Flags>, _Derived>::type,
+		_DocType, _ModelState>
+	{
+	protected:
+		using DerivedClass = typename std::conditional<std::is_same<_Derived, void>::value, DMRModel<_tw, _RandGen>, _Derived>::type;
+		using BaseClass = LDAModel<_tw, _RandGen, _Flags, _Interface, DerivedClass, _DocType, _ModelState>;
+		friend BaseClass;
+		friend typename BaseClass::BaseClass;
+		using WeightType = typename BaseClass::WeightType;
+
+		static constexpr char TMID[] = "DMR\0";
+
+		Eigen::Matrix<Float, -1, -1> lambda;
+		Eigen::Matrix<Float, -1, -1> expLambda;
+		Float sigma;
+		uint32_t F = 0;
+		uint32_t optimRepeat = 5;
+		Float alphaEps = 1e-10;
+		Float temperatureScale = 0;
+		static constexpr Float maxLambda = 10;
+		static constexpr size_t maxBFGSIteration = 10;
+
+		Dictionary metadataDict;
+		LBFGSpp::LBFGSSolver<Float, LBFGSpp::LineSearchBracketing> solver;
+
+		Float getNegativeLambdaLL(Eigen::Ref<Eigen::Matrix<Float, -1, 1>> x, Eigen::Matrix<Float, -1, 1>& g) const
+		{
+			g = (x.array() - log(this->alpha)) / pow(sigma, 2);
+			return (x.array() - log(this->alpha)).pow(2).sum() / 2 / pow(sigma, 2);
+		}
+
+		Float evaluateLambdaObj(Eigen::Ref<Eigen::Matrix<Float, -1, 1>> x, Eigen::Matrix<Float, -1, 1>& g, ThreadPool& pool, _ModelState* localData) const
+		{
+			// if one of x is greater than maxLambda, return +inf for preventing searching more
+			if ((x.array() > maxLambda).any()) return INFINITY;
+
+			const auto K = this->K;
+
+			Float fx = - static_cast<const DerivedClass*>(this)->getNegativeLambdaLL(x, g);
+			auto alphas = (x.array().exp() + alphaEps).eval();
+
+			std::vector<std::future<Eigen::Matrix<Float, -1, 1>>> res;
+			const size_t chStride = pool.getNumWorkers() * 8;
+			for (size_t ch = 0; ch < chStride; ++ch)
+			{
+				res.emplace_back(pool.enqueue([&](size_t threadId)
+				{
+					auto& tmpK = localData[threadId].tmpK;
+					if (!tmpK.size()) tmpK.resize(this->K);
+					Eigen::Matrix<Float, -1, 1> val = Eigen::Matrix<Float, -1, 1>::Zero(K * F + 1);
+					for (size_t docId = ch; docId < this->docs.size(); docId += chStride)
+					{
+						const auto& doc = this->docs[docId];
+						auto alphaDoc = alphas.segment(doc.metadata * K, K);
+						Float alphaSum = alphaDoc.sum();
+						for (Tid k = 0; k < K; ++k)
+						{
+							val[K * F] -= math::lgammaT(alphaDoc[k]) - math::lgammaT(doc.numByTopic[k] + alphaDoc[k]);
+							if (!std::isfinite(alphaDoc[k]) && alphaDoc[k] > 0) tmpK[k] = 0;
+							else tmpK[k] = -(math::digammaT(alphaDoc[k]) - math::digammaT(doc.numByTopic[k] + alphaDoc[k]));
+						}
+						//val[K * F] = -(lgammaApprox(alphaDoc.array()) - lgammaApprox(doc.numByTopic.array().cast<Float>() + alphaDoc.array())).sum();
+						//tmpK = -(digammaApprox(alphaDoc.array()) - digammaApprox(doc.numByTopic.array().cast<Float>() + alphaDoc.array()));
+						val[K * F] += math::lgammaT(alphaSum) - math::lgammaT(doc.getSumWordWeight() + alphaSum);
+						Float t = math::digammaT(alphaSum) - math::digammaT(doc.getSumWordWeight() + alphaSum);
+						if (!std::isfinite(alphaSum) && alphaSum > 0)
+						{
+							val[K * F] = -INFINITY;
+							t = 0;
+						}
+						val.segment(doc.metadata * K, K).array() -= alphaDoc.array() * (tmpK.array() + t);
+					}
+					return val;
+				}));
+			}
+			for (auto& r : res)
+			{
+				auto ret = r.get();
+				fx += ret[K * F];
+				g += ret.head(K * F);
+			}
+
+			// positive fx is an error from limited precision of float.
+			if (fx > 0) return INFINITY;
+			return -fx;
+		}
+
+		void initParameters()
+		{
+			auto dist = std::normal_distribution<Float>(log(this->alpha), sigma);
+			for (size_t i = 0; i < this->K; ++i) for (size_t j = 0; j < F; ++j)
+			{
+				lambda(i, j) = dist(this->rg);
+			}
+		}
+
+		void optimizeParameters(ThreadPool& pool, _ModelState* localData, _RandGen* rgs)
+		{
+			Eigen::Matrix<Float, -1, -1> bLambda;
+			Float fx = 0, bestFx = INFINITY;
+			for (size_t i = 0; i < optimRepeat; ++i)
+			{
+				static_cast<DerivedClass*>(this)->initParameters();
+				int ret = solver.minimize([this, &pool, localData](Eigen::Ref<Eigen::Matrix<Float, -1, 1>> x, Eigen::Matrix<Float, -1, 1>& g)
+				{
+					return static_cast<DerivedClass*>(this)->evaluateLambdaObj(x, g, pool, localData);
+				}, Eigen::Map<Eigen::Matrix<Float, -1, 1>>(lambda.data(), lambda.size()), fx);
+
+				if (fx < bestFx)
+				{
+					bLambda = lambda;
+					bestFx = fx;
+					//printf("\t(%d) %e\n", ret, fx);
+				}
+			}
+			if (!std::isfinite(bestFx))
+			{
+				throw exception::TrainingError{ "optimizing parameters has been failed!" };
+			}
+			lambda = bLambda;
+			//std::cerr << fx << std::endl;
+			expLambda = lambda.array().exp() + alphaEps;
+		}
+
+		int restoreFromTrainingError(const exception::TrainingError& e, ThreadPool& pool, _ModelState* localData, _RandGen* rgs)
+		{
+			std::cerr << "Failed to optimize! Reset prior and retry!" << std::endl;
+			lambda.setZero();
+			expLambda = lambda.array().exp() + alphaEps;
+			return 0;
+		}
+
+		template<bool _asymEta>
+		Float* getZLikelihoods(_ModelState& ld, const _DocType& doc, size_t docId, size_t vid) const
+		{
+			const size_t V = this->realV;
+			assert(vid < V);
+			auto etaHelper = this->template getEtaHelper<_asymEta>();
+			auto& zLikelihood = ld.zLikelihood;
+			zLikelihood = (doc.numByTopic.array().template cast<Float>() + this->expLambda.col(doc.metadata).array())
+				* (ld.numByTopicWord.col(vid).array().template cast<Float>() + etaHelper.getEta(vid))
+				/ (ld.numByTopic.array().template cast<Float>() + etaHelper.getEtaSum());
+
+			sample::prefixSum(zLikelihood.data(), this->K);
+			return &zLikelihood[0];
+		}
+
+		
+		double getLLDocTopic(const _DocType& doc) const
+		{
+			const size_t V = this->realV;
+			const auto K = this->K;
+
+			auto alphaDoc = expLambda.col(doc.metadata);
+
+			Float ll = 0;
+			Float alphaSum = alphaDoc.sum();
+			for (Tid k = 0; k < K; ++k)
+			{
+				ll += math::lgammaT(doc.numByTopic[k] + alphaDoc[k]);
+				ll -= math::lgammaT(alphaDoc[k]);
+			}
+			ll -= math::lgammaT(doc.getSumWordWeight() + alphaSum);
+			ll += math::lgammaT(alphaSum);
+			return ll;
+		}
+
+		template<typename _DocIter>
+		double getLLDocs(_DocIter _first, _DocIter _last) const
+		{
+			const auto K = this->K;
+
+			double ll = 0;
+			for (; _first != _last; ++_first)
+			{
+				auto& doc = *_first;
+				auto alphaDoc = expLambda.col(doc.metadata);
+				Float alphaSum = alphaDoc.sum();
+
+				for (Tid k = 0; k < K; ++k)
+				{
+					ll += math::lgammaT(doc.numByTopic[k] + alphaDoc[k]) - math::lgammaT(alphaDoc[k]);
+				}
+				ll -= math::lgammaT(doc.getSumWordWeight() + alphaSum) - math::lgammaT(alphaSum);
+			}
+			return ll;
+		}
+
+		double getLLRest(const _ModelState& ld) const
+		{
+			const auto K = this->K;
+			const auto alpha = this->alpha;
+			const auto eta = this->eta;
+			const size_t V = this->realV;
+
+			double ll = -(lambda.array() - log(alpha)).pow(2).sum() / 2 / pow(sigma, 2);
+			// topic-word distribution
+			auto lgammaEta = math::lgammaT(eta);
+			ll += math::lgammaT(V*eta) * K;
+			for (Tid k = 0; k < K; ++k)
+			{
+				ll -= math::lgammaT(ld.numByTopic[k] + V * eta);
+				for (Vid v = 0; v < V; ++v)
+				{
+					if (!ld.numByTopicWord(k, v)) continue;
+					ll += math::lgammaT(ld.numByTopicWord(k, v) + eta) - lgammaEta;
+				}
+			}
+			return ll;
+		}
+
+		void initGlobalState(bool initDocs)
+		{
+			BaseClass::initGlobalState(initDocs);
+			this->globalState.tmpK = Eigen::Matrix<Float, -1, 1>::Zero(this->K);
+			F = metadataDict.size();
+			if (initDocs)
+			{
+				lambda = Eigen::Matrix<Float, -1, -1>::Constant(this->K, F, log(this->alpha));
+			}
+			if (_Flags & flags::continuous_doc_data) this->numByTopicDoc = Eigen::Matrix<WeightType, -1, -1>::Zero(this->K, this->docs.size());
+			expLambda = lambda.array().exp();
+			LBFGSpp::LBFGSParam<Float> param;
+			param.max_iterations = maxBFGSIteration;
+			solver = decltype(solver){ param };
+		}
+
+	public:
+		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, sigma, alphaEps, metadataDict, lambda);
+		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, sigma, alphaEps, metadataDict, lambda);
+
+		DMRModel(size_t _K = 1, Float defaultAlpha = 1.0, Float _sigma = 1.0, Float _eta = 0.01, 
+			Float _alphaEps = 0, size_t _rg = std::random_device{}())
+			: BaseClass(_K, defaultAlpha, _eta, _rg), sigma(_sigma), alphaEps(_alphaEps)
+		{
+			if (_sigma <= 0) THROW_ERROR_WITH_INFO(std::runtime_error, text::format("wrong sigma value (sigma = %f)", _sigma));
+		}
+
+		template<bool _const = false>
+		_DocType& _updateDoc(_DocType& doc, const std::vector<std::string>& metadata)
+		{
+			std::string metadataJoined = text::join(metadata.begin(), metadata.end(), "_");
+			Vid xid;
+			if (_const)
+			{
+				xid = metadataDict.toWid(metadataJoined);
+				if (xid == (Vid)-1) throw std::invalid_argument("unknown metadata");
+			}
+			else
+			{
+				xid = metadataDict.add(metadataJoined);
+			}
+			doc.metadata = xid;
+			return doc;
+		}
+
+		size_t addDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) override
+		{
+			auto doc = this->_makeDoc(words);
+			return this->_addDoc(_updateDoc(doc, metadata));
+		}
+
+		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) const override
+		{
+			auto doc = as_mutable(this)->template _makeDoc<true>(words);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, metadata));
+		}
+
+		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer, 
+			const std::vector<std::string>& metadata) override
+		{
+			auto doc = this->template _makeRawDoc<false>(rawStr, tokenizer);
+			return this->_addDoc(_updateDoc(doc, metadata));
+		}
+
+		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer, 
+			const std::vector<std::string>& metadata) const override
+		{
+			auto doc = as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, metadata));
+		}
+
+		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len, 
+			const std::vector<std::string>& metadata) override
+		{
+			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
+			return this->_addDoc(_updateDoc(doc, metadata));
+		}
+
+		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len, 
+			const std::vector<std::string>& metadata) const override
+		{
+			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, metadata));
+		}
+
+		GETTER(F, size_t, F);
+		GETTER(Sigma, Float, sigma);
+		GETTER(AlphaEps, Float, alphaEps);
+		GETTER(OptimRepeat, size_t, optimRepeat);
+
+		void setAlphaEps(Float _alphaEps) override
+		{
+			alphaEps = _alphaEps;
+		}
+
+		void setOptimRepeat(size_t _optimRepeat) override
+		{
+			optimRepeat = _optimRepeat;
+		}
+
+		std::vector<Float> getTopicsByDoc(const _DocType& doc) const
+		{
+			std::vector<Float> ret(this->K);
+			auto alphaDoc = expLambda.col(doc.metadata);
+			Eigen::Map<Eigen::Matrix<Float, -1, 1>>{ret.data(), this->K}.array() =
+				(doc.numByTopic.array().template cast<Float>() + alphaDoc.array()) / (doc.getSumWordWeight() + alphaDoc.sum());
+			return ret;
+		}
+
+		std::vector<Float> getLambdaByMetadata(size_t metadataId) const override
+		{
+			assert(metadataId < metadataDict.size());
+			auto l = lambda.col(metadataId);
+			return { l.data(), l.data() + this->K };
+		}
+
+		std::vector<Float> getLambdaByTopic(Tid tid) const override
+		{
+			assert(tid < this->K);
+			auto l = lambda.row(tid);
+			return { l.data(), l.data() + F };
+		}
+
+		const Dictionary& getMetadataDict() const override { return metadataDict; }
+	};
+
+	/* This is for preventing 'undefined symbol' problem in compiling by clang. */
+	template<TermWeight _tw, typename _RandGen, size_t _Flags,
+		typename _Interface, typename _Derived, typename _DocType, typename _ModelState>
+		constexpr Float DMRModel<_tw, _RandGen, _Flags, _Interface, _Derived, _DocType, _ModelState>::maxLambda;
+
+	template<TermWeight _tw, typename _RandGen, size_t _Flags,
+		typename _Interface, typename _Derived, typename _DocType, typename _ModelState>
+		constexpr size_t DMRModel<_tw, _RandGen, _Flags, _Interface, _Derived, _DocType, _ModelState>::maxBFGSIteration;
+
+}