tomoto 0.1.0

data/vendor/tomotopy/src/TopicModel/PLDA.h

@@ -0,0 +1,17 @@
+#pragma once
+#include "LLDA.h"
+
+namespace tomoto
+{
+
+	class IPLDAModel : public ILLDAModel
+	{
+	public:
+		using DefaultDocType = DocumentLLDA<TermWeight::one>;
+		static IPLDAModel* create(TermWeight _weight, size_t _numLatentTopics = 0, size_t _numTopicsPerLabel = 1,
+			Float alpha = 0.1, Float eta = 0.01, size_t seed = std::random_device{}(),
+			bool scalarRng = false);
+
+		virtual size_t getNumLatentTopics() const = 0;
+	};
+}

data/vendor/tomotopy/src/TopicModel/PLDAModel.cpp

@@ -0,0 +1,13 @@
+#include "PLDAModel.hpp"
+
+namespace tomoto
+{
+	/*template class PLDAModel<TermWeight::one>;
+	template class PLDAModel<TermWeight::idf>;
+	template class PLDAModel<TermWeight::pmi>;*/
+
+	IPLDAModel* IPLDAModel::create(TermWeight _weight, size_t _numLatentTopics, size_t _numTopicsPerLabel, Float _alpha, Float _eta, size_t seed, bool scalarRng)
+	{
+		TMT_SWITCH_TW(_weight, scalarRng, PLDAModel, _numLatentTopics, _numTopicsPerLabel, _alpha, _eta, seed);
+	}
+}

data/vendor/tomotopy/src/TopicModel/PLDAModel.hpp

@@ -0,0 +1,214 @@
+#pragma once
+#include "LDAModel.hpp"
+#include "PLDA.h"
+
+/*
+Implementation of Labeled LDA using Gibbs sampling by bab2min
+
+* Ramage, D., Manning, C. D., & Dumais, S. (2011, August). Partially labeled topic models for interpretable text mining. In Proceedings of the 17th ACM SIGKDD international conference on Knowledge discovery and data mining (pp. 457-465). ACM.
+*/
+
+namespace tomoto
+{
+	template<TermWeight _tw, typename _RandGen,
+		typename _Interface = IPLDAModel,
+		typename _Derived = void,
+		typename _DocType = DocumentLLDA<_tw>,
+		typename _ModelState = ModelStateLDA<_tw>>
+	class PLDAModel : public LDAModel<_tw, _RandGen, flags::generator_by_doc | flags::partitioned_multisampling, _Interface,
+		typename std::conditional<std::is_same<_Derived, void>::value, PLDAModel<_tw, _RandGen>, _Derived>::type,
+		_DocType, _ModelState>
+	{
+	protected:
+		using DerivedClass = typename std::conditional<std::is_same<_Derived, void>::value, PLDAModel<_tw, _RandGen>, _Derived>::type;
+		using BaseClass = LDAModel<_tw, _RandGen, flags::generator_by_doc | flags::partitioned_multisampling, _Interface, DerivedClass, _DocType, _ModelState>;
+		friend BaseClass;
+		friend typename BaseClass::BaseClass;
+		using WeightType = typename BaseClass::WeightType;
+
+		static constexpr char TMID[] = "PLDA";
+
+		Dictionary topicLabelDict;
+
+		uint64_t numLatentTopics, numTopicsPerLabel;
+
+		template<bool _asymEta>
+		Float* getZLikelihoods(_ModelState& ld, const _DocType& doc, size_t docId, size_t vid) const
+		{
+			const size_t V = this->realV;
+			assert(vid < V);
+			auto etaHelper = this->template getEtaHelper<_asymEta>();
+			auto& zLikelihood = ld.zLikelihood;
+			zLikelihood = (doc.numByTopic.array().template cast<Float>() + this->alphas.array())
+				* (ld.numByTopicWord.col(vid).array().template cast<Float>() + etaHelper.getEta(vid))
+				/ (ld.numByTopic.array().template cast<Float>() + etaHelper.getEtaSum());
+			zLikelihood.array() *= doc.labelMask.array().template cast<Float>();
+			sample::prefixSum(zLikelihood.data(), this->K);
+			return &zLikelihood[0];
+		}
+
+		void prepareDoc(_DocType& doc, size_t docId, size_t wordSize) const
+		{
+			BaseClass::prepareDoc(doc, docId, wordSize);
+			if (doc.labelMask.size() == 0)
+			{
+				doc.labelMask.resize(this->K);
+				doc.labelMask.setZero();
+				doc.labelMask.tail(numLatentTopics).setOnes();
+			}
+			else if (doc.labelMask.size() < this->K)
+			{
+				size_t oldSize = doc.labelMask.size();
+				doc.labelMask.conservativeResize(this->K);
+				doc.labelMask.tail(this->K - oldSize).setZero();
+				doc.labelMask.tail(numLatentTopics).setOnes();
+			}
+		}
+
+		void initGlobalState(bool initDocs)
+		{
+			this->K = topicLabelDict.size() * numTopicsPerLabel + numLatentTopics;
+			this->alphas.resize(this->K);
+			this->alphas.array() = this->alpha;
+			BaseClass::initGlobalState(initDocs);
+		}
+
+		struct Generator
+		{
+			std::discrete_distribution<> theta;
+		};
+
+		Generator makeGeneratorForInit(const _DocType* doc) const
+		{
+			return Generator{ 
+				std::discrete_distribution<>{ doc->labelMask.data(), doc->labelMask.data() + doc->labelMask.size() } 
+			};
+		}
+
+		template<bool _Infer>
+		void updateStateWithDoc(Generator& g, _ModelState& ld, _RandGen& rgs, _DocType& doc, size_t i) const
+		{
+			auto& z = doc.Zs[i];
+			auto w = doc.words[i];
+			if (this->etaByTopicWord.size())
+			{
+				Eigen::Array<Float, -1, 1> col = this->etaByTopicWord.col(w);
+				for (size_t k = 0; k < col.size(); ++k) col[k] *= g.theta.probabilities()[k];
+				z = sample::sampleFromDiscrete(col.data(), col.data() + col.size(), rgs);
+			}
+			else
+			{
+				z = g.theta(rgs);
+			}
+			this->template addWordTo<1>(ld, doc, i, w, z);
+		}
+
+	public:
+		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, topicLabelDict, numLatentTopics, numTopicsPerLabel);
+		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, topicLabelDict, numLatentTopics, numTopicsPerLabel);
+
+		PLDAModel(size_t _numLatentTopics = 0, size_t _numTopicsPerLabel = 1, 
+			Float _alpha = 1.0, Float _eta = 0.01, size_t _rg = std::random_device{}())
+			: BaseClass(1, _alpha, _eta, _rg), 
+			numLatentTopics(_numLatentTopics), numTopicsPerLabel(_numTopicsPerLabel)
+		{
+			if (_numLatentTopics >= 0x80000000) 
+				THROW_ERROR_WITH_INFO(std::runtime_error, text::format("wrong numLatentTopics value (numLatentTopics = %zd)", _numLatentTopics));
+			if (_numTopicsPerLabel == 0 || _numTopicsPerLabel >= 0x80000000) 
+				THROW_ERROR_WITH_INFO(std::runtime_error, text::format("wrong numTopicsPerLabel value (numTopicsPerLabel = %zd)", _numTopicsPerLabel));
+		}
+
+		template<bool _const = false>
+		_DocType& _updateDoc(_DocType& doc, const std::vector<std::string>& labels)
+		{
+			if (_const)
+			{
+				doc.labelMask.resize(this->K);
+				doc.labelMask.setZero();
+				doc.labelMask.tail(numLatentTopics).setOnes();
+
+				std::vector<Vid> topicLabelIds;
+				for (auto& label : labels)
+				{
+					auto tid = topicLabelDict.toWid(label);
+					if (tid == (Vid)-1) continue;
+					topicLabelIds.emplace_back(tid);
+				}
+
+				for (auto tid : topicLabelIds) doc.labelMask.segment(tid * numTopicsPerLabel, numTopicsPerLabel).setOnes();
+				if (labels.empty()) doc.labelMask.setOnes();
+			}
+			else
+			{
+				if (!labels.empty())
+				{
+					std::vector<Vid> topicLabelIds;
+					for (auto& label : labels) topicLabelIds.emplace_back(topicLabelDict.add(label));
+					auto maxVal = *std::max_element(topicLabelIds.begin(), topicLabelIds.end());
+					doc.labelMask.resize((maxVal + 1) * numTopicsPerLabel);
+					doc.labelMask.setZero();
+					for (auto i : topicLabelIds) doc.labelMask.segment(i * numTopicsPerLabel, numTopicsPerLabel).setOnes();
+				}
+			}
+			return doc;
+		}
+
+		size_t addDoc(const std::vector<std::string>& words, const std::vector<std::string>& labels) override
+		{
+			auto doc = this->_makeDoc(words);
+			return this->_addDoc(_updateDoc(doc, labels));
+		}
+
+		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<std::string>& labels) const override
+		{
+			auto doc = as_mutable(this)->template _makeDoc<true>(words);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, labels));
+		}
+
+		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
+			const std::vector<std::string>& labels) override
+		{
+			auto doc = this->template _makeRawDoc<false>(rawStr, tokenizer);
+			return this->_addDoc(_updateDoc(doc, labels));
+		}
+
+		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
+			const std::vector<std::string>& labels) const override
+		{
+			auto doc = as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, labels));
+		}
+
+		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
+			const std::vector<std::string>& labels) override
+		{
+			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
+			return this->_addDoc(_updateDoc(doc, labels));
+		}
+
+		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
+			const std::vector<std::string>& labels) const override
+		{
+			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, labels));
+		}
+
+		std::vector<Float> getTopicsByDoc(const _DocType& doc) const
+		{
+			std::vector<Float> ret(this->K);
+			auto maskedAlphas = this->alphas.array() * doc.labelMask.template cast<Float>().array();
+			Eigen::Map<Eigen::Matrix<Float, -1, 1>> { ret.data(), this->K }.array() =
+				(doc.numByTopic.array().template cast<Float>() + maskedAlphas)
+				/ (doc.getSumWordWeight() + maskedAlphas.sum());
+			return ret;
+		}
+
+		const Dictionary& getTopicLabelDict() const override { return topicLabelDict; }
+
+		size_t getNumLatentTopics() const override { return numLatentTopics; }
+
+		size_t getNumTopicsPerLabel() const override { return numTopicsPerLabel; }
+	};
+}

data/vendor/tomotopy/src/TopicModel/SLDA.h

@@ -0,0 +1,54 @@
+#pragma once
+#include "LDA.h"
+
+namespace tomoto
+{
+    template<TermWeight _tw>
+	struct DocumentSLDA : public DocumentLDA<_tw>
+	{
+		using BaseDocument = DocumentLDA<_tw>;
+		using DocumentLDA<_tw>::DocumentLDA;
+		std::vector<Float> y;
+		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, y);
+		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, y);
+	};
+
+	class ISLDAModel : public ILDAModel
+	{
+	public:
+		enum class GLM
+		{
+			linear = 0,
+			binary_logistic = 1,
+		};
+
+		using DefaultDocType = DocumentSLDA<TermWeight::one>;
+		static ISLDAModel* create(TermWeight _weight, size_t _K = 1, 
+			const std::vector<ISLDAModel::GLM>& vars = {},
+			Float alpha = 0.1, Float _eta = 0.01,
+			const std::vector<Float>& _mu = {}, const std::vector<Float>& _nuSq = {},
+			const std::vector<Float>& _glmParam = {},
+			size_t seed = std::random_device{}(),
+			bool scalarRng = false);
+
+		virtual size_t addDoc(const std::vector<std::string>& words, const std::vector<Float>& y) = 0;
+		virtual std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<Float>& y) const = 0;
+		
+		virtual size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
+			const std::vector<Float>& y) = 0;
+		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
+			const std::vector<Float>& y) const = 0;
+
+		virtual size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
+			const std::vector<Float>& y) = 0;
+		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
+			const std::vector<Float>& y) const = 0;
+
+		virtual size_t getF() const = 0;
+		virtual std::vector<Float> getRegressionCoef(size_t f) const = 0;
+		virtual GLM getTypeOfVar(size_t f) const = 0;
+		virtual std::vector<Float> estimateVars(const DocumentBase* doc) const = 0;
+	};
+}

data/vendor/tomotopy/src/TopicModel/SLDAModel.cpp

@@ -0,0 +1,17 @@
+#include "SLDAModel.hpp"
+
+namespace tomoto
+{
+	/*template class SLDAModel<TermWeight::one>;
+	template class SLDAModel<TermWeight::idf>;
+	template class SLDAModel<TermWeight::pmi>;*/
+
+    ISLDAModel* ISLDAModel::create(TermWeight _weight, size_t _K, const std::vector<ISLDAModel::GLM>& vars,
+		Float _alpha, Float _eta, 
+		const std::vector<Float>& _mu, const std::vector<Float>& _nuSq,
+		const std::vector<Float>& _glmParam,
+		size_t seed, bool scalarRng)
+	{
+		TMT_SWITCH_TW(_weight, scalarRng, SLDAModel, _K, vars, _alpha, _eta, _mu, _nuSq, _glmParam, seed);
+	}
+}

data/vendor/tomotopy/src/TopicModel/SLDAModel.hpp

@@ -0,0 +1,456 @@
+#pragma once
+#include "LDAModel.hpp"
+#include "../Utils/PolyaGamma.hpp"
+#include "SLDA.h"
+
+/*
+Implementation of sLDA using Gibbs sampling by bab2min
+* Mcauliffe, J. D., & Blei, D. M. (2008). Supervised topic models. In Advances in neural information processing systems (pp. 121-128).
+* Python version implementation using Gibbs sampling : https://github.com/Savvysherpa/slda
+*/
+
+namespace tomoto
+{
+	namespace detail
+	{
+		template<typename _WeightType>
+		struct GLMFunctor
+		{
+			Eigen::Matrix<Float, -1, 1> regressionCoef; // Dim : (K)
+
+			GLMFunctor(size_t K = 0, Float mu = 0) : regressionCoef(Eigen::Matrix<Float, -1, 1>::Constant(K, mu))
+			{
+			}
+
+			virtual ISLDAModel::GLM getType() const = 0;
+
+			virtual void updateZLL(
+				Eigen::Matrix<Float, -1, 1>& zLikelihood,
+				Float y, const Eigen::Matrix<_WeightType, -1, 1>& numByTopic, size_t docId, Float docSize) const = 0;
+
+			virtual void optimizeCoef(
+				const Eigen::Matrix<Float, -1, -1>& normZ,
+				Float mu, Float nuSq,
+				Eigen::Block<Eigen::Matrix<Float, -1, -1>, -1, 1, true> ys
+			) = 0;
+
+			virtual double getLL(Float y, const Eigen::Matrix<_WeightType, -1, 1>& numByTopic,
+				Float docSize) const = 0;
+
+			virtual Float estimate(const Eigen::Matrix<_WeightType, -1, 1>& numByTopic,
+				Float docSize) const = 0;
+
+			virtual ~GLMFunctor() {};
+
+			DEFINE_SERIALIZER_VIRTUAL(regressionCoef);
+
+			static void serializerWrite(const std::unique_ptr<GLMFunctor>& p, std::ostream& ostr)
+			{
+				if (!p) serializer::writeToStream<uint32_t>(ostr, 0);
+				else
+				{
+					serializer::writeToStream<uint32_t>(ostr, (uint32_t)p->getType() + 1);
+					p->serializerWrite(ostr);
+				}
+			}
+
+			static void serializerRead(std::unique_ptr<GLMFunctor>& p, std::istream& istr);
+		};
+
+		template<typename _WeightType>
+		struct LinearFunctor : public GLMFunctor<_WeightType>
+		{
+			Float sigmaSq = 1;
+
+			LinearFunctor(size_t K = 0, Float mu = 0, Float _sigmaSq = 1)
+				: GLMFunctor<_WeightType>(K, mu), sigmaSq(_sigmaSq)
+			{
+			}
+
+			ISLDAModel::GLM getType() const override { return ISLDAModel::GLM::linear; }
+
+			void updateZLL(
+				Eigen::Matrix<Float, -1, 1>& zLikelihood,
+				Float y, const Eigen::Matrix<_WeightType, -1, 1>& numByTopic, size_t docId, Float docSize) const override
+			{
+				Float yErr = y -
+					(this->regressionCoef.array() * numByTopic.array().template cast<Float>()).sum()
+					/ docSize;
+				zLikelihood.array() *= (this->regressionCoef.array() / docSize / 2 / sigmaSq *
+					(2 * yErr - this->regressionCoef.array() / docSize)).exp();
+			}
+
+			void optimizeCoef(
+				const Eigen::Matrix<Float, -1, -1>& normZ,
+				Float mu, Float nuSq,
+				Eigen::Block<Eigen::Matrix<Float, -1, -1>, -1, 1, true> ys
+			) override
+			{
+				Eigen::Matrix<Float, -1, -1> selectedNormZ = normZ.array().rowwise() * (!ys.array().transpose().isNaN()).template cast<Float>();
+				Eigen::Matrix<Float, -1, -1> normZZT = selectedNormZ * selectedNormZ.transpose();
+				normZZT += Eigen::Matrix<Float, -1, -1>::Identity(normZZT.cols(), normZZT.cols()) / nuSq;
+				this->regressionCoef = normZZT.colPivHouseholderQr().solve(selectedNormZ * ys.array().isNaN().select(0, ys).matrix());
+			}
+
+			double getLL(Float y, const Eigen::Matrix<_WeightType, -1, 1>& numByTopic,
+				Float docSize) const override
+			{
+				Float estimatedY = estimate(numByTopic, docSize);
+				return -pow(estimatedY - y, 2) / 2 / sigmaSq;
+			}
+
+			Float estimate(const Eigen::Matrix<_WeightType, -1, 1>& numByTopic,
+				Float docSize) const override
+			{
+				return (this->regressionCoef.array() * numByTopic.array().template cast<Float>()).sum()
+					/ std::max(docSize, 0.01f);
+			}
+
+			DEFINE_SERIALIZER_AFTER_BASE(GLMFunctor<_WeightType>, sigmaSq);
+		};
+
+		template<typename _WeightType>
+		struct BinaryLogisticFunctor : public GLMFunctor<_WeightType>
+		{
+			Float b = 1;
+			Eigen::Matrix<Float, -1, 1> omega;
+
+			BinaryLogisticFunctor(size_t K = 0, Float mu = 0, Float _b = 1, size_t numDocs = 0)
+				: GLMFunctor<_WeightType>(K, mu), b(_b), omega{ Eigen::Matrix<Float, -1, 1>::Ones(numDocs) }
+			{
+			}
+
+			ISLDAModel::GLM getType() const override { return ISLDAModel::GLM::binary_logistic; }
+
+			void updateZLL(
+				Eigen::Matrix<Float, -1, 1>& zLikelihood,
+				Float y, const Eigen::Matrix<_WeightType, -1, 1>& numByTopic, size_t docId, Float docSize) const override
+			{
+				Float yErr = b * (y - 0.5f) -
+					(this->regressionCoef.array() * numByTopic.array().template cast<Float>()).sum()
+					/ docSize * omega[docId];
+				zLikelihood.array() *= (this->regressionCoef.array() / docSize *
+					(yErr - omega[docId] / 2 * this->regressionCoef.array() / docSize)).exp();
+			}
+
+			void optimizeCoef(
+				const Eigen::Matrix<Float, -1, -1>& normZ,
+				Float mu, Float nuSq,
+				Eigen::Block<Eigen::Matrix<Float, -1, -1>, -1, 1, true> ys
+			) override
+			{
+				Eigen::Matrix<Float, -1, -1> selectedNormZ = normZ.array().rowwise() * (!ys.array().transpose().isNaN()).template cast<Float>();
+				Eigen::Matrix<Float, -1, -1> normZZT = selectedNormZ * Eigen::DiagonalMatrix<Float, -1>{ omega } * selectedNormZ.transpose();
+				normZZT += Eigen::Matrix<Float, -1, -1>::Identity(normZZT.cols(), normZZT.cols()) / nuSq;
+
+				this->regressionCoef = normZZT
+					.colPivHouseholderQr().solve(selectedNormZ * ys.array().isNaN().select(0, b * (ys.array() - 0.5f)).matrix()
+						+ Eigen::Matrix<Float, -1, 1>::Constant(selectedNormZ.rows(), mu / nuSq));
+
+				RandGen rng;
+				for (size_t i = 0; i < omega.size(); ++i)
+				{
+					if (std::isnan(ys[i])) continue;
+					omega[i] = math::drawPolyaGamma(b, (this->regressionCoef.array() * normZ.col(i).array()).sum(), rng);
+				}
+			}
+
+			double getLL(Float y, const Eigen::Matrix<_WeightType, -1, 1>& numByTopic,
+				Float docSize) const override
+			{
+				Float z = (this->regressionCoef.array() * numByTopic.array().template cast<Float>()).sum()
+					/ std::max(docSize, 0.01f);
+				return b * (y * z - log(1 + exp(z)));
+			}
+
+			Float estimate(const Eigen::Matrix<_WeightType, -1, 1>& numByTopic,
+				Float docSize) const override
+			{
+				Float z = (this->regressionCoef.array() * numByTopic.array().template cast<Float>()).sum()
+					/ std::max(docSize, 0.01f);
+				return 1 / (1 + exp(-z));
+			}
+
+			DEFINE_SERIALIZER_AFTER_BASE(GLMFunctor<_WeightType>, b, omega);
+		};
+	}
+
+	template<TermWeight _tw, typename _RandGen,
+		size_t _Flags = flags::partitioned_multisampling,
+		typename _Interface = ISLDAModel,
+		typename _Derived = void,
+		typename _DocType = DocumentSLDA<_tw>,
+		typename _ModelState = ModelStateLDA<_tw>>
+	class SLDAModel : public LDAModel<_tw, _RandGen, _Flags, _Interface,
+		typename std::conditional<std::is_same<_Derived, void>::value, SLDAModel<_tw, _RandGen, _Flags>, _Derived>::type,
+		_DocType, _ModelState>
+	{
+	protected:
+		using DerivedClass = typename std::conditional<std::is_same<_Derived, void>::value, SLDAModel<_tw, _RandGen>, _Derived>::type;
+		using BaseClass = LDAModel<_tw, _RandGen, _Flags, _Interface, DerivedClass, _DocType, _ModelState>;
+		friend BaseClass;
+		friend typename BaseClass::BaseClass;
+		using WeightType = typename BaseClass::WeightType;
+
+		static constexpr char TMID[] = "SLDA";
+
+		uint64_t F; // number of response variables
+		std::vector<ISLDAModel::GLM> varTypes;
+		std::vector<Float> glmParam;
+
+		Eigen::Matrix<Float, -1, 1> mu; // Mean of regression coefficients, Dim : (F)
+		Eigen::Matrix<Float, -1, 1> nuSq; // Variance of regression coefficients, Dim : (F)
+
+		std::vector<std::unique_ptr<detail::GLMFunctor<WeightType>>> responseVars;
+		Eigen::Matrix<Float, -1, -1> normZ; // topic proportions for all docs, Dim : (K, D)
+		Eigen::Matrix<Float, -1, -1> Ys; // response variables, Dim : (D, F)
+
+		template<bool _asymEta>
+		Float* getZLikelihoods(_ModelState& ld, const _DocType& doc, size_t docId, size_t vid) const
+		{
+			const size_t V = this->realV;
+			assert(vid < V);
+			auto etaHelper = this->template getEtaHelper<_asymEta>();
+			auto& zLikelihood = ld.zLikelihood;
+			zLikelihood = (doc.numByTopic.array().template cast<Float>() + this->alphas.array())
+				* (ld.numByTopicWord.col(vid).array().template cast<Float>() + etaHelper.getEta(vid))
+				/ (ld.numByTopic.array().template cast<Float>() + etaHelper.getEtaSum());
+
+			for (size_t f = 0; f < F; ++f)
+			{
+				if (std::isnan(doc.y[f])) continue;
+				responseVars[f]->updateZLL(zLikelihood, doc.y[f], doc.numByTopic,
+					docId, doc.getSumWordWeight());
+			}
+			sample::prefixSum(zLikelihood.data(), this->K);
+			return &zLikelihood[0];
+		}
+
+		void optimizeRegressionCoef()
+		{
+			for (size_t i = 0; i < this->docs.size(); ++i)
+			{
+				normZ.col(i) = this->docs[i].numByTopic.array().template cast<Float>() / 
+					std::max((Float)this->docs[i].getSumWordWeight(), 0.01f);
+			}
+
+			for (size_t f = 0; f < F; ++f)
+			{
+				responseVars[f]->optimizeCoef(normZ, mu[f], nuSq[f], Ys.col(f));
+			}
+		}
+
+		void optimizeParameters(ThreadPool& pool, _ModelState* localData, _RandGen* rgs)
+		{
+			BaseClass::optimizeParameters(pool, localData, rgs);
+		}
+
+		void updateGlobalInfo(ThreadPool& pool, _ModelState* localData)
+		{
+			optimizeRegressionCoef();
+		}
+
+		template<typename _DocIter>
+		double getLLDocs(_DocIter _first, _DocIter _last) const
+		{
+			const auto K = this->K;
+
+			double ll = 0;
+			for (; _first != _last; ++_first)
+			{
+				auto& doc = *_first;
+				ll -= math::lgammaT(doc.getSumWordWeight() + this->alphas.sum()) - math::lgammaT(this->alphas.sum());
+				for (size_t f = 0; f < F; ++f)
+				{
+					if (std::isnan(doc.y[f])) continue;
+					ll += responseVars[f]->getLL(doc.y[f], doc.numByTopic, doc.getSumWordWeight());
+				}
+				for (Tid k = 0; k < K; ++k)
+				{
+					ll += math::lgammaT(doc.numByTopic[k] + this->alphas[k]) - math::lgammaT(this->alphas[k]);
+				}
+			}
+			return ll;
+		}
+
+		double getLLRest(const _ModelState& ld) const
+		{
+			double ll = BaseClass::getLLRest(ld);
+			for (size_t f = 0; f < F; ++f)
+			{
+				ll -= (responseVars[f]->regressionCoef.array() - mu[f]).pow(2).sum() / 2 / nuSq[f];
+			}
+			return ll;
+		}
+
+		void prepareDoc(_DocType& doc, size_t docId, size_t wordSize) const
+		{
+			BaseClass::prepareDoc(doc, docId, wordSize);
+		}
+
+		void initGlobalState(bool initDocs)
+		{
+			BaseClass::initGlobalState(initDocs);
+			if (initDocs)
+			{
+				for (size_t f = 0; f < F; ++f)
+				{
+					std::unique_ptr<detail::GLMFunctor<WeightType>> v;
+					switch (varTypes[f])
+					{
+					case ISLDAModel::GLM::linear:
+						v = make_unique<detail::LinearFunctor<WeightType>>(this->K, mu[f], 
+							f < glmParam.size() ? glmParam[f] : 1.f);
+						break;
+					case ISLDAModel::GLM::binary_logistic:
+						v = make_unique<detail::BinaryLogisticFunctor<WeightType>>(this->K, mu[f],
+							f < glmParam.size() ? glmParam[f] : 1.f, this->docs.size());
+						break;
+					}
+					responseVars.emplace_back(std::move(v));
+				}
+			}
+			Ys.resize(this->docs.size(), F);
+			normZ.resize(this->K, this->docs.size());
+			for (size_t i = 0; i < this->docs.size(); ++i)
+			{
+				Ys.row(i) = Eigen::Map<Eigen::Matrix<Float, 1, -1>>(this->docs[i].y.data(), F);
+			}
+		}
+
+	public:
+		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, F, responseVars, mu, nuSq);
+		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, F, responseVars, mu, nuSq);
+
+		SLDAModel(size_t _K = 1, const std::vector<ISLDAModel::GLM>& vars = {}, 
+			Float _alpha = 0.1, Float _eta = 0.01, 
+			const std::vector<Float>& _mu = {}, const std::vector<Float>& _nuSq = {},
+			const std::vector<Float>& _glmParam = {},
+			size_t _rg = std::random_device{}())
+			: BaseClass(_K, _alpha, _eta, _rg), F(vars.size()), varTypes(vars), 
+			glmParam(_glmParam)
+		{
+			for (auto t : varTypes)
+			{
+				if (t != ISLDAModel::GLM::linear && t != ISLDAModel::GLM::binary_logistic) THROW_ERROR_WITH_INFO(std::runtime_error, "unknown var GLM type in 'vars'");
+			}
+			mu = decltype(mu)::Zero(F);
+			std::copy(_mu.begin(), _mu.end(), mu.data());
+			nuSq = decltype(nuSq)::Ones(F);
+			std::copy(_nuSq.begin(), _nuSq.end(), nuSq.data());
+		}
+
+		std::vector<Float> getRegressionCoef(size_t f) const override
+		{
+			return { responseVars[f]->regressionCoef.data(), responseVars[f]->regressionCoef.data() + this->K };
+		}
+
+		GETTER(F, size_t, F);
+
+		ISLDAModel::GLM getTypeOfVar(size_t f) const override
+		{
+			return responseVars[f]->getType();
+		}
+
+		template<bool _const = false>
+		_DocType& _updateDoc(_DocType& doc, const std::vector<Float>& y)
+		{
+			if (_const)
+			{
+				if (y.size() > F) throw std::runtime_error{ text::format(
+					"size of 'y' is greater than the number of vars.\n"
+					"size of 'y' : %zd, number of vars: %zd", y.size(), F) };
+				doc.y = y;
+				while (doc.y.size() < F)
+				{
+					doc.y.emplace_back(NAN);
+				}
+			}
+			else
+			{
+				if (y.size() != F) throw std::runtime_error{ text::format(
+					"size of 'y' must be equal to the number of vars.\n"
+					"size of 'y' : %zd, number of vars: %zd", y.size(), F) };
+				doc.y = y;
+			}
+			return doc;
+		}
+
+		size_t addDoc(const std::vector<std::string>& words, const std::vector<Float>& y) override
+		{
+			auto doc = this->_makeDoc(words);
+			return this->_addDoc(_updateDoc(doc, y));
+		}
+
+		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<Float>& y) const override
+		{
+			auto doc = as_mutable(this)->template _makeDoc<true>(words);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, y));
+		}
+
+		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
+			const std::vector<Float>& y) override
+		{
+			auto doc = this->template _makeRawDoc<false>(rawStr, tokenizer);
+			return this->_addDoc(_updateDoc(doc, y));
+		}
+
+		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
+			const std::vector<Float>& y) const override
+		{
+			auto doc = as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, y));
+		}
+
+		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
+			const std::vector<Float>& y) override
+		{
+			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
+			return this->_addDoc(_updateDoc(doc, y));
+		}
+
+		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
+			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
+			const std::vector<Float>& y) const override
+		{
+			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, y));
+		}
+
+		std::vector<Float> estimateVars(const DocumentBase* doc) const override
+		{
+			std::vector<Float> ret;
+			auto pdoc = dynamic_cast<const _DocType*>(doc);
+			if (!pdoc) return ret;
+			for (auto& f : responseVars)
+			{
+				ret.emplace_back(f->estimate(pdoc->numByTopic, pdoc->getSumWordWeight()));
+			}
+			return ret;
+		}
+	};
+
+	template<typename _WeightType>
+	void detail::GLMFunctor<_WeightType>::serializerRead(
+		std::unique_ptr<detail::GLMFunctor<_WeightType>>& p, std::istream& istr)
+	{
+		uint32_t t = serializer::readFromStream<uint32_t>(istr);
+		if (!t) p.reset();
+		else
+		{
+			switch ((ISLDAModel::GLM)(t - 1))
+			{
+			case ISLDAModel::GLM::linear:
+				p = make_unique<LinearFunctor<_WeightType>>();
+				break;
+			case ISLDAModel::GLM::binary_logistic:
+				p = make_unique<BinaryLogisticFunctor<_WeightType>>();
+				break;
+			default:
+				throw std::ios_base::failure(text::format("wrong GLMFunctor type id %d", (t - 1)));
+			}
+			p->serializerRead(istr);
+		}
+	}
+}