tomoto 0.1.2 → 0.1.3

data/vendor/tomotopy/src/TopicModel/HDPModel.hpp

@@ -84,12 +84,12 @@ namespace tomoto
 		{
 			const size_t V = this->realV;
 			size_t pos;
-			for (pos = 0; pos < ld.numTableByTopic.size(); ++pos)
+			for (pos = 0; pos < (size_t)ld.numTableByTopic.size(); ++pos)
 			{
 				if (!ld.numTableByTopic[pos]) break;
 			}
 			
-			if (pos >= ld.numByTopic.size())
+			if (pos >= (size_t)ld.numByTopic.size())
 			{
 				size_t oldSize = ld.numByTopic.size(), newSize = pos + 1;
 				ld.numTableByTopic.conservativeResize(newSize);
@@ -302,8 +302,7 @@ namespace tomoto
 				size_t oldSize = globalState.numByTopic.size();
 				globalState.numByTopic.conservativeResize(K);
 				globalState.numByTopic.tail(K - oldSize).setZero();
-				globalState.numTableByTopic.conservativeResize(K);
-				globalState.numTableByTopic.tail(K - oldSize).setZero();
+				globalState.numTableByTopic.resize(K);
 				globalState.numByTopicWord.conservativeResize(K, Eigen::NoChange);
 				globalState.numByTopicWord.block(oldSize, 0, K - oldSize, V).setZero();
 			}
@@ -314,8 +313,6 @@ namespace tomoto
 				size_t locK = localData[i].numByTopic.size();
 				globalState.numByTopic.head(locK) 
 					+= localData[i].numByTopic.head(locK) - tState.numByTopic.head(locK);
-				globalState.numTableByTopic.head(locK)
-					+= localData[i].numTableByTopic.head(locK) - tState.numTableByTopic.head(locK);
 				globalState.numByTopicWord.block(0, 0, locK, V)
 					+= localData[i].numByTopicWord.block(0, 0, locK, V) - tState.numByTopicWord.block(0, 0, locK, V);
 			}
@@ -327,10 +324,16 @@ namespace tomoto
 				globalState.numByTopicWord = globalState.numByTopicWord.cwiseMax(0);
 			}
 
-			globalState.totalTable = accumulate(this->docs.begin(), this->docs.end(), 0, [](size_t sum, const _DocType& doc)
+
+			globalState.numTableByTopic.setZero();
+			for (auto& doc : this->docs)
 			{
-				return sum + doc.getNumTable();
-			});
+				for (auto& table : doc.numTopicByTable)
+				{
+					if (table) globalState.numTableByTopic[table.topic]++;
+				}
+			}
+			globalState.totalTable = globalState.numTableByTopic.sum();
 			
 			for (size_t i = 0; i < pool.getNumWorkers(); ++i)
 			{
@@ -368,19 +371,13 @@ namespace tomoto
 			const auto eta = this->eta;
 			double ll = 0;
 			// table partition ll
-			size_t liveK = 0;
-			for (Tid k = 0; k < K; ++k)
-			{
-				if (!isLiveTopic(k)) continue;
-				ll += math::lgammaT(ld.numTableByTopic[k]);
-				++liveK;
-			}
-			
+			size_t liveK = (ld.numTableByTopic.array() > 0).template cast<size_t>().sum();
+			Eigen::ArrayXf lg = math::lgammaApprox(ld.numTableByTopic.array().template cast<Float>());
+			ll += (ld.numTableByTopic.array() > 0).select(lg, 0).sum();
 			ll += liveK * log(gamma) - math::lgammaT(ld.totalTable + gamma) + math::lgammaT(gamma);
 
 			// topic word ll
 			ll += liveK * math::lgammaT(V * eta);
-
 			for (Tid k = 0; k < K; ++k)
 			{
 				if (!isLiveTopic(k)) continue;
@@ -545,8 +542,7 @@ namespace tomoto
 			
 			for (auto& doc : this->docs)
 			{
-				auto d = lda->_makeDoc(std::vector<std::string>{});
-				for(auto w : doc.words) d.words.emplace_back(w);
+				auto d = lda->_makeFromRawDoc(doc);
 				lda->_addDoc(d);
 			}
 			

data/vendor/tomotopy/src/TopicModel/HLDAModel.hpp

@@ -101,7 +101,7 @@ namespace tomoto
 			void addPathOne()
 			{
 				NCRPNode* node = this;
-				for (size_t i = 0; i <= level; ++i)
+				for (size_t i = 0; i <= (size_t)level; ++i)
 				{
 					++node->numCustomers;
 					node = node->getParent();
@@ -132,7 +132,7 @@ namespace tomoto
 			{
 				size_t idx = node - nodes.data();
 				const Float pNewNode = _MakeNewPath ? log(gamma / (node->numCustomers + gamma)) : -INFINITY;
-				nodeLikelihoods[idx] = weight + ((node->level < levelDepth - 1) ? pNewNode : 0);
+				nodeLikelihoods[idx] = weight + (((size_t)node->level < levelDepth - 1) ? pNewNode : 0);
 				for(auto * child = node->getChild(); child; child = child->getSibling())
 				{
 					updateNodeLikelihood(gamma, levelDepth, child, weight + log(child->numCustomers / (node->numCustomers + gamma)));
@@ -279,7 +279,7 @@ namespace tomoto
 					nodes[idx].level = l;
 				}
 
-				if (ld.numByTopic.size() < nodes.size())
+				if ((size_t)ld.numByTopic.size() < nodes.size())
 				{
 					size_t oldSize = ld.numByTopic.rows();
 					size_t newSize = std::max(nodes.size(), ((oldSize + oldSize / 2 + 7) / 8) * 8);

data/vendor/tomotopy/src/TopicModel/LDA.h

@@ -117,17 +117,6 @@ namespace tomoto
 			Float _alpha = 0.1, Float _eta = 0.01, size_t seed = std::random_device{}(),
 			bool scalarRng = false);
 
-		virtual size_t addDoc(const std::vector<std::string>& words) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len) const = 0;
-
 		virtual TermWeight getTermWeight() const = 0;
 		virtual size_t getOptimInterval() const = 0;
 		virtual void setOptimInterval(size_t) = 0;

data/vendor/tomotopy/src/TopicModel/LDAModel.hpp

@@ -609,7 +609,7 @@ namespace tomoto
 
 		double getLL() const
 		{
-			return static_cast<const DerivedClass*>(this)->template getLLDocs<>(this->docs.begin(), this->docs.end())
+			return static_cast<const DerivedClass*>(this)->getLLDocs(this->docs.begin(), this->docs.end())
 				+ static_cast<const DerivedClass*>(this)->getLLRest(this->globalState);
 		}
 
@@ -898,36 +898,24 @@ namespace tomoto
 			burnIn = iteration;
 		}
 
-		size_t addDoc(const std::vector<std::string>& words) override
+		size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) override
 		{
-			return this->_addDoc(this->_makeDoc(words));
+			return this->_addDoc(this->template _makeFromRawDoc<false>(rawDoc, tokenizer));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) const override
 		{
-			return make_unique<_DocType>(as_mutable(this)->template _makeDoc<true>(words));
+			return make_unique<_DocType>(as_mutable(this)->template _makeFromRawDoc<true>(rawDoc, tokenizer));
 		}
 
-		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer) override
+		size_t addDoc(const RawDoc& rawDoc) override
 		{
-			return this->_addDoc(this->template _makeRawDoc<false>(rawStr, tokenizer));
+			return this->_addDoc(this->_makeFromRawDoc(rawDoc));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc) const override
 		{
-			return make_unique<_DocType>(as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer));
-		}
-
-		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len) override
-		{
-			return this->_addDoc(this->_makeRawDoc(rawStr, words, pos, len));
-		}
-
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len) const override
-		{
-			return make_unique<_DocType>(this->_makeRawDoc(rawStr, words, pos, len));
+			return make_unique<_DocType>(as_mutable(this)->template _makeFromRawDoc<true>(rawDoc));
 		}
 
 		void setWordPrior(const std::string& word, const std::vector<Float>& priors) override

data/vendor/tomotopy/src/TopicModel/LLDA.h

@@ -23,21 +23,6 @@ namespace tomoto
 			Float alpha = 0.1, Float eta = 0.01, size_t seed = std::random_device{}(),
 			bool scalarRng = false);
 
-		virtual size_t addDoc(const std::vector<std::string>& words, const std::vector<std::string>& label) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<std::string>& label) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer, 
-			const std::vector<std::string>& label) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<std::string>& label) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len, 
-			const std::vector<std::string>& label) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<std::string>& label) const = 0;
-
 		virtual const Dictionary& getTopicLabelDict() const = 0;
 
 		virtual size_t getNumTopicsPerLabel() const = 0;

data/vendor/tomotopy/src/TopicModel/LLDAModel.hpp

@@ -144,46 +144,28 @@ namespace tomoto
 			return doc;
 		}
 
-		size_t addDoc(const std::vector<std::string>& words, const std::vector<std::string>& labels) override
+		size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) override
 		{
-			auto doc = this->_makeDoc(words);
-			return this->_addDoc(_updateDoc(doc, labels));
+			auto doc = this->template _makeFromRawDoc<false>(rawDoc, tokenizer);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<std::string>& labels) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) const override
 		{
-			auto doc = as_mutable(this)->template _makeDoc<true>(words);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, labels));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc, tokenizer);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
-		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<std::string>& labels) override
+		size_t addDoc(const RawDoc& rawDoc) override
 		{
-			auto doc = this->template _makeRawDoc<false>(rawStr, tokenizer);
-			return this->_addDoc(_updateDoc(doc, labels));
+			auto doc = this->_makeFromRawDoc(rawDoc);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<std::string>& labels) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc) const override
 		{
-			auto doc = as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, labels));
-		}
-
-		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<std::string>& labels) override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return this->_addDoc(_updateDoc(doc, labels));
-		}
-
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<std::string>& labels) const override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, labels));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
 		std::vector<Float> getTopicsByDoc(const _DocType& doc) const

data/vendor/tomotopy/src/TopicModel/MGLDA.h

@@ -37,21 +37,6 @@ namespace tomoto
 			Float _etaG = 0.01, Float _etaL = 0.01, Float _gamma = 0.1, size_t seed = std::random_device{}(),
 			bool scalarRng = false);
 
-		virtual size_t addDoc(const std::vector<std::string>& words, const std::string& delimiter) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::string& delimiter) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::string& delimiter) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::string& delimiter) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::string& delimiter) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::string& delimiter) const = 0;
-
 		virtual size_t getKL() const = 0;
 		virtual size_t getT() const = 0;
 		virtual Float getAlphaL() const = 0;

data/vendor/tomotopy/src/TopicModel/MGLDAModel.hpp

@@ -385,53 +385,12 @@ namespace tomoto
 			if (_etaL <= 0) THROW_ERROR_WITH_INFO(std::runtime_error, text::format("wrong etaL value (etaL = %f)", _etaL));
 		}
 
-
-		template<bool _const = false>
-		_DocType _makeDoc(const std::vector<std::string>& words, const std::string& delimiter)
-		{
-			_DocType doc{ 1.f };
-			size_t numSent = 0;
-			for (auto& w : words)
-			{
-				if (w == delimiter)
-				{
-					++numSent;
-					continue;
-				}
-
-				Vid id;
-				if (_const)
-				{
-					id = this->dict.toWid(w);
-					if (id == (Vid)-1) continue;
-				}
-				else
-				{
-					id = this->dict.add(w);
-				}
-				doc.words.emplace_back(id);
-				doc.sents.emplace_back(numSent);
-			}
-			doc.numBySent.resize(doc.sents.empty() ? 0 : (doc.sents.back() + 1));
-			return doc;
-		}
-
-		size_t addDoc(const std::vector<std::string>& words, const std::string& delimiter) override
-		{
-			return this->_addDoc(_makeDoc(words, delimiter));
-		}
-
-		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::string& delimiter) const override
-		{
-			return make_unique<_DocType>(as_mutable(this)->template _makeDoc<true>(words, delimiter));
-		}
-
 		template<bool _const, typename _FnTokenizer>
-		_DocType _makeRawDoc(const std::string& rawStr, _FnTokenizer&& tokenizer, const std::string& delimiter)
+		_DocType _makeFromRawDoc(const RawDoc& rawDoc, _FnTokenizer&& tokenizer, const std::string& delimiter)
 		{
-			_DocType doc{ 1.f };
+			_DocType doc;
 			size_t numSent = 0;
-			doc.rawStr = rawStr;
+			doc.rawStr = rawDoc.rawStr;
 			for (auto& p : tokenizer(doc.rawStr))
 			{
 				if (std::get<0>(p) == delimiter)
@@ -461,57 +420,85 @@ namespace tomoto
 			return doc;
 		}
 
-		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::string& delimiter)
+		size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer)
 		{
-			return this->_addDoc(_makeRawDoc<false>(rawStr, tokenizer, delimiter));
+			return this->_addDoc(_makeFromRawDoc<false>(rawDoc, tokenizer, rawDoc.template getMisc<std::string>("delimiter")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::string& delimiter) const
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) const
 		{
-			return make_unique<_DocType>(as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer, delimiter));
+			return make_unique<_DocType>(as_mutable(this)->template _makeFromRawDoc<true>(rawDoc, tokenizer, rawDoc.template getMisc<std::string>("delimiter")));
 		}
 
-		_DocType _makeRawDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len, const std::string& delimiter) const
+		template<bool _const = false>
+		_DocType _makeFromRawDoc(const RawDoc& rawDoc)
 		{
-			_DocType doc{ 1.f };
-			doc.rawStr = rawStr;
+			_DocType doc;
+			doc.rawStr = rawDoc.rawStr;
+			auto delimiter = rawDoc.template getMisc<std::string>("delimiter");
 			size_t numSent = 0;
 			Vid delimiterId = this->dict.toWid(delimiter);
-			for (size_t i = 0; i < words.size(); ++i)
+			if (!rawDoc.rawWords.empty())
 			{
-				auto& w = words[i];
-				if (w == delimiterId)
+				for (size_t i = 0; i < rawDoc.rawWords.size(); ++i)
 				{
-					++numSent;
-					continue;
+					auto& w = rawDoc.rawWords[i];
+					if (w == delimiter)
+					{
+						++numSent;
+						continue;
+					}
+
+					Vid id;
+					if (_const)
+					{
+						id = this->dict.toWid(w);
+						if (id == (Vid)-1) continue;
+					}
+					else
+					{
+						id = this->dict.add(w);
+					}
+					doc.words.emplace_back(id);
+					doc.sents.emplace_back(numSent);
+					if (rawDoc.rawWords.size() == rawDoc.origWordPos.size())
+					{
+						doc.origWordPos.emplace_back(rawDoc.origWordPos[i]);
+						doc.origWordLen.emplace_back(rawDoc.origWordLen[i]);
+					}
 				}
-				doc.words.emplace_back(w);
-				doc.sents.emplace_back(numSent);
-				if (words.size() == pos.size())
+			}
+			else if (!rawDoc.words.empty())
+			{
+				for (size_t i = 0; i < rawDoc.words.size(); ++i)
 				{
-					doc.origWordPos.emplace_back(pos[i]);
-					doc.origWordLen.emplace_back(len[i]);
+					auto& w = rawDoc.words[i];
+					if (w == delimiterId)
+					{
+						++numSent;
+						continue;
+					}
+					doc.words.emplace_back(w);
+					doc.sents.emplace_back(numSent);
+					if (rawDoc.words.size() == rawDoc.origWordPos.size())
+					{
+						doc.origWordPos.emplace_back(rawDoc.origWordPos[i]);
+						doc.origWordLen.emplace_back(rawDoc.origWordLen[i]);
+					}
 				}
 			}
 			doc.numBySent.resize(doc.sents.empty() ? 0 : (doc.sents.back() + 1));
 			return doc;
 		}
 
-		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::string& delimiter)
+		size_t addDoc(const RawDoc& rawDoc)
 		{
-			return this->_addDoc(_makeRawDoc(rawStr, words, pos, len, delimiter));
+			return this->_addDoc(_makeFromRawDoc(rawDoc));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::string& delimiter) const
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc) const
 		{
-			return make_unique<_DocType>(_makeRawDoc(rawStr, words, pos, len, delimiter));
+			return make_unique<_DocType>(as_mutable(this)->template _makeFromRawDoc<true>(rawDoc));
 		}
 
 		void setWordPrior(const std::string& word, const std::vector<Float>& priors) override