tomoto 0.1.2 → 0.1.3

data/vendor/tomotopy/src/TopicModel/PLDAModel.hpp

@@ -153,46 +153,28 @@ namespace tomoto
 			return doc;
 		}
 
-		size_t addDoc(const std::vector<std::string>& words, const std::vector<std::string>& labels) override
+		size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) override
 		{
-			auto doc = this->_makeDoc(words);
-			return this->_addDoc(_updateDoc(doc, labels));
+			auto doc = this->template _makeFromRawDoc<false>(rawDoc, tokenizer);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<std::string>& labels) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) const override
 		{
-			auto doc = as_mutable(this)->template _makeDoc<true>(words);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, labels));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc, tokenizer);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
-		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<std::string>& labels) override
+		size_t addDoc(const RawDoc& rawDoc) override
 		{
-			auto doc = this->template _makeRawDoc<false>(rawStr, tokenizer);
-			return this->_addDoc(_updateDoc(doc, labels));
+			auto doc = this->_makeFromRawDoc(rawDoc);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<std::string>& labels) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc) const override
 		{
-			auto doc = as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, labels));
-		}
-
-		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<std::string>& labels) override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return this->_addDoc(_updateDoc(doc, labels));
-		}
-
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<std::string>& labels) const override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, labels));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
 		std::vector<Float> getTopicsByDoc(const _DocType& doc) const

data/vendor/tomotopy/src/TopicModel/SLDA.h

@@ -31,21 +31,6 @@ namespace tomoto
 			size_t seed = std::random_device{}(),
 			bool scalarRng = false);
 
-		virtual size_t addDoc(const std::vector<std::string>& words, const std::vector<Float>& y) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<Float>& y) const = 0;
-		
-		virtual size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<Float>& y) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<Float>& y) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<Float>& y) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<Float>& y) const = 0;
-
 		virtual size_t getF() const = 0;
 		virtual std::vector<Float> getRegressionCoef(size_t f) const = 0;
 		virtual GLM getTypeOfVar(size_t f) const = 0;

data/vendor/tomotopy/src/TopicModel/SLDAModel.hpp

@@ -148,7 +148,7 @@ namespace tomoto
 						+ Eigen::Matrix<Float, -1, 1>::Constant(selectedNormZ.rows(), mu / nuSq));
 
 				RandGen rng;
-				for (size_t i = 0; i < omega.size(); ++i)
+				for (size_t i = 0; i < (size_t)omega.size(); ++i)
 				{
 					if (std::isnan(ys[i])) continue;
 					omega[i] = math::drawPolyaGamma(b, (this->regressionCoef.array() * normZ.col(i).array()).sum(), rng);
@@ -358,8 +358,8 @@ namespace tomoto
 			if (_const)
 			{
 				if (y.size() > F) throw std::runtime_error{ text::format(
-					"size of 'y' is greater than the number of vars.\n"
-					"size of 'y' : %zd, number of vars: %zd", y.size(), F) };
+					"size of `y` is greater than the number of vars.\n"
+					"size of `y` : %zd, number of vars: %zd", y.size(), F) };
 				doc.y = y;
 				while (doc.y.size() < F)
 				{
@@ -369,53 +369,35 @@ namespace tomoto
 			else
 			{
 				if (y.size() != F) throw std::runtime_error{ text::format(
-					"size of 'y' must be equal to the number of vars.\n"
-					"size of 'y' : %zd, number of vars: %zd", y.size(), F) };
+					"size of `y` must be equal to the number of vars.\n"
+					"size of `y` : %zd, number of vars: %zd", y.size(), F) };
 				doc.y = y;
 			}
 			return doc;
 		}
 
-		size_t addDoc(const std::vector<std::string>& words, const std::vector<Float>& y) override
+		size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) override
 		{
-			auto doc = this->_makeDoc(words);
-			return this->_addDoc(_updateDoc(doc, y));
+			auto doc = this->template _makeFromRawDoc<false>(rawDoc, tokenizer);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMiscDefault<std::vector<Float>>("y")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<Float>& y) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) const override
 		{
-			auto doc = as_mutable(this)->template _makeDoc<true>(words);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, y));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc, tokenizer);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMiscDefault<std::vector<Float>>("y")));
 		}
 
-		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<Float>& y) override
+		size_t addDoc(const RawDoc& rawDoc) override
 		{
-			auto doc = this->template _makeRawDoc<false>(rawStr, tokenizer);
-			return this->_addDoc(_updateDoc(doc, y));
+			auto doc = this->_makeFromRawDoc(rawDoc);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMiscDefault<std::vector<Float>>("y")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<Float>& y) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc) const override
 		{
-			auto doc = as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, y));
-		}
-
-		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<Float>& y) override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return this->_addDoc(_updateDoc(doc, y));
-		}
-
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<Float>& y) const override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, y));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMiscDefault<std::vector<Float>>("y")));
 		}
 
 		std::vector<Float> estimateVars(const DocumentBase* doc) const override

data/vendor/tomotopy/src/TopicModel/TopicModel.hpp

@@ -7,27 +7,112 @@
 #include "../Utils/ThreadPool.hpp"
 #include "../Utils/serializer.hpp"
 #include "../Utils/exception.h"
+#include "../Utils/SharedString.hpp"
 #include <EigenRand/EigenRand>
+#include <mapbox/variant.hpp>
 
 namespace tomoto
 {
 	using RandGen = Eigen::Rand::P8_mt19937_64<uint32_t>;
 	using ScalarRandGen = Eigen::Rand::UniversalRandomEngine<uint32_t, std::mt19937_64>;
 
-	class DocumentBase
+	struct RawDocKernel
 	{
-	public:
 		Float weight = 1;
+		SharedString docUid;
+		SharedString rawStr;
+		std::vector<uint32_t> origWordPos;
+		std::vector<uint16_t> origWordLen;
+
+		RawDocKernel(const RawDocKernel&) = default;
+		RawDocKernel(RawDocKernel&&) = default;
+
+		RawDocKernel(Float _weight = 1, const SharedString& _docUid = {})
+			: weight{ _weight }, docUid{ _docUid }
+		{
+		}
+	};
+
+	struct RawDoc : public RawDocKernel
+	{
+		using Var = mapbox::util::variant<
+			std::string, uint32_t, Float,
+			std::vector<std::string>, std::vector<uint32_t>, std::vector<Float>,
+			std::shared_ptr<void>
+		>;
+		using MiscType = std::unordered_map<std::string, Var>;
+
+		std::vector<Vid> words;
+		std::vector<std::string> rawWords;
+		MiscType misc;
+
+		RawDoc() = default;
+		RawDoc(const RawDoc&) = default;
+		RawDoc(RawDoc&&) = default;
+
+		RawDoc(const RawDocKernel& o) 
+			: RawDocKernel{ o }
+		{
+		}
+
+		template<typename _Ty>
+		const _Ty& getMisc(const std::string& name) const
+		{
+			auto it = misc.find(name);
+			if (it == misc.end()) throw std::invalid_argument{ "There is no value named `" + name + "` in misc data" };
+			if (!it->second.template is<_Ty>()) throw std::invalid_argument{ "Value named `" + name + "` is not in right type." };
+			return it->second.template get<_Ty>();
+		}
+
+		template<typename _Ty>
+		_Ty getMiscDefault(const std::string& name) const
+		{
+			auto it = misc.find(name);
+			if (it == misc.end()) return {};
+			if (!it->second.template is<_Ty>()) throw std::invalid_argument{ "Value named `" + name + "` is not in right type." };
+			return it->second.template get<_Ty>();
+		}
+	};
+
+	class DocumentBase : public RawDocKernel
+	{
+	public:
 		tvector<Vid> words; // word id of each word
 		std::vector<uint32_t> wOrder; // original word order (optional)
+		
+		DocumentBase(const DocumentBase&) = default;
+		DocumentBase(DocumentBase&&) = default;
+
+		DocumentBase(const RawDocKernel& o) 
+			: RawDocKernel{ o }
+		{
+		}
+
+		DocumentBase(Float _weight = 1, const SharedString& _docUid = {})
+			: RawDocKernel{ _weight, _docUid }
+		{
+		}
 
-		std::string docUid;
-		std::string rawStr;
-		std::vector<uint32_t> origWordPos;
-		std::vector<uint16_t> origWordLen;
-		DocumentBase(Float _weight = 1) : weight(_weight) {}
 		virtual ~DocumentBase() {}
 
+		virtual operator RawDoc() const
+		{
+			RawDoc raw{ *this };
+			if (wOrder.empty())
+			{
+				raw.words.insert(raw.words.begin(), words.begin(), words.end());
+			}
+			else
+			{
+				raw.words.resize(words.size());
+				for (size_t i = 0; i < words.size(); ++i)
+				{
+					raw.words[i] = words[wOrder[i]];
+				}
+			}
+			return raw;
+		}
+
 		DEFINE_SERIALIZER_WITH_VERSION(0, serializer::to_key("Docu"), weight, words, wOrder);
 		DEFINE_TAGGED_SERIALIZER_WITH_VERSION(1, 0x00010001, weight, words, wOrder, 
 			rawStr, origWordPos, origWordLen,
@@ -127,8 +212,20 @@ namespace tomoto
 		virtual void loadModel(std::istream& reader, 
 			std::vector<uint8_t>* extra_data = nullptr) = 0;
 		virtual const DocumentBase* getDoc(size_t docId) const = 0;
+		virtual size_t getDocIdByUid(const std::string& docUid) const = 0;
+
+		// it tokenizes rawDoc.rawStr to get words, pos and len of the document
+		virtual size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) = 0;
+		virtual std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) const = 0;
+
+		// it uses words, pos and len of rawDoc itself.
+		virtual size_t addDoc(const RawDoc& rawDoc) = 0;
+		virtual std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc) const = 0;
+
+		virtual bool updateVocab(const std::vector<std::string>& words) = 0;
 
-		virtual void updateVocab(const std::vector<std::string>& words) = 0;
+		virtual double getDocLL(const DocumentBase* doc) const = 0;
+		virtual double getStateLL() const = 0;
 
 		virtual double getLLPerWord() const = 0;
 		virtual double getPerplexity() const = 0;
@@ -201,6 +298,7 @@ namespace tomoto
 		std::vector<DocType> docs;
 		std::vector<uint64_t> vocabCf;
 		std::vector<uint64_t> vocabDf;
+		std::unordered_map<SharedString, size_t> uidMap;
 		size_t globalStep = 0;
 		_ModelState globalState, tState;
 		Dictionary dict;
@@ -273,6 +371,8 @@ namespace tomoto
 		>::value, size_t>::type _addDoc(_DocTy&& doc)
 		{
 			if (doc.words.empty()) return -1;
+			if (!doc.docUid.empty() && uidMap.count(doc.docUid))
+				throw exception::InvalidArgument{ "there is a document with uid = " + std::string{ doc.docUid } + " already." };
 			size_t maxWid = *std::max_element(doc.words.begin(), doc.words.end());
 			if (vocabCf.size() <= maxWid)
 			{
@@ -282,47 +382,48 @@ namespace tomoto
 			for (auto w : doc.words) ++vocabCf[w];
 			std::unordered_set<Vid> uniq{ doc.words.begin(), doc.words.end() };
 			for (auto w : uniq) ++vocabDf[w];
+			uidMap.emplace(doc.docUid, docs.size());
 			docs.emplace_back(std::forward<_DocTy>(doc));
 			return docs.size() - 1;
 		}
 
 		template<bool _const = false>
-		DocType _makeDoc(const std::vector<std::string>& words, Float weight = 1)
+		DocType _makeFromRawDoc(const RawDoc& rawDoc)
 		{
-			DocType doc{ weight };
-			for (auto& w : words)
+			DocType doc{ rawDoc };
+			if (!rawDoc.rawWords.empty())
 			{
-				Vid id;
-				if (_const)
+				for (auto& w : rawDoc.rawWords)
 				{
-					id = dict.toWid(w);
-					if (id == (Vid)-1) continue;
-				}
-				else
-				{
-					id = dict.add(w);
+					Vid id;
+					if (_const)
+					{
+						id = dict.toWid(w);
+						if (id == (Vid)-1) continue;
+					}
+					else
+					{
+						id = dict.add(w);
+					}
+					doc.words.emplace_back(id);
 				}
-				doc.words.emplace_back(id);
 			}
-			return doc;
-		}
-
-		DocType _makeRawDoc(const std::string& rawStr, const std::vector<Vid>& words, 
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len, Float weight = 1) const
-		{
-			DocType doc{ weight };
-			doc.rawStr = rawStr;
-			for (auto& w : words) doc.words.emplace_back(w);
-			doc.origWordPos = pos;
-			doc.origWordLen = len;
+			else if(!rawDoc.words.empty())
+			{
+				for (auto& w : rawDoc.words) doc.words.emplace_back(w);
+			}
+			else
+			{
+				throw std::invalid_argument{ "Either `words` or `rawWords` must be filled." };
+			}
 			return doc;
 		}
 
 		template<bool _const, typename _FnTokenizer>
-		DocType _makeRawDoc(const std::string& rawStr, _FnTokenizer&& tokenizer, Float weight = 1)
+		DocType _makeFromRawDoc(const RawDoc& rawDoc, _FnTokenizer&& tokenizer)
 		{
-			DocType doc{ weight };
-			doc.rawStr = rawStr;
+			DocType doc{ rawDoc };
+			doc.rawStr = rawDoc.rawStr;
 			for (auto& p : tokenizer(doc.rawStr))
 			{
 				Vid wid;
@@ -452,10 +553,11 @@ namespace tomoto
 			return realV;
 		}
 
-		void updateVocab(const std::vector<std::string>& words) override
+		bool updateVocab(const std::vector<std::string>& words) override
 		{
-			if(dict.size()) THROW_ERROR_WITH_INFO(exception::InvalidArgument, "updateVocab after addDoc");
-			for(auto& w : words) dict.add(w);
+			bool empty = dict.size() == 0;
+			for (auto& w : words) dict.add(w);
+			return empty;
 		}
 
 		void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0) override
@@ -606,6 +708,18 @@ namespace tomoto
 			return vid2String(getWidsByDocSorted(doc, topN));
 		}
 
+		double getDocLL(const DocumentBase* doc) const override
+		{
+			auto* p = dynamic_cast<const DocType*>(doc);
+			if (!p) throw std::invalid_argument{ "wrong `doc` type." };
+			return static_cast<const _Derived*>(this)->getLLDocs(p, p + 1);
+		}
+
+		double getStateLL() const override
+		{
+			return static_cast<const _Derived*>(this)->getLLRest(this->globalState);
+		}
+
 		std::vector<double> infer(const std::vector<DocumentBase*>& docs, size_t maxIter, Float tolerance, size_t numWorkers, ParallelScheme ps, bool together) const override
 		{
 			if (!numWorkers) numWorkers = std::thread::hardware_concurrency();
@@ -651,12 +765,18 @@ namespace tomoto
 			return extractTopN<Tid>(getTopicsByDoc(doc), topN);
 		}
 
-
 		const DocumentBase* getDoc(size_t docId) const override
 		{
 			return &_getDoc(docId);
 		}
 
+		size_t getDocIdByUid(const std::string& docUid) const override
+		{
+			auto it = uidMap.find(SharedString{ docUid });
+			if (it == uidMap.end()) return -1;
+			return it->second;
+		}
+
 		size_t getGlobalStep() const override
 		{
 			return globalStep;