tomoto 0.1.4 → 0.2.0

data/vendor/tomotopy/src/TopicModel/TopicModel.hpp

@@ -16,6 +16,9 @@ namespace tomoto
 	using RandGen = Eigen::Rand::P8_mt19937_64<uint32_t>;
 	using ScalarRandGen = Eigen::Rand::UniversalRandomEngine<uint32_t, std::mt19937_64>;
 
+	using Vector = Eigen::Matrix<Float, -1, 1>;
+	using Matrix = Eigen::Matrix<Float, -1, -1>;
+
 	struct RawDocKernel
 	{
 		Float weight = 1;
@@ -59,8 +62,8 @@ namespace tomoto
 		const _Ty& getMisc(const std::string& name) const
 		{
 			auto it = misc.find(name);
-			if (it == misc.end()) throw std::invalid_argument{ "There is no value named `" + name + "` in misc data" };
-			if (!it->second.template is<_Ty>()) throw std::invalid_argument{ "Value named `" + name + "` is not in right type." };
+			if (it == misc.end()) throw exc::InvalidArgument{ "There is no value named `" + name + "` in misc data" };
+			if (!it->second.template is<_Ty>()) throw exc::InvalidArgument{ "Value named `" + name + "` is not in right type." };
 			return it->second.template get<_Ty>();
 		}
 
@@ -69,11 +72,13 @@ namespace tomoto
 		{
 			auto it = misc.find(name);
 			if (it == misc.end()) return {};
-			if (!it->second.template is<_Ty>()) throw std::invalid_argument{ "Value named `" + name + "` is not in right type." };
+			if (!it->second.template is<_Ty>()) throw exc::InvalidArgument{ "Value named `" + name + "` is not in right type." };
 			return it->second.template get<_Ty>();
 		}
 	};
 
+	class ITopicModel;
+
 	class DocumentBase : public RawDocKernel
 	{
 	public:
@@ -95,6 +100,11 @@ namespace tomoto
 
 		virtual ~DocumentBase() {}
 
+		virtual RawDoc::MiscType makeMisc(const ITopicModel*) const
+		{
+			return {};
+		}
+
 		virtual operator RawDoc() const
 		{
 			RawDoc raw{ *this };
@@ -110,6 +120,7 @@ namespace tomoto
 					raw.words[i] = words[wOrder[i]];
 				}
 			}
+			//raw.misc = makeMisc();
 			return raw;
 		}
 
@@ -212,6 +223,9 @@ namespace tomoto
 			const std::vector<uint8_t>* extra_data = nullptr) const = 0;
 		virtual void loadModel(std::istream& reader, 
 			std::vector<uint8_t>* extra_data = nullptr) = 0;
+
+		virtual std::unique_ptr<ITopicModel> copy() const = 0;
+
 		virtual const DocumentBase* getDoc(size_t docId) const = 0;
 		virtual size_t getDocIdByUid(const std::string& docUid) const = 0;
 
@@ -242,12 +256,12 @@ namespace tomoto
 		virtual void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0) = 0;
 		
 		virtual size_t getK() const = 0;
-		virtual std::vector<Float> getWidsByTopic(size_t tid) const = 0;
+		virtual std::vector<Float> getWidsByTopic(size_t tid, bool normalize = true) const = 0;
 		virtual std::vector<std::pair<std::string, Float>> getWordsByTopicSorted(size_t tid, size_t topN) const = 0;
 
 		virtual std::vector<std::pair<std::string, Float>> getWordsByDocSorted(const DocumentBase* doc, size_t topN) const = 0;
 		
-		virtual std::vector<Float> getTopicsByDoc(const DocumentBase* doc) const = 0;
+		virtual std::vector<Float> getTopicsByDoc(const DocumentBase* doc, bool normalize = true) const = 0;
 		virtual std::vector<std::pair<Tid, Float>> getTopicsByDocSorted(const DocumentBase* doc, size_t topN) const = 0;
 		virtual std::vector<double> infer(const std::vector<DocumentBase*>& docs, size_t maxIter, Float tolerance, size_t numWorkers, ParallelScheme ps, bool together) const = 0;
 		virtual ~ITopicModel() {}
@@ -308,7 +322,7 @@ namespace tomoto
 		size_t maxThreads[(size_t)ParallelScheme::size] = { 0, };
 		size_t minWordCf = 0, minWordDf = 0, removeTopN = 0;
 
-		std::unique_ptr<ThreadPool> cachedPool;
+		PreventCopy<std::unique_ptr<ThreadPool>> cachedPool;
 
 		void _saveModel(std::ostream& writer, bool fullModel, const std::vector<uint8_t>* extra_data) const
 		{
@@ -373,7 +387,7 @@ namespace tomoto
 		{
 			if (doc.words.empty()) return -1;
 			if (!doc.docUid.empty() && uidMap.count(doc.docUid))
-				throw exception::InvalidArgument{ "there is a document with uid = " + std::string{ doc.docUid } + " already." };
+				throw exc::InvalidArgument{ "there is a document with uid = '" + std::string{ doc.docUid } + "' already." };
 			size_t maxWid = *std::max_element(doc.words.begin(), doc.words.end());
 			if (vocabCf.size() <= maxWid)
 			{
@@ -383,7 +397,7 @@ namespace tomoto
 			for (auto w : doc.words) ++vocabCf[w];
 			std::unordered_set<Vid> uniq{ doc.words.begin(), doc.words.end() };
 			for (auto w : uniq) ++vocabDf[w];
-			uidMap.emplace(doc.docUid, docs.size());
+			if (!doc.docUid.empty()) uidMap.emplace(doc.docUid, docs.size());
 			docs.emplace_back(std::forward<_DocTy>(doc));
 			return docs.size() - 1;
 		}
@@ -415,7 +429,7 @@ namespace tomoto
 			}
 			else
 			{
-				throw std::invalid_argument{ "Either `words` or `rawWords` must be filled." };
+				throw exc::InvalidArgument{ "Either `words` or `rawWords` must be filled." };
 			}
 			return doc;
 		}
@@ -461,7 +475,19 @@ namespace tomoto
 			auto tx = [](_DocType& doc) { return &doc.words; };
 			tvector<Vid>::trade(words, 
 				makeTransformIter(docs.begin(), tx),
-				makeTransformIter(docs.end(), tx));
+				makeTransformIter(docs.end(), tx)
+			);
+		}
+
+		void updateForCopy()
+		{
+			size_t offset = 0;
+			for (auto& doc : docs)
+			{
+				size_t size = doc.words.size();
+				doc.words = tvector<Vid>{ words.data() + offset, size };
+				offset += size;
+			}
 		}
 
 		size_t countRealN() const
@@ -529,7 +555,7 @@ namespace tomoto
 			}
 		}
 
-		int restoreFromTrainingError(const exception::TrainingError& e, ThreadPool& pool, _ModelState* localData, _RandGen* rgs)
+		int restoreFromTrainingError(const exc::TrainingError& e, ThreadPool& pool, _ModelState* localData, _RandGen* rgs)
 		{
 			throw e;
 		}
@@ -539,6 +565,15 @@ namespace tomoto
 		{
 		}
 
+		TopicModel(const TopicModel&) = default;
+
+		std::unique_ptr<ITopicModel> copy() const override
+		{
+			auto ret = std::make_unique<_Derived>(*static_cast<const _Derived*>(this));
+			ret->updateForCopy();
+			return ret;
+		}
+
 		size_t getNumDocs() const override
 		{ 
 			return docs.size(); 
@@ -578,11 +613,11 @@ namespace tomoto
 				if ((_Flags & flags::shared_state)) return ParallelScheme::none;
 				return ParallelScheme::copy_merge;
 			case ParallelScheme::copy_merge:
-				if ((_Flags & flags::shared_state)) THROW_ERROR_WITH_INFO(exception::InvalidArgument, 
+				if ((_Flags & flags::shared_state)) THROW_ERROR_WITH_INFO(exc::InvalidArgument, 
 					std::string{ "This model doesn't provide ParallelScheme::" } + toString(ps));
 				break;
 			case ParallelScheme::partition:
-				if (!(_Flags & flags::partitioned_multisampling)) THROW_ERROR_WITH_INFO(exception::InvalidArgument,
+				if (!(_Flags & flags::partitioned_multisampling)) THROW_ERROR_WITH_INFO(exc::InvalidArgument,
 					std::string{ "This model doesn't provide ParallelScheme::" } + toString(ps));
 				break;
 			}
@@ -597,7 +632,7 @@ namespace tomoto
 			if (numWorkers == 1 || (_Flags & flags::shared_state)) ps = ParallelScheme::none;
 			if (!cachedPool || cachedPool->getNumWorkers() != numWorkers)
 			{
-				cachedPool = make_unique<ThreadPool>(numWorkers);
+				cachedPool = std::make_unique<ThreadPool>(numWorkers);
 			}
 
 			std::vector<_ModelState> localData;
@@ -647,7 +682,7 @@ namespace tomoto
 						}
 						break;
 					}
-					catch (const exception::TrainingError& e)
+					catch (const exc::TrainingError& e)
 					{
 						std::cerr << e.what() << std::endl;
 						int ret = static_cast<_Derived*>(this)->restoreFromTrainingError(
@@ -675,14 +710,14 @@ namespace tomoto
 			return 0;
 		}
 
-		std::vector<Float> getWidsByTopic(size_t tid) const override
+		std::vector<Float> getWidsByTopic(size_t tid, bool normalize) const override
 		{
-			return static_cast<const _Derived*>(this)->_getWidsByTopic(tid);
+			return static_cast<const _Derived*>(this)->_getWidsByTopic(tid, normalize);
 		}
 
 		std::vector<std::pair<Vid, Float>> getWidsByTopicSorted(size_t tid, size_t topN) const
 		{
-			return extractTopN<Vid>(static_cast<const _Derived*>(this)->_getWidsByTopic(tid), topN);
+			return extractTopN<Vid>(static_cast<const _Derived*>(this)->_getWidsByTopic(tid, true), topN);
 		}
 
 		std::vector<std::pair<std::string, Float>> vid2String(const std::vector<std::pair<Vid, Float>>& vids) const
@@ -716,7 +751,7 @@ namespace tomoto
 		double getDocLL(const DocumentBase* doc) const override
 		{
 			auto* p = dynamic_cast<const DocType*>(doc);
-			if (!p) throw std::invalid_argument{ "wrong `doc` type." };
+			if (!p) throw exc::InvalidArgument{ "wrong `doc` type." };
 			return static_cast<const _Derived*>(this)->getLLDocs(p, p + 1);
 		}
 
@@ -757,17 +792,17 @@ namespace tomoto
 					return static_cast<const _Derived*>(this)->template _infer<false, ParallelScheme::partition>(b, e, maxIter, tolerance, numWorkers);
 				}
 			}
-			THROW_ERROR_WITH_INFO(exception::InvalidArgument, "invalid ParallelScheme");
+			THROW_ERROR_WITH_INFO(exc::InvalidArgument, "invalid ParallelScheme");
 		}
 
-		std::vector<Float> getTopicsByDoc(const DocumentBase* doc) const override
+		std::vector<Float> getTopicsByDoc(const DocumentBase* doc, bool normalize) const override
 		{
-			return static_cast<const _Derived*>(this)->getTopicsByDoc(*static_cast<const DocType*>(doc));
+			return static_cast<const _Derived*>(this)->getTopicsByDoc(*static_cast<const DocType*>(doc), normalize);
 		}
 
 		std::vector<std::pair<Tid, Float>> getTopicsByDocSorted(const DocumentBase* doc, size_t topN) const override
 		{
-			return extractTopN<Tid>(getTopicsByDoc(doc), topN);
+			return extractTopN<Tid>(getTopicsByDoc(doc, true), topN);
 		}
 
 		const DocumentBase* getDoc(size_t docId) const override

data/vendor/tomotopy/src/Utils/AliasMethod.hpp

@@ -35,8 +35,8 @@ namespace tomoto
 				bitsize = o.bitsize;
 				if (msize)
 				{
-					arr = make_unique<_Precision[]>(1 << bitsize);
-					alias = make_unique<size_t[]>(1 << bitsize);
+					arr = std::make_unique<_Precision[]>(1 << bitsize);
+					alias = std::make_unique<size_t[]>(1 << bitsize);
 
 					std::copy(o.arr.get(), o.arr.get() + (1 << bitsize), arr.get());
 					std::copy(o.alias.get(), o.alias.get() + (1 << bitsize), alias.get());
@@ -70,7 +70,7 @@ namespace tomoto
 					sum += *it;
 				}
 
-				if (!std::isfinite(sum)) THROW_ERROR_WITH_INFO(exception::InvalidArgument, "cannot build NaN value distribution");
+				if (!std::isfinite(sum)) THROW_ERROR_WITH_INFO(exc::InvalidArgument, "cannot build NaN value distribution");
 
 				// ceil to power of 2
 				nbsize = log2_ceil(msize);
@@ -78,15 +78,15 @@ namespace tomoto
 
 				if (nbsize != bitsize)
 				{
-					arr = make_unique<_Precision[]>(psize);
+					arr = std::make_unique<_Precision[]>(psize);
 					std::fill(arr.get(), arr.get() + psize, 0);
-					alias = make_unique<size_t[]>(psize);
+					alias = std::make_unique<size_t[]>(psize);
 					bitsize = nbsize;
 				}
 				
 				sum /= psize;
 
-				auto f = make_unique<double[]>(psize);
+				auto f = std::make_unique<double[]>(psize);
 				auto pf = f.get();
 				for (auto it = first; it != last; ++it, ++pf)
 				{

data/vendor/tomotopy/src/Utils/Dictionary.h

@@ -13,6 +13,7 @@ namespace tomoto
 	using Vid = uint32_t;
 	static constexpr Vid non_vocab_id = (Vid)-1;
 	using Tid = uint16_t;
+	static constexpr Vid non_topic_id = (Tid)-1;
 	using Float = float;
 
 	struct VidPair : public std::pair<Vid, Vid>
@@ -101,6 +102,16 @@ namespace tomoto
 			}
 			return r;
 		}
+
+		std::vector<Vid> mapToNewDictAdd(const std::vector<Vid>& v, Dictionary& newDict) const
+		{
+			std::vector<Vid> r(v.size());
+			for (size_t i = 0; i < v.size(); ++i)
+			{
+				r[i] = mapToNewDict(v[i], newDict);
+			}
+			return r;
+		}
 	};
 
 }

data/vendor/tomotopy/src/Utils/SharedString.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <string>
+#include "serializer.hpp"
 
 namespace tomoto
 {
@@ -109,7 +110,7 @@ namespace tomoto
 
 		bool empty() const
 		{
-			return ptr == nullptr;
+			return ptr == nullptr || size() == 0;
 		}
 
 		operator std::string() const
@@ -167,6 +168,30 @@ namespace tomoto
 			return !operator==(o);
 		}
 	};
+
+	namespace serializer
+	{
+		template<>
+		struct Serializer<SharedString>
+		{
+			using VTy = SharedString;
+			void write(std::ostream& ostr, const VTy& v)
+			{
+				writeToStream(ostr, (uint32_t)v.size());
+				if (!ostr.write((const char*)v.data(), v.size()))
+					throw std::ios_base::failure(std::string("writing type 'SharedString' is failed"));
+			}
+
+			void read(std::istream& istr, VTy& v)
+			{
+				auto size = readFromStream<uint32_t>(istr);
+				std::vector<char> t(size);
+				if (!istr.read((char*)t.data(), t.size()))
+					throw std::ios_base::failure(std::string("reading type 'SharedString' is failed"));
+				v = SharedString{ t.data(), t.data() + t.size() };
+			}
+		};
+	}
 }
 
 namespace std

data/vendor/tomotopy/src/Utils/Trie.hpp

@@ -4,6 +4,7 @@
 #include <deque>
 #include <functional>
 #include <iterator>
+#include "serializer.hpp"
 
 namespace tomoto
 {
@@ -24,6 +25,17 @@ namespace tomoto
 			if (it == this->end()) return this->emplace(key, typename _Map::mapped_type{}).first->second;
 			else return it->second;
 		}
+
+		void serializerWrite(std::ostream& os) const
+		{
+			serializer::writeMany(os, (const _Map&)*this);
+		}
+
+		template<typename _Istr>
+		void serializerRead(_Istr& is)
+		{
+			serializer::readMany(is, (_Map&)*this);
+		}
 	};
 
 	template<class _Map, class _Node>
@@ -50,10 +62,13 @@ namespace tomoto
 	struct Trie
 	{
 		using Node = typename std::conditional<std::is_same<_Trie, void>::value, Trie, _Trie>::type;
+		using Key = _Key;
+		using KeyStore = _KeyStore;
 		using iterator = TrieIterator<_KeyStore, Node>;
 		_KeyStore next = {};
-		int32_t fail = 0;
 		_Value val = {};
+		int32_t fail = 0;
+		uint32_t depth = 0;
 
 		Trie() {}
 		~Trie() {}
@@ -84,13 +99,14 @@ namespace tomoto
 			if (first == last)
 			{
 				if (!val) val = _val;
-				return (Node*)this;
+				return this;
 			}
 
 			auto v = *first;
 			if (!getNext(v))
 			{
 				next[v] = alloc() - this;
+				getNext(v)->depth = depth + 1;
 			}
 			return getNext(v)->build(++first, last, _val, alloc);
 		}
@@ -104,50 +120,48 @@ namespace tomoto
 			return nullptr;
 		}
 
-		template<class _Func>
-		void traverse(_Func&& func)
+		template<typename _Fn>
+		void traverse_with_keys(_Fn&& fn, std::vector<_Key>& rkeys) const
 		{
-			if (val)
-			{
-				if (func(val)) return;
-			}
+			fn((Node*)this, rkeys);
+
 			for (auto& p : next)
 			{
-				if (getNext(p.first))
+				if (p.second)
 				{
-					getNext(p.first)->traverse(std::forward<_Func>(func));
+					rkeys.emplace_back(p.first);
+					getNext(p.first)->traverse_with_keys(fn, rkeys);
+					rkeys.pop_back();
 				}
 			}
-			return;
 		}
 
 		template<typename _Fn>
-		void traverse_with_keys(_Fn&& fn, std::vector<_Key>& rkeys)
+		void traverse_with_keys_post(_Fn&& fn, std::vector<_Key>& rkeys) const
 		{
-			fn((Node*)this, rkeys);
-
 			for (auto& p : next)
 			{
-				if (p.first)
+				if (p.second)
 				{
 					rkeys.emplace_back(p.first);
-					getNext(p.first)->traverse_with_keys(fn, rkeys);
+					getNext(p.first)->traverse_with_keys_post(fn, rkeys);
 					rkeys.pop_back();
 				}
 			}
+			fn((Node*)this, rkeys);
 		}
 
 		template<class _Iterator>
-		std::pair<_Value*, size_t> findMaximumMatch(_Iterator begin, _Iterator end, size_t idxCnt = 0)
+		std::pair<Node*, size_t> findMaximumMatch(_Iterator begin, _Iterator end, size_t idxCnt = 0) const
 		{
-			if (begin == end) return std::make_pair(val ? &val : nullptr, idxCnt);
+			if (begin == end) return std::make_pair((Node*)this, idxCnt);
 			auto n = getNext(*begin);
 			if (n)
 			{
 				auto v = n->findMaximumMatch(++begin, end, idxCnt + 1);
-				if (v.first) return v;
+				if (v.first->val) return v;
 			}
-			return std::make_pair(val ? &val : nullptr, idxCnt);
+			return std::make_pair((Node*)this, idxCnt);
 		}
 
 		Node* findFail(_Key i) const
@@ -172,7 +186,7 @@ namespace tomoto
 		void fillFail()
 		{
 			std::deque<Node*> dq;
-			for (dq.emplace_back(this); !dq.empty(); dq.pop_front())
+			for (dq.emplace_back((Node*)this); !dq.empty(); dq.pop_front())
 			{
 				auto p = dq.front();
 				for (auto&& kv : p->next)
@@ -196,6 +210,17 @@ namespace tomoto
 				}
 			}
 		}
+
+		void serializerWrite(std::ostream& os) const
+		{
+			serializer::writeMany(os, next, val, fail, depth);
+		}
+
+		template<typename _Istr>
+		void serializerRead(_Istr& is)
+		{
+			serializer::readMany(is, next, val, fail, depth);
+		}
 	};
 
 	template<class _Key, class _Value, class _KeyStore = ConstAccess<std::map<_Key, int32_t>>>