tomoto 0.1.2 → 0.1.3

data/vendor/tomotopy/src/Labeling/FoRelevance.h

@@ -16,12 +16,163 @@ namespace tomoto
 {
 	namespace label
 	{
+		template<typename _DocIter, typename _Freqs>
+		std::vector<Candidate> extractPMINgrams(_DocIter docBegin, _DocIter docEnd, 
+			_Freqs&& vocabFreqs, _Freqs&& vocabDf,
+			size_t candMinCnt, size_t candMinDf, size_t minNgrams, size_t maxNgrams, size_t maxCandidates, float minScore)
+		{
+			struct vvhash
+			{
+				size_t operator()(const std::pair<Vid, Vid>& k) const
+				{
+					return std::hash<Vid>{}(k.first) ^ std::hash<Vid>{}(k.second);
+				}
+			};
+
+			// counting unigrams & bigrams
+			std::unordered_map<std::pair<Vid, Vid>, size_t, vvhash> bigramCnt, bigramDf;
+
+			for(auto docIt = docBegin; docIt != docEnd; ++docIt)
+			{
+				std::unordered_set<std::pair<Vid, Vid>, vvhash> uniqBigram;
+				auto doc = *docIt;
+				Vid prevWord = doc[0];
+				for (size_t j = 1; j < doc.size(); ++j)
+				{
+					Vid curWord = doc[j];
+					if (curWord != non_vocab_id && vocabFreqs[curWord] >= candMinCnt && vocabDf[curWord] >= candMinDf)
+					{
+						if (prevWord != non_vocab_id && vocabFreqs[prevWord] >= candMinCnt && vocabDf[prevWord] >= candMinDf)
+						{
+							bigramCnt[std::make_pair(prevWord, curWord)]++;
+							uniqBigram.emplace(prevWord, curWord);
+						}
+					}
+					prevWord = curWord;
+				}
+
+				for (auto& p : uniqBigram) bigramDf[p]++;
+			}
+
+
+			// counting ngrams
+			std::vector<TrieEx<Vid, size_t>> trieNodes;
+
+			if (maxNgrams > 2)
+			{
+				std::unordered_set<std::pair<Vid, Vid>, vvhash> validPair;
+				for (auto& p : bigramCnt)
+				{
+					if (p.second >= candMinCnt) validPair.emplace(p.first);
+				}
+
+				trieNodes.resize(1);
+				auto allocNode = [&]() { return trieNodes.emplace_back(), & trieNodes.back(); };
+
+				for (auto docIt = docBegin; docIt != docEnd; ++docIt)
+				{
+					auto doc = *docIt;
+					if (trieNodes.capacity() < trieNodes.size() + doc.size() * maxNgrams)
+					{
+						trieNodes.reserve(std::max(trieNodes.size() + doc.size() * maxNgrams, trieNodes.capacity() * 2));
+					}
+
+					Vid prevWord = doc[0];
+					size_t labelLen = 0;
+					auto node = &trieNodes[0];
+					if (prevWord != non_vocab_id && vocabFreqs[prevWord] >= candMinCnt)
+					{
+						node = trieNodes[0].makeNext(prevWord, allocNode);
+						node->val++;
+						labelLen = 1;
+					}
+
+					for (size_t j = 1; j < doc.size(); ++j)
+					{
+						Vid curWord = doc[j];
+
+						if (curWord != non_vocab_id && vocabFreqs[curWord] < candMinCnt)
+						{
+							node = &trieNodes[0];
+							labelLen = 0;
+						}
+						else
+						{
+							if (labelLen >= maxNgrams)
+							{
+								node = node->getFail();
+								labelLen--;
+							}
+
+							if (validPair.count(std::make_pair(prevWord, curWord)))
+							{
+								auto nnode = node->makeNext(curWord, allocNode);
+								node = nnode;
+								do
+								{
+									nnode->val++;
+								} while (nnode = nnode->getFail());
+								labelLen++;
+							}
+							else
+							{
+								node = trieNodes[0].makeNext(curWord, allocNode);
+								node->val++;
+								labelLen = 1;
+							}
+						}
+						prevWord = curWord;
+					}
+				}
+			}
+
+			float totN = std::accumulate(vocabFreqs.begin(), vocabFreqs.end(), (size_t)0);
+
+			// calculating PMIs
+			std::vector<Candidate> candidates;
+			for (auto& p : bigramCnt)
+			{
+				auto& bigram = p.first;
+				if (p.second < candMinCnt) continue;
+				if (bigramDf[bigram] < candMinDf) continue;
+				auto pmi = std::log(p.second * totN
+					/ vocabFreqs[bigram.first] / vocabFreqs[bigram.second]);
+				if (pmi <= 0) continue;
+				candidates.emplace_back(pmi, bigram.first, bigram.second);
+			}
+
+			if (maxNgrams > 2)
+			{
+				std::vector<Vid> rkeys;
+				trieNodes[0].traverse_with_keys([&](const TrieEx<Vid, size_t>* node, const std::vector<Vid>& rkeys)
+				{
+					if (rkeys.size() <= 2 || rkeys.size() < minNgrams || node->val < candMinCnt) return;
+					auto pmi = node->val / totN;
+					for (auto k : rkeys)
+					{
+						pmi *= totN / vocabFreqs[k];
+					}
+					pmi = std::log(pmi);
+					if (pmi < minScore) return;
+					candidates.emplace_back(pmi, rkeys);
+				}, rkeys);
+			}
+
+			std::sort(candidates.begin(), candidates.end(), [](const Candidate& a, const Candidate& b)
+			{
+				return a.score > b.score;
+			});
+			if (candidates.size() > maxCandidates) candidates.erase(candidates.begin() + maxCandidates, candidates.end());
+			return candidates;
+		}
+
+
 		class PMIExtractor : public IExtractor
 		{
-			size_t candMinCnt, candMinDf, maxLabelLen, maxCandidates;
+			size_t candMinCnt, candMinDf, minLabelLen, maxLabelLen, maxCandidates;
 		public:
-			PMIExtractor(size_t _candMinCnt = 10, size_t _candMinDf = 2, size_t _maxLabelLen = 5, size_t _maxCandidates = 1000)
-				: candMinCnt{ _candMinCnt }, candMinDf{ _candMinDf }, maxLabelLen{ _maxLabelLen }, maxCandidates{ _maxCandidates }
+			PMIExtractor(size_t _candMinCnt = 10, size_t _candMinDf = 2, size_t _minLabelLen = 1, size_t _maxLabelLen = 5, size_t _maxCandidates = 1000)
+				: candMinCnt{ _candMinCnt }, candMinDf{ _candMinDf }, minLabelLen{ _minLabelLen}, maxLabelLen{ _maxLabelLen }, maxCandidates{ _maxCandidates }
 			{
 			}
 			
@@ -33,7 +184,7 @@ namespace tomoto
 			struct CandidateEx : public Candidate
 			{
 				std::unordered_map<std::string, size_t> names;
-				std::vector<size_t> docIds;
+				std::set<size_t> docIds;
 				Eigen::Array<Float, -1, 1> scores;
 
 				CandidateEx()
@@ -49,6 +200,7 @@ namespace tomoto
 			const ITopicModel* tm;
 			size_t candMinDf;
 			float smoothing, lambda, mu;
+			size_t windowSize;
 			std::unique_ptr<ThreadPool> pool;
 			std::unique_ptr<std::mutex[]> mtx;
 			std::vector<CandidateEx> candidates;
@@ -63,9 +215,10 @@ namespace tomoto
 			FoRelevance(const ITopicModel* _tm, 
 				_Iter candFirst, _Iter candEnd,
 				size_t _candMinDf = 2, float _smoothing = 0.1f, float _lambda = 0.1f, float _mu = 0.1f,
+				size_t _windowSize = (size_t)-1,
 				size_t numWorkers = 0)
 				: tm{ _tm }, candMinDf{ _candMinDf },
-				smoothing{ _smoothing }, lambda{ _lambda }, mu{ _mu }
+				smoothing{ _smoothing }, lambda{ _lambda }, mu{ _mu }, windowSize{ _windowSize }
 			{
 				if (!numWorkers) numWorkers = std::thread::hardware_concurrency();
 				if (numWorkers > 1)

data/vendor/tomotopy/src/TopicModel/DMR.h

@@ -8,7 +8,7 @@ namespace tomoto
 	{
 		using BaseDocument = DocumentLDA<_tw>;
 		using DocumentLDA<_tw>::DocumentLDA;
-		size_t metadata = 0;
+		uint64_t metadata = 0;
 
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 0, metadata);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseDocument, 1, 0x00010001, metadata);
@@ -23,21 +23,6 @@ namespace tomoto
 			size_t seed = std::random_device{}(),
 			bool scalarRng = false);
 
-		virtual size_t addDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) const = 0;
-		
-		virtual size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<std::string>& metadata) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<std::string>& metadata) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<std::string>& metadata) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<std::string>& metadata) const = 0;
-
 		virtual void setAlphaEps(Float _alphaEps) = 0;
 		virtual Float getAlphaEps() const = 0;
 		virtual void setOptimRepeat(size_t repeat) = 0;

data/vendor/tomotopy/src/TopicModel/DMRModel.hpp

@@ -262,63 +262,44 @@ namespace tomoto
 		}
 
 		template<bool _const = false>
-		_DocType& _updateDoc(_DocType& doc, const std::vector<std::string>& metadata)
+		_DocType& _updateDoc(_DocType& doc, const std::string& metadata)
 		{
-			std::string metadataJoined = text::join(metadata.begin(), metadata.end(), "_");
 			Vid xid;
 			if (_const)
 			{
-				xid = metadataDict.toWid(metadataJoined);
+				xid = metadataDict.toWid(metadata);
 				if (xid == (Vid)-1) throw std::invalid_argument("unknown metadata");
 			}
 			else
 			{
-				xid = metadataDict.add(metadataJoined);
+				xid = metadataDict.add(metadata);
 			}
 			doc.metadata = xid;
 			return doc;
 		}
 
-		size_t addDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) override
+		size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) override
 		{
-			auto doc = this->_makeDoc(words);
-			return this->_addDoc(_updateDoc(doc, metadata));
+			auto doc = this->template _makeFromRawDoc<false>(rawDoc, tokenizer);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMisc<std::string>("metadata")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) const override
 		{
-			auto doc = as_mutable(this)->template _makeDoc<true>(words);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, metadata));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc, tokenizer);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMisc<std::string>("metadata")));
 		}
 
-		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer, 
-			const std::vector<std::string>& metadata) override
+		size_t addDoc(const RawDoc& rawDoc) override
 		{
-			auto doc = this->template _makeRawDoc<false>(rawStr, tokenizer);
-			return this->_addDoc(_updateDoc(doc, metadata));
+			auto doc = this->_makeFromRawDoc(rawDoc);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMisc<std::string>("metadata")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer, 
-			const std::vector<std::string>& metadata) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc) const override
 		{
-			auto doc = as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, metadata));
-		}
-
-		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len, 
-			const std::vector<std::string>& metadata) override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return this->_addDoc(_updateDoc(doc, metadata));
-		}
-
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len, 
-			const std::vector<std::string>& metadata) const override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, metadata));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMisc<std::string>("metadata")));
 		}
 
 		GETTER(F, size_t, F);

data/vendor/tomotopy/src/TopicModel/DT.h

@@ -10,7 +10,7 @@ namespace tomoto
 		using BaseDocument = DocumentLDA<_tw>;
 		using DocumentLDA<_tw>::DocumentLDA;
 
-		size_t timepoint = 0;
+		uint64_t timepoint = 0;
 		ShareableVector<Float> eta;
 		sample::AliasMethod<> aliasTable;
 
@@ -29,21 +29,6 @@ namespace tomoto
 			size_t seed = std::random_device{}(),
 			bool scalarRng = false);
 		
-		virtual size_t addDoc(const std::vector<std::string>& words, size_t timepoint) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, size_t timepoint) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			size_t timepoint) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			size_t timepoint) const = 0;
-
-		virtual size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			size_t timepoint) = 0;
-		virtual std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			size_t timepoint) const = 0;
-
 		virtual size_t getT() const = 0;
 		virtual std::vector<uint32_t> getNumDocsByT() const = 0;
 

data/vendor/tomotopy/src/TopicModel/DTModel.hpp

@@ -468,7 +468,7 @@ namespace tomoto
 			return ret;
 		}
 
-		_DocType& _updateDoc(_DocType& doc, size_t timepoint) const
+		_DocType& _updateDoc(_DocType& doc, uint32_t timepoint) const
 		{
 			if (timepoint >= T) THROW_ERROR_WITH_INFO(exception::InvalidArgument, "timepoint must < T");
 			doc.timepoint = timepoint;
@@ -512,51 +512,34 @@ namespace tomoto
 		{
 		}
 
-		size_t addDoc(const std::vector<std::string>& words, size_t timepoint) override
+		size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) override
 		{
-			auto doc = this->_makeDoc(words);
-			return this->_addDoc(_updateDoc(doc, timepoint));
+			auto doc = this->template _makeFromRawDoc<false>(rawDoc, tokenizer);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMisc<uint32_t>("timepoint")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, size_t timepoint) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) const override
 		{
-			auto doc = as_mutable(this)->template _makeDoc<true>(words);
-			return make_unique<_DocType>(_updateDoc(doc, timepoint));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc, tokenizer);
+			return make_unique<_DocType>(_updateDoc(doc, rawDoc.template getMisc<uint32_t>("timepoint")));
 		}
 
-		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			size_t timepoint) override
+		size_t addDoc(const RawDoc& rawDoc) override
 		{
-			auto doc = this->template _makeRawDoc<false>(rawStr, tokenizer);
-			return this->_addDoc(_updateDoc(doc, timepoint));
+			auto doc = this->_makeFromRawDoc(rawDoc);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMisc<uint32_t>("timepoint")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			size_t timepoint) const override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc) const override
 		{
-			auto doc = as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer);
-			return make_unique<_DocType>(_updateDoc(doc, timepoint));
-		}
-
-		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			size_t timepoint) override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return this->_addDoc(_updateDoc(doc, timepoint));
-		}
-
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			size_t timepoint) const override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return make_unique<_DocType>(_updateDoc(doc, timepoint));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc);
+			return make_unique<_DocType>(_updateDoc(doc, rawDoc.template getMisc<uint32_t>("timepoint")));
 		}
 
 		Float getAlpha(size_t k, size_t t) const override
 		{
-			return alphas(k, t);
+			if (alphas.size()) return alphas(k, t);
+			return 0;
 		}
 
 		std::vector<Float> getPhi(size_t k, size_t t) const override

data/vendor/tomotopy/src/TopicModel/GDMRModel.hpp

@@ -353,58 +353,36 @@ namespace tomoto
 		}
 
 		template<bool _const = false>
-		_DocType& _updateDoc(_DocType& doc, const std::vector<std::string>& metadata) const
+		_DocType& _updateDoc(_DocType& doc, const std::vector<Float>& metadata) const
 		{
 			if (metadata.size() != degreeByF.size()) 
 				throw std::invalid_argument{ "a length of `metadata` should be equal to a length of `degrees`" };
-
-			std::transform(metadata.begin(), metadata.end(), back_inserter(doc.metadataOrg), [](const std::string& w)
-			{
-				return std::stof(w);
-			});
+			doc.metadataOrg = metadata;
 			return doc;
 		}
 
-		size_t addDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) override
-		{
-			auto doc = this->_makeDoc(words);
-			return this->_addDoc(_updateDoc(doc, metadata));
-		}
-
-		std::unique_ptr<DocumentBase> makeDoc(const std::vector<std::string>& words, const std::vector<std::string>& metadata) const override
+		size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) override
 		{
-			auto doc = as_mutable(this)->template _makeDoc<true>(words);
-			return make_unique<_DocType>(_updateDoc(doc, metadata));
+			auto doc = this->template _makeFromRawDoc<false>(rawDoc, tokenizer);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMisc<std::vector<Float>>("metadata")));
 		}
 
-		size_t addDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<std::string>& metadata) override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) const override
 		{
-			auto doc = this->template _makeRawDoc<false>(rawStr, tokenizer);
-			return this->_addDoc(_updateDoc(doc, metadata));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc, tokenizer);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMisc<std::vector<Float>>("metadata")));
 		}
 
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const RawDocTokenizer::Factory& tokenizer,
-			const std::vector<std::string>& metadata) const override
+		size_t addDoc(const RawDoc& rawDoc) override
 		{
-			auto doc = as_mutable(this)->template _makeRawDoc<true>(rawStr, tokenizer);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, metadata));
+			auto doc = this->_makeFromRawDoc(rawDoc);
+			return this->_addDoc(_updateDoc(doc, rawDoc.template getMisc<std::vector<Float>>("metadata")));
 		}
 
-		size_t addDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<std::string>& metadata) override
+		std::unique_ptr<DocumentBase> makeDoc(const RawDoc& rawDoc) const override
 		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return this->_addDoc(_updateDoc(doc, metadata));
-		}
-
-		std::unique_ptr<DocumentBase> makeDoc(const std::string& rawStr, const std::vector<Vid>& words,
-			const std::vector<uint32_t>& pos, const std::vector<uint16_t>& len,
-			const std::vector<std::string>& metadata) const override
-		{
-			auto doc = this->_makeRawDoc(rawStr, words, pos, len);
-			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, metadata));
+			auto doc = as_mutable(this)->template _makeFromRawDoc<true>(rawDoc);
+			return make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMisc<std::vector<Float>>("metadata")));
 		}
 
 		std::vector<Float> getTopicsByDoc(const _DocType& doc) const
@@ -428,7 +406,10 @@ namespace tomoto
 		std::vector<Float> getLambdaByTopic(Tid tid) const override
 		{
 			std::vector<Float> ret(this->F);
-			for (size_t f = 0; f < this->F; ++f) ret[f] = this->lambda.row(tid)[f];
+			if (this->lambda.size())
+			{
+				Eigen::Map<Eigen::Matrix<Float, -1, 1>>{ ret.data(), (Eigen::Index)ret.size() } = this->lambda.row(tid);
+			}
 			return ret;
 		}