tomoto 0.2.2 → 0.3.0

data/vendor/tomotopy/src/TopicModel/LDAModel.hpp

@@ -117,21 +117,36 @@ namespace tomoto
 	template<>
 	struct TwId<TermWeight::one>
 	{
-		static constexpr char TWID[] = "one\0";
+		static constexpr auto twid()
+		{
+			return serializer::to_key("one\0");
+		}
 	};
 
 	template<>
 	struct TwId<TermWeight::idf>
 	{
-		static constexpr char TWID[] = "idf\0";
+		static constexpr auto twid()
+		{
+			return serializer::to_key("idf\0");
+		}
 	};
 
 	template<>
 	struct TwId<TermWeight::pmi>
 	{
-		static constexpr char TWID[] = "pmi\0";
+		static constexpr auto twid()
+		{
+			return serializer::to_key("pmi\0");
+		}
 	};
 
+	inline Float floorBit(Float x, int bitsUnderPoint = 8)
+	{
+		Float s = (1 << bitsUnderPoint);
+		return floor(x * s) / s;
+	}
+
 	// to make HDP friend of LDA for HDPModel::converToLDA
 	template<TermWeight _tw,
 		typename _RandGen,
@@ -169,7 +184,11 @@ namespace tomoto
 			typename>
 		friend class HDPModel;
 
-		static constexpr char TMID[] = "LDA\0";
+		static constexpr auto tmid()
+		{
+			return serializer::to_key("LDA\0");
+		}
+
 		using WeightType = typename std::conditional<_tw == TermWeight::one, int32_t, float>::type;
 
 		enum { m_flags = _Flags };
@@ -189,7 +208,7 @@ namespace tomoto
 		struct ExtraDocData
 		{
 			std::vector<Vid> vChunkOffset;
-			Eigen::Matrix<uint32_t, -1, -1> chunkOffsetByDoc;
+			Eigen::Matrix<size_t, -1, -1> chunkOffsetByDoc;
 		};
 
 		ExtraDocData eddTrain;
@@ -261,7 +280,7 @@ namespace tomoto
 		}
 
 		template<int _inc>
-		inline void addWordTo(_ModelState& ld, _DocType& doc, uint32_t pid, Vid vid, Tid tid) const
+		inline void addWordTo(_ModelState& ld, _DocType& doc, size_t pid, Vid vid, Tid tid) const
 		{
 			assert(tid < K);
 			assert(vid < this->realV);
@@ -469,11 +488,11 @@ namespace tomoto
 		{
 			if (_ps == ParallelScheme::partition)
 			{
-				return (this->realV + 3) / 4;
+				return std::max(((size_t)this->realV + 3) / 4, (size_t)1);
 			}
 			if (_ps == ParallelScheme::copy_merge)
 			{
-				return (this->docs.size() + 1) / 2;
+				return std::max((this->docs.size() + 1) / 2, (size_t)1);
 			}
 			return (size_t)-1;
 		}
@@ -620,7 +639,7 @@ namespace tomoto
 					for (Vid v = 0; v < V; ++v)
 					{
 						if (!ld.numByTopicWord(k, v)) continue;
-						ll += math::lgammaT(ld.numByTopicWord(k, v) + etaByTopicWord(v, k)) - math::lgammaT(etaByTopicWord(v, k));
+						ll += math::lgammaT(ld.numByTopicWord(k, v) + etaByTopicWord(k, v)) - math::lgammaT(etaByTopicWord(k, v));
 						assert(std::isfinite(ll));
 					}
 				}
@@ -972,12 +991,14 @@ namespace tomoto
 
 		void setOptimInterval(size_t _optimInterval) override
 		{
-			optimInterval = _optimInterval;
+			if (_optimInterval > 0x7FFFFFFF) THROW_ERROR_WITH_INFO(exc::InvalidArgument, "wrong value");
+			optimInterval = (uint32_t)_optimInterval;
 		}
 
 		void setBurnInIteration(size_t iteration) override
 		{
-			burnIn = iteration;
+			if (iteration > 0x7FFFFFFF) THROW_ERROR_WITH_INFO(exc::InvalidArgument, "wrong value");
+			burnIn = (uint32_t)iteration;
 		}
 
 		size_t addDoc(const RawDoc& rawDoc, const RawDocTokenizer::Factory& tokenizer) override
@@ -1008,6 +1029,11 @@ namespace tomoto
 				if (p < 0) THROW_ERROR_WITH_INFO(exc::InvalidArgument, "priors must not be less than 0.");
 			}
 			this->dict.add(word);
+			if (this->dict.size() > this->vocabCf.size())
+			{
+				this->vocabCf.resize(this->dict.size());
+				this->vocabDf.resize(this->dict.size());
+			}
 			etaByWord.emplace(word, priors);
 		}
 
@@ -1037,19 +1063,23 @@ namespace tomoto
 			}
 		}
 
-		void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0) override
+		void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0, bool updateStopwords = true) override
 		{
-			if (initDocs) this->removeStopwords(minWordCnt, minWordDf, removeTopN);
+			if (initDocs && updateStopwords) this->removeStopwords(minWordCnt, minWordDf, removeTopN);
 			static_cast<DerivedClass*>(this)->updateWeakArray();
 			static_cast<DerivedClass*>(this)->initGlobalState(initDocs);
 			static_cast<DerivedClass*>(this)->prepareWordPriors();
 
 			const size_t V = this->realV;
+			if (V == 0) 
+			{
+				std::cerr << "[warn] No valid vocabs in the model!" << std::endl;
+			}
 
 			if (initDocs)
 			{
 				std::vector<uint32_t> df, cf, tf;
-				uint32_t totCf;
+				size_t totCf;
 
 				// calculate weighting
 				if (_tw != TermWeight::one)
@@ -1064,14 +1094,14 @@ namespace tomoto
 							++df[w];
 						}
 					}
-					totCf = accumulate(this->vocabCf.begin(), this->vocabCf.end(), 0);
+					totCf = std::accumulate(this->vocabCf.begin(), this->vocabCf.end(), 0);
 				}
 				if (_tw == TermWeight::idf)
 				{
 					vocabWeights.resize(V);
 					for (size_t i = 0; i < V; ++i)
 					{
-						vocabWeights[i] = log(this->docs.size() / (Float)df[i]);
+						vocabWeights[i] = (Float)log(this->docs.size() / (double)df[i]);
 					}
 				}
 				else if (_tw == TermWeight::pmi)
@@ -1079,7 +1109,7 @@ namespace tomoto
 					vocabWeights.resize(V);
 					for (size_t i = 0; i < V; ++i)
 					{
-						vocabWeights[i] = this->vocabCf[i] / (float)totCf;
+						vocabWeights[i] = (Float)(this->vocabCf[i] / (double)totCf);
 					}
 				}
 
@@ -1096,7 +1126,7 @@ namespace tomoto
 				for (auto& doc : this->docs) doc.updateSumWordWeight(this->realV);
 			}
 			static_cast<DerivedClass*>(this)->prepareShared();
-			BaseClass::prepare(initDocs, minWordCnt, minWordDf, removeTopN);
+			BaseClass::prepare(initDocs, minWordCnt, minWordDf, removeTopN, updateStopwords);
 		}
 
 		std::vector<uint64_t> getCountByTopic() const override
@@ -1104,8 +1134,9 @@ namespace tomoto
 			return static_cast<const DerivedClass*>(this)->_getTopicsCount();
 		}
 
-		std::vector<Float> getTopicsByDoc(const _DocType& doc, bool normalize) const
+		std::vector<Float> _getTopicsByDoc(const _DocType& doc, bool normalize) const
 		{
+			if (!doc.numByTopic.size()) return {};
 			std::vector<Float> ret(K);
 			Eigen::Map<Eigen::Array<Float, -1, 1>> m{ ret.data(), K };
 			if (normalize)

data/vendor/tomotopy/src/TopicModel/LLDAModel.hpp

@@ -26,7 +26,10 @@ namespace tomoto
 		friend typename BaseClass::BaseClass;
 		using WeightType = typename BaseClass::WeightType;
 
-		static constexpr char TMID[] = "LLDA";
+		static constexpr auto tmid()
+		{
+			return serializer::to_key("LLDA");
+		}
 
 		Dictionary topicLabelDict;
 
@@ -171,8 +174,9 @@ namespace tomoto
 			return std::make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
-		std::vector<Float> getTopicsByDoc(const _DocType& doc, bool normalize) const
+		std::vector<Float> _getTopicsByDoc(const _DocType& doc, bool normalize) const
 		{
+			if (!doc.numByTopic.size()) return {};
 			std::vector<Float> ret(this->K);
 			auto maskedAlphas = this->alphas.array() * doc.labelMask.template cast<Float>().array();
 			Eigen::Map<Eigen::Array<Float, -1, 1>> m{ ret.data(), this->K };

data/vendor/tomotopy/src/TopicModel/MGLDAModel.hpp

@@ -63,7 +63,7 @@ namespace tomoto
 		}
 
 		template<int _inc> 
-		inline void addWordTo(_ModelState& ld, _DocType& doc, uint32_t pid, Vid vid, Tid tid, uint16_t s, uint8_t w, uint8_t r) const
+		inline void addWordTo(_ModelState& ld, _DocType& doc, size_t pid, Vid vid, Tid tid, uint16_t s, uint8_t w, uint8_t r) const
 		{
 			const auto K = this->K;
 
@@ -527,8 +527,9 @@ namespace tomoto
 			this->etaByWord.emplace(word, priors);
 		}
 
-		std::vector<Float> getTopicsByDoc(const _DocType& doc, bool normalize) const
+		std::vector<Float> _getTopicsByDoc(const _DocType& doc, bool normalize) const
 		{
+			if (!doc.numByTopic.size()) return {};
 			std::vector<Float> ret(this->K + KL);
 			Eigen::Map<Eigen::Array<Float, -1, 1>> m{ ret.data(), this->K + KL };
 			if (normalize)

data/vendor/tomotopy/src/TopicModel/PAModel.hpp

@@ -90,7 +90,7 @@ namespace tomoto
 		}
 
 		template<int _inc> 
-		inline void addWordTo(_ModelState& ld, _DocType& doc, uint32_t pid, Vid vid, Tid z1, Tid z2) const
+		inline void addWordTo(_ModelState& ld, _DocType& doc, size_t pid, Vid vid, Tid z1, Tid z2) const
 		{
 			assert(vid < this->realV);
 			constexpr bool _dec = _inc < 0 && _tw != TermWeight::one;

data/vendor/tomotopy/src/TopicModel/PLDAModel.hpp

@@ -26,7 +26,10 @@ namespace tomoto
 		friend typename BaseClass::BaseClass;
 		using WeightType = typename BaseClass::WeightType;
 
-		static constexpr char TMID[] = "PLDA";
+		static constexpr auto tmid()
+		{
+			return serializer::to_key("PLDA");
+		}
 
 		Dictionary topicLabelDict;
 
@@ -178,8 +181,9 @@ namespace tomoto
 			return std::make_unique<_DocType>(as_mutable(this)->template _updateDoc<true>(doc, rawDoc.template getMiscDefault<std::vector<std::string>>("labels")));
 		}
 
-		std::vector<Float> getTopicsByDoc(const _DocType& doc, bool normalize) const
+		std::vector<Float> _getTopicsByDoc(const _DocType& doc, bool normalize) const
 		{
+			if (!doc.numByTopic.size()) return {};
 			std::vector<Float> ret(this->K);
 			auto maskedAlphas = this->alphas.array() * doc.labelMask.template cast<Float>().array();
 			Eigen::Map<Eigen::Array<Float, -1, 1>> m{ ret.data(), this->K };

data/vendor/tomotopy/src/TopicModel/PT.h

@@ -18,7 +18,7 @@ namespace tomoto
 
 	struct PTArgs : public LDAArgs
 	{
-		size_t p = 100;
+		size_t p = 0;
 		Float lambda = 0.01;
 	};
 
@@ -30,5 +30,7 @@ namespace tomoto
 			bool scalarRng = false);
 
 		virtual size_t getP() const = 0;
+		virtual std::vector<Float> getTopicsFromPseudoDoc(const DocumentBase* doc, bool normalize = true) const = 0;
+		virtual std::vector<std::pair<Tid, Float>> getTopicsFromPseudoDocSorted(const DocumentBase* doc, size_t topN) const = 0;
 	};
 }

data/vendor/tomotopy/src/TopicModel/PTModel.hpp

@@ -38,7 +38,10 @@ namespace tomoto
 		friend typename BaseClass::BaseClass;
 		using WeightType = typename BaseClass::WeightType;
 
-		static constexpr char TMID[] = "PTM";
+		static constexpr auto tmid()
+		{
+			return serializer::to_key("PTM");
+		}
 
 		uint64_t numPDocs;
 		Float lambda;
@@ -72,7 +75,7 @@ namespace tomoto
 			}
 			--ld.numDocsByPDoc[doc.pseudoDoc];
 			
-			if (pool)
+			if (pool && pool->getNumWorkers() > 1)
 			{
 				std::vector<std::future<void>> futures;
 				for (size_t w = 0; w < pool->getNumWorkers(); ++w)
@@ -250,6 +253,16 @@ namespace tomoto
 			}
 		}
 
+		void updateForCopy()
+		{
+			BaseClass::updateForCopy();
+			size_t offset = 0;
+			for (auto& doc : this->docs)
+			{
+				doc.template update<>(this->globalState.numByTopicPDoc.col(doc.pseudoDoc).data(), *static_cast<DerivedClass*>(this));
+			}
+		}
+
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, numPDocs, lambda);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, numPDocs, lambda);
@@ -261,8 +274,9 @@ namespace tomoto
 		{
 		}
 
-		std::vector<Float> getTopicsByDoc(const _DocType& doc, bool normalize) const
+		std::vector<Float> _getTopicsByDoc(const _DocType& doc, bool normalize) const
 		{
+			if (doc.Zs.empty()) return {};
 			std::vector<Float> ret(this->K);
 			Eigen::Map<Eigen::Array<Float, -1, 1>> m{ ret.data(), this->K };
 			m = this->alphas.array();
@@ -277,6 +291,25 @@ namespace tomoto
 			return ret;
 		}
 
+		std::vector<Float> getTopicsFromPseudoDoc(const DocumentBase* _doc, bool normalize) const override
+		{
+			auto& doc = *static_cast<const _DocType*>(_doc);
+			if (!doc.numByTopic.size()) return {};
+			std::vector<Float> ret(this->K);
+			Eigen::Map<Eigen::Array<Float, -1, 1>> m{ ret.data(), this->K };
+			m = doc.numByTopic.array().template cast<Float>() + this->alphas.array();
+			if (normalize)
+			{
+				m /= m.sum();
+			}
+			return ret;
+		}
+
+		std::vector<std::pair<Tid, Float>> getTopicsFromPseudoDocSorted(const DocumentBase* doc, size_t topN) const override
+		{
+			return extractTopN<Tid>(getTopicsFromPseudoDoc(doc, true), topN);
+		}
+
 		void updateDocs()
 		{
 			for (auto& doc : this->docs)

data/vendor/tomotopy/src/TopicModel/SLDAModel.hpp

@@ -216,7 +216,10 @@ namespace tomoto
 		friend typename BaseClass::BaseClass;
 		using WeightType = typename BaseClass::WeightType;
 
-		static constexpr char TMID[] = "SLDA";
+		static constexpr auto tmid()
+		{
+			return serializer::to_key("SLDA");
+		}
 
 		uint64_t F; // number of response variables
 		std::vector<ISLDAModel::GLM> varTypes;
@@ -376,11 +379,11 @@ namespace tomoto
 			{
 				nuSq = Vector::Ones(F);
 			}
-			else if (args.mu.size() == 1)
+			else if (args.nuSq.size() == 1)
 			{
 				nuSq = Vector::Constant(F, args.nuSq[0]);
 			}
-			else if (args.mu.size() == F)
+			else if (args.nuSq.size() == F)
 			{
 				nuSq = Eigen::Map<const Vector>(args.nuSq.data(), args.nuSq.size());
 			}

data/vendor/tomotopy/src/TopicModel/TopicModel.hpp

@@ -13,7 +13,7 @@
 
 namespace tomoto
 {
-	using RandGen = Eigen::Rand::P8_mt19937_64<uint32_t>;
+	using RandGen = Eigen::Rand::P8_mt19937_64_32;
 	using ScalarRandGen = Eigen::Rand::UniversalRandomEngine<uint32_t, std::mt19937_64>;
 
 	using Vector = Eigen::Matrix<Float, -1, 1>;
@@ -249,11 +249,12 @@ namespace tomoto
 		virtual size_t getNumDocs() const = 0;
 		virtual const Dictionary& getVocabDict() const = 0;
 		virtual const std::vector<uint64_t>& getVocabCf() const = 0;
+		virtual std::vector<double> getVocabWeightedCf() const = 0;
 		virtual const std::vector<uint64_t>& getVocabDf() const = 0;
 
 		virtual int train(size_t iteration, size_t numWorkers, ParallelScheme ps = ParallelScheme::default_, bool freeze_topics = false) = 0;
 		virtual size_t getGlobalStep() const = 0;
-		virtual void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0) = 0;
+		virtual void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0, bool updateStopwords = true) = 0;
 		
 		virtual size_t getK() const = 0;
 		virtual std::vector<Float> getWidsByTopic(size_t tid, bool normalize = true) const = 0;
@@ -319,6 +320,7 @@ namespace tomoto
 		Dictionary dict;
 		uint64_t realV = 0; // vocab size after removing stopwords
 		uint64_t realN = 0; // total word size after removing stopwords
+		double weightedN = 0;
 		size_t maxThreads[(size_t)ParallelScheme::size] = { 0, };
 		size_t minWordCf = 0, minWordDf = 0, removeTopN = 0;
 
@@ -327,15 +329,17 @@ namespace tomoto
 		void _saveModel(std::ostream& writer, bool fullModel, const std::vector<uint8_t>* extra_data) const
 		{
 			serializer::writeMany(writer,
-				serializer::to_keyz(static_cast<const _Derived*>(this)->TMID),
-				serializer::to_keyz(static_cast<const _Derived*>(this)->TWID));
+				serializer::to_keyz(static_cast<const _Derived*>(this)->tmid()),
+				serializer::to_keyz(static_cast<const _Derived*>(this)->twid())
+			);
 			serializer::writeTaggedMany(writer, 0x00010001,
 				serializer::to_keyz("dict"), dict, 
 				serializer::to_keyz("vocabCf"), vocabCf,
 				serializer::to_keyz("vocabDf"), vocabDf,
 				serializer::to_keyz("realV"), realV,
 				serializer::to_keyz("globalStep"), globalStep,
-				serializer::to_keyz("extra"), extra_data ? *extra_data : std::vector<uint8_t>(0));
+				serializer::to_keyz("extra"), extra_data ? *extra_data : std::vector<uint8_t>(0)
+			);
 			serializer::writeMany(writer, *static_cast<const _Derived*>(this));
 			globalState.serializerWrite(writer);
 			if (fullModel)
@@ -355,8 +359,9 @@ namespace tomoto
 			{
 				std::vector<uint8_t> extra;
 				serializer::readMany(reader, 
-					serializer::to_keyz(static_cast<_Derived*>(this)->TMID),
-					serializer::to_keyz(static_cast<_Derived*>(this)->TWID));
+					serializer::to_keyz(static_cast<_Derived*>(this)->tmid()),
+					serializer::to_keyz(static_cast<_Derived*>(this)->twid())
+				);
 				serializer::readTaggedMany(reader, 0x00010001, 
 					serializer::to_keyz("dict"), dict,
 					serializer::to_keyz("vocabCf"), vocabCf,
@@ -370,14 +375,17 @@ namespace tomoto
 			{
 				reader.seekg(start_pos);
 				serializer::readMany(reader,
-					serializer::to_key(static_cast<_Derived*>(this)->TMID),
-					serializer::to_key(static_cast<_Derived*>(this)->TWID),
-					dict, vocabCf, realV);
+					serializer::to_key(static_cast<_Derived*>(this)->tmid()),
+					serializer::to_key(static_cast<_Derived*>(this)->twid()),
+					dict, vocabCf, realV
+				);
 			}
 			serializer::readMany(reader, *static_cast<_Derived*>(this));
 			globalState.serializerRead(reader);
 			serializer::readMany(reader, docs);
-			realN = countRealN();
+			auto p = countRealN();
+			realN = p.first;
+			weightedN = p.second;
 		}
 
 		template<typename _DocTy>
@@ -429,7 +437,7 @@ namespace tomoto
 			}
 			else
 			{
-				throw exc::InvalidArgument{ "Either `words` or `rawWords` must be filled." };
+				throw exc::EmptyWordArgument{ "Either `words` or `rawWords` must be filled." };
 			}
 			return doc;
 		}
@@ -490,17 +498,23 @@ namespace tomoto
 			}
 		}
 
-		size_t countRealN() const
+		std::pair<size_t, double> countRealN() const
 		{
 			size_t n = 0;
+			double weighted = 0;
 			for (auto& doc : docs)
 			{
-				for (auto& w : doc.words)
+				for (size_t i = 0; i < doc.words.size(); ++i)
 				{
-					if (w < realV) ++n;
+					auto w = doc.words[i];
+					if (w < realV)
+					{
+						++n;
+						weighted += doc.wordWeights.empty() ? 1 : doc.wordWeights[i];
+					}
 				}
 			}
-			return n;
+			return std::make_pair(n, weighted);
 		}
 
 		void removeStopwords(size_t minWordCnt, size_t minWordDf, size_t removeTopN)
@@ -544,14 +558,9 @@ namespace tomoto
 			}
 
 			dict.reorder(order);
-			realN = 0;
 			for (auto& doc : docs)
 			{
-				for (auto& w : doc.words)
-				{
-					w = order[w];
-					if (w < realV) ++realN;
-				}
+				for (auto& w : doc.words) w = order[w];
 			}
 		}
 
@@ -596,8 +605,12 @@ namespace tomoto
 			return empty;
 		}
 
-		void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0) override
+		void prepare(bool initDocs = true, size_t minWordCnt = 0, size_t minWordDf = 0, size_t removeTopN = 0, bool updateStopwords = true) override
 		{
+			auto p = countRealN();
+			realN = p.first;
+			weightedN = p.second;
+
 			maxThreads[(size_t)ParallelScheme::default_] = -1;
 			maxThreads[(size_t)ParallelScheme::none] = -1;
 			maxThreads[(size_t)ParallelScheme::copy_merge] = static_cast<_Derived*>(this)->template estimateMaxThreads<ParallelScheme::copy_merge>();
@@ -661,7 +674,8 @@ namespace tomoto
 			auto state = ps == ParallelScheme::none ? &globalState : localData.data();
 			for (size_t i = 0; i < iteration; ++i)
 			{
-				while (1)
+				size_t retry;
+				for (retry = 0; retry < 10; ++retry)
 				{
 					try
 					{
@@ -690,6 +704,7 @@ namespace tomoto
 						if(ret < 0) return ret;
 					}
 				}
+				if (retry >= 10) return -1;
 				++globalStep;
 			}
 			return 0;
@@ -697,7 +712,7 @@ namespace tomoto
 
 		double getLLPerWord() const override
 		{
-			return words.empty() ? 0 : static_cast<const _Derived*>(this)->getLL() / realN;
+			return words.empty() ? 0 : static_cast<const _Derived*>(this)->getLL() / weightedN;
 		}
 
 		double getPerplexity() const override
@@ -797,7 +812,7 @@ namespace tomoto
 
 		std::vector<Float> getTopicsByDoc(const DocumentBase* doc, bool normalize) const override
 		{
-			return static_cast<const _Derived*>(this)->getTopicsByDoc(*static_cast<const DocType*>(doc), normalize);
+			return static_cast<const _Derived*>(this)->_getTopicsByDoc(*static_cast<const DocType*>(doc), normalize);
 		}
 
 		std::vector<std::pair<Tid, Float>> getTopicsByDocSorted(const DocumentBase* doc, size_t topN) const override
@@ -832,6 +847,20 @@ namespace tomoto
 			return vocabCf;
 		}
 
+		std::vector<double> getVocabWeightedCf() const override
+		{
+			std::vector<double> ret(realV);
+			for (auto& doc : docs)
+			{
+				for (size_t i = 0; i < doc.words.size(); ++i)
+				{
+					if (doc.words[i] >= realV) continue;
+					ret[doc.words[i]] += doc.wordWeights.empty() ? 1 : doc.wordWeights[i];
+				}
+			}
+			return ret;
+		}
+
 		const std::vector<uint64_t>& getVocabDf() const override
 		{
 			return vocabDf;

data/vendor/tomotopy/src/Utils/AliasMethod.hpp

@@ -35,11 +35,12 @@ namespace tomoto
 				bitsize = o.bitsize;
 				if (msize)
 				{
-					arr = std::make_unique<_Precision[]>(1 << bitsize);
-					alias = std::make_unique<size_t[]>(1 << bitsize);
+					size_t n = (size_t)1 << bitsize;
+					arr = std::make_unique<_Precision[]>(n);
+					alias = std::make_unique<size_t[]>(n);
 
-					std::copy(o.arr.get(), o.arr.get() + (1 << bitsize), arr.get());
-					std::copy(o.alias.get(), o.alias.get() + (1 << bitsize), alias.get());
+					std::copy(o.arr.get(), o.arr.get() + n, arr.get());
+					std::copy(o.alias.get(), o.alias.get() + n, alias.get());
 				}
 				return *this;
 			}

data/vendor/tomotopy/src/Utils/Dictionary.h

@@ -32,7 +32,7 @@ namespace tomoto
 			auto it = dict.find(word);
 			if (it == dict.end())
 			{
-				dict.emplace(std::make_pair(word, dict.size()));
+				dict.emplace(word, (Vid)dict.size());
 				id2word.emplace_back(word);
 				return (Vid)(dict.size() - 1);
 			}
@@ -64,7 +64,7 @@ namespace tomoto
 			serializer::readMany(reader, serializer::to_key("Dict"), id2word);
 			for (size_t i = 0; i < id2word.size(); ++i)
 			{
-				dict.emplace(id2word[i], i);
+				dict.emplace(id2word[i], (Vid)i);
 			}
 		}
 

data/vendor/tomotopy/src/Utils/EigenAddonOps.hpp

@@ -56,7 +56,7 @@ namespace Eigen
 	}
 }
 #endif
-#if defined(EIGEN_VECTORIZE_SSE2)
+#ifdef EIGEN_VECTORIZE_SSE2
 #include <xmmintrin.h>
 #include "sse_gamma.h"
 
@@ -91,6 +91,41 @@ namespace Eigen
 	}
 }
 #endif
+#ifdef EIGEN_VECTORIZE_NEON
+#include <arm_neon.h>
+
+
+namespace Eigen
+{
+	namespace internal
+	{
+		template<> struct to_int_packet<Packet4f>
+		{
+			typedef Packet4i type;
+		};
+
+		template<> struct to_float_packet<Packet4i>
+		{
+			typedef Packet4f type;
+		};
+
+		EIGEN_STRONG_INLINE Packet4f p_to_f32(const Packet4i& a)
+		{
+			return vcvtq_f32_s32(a);
+		}
+
+		EIGEN_STRONG_INLINE Packet4f p_bool2float(const Packet4f& a)
+		{
+			return vcvtq_f32_s32(vandq_s32(a, vdupq_n_s32(1)));
+		}
+
+		EIGEN_STRONG_INLINE Packet4f p_bool2float(const Packet4i& a)
+		{
+			return p_bool2float((Packet4f)vreinterpretq_f32_s32((int32x4_t)a));
+		}
+	}
+}
+#endif
 
 namespace Eigen
 {