tomoto 0.4.0 → 0.4.1

data/vendor/tomotopy/src/Utils/serializer.hpp

@@ -9,6 +9,7 @@
 #include <unordered_map>
 #include <Eigen/Dense>
 #include "text.hpp"
+#include "Utils.hpp"
 
 /*
 
@@ -30,36 +31,51 @@ namespace tomoto
 {
 	namespace serializer
 	{
-		struct membuf : std::streambuf 
+		struct membuf : public std::streambuf
 		{
-			membuf(char* base, std::ptrdiff_t n) 
+			membuf(bool read, bool write, char* base, std::ptrdiff_t n);
+			~membuf();
+
+			std::streampos seekpos(pos_type sp, std::ios_base::openmode which) override;
+
+			std::streampos seekoff(off_type off,
+				std::ios_base::seekdir dir,
+				std::ios_base::openmode which = std::ios_base::in
+			) override;
+
+			const char* curptr() const
 			{
-				this->setg(base, base, base + n);
+				return this->gptr();
 			}
+		};
+
+		class imstream : public std::istream
+		{
+			membuf buf;
+		public:
+			imstream(const char* base, std::ptrdiff_t n);
+			~imstream();
 
-			pos_type seekpos(pos_type sp, std::ios_base::openmode which) override {
-				return seekoff(sp - pos_type(off_type(0)), std::ios_base::beg, which);
+			template<class Ty>
+			imstream(const Ty& m) : imstream(m.get(), m.size())
+			{
 			}
 
-			pos_type seekoff(off_type off,
-				std::ios_base::seekdir dir,
-				std::ios_base::openmode which = std::ios_base::in) override {
-				if (dir == std::ios_base::cur)
-					gbump(off);
-				else if (dir == std::ios_base::end)
-					setg(eback(), egptr() + off, egptr());
-				else if (dir == std::ios_base::beg)
-					setg(eback(), eback() + off, egptr());
-				return gptr() - eback();
+			const char* curptr() const
+			{
+				return buf.curptr();
 			}
 		};
 
-		class imstream : public std::istream
+		class omstream : public std::ostream
 		{
 			membuf buf;
 		public:
-			imstream(const char* base, std::ptrdiff_t n)
-				: std::istream(&buf), buf((char*)base, n)
+			omstream(char* base, std::ptrdiff_t n);
+			~omstream();
+
+			template<class Ty>
+			omstream(const Ty& m) : omstream(m.get(), m.size())
 			{
 			}
 		};
@@ -224,6 +240,16 @@ namespace tomoto
 			return v;
 		}
 
+		template<typename _Ty>
+		inline uint64_t computeHash(uint64_t seed, const _Ty& v)
+		{
+			return Serializer<
+				typename std::remove_const<typename std::remove_reference<_Ty>::type>::type
+			>{}.hash(seed, v);
+		}
+
+		uint64_t computeFastHash(const void* data, size_t size, uint64_t seed = 0);
+
 		inline void writeMany(std::ostream& ostr)
 		{
 			// do nothing
@@ -279,6 +305,22 @@ namespace tomoto
 			return m == first.m;
 		}
 
+		inline uint64_t computeHashMany(uint64_t seed)
+		{
+			return seed;
+		}
+
+		template<typename _FirstTy, typename ... _RestTy>
+		inline typename std::enable_if<
+			!is_key<typename std::remove_reference<_FirstTy>::type>::value,
+			uint64_t
+		>::type computeHashMany(uint64_t seed, _FirstTy&& first, _RestTy&&... rest)
+		{
+			seed = computeHash(seed, std::forward<_FirstTy>(first));
+			seed = computeHashMany(seed, std::forward<_RestTy>(rest)...);
+			return seed;
+		}
+
 		template<size_t>
 		struct version_holder {};
 
@@ -343,6 +385,11 @@ namespace tomoto
 				if (!istr.read((char*)&v, sizeof(_Ty)))
 					throw std::ios_base::failure(std::string("reading type '") + typeid(_Ty).name() + std::string("' is failed"));
 			}
+
+			uint64_t hash(uint64_t seed, const _Ty& v)
+			{
+				return computeFastHash(&v, sizeof(_Ty), seed);
+			}
 		};
 
 		template<typename _Ty>
@@ -357,6 +404,11 @@ namespace tomoto
 			{
 				v.serializerRead(istr);
 			}
+
+			uint64_t hash(uint64_t seed, const _Ty& v)
+			{
+				return v.computeHash(seed);
+			}
 		};
 
 		template<typename _Ty>
@@ -435,6 +487,13 @@ namespace tomoto
 				if (!istr.read((char*)v.data(), sizeof(_Ty) * rows * cols))
 					throw std::ios_base::failure(std::string("reading type '") + typeid(_Ty).name() + std::string("' is failed"));
 			}
+
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				seed = computeHash(seed, (uint32_t)v.rows());
+				seed = computeHash(seed, (uint32_t)v.cols());
+				return computeFastHash(v.data(), sizeof(_Ty) * v.size(), seed);
+			}
 		};
 
 		template<typename _Ty>
@@ -457,6 +516,13 @@ namespace tomoto
 				if (!istr.read((char*)v.data(), sizeof(_Ty) * rows * cols))
 					throw std::ios_base::failure(std::string("reading type '") + typeid(_Ty).name() + std::string("' is failed"));
 			}
+
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				seed = computeHash(seed, (uint32_t)v.rows());
+				seed = computeHash(seed, (uint32_t)v.cols());
+				return computeFastHash(v.data(), sizeof(_Ty) * v.size(), seed);
+			}
 		};
 
 		template<typename _Ty>
@@ -487,6 +553,12 @@ namespace tomoto
 				if (!istr.read((char*)v.data(), sizeof(_Ty) * size))
 					throw std::ios_base::failure(std::string("reading type '") + typeid(_Ty).name() + std::string("' is failed"));
 			}
+
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				seed = computeHash(seed, (uint32_t)v.size());
+				return computeFastHash(v.data(), sizeof(_Ty) * v.size(), seed);
+			}
 		};
 
 		template<typename _Ty>
@@ -505,6 +577,16 @@ namespace tomoto
 				v.resize(size);
 				for (auto& e : v) Serializer<_Ty>{}.read(istr, e);
 			}
+
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				seed = computeHash(seed, (uint32_t)v.size());
+				for (auto& e : v)
+				{
+					seed = computeHash(seed, e);
+				}
+				return seed;
+			}
 		};
 
 		template<typename _Ty, size_t n>
@@ -525,6 +607,12 @@ namespace tomoto
 				if (!istr.read((char*)v.data(), sizeof(_Ty) * size))
 					throw std::ios_base::failure(std::string("reading type '") + typeid(_Ty).name() + std::string("' is failed"));
 			}
+
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				seed = computeHash(seed, (uint32_t)v.size());
+				return computeFastHash(v.data(), sizeof(_Ty) * v.size(), seed);
+			}
 		};
 
 		template<typename _Ty, size_t n>
@@ -543,6 +631,16 @@ namespace tomoto
 				if (n != size) throw std::ios_base::failure(text::format("the size of array must be %zd, not %zd", n, size));
 				for (auto& e : v) Serializer<_Ty>{}.read(istr, e);
 			}
+
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				seed = computeHash(seed, (uint32_t)v.size());
+				for (auto& e : v)
+				{
+					seed = computeHash(seed, e);
+				}
+				return seed;
+			}
 		};
 
 		template<typename _Ty>
@@ -563,6 +661,12 @@ namespace tomoto
 				if (!istr.read((char*)v.data(), sizeof(_Ty) * size))
 					throw std::ios_base::failure(std::string("reading type '") + typeid(_Ty).name() + std::string("' is failed"));
 			}
+
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				seed = computeHash(seed, (uint32_t)v.size());
+				return computeFastHash(v.data(), sizeof(_Ty) * v.size(), seed);
+			}
 		};
 
 		template<typename _Ty1, typename _Ty2>
@@ -578,6 +682,13 @@ namespace tomoto
 			{
 				readMany(istr, v.first, v.second);
 			}
+
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				seed = computeHash(seed, v.first);
+				seed = computeHash(seed, v.second);
+				return seed;
+			}
 		};
 
 		template<typename _Ty1, typename _Ty2>
@@ -599,6 +710,8 @@ namespace tomoto
 					v.emplace(readFromStream<std::pair<_Ty1, _Ty2>>(istr));
 				}
 			}
+
+			// not support hash
 		};
 
 		template<typename _Ty1, typename _Ty2>
@@ -620,6 +733,16 @@ namespace tomoto
 					v.emplace(readFromStream<std::pair<_Ty1, _Ty2>>(istr));
 				}
 			}
+
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				seed = computeHash(seed, (uint32_t)v.size());
+				for (auto& e : v)
+				{
+					seed = computeHash(seed, e);
+				}
+				return seed;
+			}
 		};
 
 		template<typename _Ty>
@@ -635,91 +758,98 @@ namespace tomoto
 			{
 				_Ty::serializerRead(v, istr);
 			}
-		};
 
-		static auto taggedDataKey = to_key("TPTK");
+			uint64_t hash(uint64_t seed, const VTy& v)
+			{
+				return _Ty::serializerHash(v, seed);
+			}
+		};
 
-		template<size_t _len, typename _Ty>
-		inline void writeTaggedData(std::ostream& ostr, uint32_t version, uint32_t trailing_cnt, const Key<_len>& key, const _Ty& data)
+		class BlockStreamBuffer : public std::basic_streambuf<char>
 		{
-			uint16_t major = version >> 16, minor = version & 0xFFFF;
-			writeMany(ostr, taggedDataKey, version);
-			std::streampos totsize_pos = ostr.tellp();
-			writeMany(ostr, (uint64_t)0, (uint32_t)_len, trailing_cnt, key, data);
-			std::streampos end_pos = ostr.tellp();
-			ostr.seekp(totsize_pos);
-			writeMany(ostr, (uint64_t)(end_pos - totsize_pos));
-			ostr.seekp(end_pos);
-		}
+			std::vector<std::unique_ptr<uint8_t[]>> buffers;
+			size_t block_size = 0;
+		public:
+			BlockStreamBuffer(size_t _block_size = 4096);
+			~BlockStreamBuffer();
 
-		template<size_t _len, typename _Ty>
-		inline std::pair<bool, std::streampos> readTaggedData(std::istream& istr, uint32_t version, uint32_t& trailing_cnt, const Key<_len>& key, _Ty& data)
+			int overflow(int c) override;
+
+			std::streamsize xsputn(const char* s, std::streamsize n) override;
+
+			size_t totalSize() const;
+
+			template<class Fn>
+			void iterateBuffers(Fn fn) const
+			{
+				for (size_t i = 0; i < buffers.size() - 1; ++i)
+				{
+					fn(buffers[i].get(), block_size);
+				}
+				fn(buffers.back().get(), this->pptr() - this->pbase());
+			}
+		};
+
+		static constexpr uint32_t taggedDataKeyUint = 0x4b545054; // "TPTK"
+
+		struct TaggedDataHeader
 		{
-			uint16_t major = version >> 16, minor = version & 0xFFFF;
+			uint32_t key;
+			uint32_t version;
 			uint64_t totsize;
 			uint32_t keysize;
-			std::streampos start_pos = istr.tellg();
-			readMany(istr, taggedDataKey, version);
-			std::streampos totsize_pos = istr.tellg();
-			readMany(istr, totsize, keysize, trailing_cnt);
-			std::streampos end_pos = totsize_pos + (std::streamoff)totsize;
-			if (_len != keysize)
-			{
-				istr.seekg(start_pos);
-				return std::make_pair(false, end_pos);
-			}
-			
-			if (!readTest(istr, key))
-			{
-				istr.seekg(start_pos);
-				return std::make_pair(false, end_pos);
-			}
-			
-			readMany(istr, data);
-			if (end_pos != istr.tellg())
+			uint32_t trailing_cnt;
+		};
+
+		template<size_t _len, typename _Ty>
+		inline void writeTaggedData(std::ostream& ostr, uint32_t version, uint32_t trailing_cnt, const Key<_len>& key, const _Ty& data)
+		{
+			BlockStreamBuffer buf;
+			std::ostream serialized_data(&buf);
+			writeMany(serialized_data, key, data);
+			const auto key_data_size = buf.totalSize();
+
+			TaggedDataHeader h;
+			h.key = taggedDataKeyUint;
+			h.version = version;
+			h.totsize = key_data_size + 16;
+			h.keysize = key.size();
+			h.trailing_cnt = trailing_cnt;
+
+			ostr.write((const char*)&h, sizeof(h));
+			buf.iterateBuffers([&](const void* data, size_t size)
 			{
-				istr.seekg(start_pos);
-				return std::make_pair(false, end_pos);
-			}
-			return std::make_pair(true, end_pos);
+				ostr.write((const char*)data, size);
+			});
 		}
 
-		inline void readTaggedMany(std::istream& istr, uint32_t version)
+		using TaggedDataMap = std::unordered_map<std::string, std::pair<std::streampos, std::streampos>>;
+
+		TaggedDataMap readTaggedDataMap(std::istream& istr, uint32_t version);
+
+		inline void readTaggedMany(std::istream& istr, const TaggedDataMap& data_map, uint32_t version)
 		{
 			// seek to the end of tagged data list
-			uint32_t trailing_cnt;
-			do
-			{
-				uint64_t totsize;
-				uint32_t keysize;
-				readMany(istr, taggedDataKey, version);
-				std::streampos totsize_pos = istr.tellg();
-				readMany(istr, totsize, keysize, trailing_cnt);
-				istr.seekg(totsize_pos + (std::streamoff)totsize);
-
-			} while (trailing_cnt);
+			istr.seekg(data_map.find("")->second.second);
 		}
 
 		template<size_t _len, typename _Ty, typename ... _Rest>
-		inline void readTaggedMany(std::istream& istr, uint32_t version, const Key<_len>& key, _Ty& data, _Rest&&... rest)
+		inline void readTaggedMany(std::istream& istr, const TaggedDataMap& data_map, uint32_t version, const Key<_len>& key, _Ty& data, _Rest&&... rest)
 		{
-			auto start_pos = istr.tellg();
-			uint32_t trailing_cnt;
-			do
+			auto it = data_map.find(key.str());
+			if (it != data_map.end())
 			{
-				std::pair<bool, std::streampos> p = readTaggedData(istr, version, trailing_cnt, key, data);
-				if (p.first)
-				{
-					break;
-				}
-				else
-				{
-					istr.seekg(p.second);
-				}
-			} while (trailing_cnt);
+				istr.seekg(it->second.first);
+				readMany(istr, data);
+			}
+			readTaggedMany(istr, data_map, version, std::forward<_Rest>(rest)...);
+		}
 
-			istr.seekg(start_pos);
-			readTaggedMany(istr, version, std::forward<_Rest>(rest)...);
+		template<typename ... _Rest>
+		inline void readTaggedMany(std::istream& istr, uint32_t version, _Rest&&... rest)
+		{
+			const auto data_map = readTaggedDataMap(istr, version);
+			readTaggedMany(istr, data_map, version, std::forward<_Rest>(rest)...);
 		}
 
 		inline void writeTaggedMany(std::ostream& ostr, uint32_t version)
@@ -745,6 +875,11 @@ void serializerWrite(std::ostream& ostr) const\
 	tomoto::serializer::writeMany(ostr, __VA_ARGS__);\
 }
 
+#define DEFINE_HASHER(...) uint64_t computeHash(uint64_t seed) const\
+{\
+	return tomoto::serializer::computeHashMany(seed, __VA_ARGS__);\
+}
+
 #define DEFINE_SERIALIZER_WITH_VERSION(v,...) void serializerRead(tomoto::serializer::version_holder<v>, std::istream& istr)\
 {\
 	tomoto::serializer::readMany(istr, __VA_ARGS__);\
@@ -775,6 +910,12 @@ void serializerWrite(std::ostream& ostr) const\
 	tomoto::serializer::writeMany(ostr, __VA_ARGS__);\
 }
 
+#define DEFINE_HASHER_AFTER_BASE(base, ...) uint64_t computeHash(uint64_t seed) const\
+{\
+	seed = base::computeHash(seed);\
+	return tomoto::serializer::computeHashMany(seed, __VA_ARGS__);\
+}\
+
 #define DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(base, v, ...) void serializerRead(tomoto::serializer::version_holder<v> _v, std::istream& istr)\
 {\
 	base::serializerRead(_v, istr);\
@@ -786,6 +927,17 @@ void serializerWrite(tomoto::serializer::version_holder<v> _v, std::ostream& ost
 	tomoto::serializer::writeMany(ostr, __VA_ARGS__);\
 }
 
+#define DEFINE_OUT_SERIALIZER_AFTER_BASE_WITH_VERSION(cls, base, v, ...) template<TermWeight _tw> void cls<_tw>::serializerRead(tomoto::serializer::version_holder<v> _v, std::istream& istr)\
+{\
+	base::serializerRead(_v, istr);\
+	tomoto::serializer::readMany(istr, __VA_ARGS__);\
+}\
+template<TermWeight _tw> void cls<_tw>::serializerWrite(tomoto::serializer::version_holder<v> _v, std::ostream& ostr) const\
+{\
+	base::serializerWrite(_v, ostr);\
+	tomoto::serializer::writeMany(ostr, __VA_ARGS__);\
+}
+
 #define DEFINE_SERIALIZER_BASE_WITH_VERSION(base, v) void serializerRead(tomoto::serializer::version_holder<v> _v, std::istream& istr)\
 {\
 	base::serializerRead(_v, istr);\
@@ -795,6 +947,19 @@ void serializerWrite(tomoto::serializer::version_holder<v> _v, std::ostream& ost
 	base::serializerWrite(_v, ostr);\
 }
 
+#define DEFINE_OUT_SERIALIZER_BASE_WITH_VERSION(cls, base, v) template<TermWeight _tw> void cls<_tw>::serializerRead(tomoto::serializer::version_holder<v> _v, std::istream& istr)\
+{\
+	base::serializerRead(_v, istr);\
+}\
+template<TermWeight _tw> void cls<_tw>::serializerWrite(tomoto::serializer::version_holder<v> _v, std::ostream& ostr) const\
+{\
+	base::serializerWrite(_v, ostr);\
+}
+
+
+#define DECLARE_SERIALIZER_WITH_VERSION(v) void serializerRead(tomoto::serializer::version_holder<v> _v, std::istream& istr);\
+void serializerWrite(tomoto::serializer::version_holder<v> _v, std::ostream& ostr) const;
+
 #define DEFINE_SERIALIZER_AFTER_BASE_CALLBACK(base, onRead, ...) void serializerRead(std::istream& istr)\
 {\
 	base::serializerRead(istr);\
@@ -863,3 +1028,14 @@ void serializerWrite(tomoto::serializer::version_holder<v> _v, std::ostream& ost
 	base::serializerWrite(_v, ostr);\
 	tomoto::serializer::writeTaggedMany(ostr, t, _TO_KEY_VALUE(__VA_ARGS__));\
 }
+
+#define DEFINE_OUT_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(cls, base, v, t,...) template<TermWeight _tw> void cls<_tw>::serializerRead(tomoto::serializer::version_holder<v> _v, std::istream& istr)\
+{\
+	base::serializerRead(_v, istr);\
+	tomoto::serializer::readTaggedMany(istr, t, _TO_KEY_VALUE(__VA_ARGS__));\
+}\
+template<TermWeight _tw> void cls<_tw>::serializerWrite(tomoto::serializer::version_holder<v> _v, std::ostream& ostr) const\
+{\
+	base::serializerWrite(_v, ostr);\
+	tomoto::serializer::writeTaggedMany(ostr, t, _TO_KEY_VALUE(__VA_ARGS__));\
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tomoto
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-28 00:00:00.000000000 Z
+date: 2024-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -499,6 +499,7 @@ files:
 - vendor/tomotopy/src/TopicModel/SLDAModel.hpp
 - vendor/tomotopy/src/TopicModel/TopicModel.hpp
 - vendor/tomotopy/src/Utils/AliasMethod.hpp
+- vendor/tomotopy/src/Utils/Dictionary.cpp
 - vendor/tomotopy/src/Utils/Dictionary.h
 - vendor/tomotopy/src/Utils/EigenAddonOps.hpp
 - vendor/tomotopy/src/Utils/LBFGS.h
@@ -506,10 +507,13 @@ files:
 - vendor/tomotopy/src/Utils/LBFGS/LineSearchBracketing.h
 - vendor/tomotopy/src/Utils/LBFGS/Param.h
 - vendor/tomotopy/src/Utils/LUT.hpp
+- vendor/tomotopy/src/Utils/Mmap.cpp
+- vendor/tomotopy/src/Utils/Mmap.h
 - vendor/tomotopy/src/Utils/MultiNormalDistribution.hpp
 - vendor/tomotopy/src/Utils/PolyaGamma.hpp
 - vendor/tomotopy/src/Utils/PolyaGammaHybrid.hpp
-- vendor/tomotopy/src/Utils/SharedString.hpp
+- vendor/tomotopy/src/Utils/SharedString.cpp
+- vendor/tomotopy/src/Utils/SharedString.h
 - vendor/tomotopy/src/Utils/ThreadPool.hpp
 - vendor/tomotopy/src/Utils/Trie.hpp
 - vendor/tomotopy/src/Utils/TruncMultiNormal.hpp
@@ -521,6 +525,7 @@ files:
 - vendor/tomotopy/src/Utils/neon_gamma.h
 - vendor/tomotopy/src/Utils/rtnorm.hpp
 - vendor/tomotopy/src/Utils/sample.hpp
+- vendor/tomotopy/src/Utils/serializer.cpp
 - vendor/tomotopy/src/Utils/serializer.hpp
 - vendor/tomotopy/src/Utils/slp.hpp
 - vendor/tomotopy/src/Utils/sse_gamma.h
@@ -553,7 +558,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: High performance topic modeling for Ruby