tomoto 0.1.2 → 0.1.3

data/vendor/tomotopy/src/Utils/Dictionary.h

@@ -11,9 +11,15 @@
 namespace tomoto
 {
 	using Vid = uint32_t;
+	static constexpr Vid non_vocab_id = (Vid)-1;
 	using Tid = uint16_t;
 	using Float = float;
 
+	struct VidPair : public std::pair<Vid, Vid>
+	{
+		using std::pair<Vid, Vid>::pair;
+	};
+
 	class Dictionary
 	{
 	protected:
@@ -34,7 +40,7 @@ namespace tomoto
 
 		size_t size() const { return dict.size(); }
 		
-		std::string toWord(Vid vid) const
+		const std::string& toWord(Vid vid) const
 		{
 			assert(vid < id2word.size());
 			return id2word[vid];
@@ -43,7 +49,7 @@ namespace tomoto
 		Vid toWid(const std::string& word) const
 		{
 			auto it = dict.find(word);
-			if (it == dict.end()) return (Vid)-1;
+			if (it == dict.end()) return non_vocab_id;
 			return it->second;
 		}
 
@@ -75,6 +81,38 @@ namespace tomoto
 				id2word[p.second] = p.first;
 			}
 		}
+
+		const std::vector<std::string>& getRaw() const
+		{
+			return id2word;
+		}
+
+		Vid mapToNewDict(Vid v, const Dictionary& newDict) const
+		{
+			return newDict.toWid(toWord(v));
+		}
+
+		std::vector<Vid> mapToNewDict(const std::vector<Vid>& v, const Dictionary& newDict) const
+		{
+			std::vector<Vid> r(v.size());
+			for (size_t i = 0; i < v.size(); ++i)
+			{
+				r[i] = mapToNewDict(v[i], newDict);
+			}
+			return r;
+		}
 	};
 
 }
+
+namespace std
+{
+	template<>
+	struct hash<tomoto::VidPair>
+	{
+		size_t operator()(const tomoto::VidPair& p) const
+		{
+			return hash<size_t>{}(p.first) ^ hash<size_t>{}(p.second);
+		}
+	};
+}

data/vendor/tomotopy/src/Utils/EigenAddonOps.hpp

@@ -12,6 +12,12 @@ namespace Eigen
 		{
 			typedef PacketType type;
 		};
+
+		template<typename PacketType>
+		struct to_float_packet
+		{
+			typedef PacketType type;
+		};
 	}
 }
 
@@ -28,15 +34,29 @@ namespace Eigen
 			typedef Packet8i type;
 		};
 
+		template<> struct to_float_packet<Packet8i>
+		{
+			typedef Packet8f type;
+		};
+
 		EIGEN_STRONG_INLINE Packet8f p_to_f32(const Packet8i& a)
 		{
 			return _mm256_cvtepi32_ps(a);
 		}
 
+		EIGEN_STRONG_INLINE Packet8f p_bool2float(const Packet8f& a)
+		{
+			return _mm256_and_ps(_mm256_cmp_ps(a, _mm256_set1_ps(0), _CMP_NEQ_OQ), _mm256_set1_ps(1));
+		}
+
+		EIGEN_STRONG_INLINE Packet8f p_bool2float(const Packet8i& a)
+		{
+			return p_bool2float(_mm256_castsi256_ps(a));
+		}
 	}
 }
-
-#elif defined(EIGEN_VECTORIZE_SSE2)
+#endif
+#if defined(EIGEN_VECTORIZE_SSE2)
 #include <xmmintrin.h>
 #include "sse_gamma.h"
 
@@ -49,11 +69,25 @@ namespace Eigen
 			typedef Packet4i type;
 		};
 
+		template<> struct to_float_packet<Packet4i>
+		{
+			typedef Packet4f type;
+		};
+
 		EIGEN_STRONG_INLINE Packet4f p_to_f32(const Packet4i& a)
 		{
 			return _mm_cvtepi32_ps(a);
 		}
+		
+		EIGEN_STRONG_INLINE Packet4f p_bool2float(const Packet4f& a)
+		{
+			return _mm_and_ps(_mm_cmpneq_ps(a, _mm_set1_ps(0)), _mm_set1_ps(1));
+		}
 
+		EIGEN_STRONG_INLINE Packet4f p_bool2float(const Packet4i& a)
+		{
+			return p_bool2float(_mm_castsi128_ps(a));
+		}
 	}
 }
 #endif
@@ -84,7 +118,7 @@ namespace Eigen
 		template<>
 		struct scalar_cast_op<int32_t, float> {
 			EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op)
-				typedef float result_type;
+			typedef float result_type;
 			EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const float operator() (const int32_t& a) const { return cast<int32_t, float>(a); }
 			
 			template<typename Packet>
@@ -156,6 +190,81 @@ namespace Eigen
 			evaluator<ArgType> m_argImpl;
 		};
 
+		struct scalar_bool2float
+		{
+			EIGEN_EMPTY_STRUCT_CTOR(scalar_bool2float)
+			typedef float result_type;
+			EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const float operator() (const int32_t& a) const { return a ? 1.f : 0.f; }
+
+			template<typename Packet>
+			EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename to_float_packet<Packet>::type packetOp(const Packet& a) const
+			{
+				return p_bool2float(a);
+			}
+		};
+
+		template<>
+		struct functor_traits<scalar_bool2float>
+		{
+			enum { Cost = NumTraits<float>::AddCost, PacketAccess = 1 };
+		};
+
+		template<typename ArgType>
+		struct unary_evaluator<CwiseUnaryOp<scalar_bool2float, ArgType>, IndexBased >
+			: evaluator_base<CwiseUnaryOp<scalar_bool2float, ArgType> >
+		{
+			typedef CwiseUnaryOp<scalar_bool2float, ArgType> XprType;
+
+			enum {
+				CoeffReadCost = evaluator<ArgType>::CoeffReadCost + functor_traits<scalar_bool2float>::Cost,
+
+				Flags = evaluator<ArgType>::Flags
+				& (HereditaryBits | LinearAccessBit | (functor_traits<scalar_bool2float>::PacketAccess ? PacketAccessBit : 0)),
+				Alignment = evaluator<ArgType>::Alignment
+			};
+
+			EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
+				explicit unary_evaluator(const XprType& op)
+				: m_functor(op.functor()),
+				m_argImpl(op.nestedExpression())
+			{
+				EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<float>::AddCost);
+				EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
+			}
+
+			typedef typename XprType::CoeffReturnType CoeffReturnType;
+
+			EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
+				CoeffReturnType coeff(Index row, Index col) const
+			{
+				return m_functor(m_argImpl.coeff(row, col));
+			}
+
+			EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
+				CoeffReturnType coeff(Index index) const
+			{
+				return m_functor(m_argImpl.coeff(index));
+			}
+
+			template<int LoadMode, typename PacketType>
+			EIGEN_STRONG_INLINE
+				PacketType packet(Index row, Index col) const
+			{
+				return m_functor.packetOp(m_argImpl.template packet<LoadMode, typename to_int_packet<PacketType>::type>(row, col));
+			}
+
+			template<int LoadMode, typename PacketType>
+			EIGEN_STRONG_INLINE
+				PacketType packet(Index index) const
+			{
+				return m_functor.packetOp(m_argImpl.template packet<LoadMode, typename to_int_packet<PacketType>::type>(index));
+			}
+
+		protected:
+			const scalar_bool2float m_functor;
+			evaluator<ArgType> m_argImpl;
+		};
+
 	}
 
 	template <typename Derived, typename T> EIGEN_DEVICE_FUNC inline 
@@ -178,4 +287,14 @@ namespace Eigen
 			y.derived()
 		);
 	}
+
+
+	template<typename Derived>
+	inline const CwiseUnaryOp<internal::scalar_bool2float, const Derived>
+		bool2float(const Eigen::ArrayBase<Derived>& x)
+	{
+		return CwiseUnaryOp<internal::scalar_bool2float, const Derived>(
+			x.derived()
+		);
+	}
 }

data/vendor/tomotopy/src/Utils/SharedString.hpp

@@ -0,0 +1,181 @@
+#pragma once
+
+#include <string>
+
+namespace tomoto
+{
+	class SharedString
+	{
+		const char* ptr = nullptr;
+		size_t len = 0;
+
+		void incref()
+		{
+			if (ptr)
+			{
+				++*(size_t*)ptr;
+			}
+		}
+
+		void decref()
+		{
+			if (ptr)
+			{
+				if (--*(size_t*)ptr == 0)
+				{
+					delete[] ptr;
+					ptr = nullptr;
+				}
+			}
+		}
+
+		void init(const char* _begin, const char* _end)
+		{
+			ptr = new char[_end - _begin + 9];
+			*(size_t*)ptr = 1;
+			len = _end - _begin;
+			std::memcpy((void*)(ptr + 8), _begin, _end - _begin);
+			((char*)ptr)[_end - _begin + 8] = 0;
+		}
+
+	public:
+
+		SharedString()
+		{
+		}
+
+		explicit SharedString(const char* _begin, const char* _end)
+		{
+			init(_begin, _end);
+		}
+
+		explicit SharedString(const char* _ptr)
+		{
+			if (_ptr)
+			{
+				init(_ptr, _ptr + std::strlen(_ptr));
+			}
+		}
+
+		explicit SharedString(const std::string& str)
+		{
+			if (!str.empty())
+			{
+				init(str.data(), str.data() + str.size());
+			}
+		}
+
+		SharedString(const SharedString& o) noexcept
+			: ptr{ o.ptr }, len{ o.len }
+		{
+			incref();
+		}
+
+		SharedString(SharedString&& o) noexcept
+		{
+			std::swap(ptr, o.ptr);
+			std::swap(len, o.len);
+		}
+
+		~SharedString()
+		{
+			decref();
+		}
+
+		SharedString& operator=(const SharedString& o)
+		{
+			if (this != &o)
+			{
+				decref();
+				ptr = o.ptr;
+				len = o.len;
+				incref();
+			}
+			return *this;
+		}
+
+		SharedString& operator=(SharedString&& o) noexcept
+		{
+			std::swap(ptr, o.ptr);
+			std::swap(len, o.len);
+			return *this;
+		}
+
+		size_t size() const
+		{
+			if (ptr) return len;
+			return 0;
+		}
+
+		bool empty() const
+		{
+			return ptr == nullptr;
+		}
+
+		operator std::string() const
+		{
+			if (!ptr) return {};
+			return { ptr + 8, ptr + 8 + len };
+		}
+
+		const char* c_str() const
+		{
+			if (!ptr) return "";
+			return ptr + 8;
+		}
+
+		const char* data() const
+		{
+			return c_str();
+		}
+
+		const char* begin() const
+		{
+			return data();
+		}
+
+		const char* end() const
+		{
+			return data() + size();
+		}
+
+		std::string substr(size_t start, size_t len) const
+		{
+			return { c_str() + start, c_str() + start + len };
+		}
+
+		bool operator==(const SharedString& o) const
+		{
+			if (ptr == o.ptr) return true;
+			if (size() != o.size()) return false;
+			return std::equal(begin(), end(), o.begin());
+		}
+
+		bool operator==(const std::string& o) const
+		{
+			if (size() != o.size()) return false;
+			return std::equal(begin(), end(), o.begin());
+		}
+
+		bool operator!=(const SharedString& o) const
+		{
+			return !operator==(o);
+		}
+
+		bool operator!=(const std::string& o) const
+		{
+			return !operator==(o);
+		}
+	};
+}
+
+namespace std
+{
+	template <> struct hash<tomoto::SharedString>
+	{
+		size_t operator()(const tomoto::SharedString& x) const
+		{
+			return hash<string>{}(x);
+		}
+	};
+}

data/vendor/tomotopy/src/Utils/math.h

@@ -154,7 +154,7 @@ namespace tomoto
 		inline float digammaT(float x) { return detail::LUT_digamma::get(x); }
 
 		template<class _T>
-		inline _T lgammaApprox(_T z)
+		inline auto lgammaApprox(_T z) -> decltype((z + 2.5)* log(z + 3) - (z + 3) + 0.91893853 + 1. / 12. / (z + 3) - log(z * (z + 1) * (z + 2)))
 		{
 			// approximation : lgamma(z) ~= (z+2.5)ln(z+3) - z - 3 + 0.5 ln (2pi) + 1/12/(z + 3) - ln (z(z+1)(z+2))
 			return (z + 2.5) * log(z + 3) - (z + 3) + 0.91893853 + 1. / 12. / (z + 3) - log(z * (z + 1) * (z + 2));

data/vendor/tomotopy/src/Utils/sample.hpp

@@ -77,7 +77,7 @@ namespace tomoto
 				offset = _mm_shuffle_ps(out, out, _MM_SHUFFLE(3, 3, 3, 3));
 			}
 			if (!n4) n4 = 1;
-			for (size_t i = n4; i < n; ++i)
+			for (int i = n4; i < n; ++i)
 			{
 				arr[i] += arr[i - 1];
 			}

data/vendor/tomotopy/src/Utils/serializer.hpp

@@ -8,6 +8,7 @@
 #include <vector>
 #include "tvector.hpp"
 #include "text.hpp"
+#include "SharedString.hpp"
 
 /*
 
@@ -435,6 +436,22 @@ namespace tomoto
 				throw std::ios_base::failure( std::string("reading type '") + typeid(_Ty).name() + std::string("' is failed") );
 		}
 
+		inline void writeToBinStreamImpl(std::ostream& ostr, const SharedString& v)
+		{
+			writeToStream<uint32_t>(ostr, (uint32_t)v.size());
+			if (!ostr.write((const char*)v.data(), v.size()))
+				throw std::ios_base::failure(std::string("writing type 'SharedString' is failed"));
+		}
+
+		inline void readFromBinStreamImpl(std::istream& istr, SharedString& v)
+		{
+			uint32_t size = readFromStream<uint32_t>(istr);
+			std::vector<char> t(size);
+			if (!istr.read((char*)t.data(), t.size()))
+				throw std::ios_base::failure(std::string("reading type 'SharedString' is failed"));
+			v = SharedString{ t.data(), t.data() + t.size() };
+		}
+
 		template<class _Ty>
 		inline typename std::enable_if<std::is_abstract<_Ty>::value>::type writeToBinStreamImpl(std::ostream& ostr, const std::unique_ptr<_Ty>& v)
 		{