tomoto 0.2.3 → 0.3.0

data/vendor/EigenRand/EigenRand/arch/AVX/PacketFilter.h

@@ -0,0 +1,149 @@
+/**
+ * @file PacketFilter.h
+ * @author bab2min (bab2min@gmail.com)
+ * @brief
+ * @version 0.4.0
+ * @date 2021-09-17
+ *
+ * @copyright Copyright (c) 2020-2021
+ *
+ */
+
+#ifndef EIGENRAND_PACKET_FILTER_AVX_H
+#define EIGENRAND_PACKET_FILTER_AVX_H
+
+#include <immintrin.h>
+
+namespace Eigen
+{
+	namespace Rand
+	{
+		namespace detail
+		{
+			template<>
+			class CompressMask<32>
+			{
+				std::array<std::array<internal::Packet8i, 256>, 15> idx;
+				std::array<internal::Packet8f, 8> selector;
+				std::array<uint8_t, 256> cnt;
+
+				static internal::Packet8i make_compress(int mask, int offset = 0)
+				{
+					int32_t ret[8] = { 0, };
+					int n = offset;
+					for (int i = 0; i < 8; ++i)
+					{
+						int l = mask & 1;
+						mask >>= 1;
+						if (l)
+						{
+							if (n >= 0) ret[n] = i;
+							if (++n >= 8) break;
+						}
+					}
+					return _mm256_loadu_si256((internal::Packet8i*)ret);
+				}
+
+				static uint8_t count(int mask)
+				{
+					uint8_t ret = 0;
+					for (int i = 0; i < 8; ++i)
+					{
+						ret += mask & 1;
+						mask >>= 1;
+					}
+					return ret;
+				}
+
+				CompressMask()
+				{
+					for (int i = 0; i < 256; ++i)
+					{
+						for (int o = 0; o < 15; ++o)
+						{
+							idx[o][i] = make_compress(i, o < 8 ? o : o - 15);
+						}
+
+						cnt[i] = count(i);
+					}
+
+					selector[0] = _mm256_castsi256_ps(_mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, 0));
+					selector[1] = _mm256_castsi256_ps(_mm256_setr_epi32(-1, 0, 0, 0, 0, 0, 0, 0));
+					selector[2] = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, 0, 0, 0, 0, 0, 0));
+					selector[3] = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, -1, 0, 0, 0, 0, 0));
+					selector[4] = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, -1, -1, 0, 0, 0, 0));
+					selector[5] = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, -1, -1, -1, 0, 0, 0));
+					selector[6] = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, -1, -1, -1, -1, 0, 0));
+					selector[7] = _mm256_castsi256_ps(_mm256_setr_epi32(-1, -1, -1, -1, -1, -1, -1, 0));
+				}
+
+				static EIGEN_STRONG_INLINE internal::Packet8f permute(const internal::Packet8f& p, const internal::Packet8i& i)
+				{
+#ifdef EIGEN_VECTORIZE_AVX2
+					return _mm256_permutevar8x32_ps(p, i);
+#else
+					auto l = _mm256_permutevar_ps(p, i);
+					auto h = _mm256_permutevar_ps(_mm256_permute2f128_ps(p, p, 0x01), i);
+					internal::Packet4i i1, i2;
+					internal::split_two(i, i1, i2);
+					i1 = _mm_slli_epi32(i1, 29);
+					i2 = _mm_slli_epi32(i2, 29);
+					auto c = _mm256_castsi256_ps(
+						internal::combine_two(
+							_mm_cmplt_epi32(i1, internal::pset1<internal::Packet4i>(0)),
+							_mm_cmplt_epi32(internal::pset1<internal::Packet4i>(-1), i2)
+						)
+					);
+					return internal::pblendv(c, h, l);
+#endif
+				}
+
+			public:
+				enum { full_size = 8 };
+				static const CompressMask& get_inst()
+				{
+					static CompressMask cm;
+					return cm;
+				}
+
+				template<typename Packet>
+				EIGEN_STRONG_INLINE int compress_append(Packet& _value, const Packet& _mask,
+					Packet& _rest, int rest_cnt, bool& full) const
+				{
+					auto& value = reinterpret_cast<internal::Packet8f&>(_value);
+					auto& mask = reinterpret_cast<const internal::Packet8f&>(_mask);
+					auto& rest = reinterpret_cast<internal::Packet8f&>(_rest);
+
+					int m = _mm256_movemask_ps(mask);
+					if (cnt[m] == full_size)
+					{
+						full = true;
+						return rest_cnt;
+					}
+
+					auto p1 = permute(value, idx[rest_cnt][m]);
+					p1 = internal::pblendv(selector[rest_cnt], rest, p1);
+
+					auto new_cnt = rest_cnt + cnt[m];
+					if (new_cnt >= full_size)
+					{
+						if (new_cnt > full_size)
+						{
+							rest = permute(value, idx[new_cnt - cnt[m] + full_size - 1][m]);
+						}
+						value = p1;
+						full = true;
+						return new_cnt - full_size;
+					}
+					else
+					{
+						rest = p1;
+						full = false;
+						return new_cnt;
+					}
+				}
+			};
+		}
+	}
+}
+#endif

data/vendor/EigenRand/EigenRand/arch/AVX/RandUtils.h

@@ -0,0 +1,228 @@
+/**
+ * @file RandUtils.h
+ * @author bab2min (bab2min@gmail.com)
+ * @brief
+ * @version 0.4.0
+ * @date 2021-09-17
+ *
+ * @copyright Copyright (c) 2020-2021
+ *
+ */
+
+#ifndef EIGENRAND_RAND_UTILS_AVX_H
+#define EIGENRAND_RAND_UTILS_AVX_H
+
+#include <immintrin.h>
+
+namespace Eigen
+{
+	namespace internal
+	{
+		template<typename Rng>
+		struct RawbitsMaker<Packet4i, Rng, Packet8i, Rand::RandomEngineType::packet>
+		{
+			EIGEN_STRONG_INLINE Packet4i rawbits(Rng& rng)
+			{
+				return rng.half();
+			}
+
+			EIGEN_STRONG_INLINE Packet4i rawbits_34(Rng& rng)
+			{
+				return rng.half();
+			}
+
+			EIGEN_STRONG_INLINE Packet4i rawbits_half(Rng& rng)
+			{
+				return rng.half();
+			}
+		};
+
+		template<typename Rng>
+		struct RawbitsMaker<Packet8i, Rng, Packet4i, Rand::RandomEngineType::packet>
+		{
+			EIGEN_STRONG_INLINE Packet8i rawbits(Rng& rng)
+			{
+				return _mm256_insertf128_si256(_mm256_castsi128_si256(rng()), rng(), 1);
+			}
+
+			EIGEN_STRONG_INLINE Packet8i rawbits_34(Rng& rng)
+			{
+				return _mm256_insertf128_si256(_mm256_castsi128_si256(rng()), rng(), 1);
+			}
+
+			EIGEN_STRONG_INLINE Packet4i rawbits_half(Rng& rng)
+			{
+				return rng();
+			}
+		};
+
+		template<typename Rng, typename RngResult>
+		struct RawbitsMaker<Packet8i, Rng, RngResult, Rand::RandomEngineType::scalar>
+		{
+			EIGEN_STRONG_INLINE Packet8i rawbits(Rng& rng)
+			{
+				if (sizeof(decltype(rng())) == 8)
+				{
+					return _mm256_set_epi64x(rng(), rng(), rng(), rng());
+				}
+				else
+				{
+					return _mm256_set_epi32(rng(), rng(), rng(), rng(),
+						rng(), rng(), rng(), rng());
+				}
+			}
+
+			EIGEN_STRONG_INLINE Packet8i rawbits_34(Rng& rng)
+			{
+				Packet8i p;
+				if (sizeof(decltype(rng())) == 8)
+				{
+#ifdef EIGEN_VECTORIZE_AVX2
+					p = _mm256_setr_epi64x(rng(), rng(), rng(), 0);
+					p = _mm256_permutevar8x32_epi32(p, _mm256_setr_epi32(0, 1, 2, 7, 3, 4, 5, 7));
+					p = _mm256_shuffle_epi8(p, _mm256_setr_epi8(
+						0, 1, 2, 3,
+						4, 5, 6, 7,
+						8, 9, 10, 11,
+						3, 7, 11, 11,
+						0, 1, 2, 3,
+						4, 5, 6, 7,
+						8, 9, 10, 11,
+						3, 7, 11, 11
+					));
+
+#else
+					auto v = rng();
+					p = _mm256_setr_epi64x(rng(), v, rng(), v >> 32);
+					Packet4i p1, p2, o = _mm_setr_epi8(
+						0, 1, 2, 3,
+						4, 5, 6, 7,
+						8, 9, 10, 11,
+						3, 7, 11, 11);
+					split_two(p, p1, p2);
+					p = combine_two(_mm_shuffle_epi8(p1, o), _mm_shuffle_epi8(p2, o));
+#endif
+				}
+				else
+				{
+					p = _mm256_setr_epi32(rng(), rng(), rng(), 0, rng(), rng(), rng(), 0);
+#ifdef EIGEN_VECTORIZE_AVX2
+					p = _mm256_shuffle_epi8(p, _mm256_setr_epi8(
+						0, 1, 2, 3,
+						4, 5, 6, 7,
+						8, 9, 10, 11,
+						3, 7, 11, 11,
+						0, 1, 2, 3,
+						4, 5, 6, 7,
+						8, 9, 10, 11,
+						3, 7, 11, 11
+					));
+#else
+					Packet4i p1, p2, o = _mm_setr_epi8(
+						0, 1, 2, 3,
+						4, 5, 6, 7,
+						8, 9, 10, 11,
+						3, 7, 11, 11);
+					split_two(p, p1, p2);
+					p = combine_two(_mm_shuffle_epi8(p1, o), _mm_shuffle_epi8(p2, o));
+#endif
+				}
+				return p;
+			}
+
+			EIGEN_STRONG_INLINE Packet4i rawbits_half(Rng& rng)
+			{
+				if (sizeof(decltype(rng())) == 8)
+				{
+					return _mm_set_epi64x(rng(), rng());
+				}
+				else
+				{
+					return _mm_set_epi32(rng(), rng(), rng(), rng());
+				}
+			}
+		};
+
+		template<typename Rng>
+		struct RawbitsMaker<Packet8i, Rng, Packet8i, Rand::RandomEngineType::packet>
+		{
+			EIGEN_STRONG_INLINE Packet8i rawbits(Rng& rng)
+			{
+				return rng();
+			}
+
+			EIGEN_STRONG_INLINE Packet8i rawbits_34(Rng& rng)
+			{
+				return rng();
+			}
+
+			EIGEN_STRONG_INLINE Packet4i rawbits_half(Rng& rng)
+			{
+				return rng.half();
+			}
+		};
+
+#ifndef EIGEN_VECTORIZE_AVX2
+		template<>
+		EIGEN_STRONG_INLINE Packet8f bit_to_ur_float<Packet8i>(const Packet8i& x)
+		{
+			const Packet4i lower = pset1<Packet4i>(0x7FFFFF),
+				upper = pset1<Packet4i>(127 << 23);
+			const Packet8f one = pset1<Packet8f>(1);
+
+			Packet4i x1, x2;
+			split_two(x, x1, x2);
+
+			return psub(reinterpret_to_float(
+				combine_two(por(pand(x1, lower), upper), por(pand(x2, lower), upper)
+				)), one);
+		}
+
+		template<>
+		EIGEN_STRONG_INLINE Packet4d bit_to_ur_double<Packet8i>(const Packet8i& x)
+		{
+			const Packet4i lower = pseti64<Packet4i>(0xFFFFFFFFFFFFFull),
+				upper = pseti64<Packet4i>(1023ull << 52);
+			const Packet4d one = pset1<Packet4d>(1);
+
+			Packet4i x1, x2;
+			split_two(x, x1, x2);
+
+			return psub(reinterpret_to_double(
+				combine_two(por(pand(x1, lower), upper), por(pand(x2, lower), upper)
+				)), one);
+		}
+#endif
+
+		template<typename Rng>
+		struct UniformRealUtils<Packet8f, Rng> : public RawbitsMaker<Packet8i, Rng>
+		{
+			EIGEN_STRONG_INLINE Packet8f zero_to_one(Rng& rng)
+			{
+				return pdiv(_mm256_cvtepi32_ps(pand(this->rawbits(rng), pset1<Packet8i>(0x7FFFFFFF))),
+					pset1<Packet8f>(0x7FFFFFFF));
+			}
+
+			EIGEN_STRONG_INLINE Packet8f uniform_real(Rng& rng)
+			{
+				return bit_to_ur_float(this->rawbits_34(rng));
+			}
+		};
+
+		template<typename Rng>
+		struct UniformRealUtils<Packet4d, Rng> : public RawbitsMaker<Packet8i, Rng>
+		{
+			EIGEN_STRONG_INLINE Packet4d zero_to_one(Rng& rng)
+			{
+				return pdiv(_mm256_cvtepi32_pd(pand(this->rawbits_half(rng), pset1<Packet4i>(0x7FFFFFFF))),
+					pset1<Packet4d>(0x7FFFFFFF));
+			}
+
+			EIGEN_STRONG_INLINE Packet4d uniform_real(Rng& rng)
+			{
+				return bit_to_ur_double(this->rawbits(rng));
+			}
+		};
+	}
+}
+#endif