faiss 0.6.0 → 0.6.1

data/vendor/faiss/faiss/{utils → impl/simdlib}/simdlib_neon.h

@@ -19,6 +19,7 @@
 #include <arm_neon.h>
 
 #include <faiss/impl/FaissAssert.h>
+#include <faiss/impl/simdlib/simdlib.h>
 
 namespace faiss {
 
@@ -269,18 +270,20 @@ static inline uint16x8_t vshrq(uint16x8_t vec) {
 } // namespace detail
 
 /// vector of 16 elements in uint16
-struct simd16uint16 {
+template <>
+struct simd16uint16_tpl<SIMDLevel::ARM_NEON> {
     uint16x8x2_t data;
 
-    simd16uint16() = default;
+    simd16uint16_tpl() = default;
 
-    explicit simd16uint16(int x) : data{vdupq_n_u16(x), vdupq_n_u16(x)} {}
+    explicit simd16uint16_tpl(int x) : data{vdupq_n_u16(x), vdupq_n_u16(x)} {}
 
-    explicit simd16uint16(uint16_t x) : data{vdupq_n_u16(x), vdupq_n_u16(x)} {}
+    explicit simd16uint16_tpl(uint16_t x)
+            : data{vdupq_n_u16(x), vdupq_n_u16(x)} {}
 
-    explicit simd16uint16(const uint16x8x2_t& v) : data{v} {}
+    explicit simd16uint16_tpl(const uint16x8x2_t& v) : data{v} {}
 
-    explicit simd16uint16(
+    explicit simd16uint16_tpl(
             uint16_t u0,
             uint16_t u1,
             uint16_t u2,
@@ -323,10 +326,10 @@ struct simd16uint16 {
             typename std::enable_if<
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
-    explicit simd16uint16(const T& x)
+    explicit simd16uint16_tpl(const T& x)
             : data{detail::simdlib::reinterpret_u16(x.data)} {}
 
-    explicit simd16uint16(const uint16_t* x)
+    explicit simd16uint16_tpl(const uint16_t* x)
             : data{vld1q_u16(x), vld1q_u16(x + 8)} {}
 
     void clear() {
@@ -372,65 +375,74 @@ struct simd16uint16 {
         detail::simdlib::set1(data, x).call<&detail::simdlib::vdupq_n_u16>();
     }
 
-    simd16uint16 operator*(const simd16uint16& other) const {
-        return simd16uint16{detail::simdlib::binary_func(data, other.data)
-                                    .call<&vmulq_u16>()};
+    simd16uint16_tpl operator*(const simd16uint16_tpl& other) const {
+        return simd16uint16_tpl{detail::simdlib::binary_func(data, other.data)
+                                        .call<&vmulq_u16>()};
     }
 
     // shift must be known at compile time
-    simd16uint16 operator>>(const int shift) const {
+    simd16uint16_tpl operator>>(const int shift) const {
         switch (shift) {
             case 0:
                 return *this;
             case 1:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshrq<1>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<1>>()};
             case 2:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshrq<2>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<2>>()};
             case 3:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshrq<3>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<3>>()};
             case 4:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshrq<4>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<4>>()};
             case 5:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshrq<5>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<5>>()};
             case 6:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshrq<6>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<6>>()};
             case 7:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshrq<7>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<7>>()};
             case 8:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshrq<8>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<8>>()};
             case 9:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshrq<9>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<9>>()};
             case 10:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshrq<10>>()};
             case 11:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshrq<11>>()};
             case 12:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshrq<12>>()};
             case 13:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshrq<13>>()};
             case 14:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshrq<14>>()};
             case 15:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshrq<15>>()};
             default:
@@ -439,59 +451,68 @@ struct simd16uint16 {
     }
 
     // shift must be known at compile time
-    simd16uint16 operator<<(const int shift) const {
+    simd16uint16_tpl operator<<(const int shift) const {
         switch (shift) {
             case 0:
                 return *this;
             case 1:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshlq<1>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<1>>()};
             case 2:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshlq<2>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<2>>()};
             case 3:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshlq<3>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<3>>()};
             case 4:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshlq<4>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<4>>()};
             case 5:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshlq<5>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<5>>()};
             case 6:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshlq<6>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<6>>()};
             case 7:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshlq<7>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<7>>()};
             case 8:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshlq<8>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<8>>()};
             case 9:
-                return simd16uint16{detail::simdlib::unary_func(data)
-                                            .call<detail::simdlib::vshlq<9>>()};
+                return simd16uint16_tpl{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<9>>()};
             case 10:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshlq<10>>()};
             case 11:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshlq<11>>()};
             case 12:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshlq<12>>()};
             case 13:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshlq<13>>()};
             case 14:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshlq<14>>()};
             case 15:
-                return simd16uint16{
+                return simd16uint16_tpl{
                         detail::simdlib::unary_func(data)
                                 .call<detail::simdlib::vshlq<15>>()};
             default:
@@ -499,24 +520,24 @@ struct simd16uint16 {
         }
     }
 
-    simd16uint16 operator+=(const simd16uint16& other) {
+    simd16uint16_tpl& operator+=(const simd16uint16_tpl& other) {
         *this = *this + other;
         return *this;
     }
 
-    simd16uint16 operator-=(const simd16uint16& other) {
+    simd16uint16_tpl& operator-=(const simd16uint16_tpl& other) {
         *this = *this - other;
         return *this;
     }
 
-    simd16uint16 operator+(const simd16uint16& other) const {
-        return simd16uint16{detail::simdlib::binary_func(data, other.data)
-                                    .call<&vaddq_u16>()};
+    simd16uint16_tpl operator+(const simd16uint16_tpl& other) const {
+        return simd16uint16_tpl{detail::simdlib::binary_func(data, other.data)
+                                        .call<&vaddq_u16>()};
     }
 
-    simd16uint16 operator-(const simd16uint16& other) const {
-        return simd16uint16{detail::simdlib::binary_func(data, other.data)
-                                    .call<&vsubq_u16>()};
+    simd16uint16_tpl operator-(const simd16uint16_tpl& other) const {
+        return simd16uint16_tpl{detail::simdlib::binary_func(data, other.data)
+                                        .call<&vsubq_u16>()};
     }
 
     template <
@@ -524,8 +545,8 @@ struct simd16uint16 {
             typename std::enable_if<
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
-    simd16uint16 operator&(const T& other) const {
-        return simd16uint16{
+    simd16uint16_tpl operator&(const T& other) const {
+        return simd16uint16_tpl{
                 detail::simdlib::binary_func(
                         data, detail::simdlib::reinterpret_u16(other.data))
                         .template call<&vandq_u16>()};
@@ -536,8 +557,8 @@ struct simd16uint16 {
             typename std::enable_if<
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
-    simd16uint16 operator|(const T& other) const {
-        return simd16uint16{
+    simd16uint16_tpl operator|(const T& other) const {
+        return simd16uint16_tpl{
                 detail::simdlib::binary_func(
                         data, detail::simdlib::reinterpret_u16(other.data))
                         .template call<&vorrq_u16>()};
@@ -548,17 +569,17 @@ struct simd16uint16 {
             typename std::enable_if<
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
-    simd16uint16 operator^(const T& other) const {
-        return simd16uint16{
+    simd16uint16_tpl operator^(const T& other) const {
+        return simd16uint16_tpl{
                 detail::simdlib::binary_func(
                         data, detail::simdlib::reinterpret_u16(other.data))
                         .template call<&veorq_u16>()};
     }
 
     // returns binary masks
-    simd16uint16 operator==(const simd16uint16& other) const {
-        return simd16uint16{detail::simdlib::binary_func(data, other.data)
-                                    .call<&vceqq_u16>()};
+    simd16uint16_tpl operator==(const simd16uint16_tpl& other) const {
+        return simd16uint16_tpl{detail::simdlib::binary_func(data, other.data)
+                                        .call<&vceqq_u16>()};
     }
 
     // Checks whether the other holds exactly the same bytes.
@@ -571,8 +592,8 @@ struct simd16uint16 {
         return vminvq_u16(equal) == 0xffffu;
     }
 
-    simd16uint16 operator~() const {
-        return simd16uint16{
+    simd16uint16_tpl operator~() const {
+        return simd16uint16_tpl{
                 detail::simdlib::unary_func(data).call<&vmvnq_u16>()};
     }
 
@@ -583,7 +604,7 @@ struct simd16uint16 {
 
     // mask of elements where this >= thresh
     // 2 bit per component: 16 * 2 = 32 bit
-    uint32_t ge_mask(const simd16uint16& thresh) const {
+    uint32_t ge_mask(const simd16uint16_tpl& thresh) const {
         const auto input = detail::simdlib::binary_func(data, thresh.data)
                                    .call<&vcgeq_u16>();
         const auto vmovmask_u16 = [](uint16x8_t v) -> uint16_t {
@@ -597,15 +618,15 @@ struct simd16uint16 {
                 vmovmask_u16(input.val[0]);
     }
 
-    uint32_t le_mask(const simd16uint16& thresh) const {
+    uint32_t le_mask(const simd16uint16_tpl& thresh) const {
         return thresh.ge_mask(*this);
     }
 
-    uint32_t gt_mask(const simd16uint16& thresh) const {
+    uint32_t gt_mask(const simd16uint16_tpl& thresh) const {
         return ~le_mask(thresh);
     }
 
-    bool all_gt(const simd16uint16& thresh) const {
+    bool all_gt(const simd16uint16_tpl& thresh) const {
         return le_mask(thresh) == 0;
     }
 
@@ -617,33 +638,39 @@ struct simd16uint16 {
         return tab[i - high * 8];
     }
 
-    void accu_min(const simd16uint16& incoming) {
+    void accu_min(const simd16uint16_tpl& incoming) {
         data = detail::simdlib::binary_func(incoming.data, data)
                        .call<&vminq_u16>();
     }
 
-    void accu_max(const simd16uint16& incoming) {
+    void accu_max(const simd16uint16_tpl& incoming) {
         data = detail::simdlib::binary_func(incoming.data, data)
                        .call<&vmaxq_u16>();
     }
 };
 
 // not really a std::min because it returns an elementwise min
-inline simd16uint16 min(const simd16uint16& av, const simd16uint16& bv) {
-    return simd16uint16{
+inline simd16uint16_tpl<SIMDLevel::ARM_NEON> min(
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& av,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& bv) {
+    return simd16uint16_tpl<SIMDLevel::ARM_NEON>{
             detail::simdlib::binary_func(av.data, bv.data).call<&vminq_u16>()};
 }
 
-inline simd16uint16 max(const simd16uint16& av, const simd16uint16& bv) {
-    return simd16uint16{
+inline simd16uint16_tpl<SIMDLevel::ARM_NEON> max(
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& av,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& bv) {
+    return simd16uint16_tpl<SIMDLevel::ARM_NEON>{
             detail::simdlib::binary_func(av.data, bv.data).call<&vmaxq_u16>()};
 }
 
 // decompose in 128-lanes: a = (a0, a1), b = (b0, b1)
 // return (a0 + a1, b0 + b1)
 // TODO find a better name
-inline simd16uint16 combine2x2(const simd16uint16& a, const simd16uint16& b) {
-    return simd16uint16{uint16x8x2_t{
+inline simd16uint16_tpl<SIMDLevel::ARM_NEON> combine2x2(
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd16uint16_tpl<SIMDLevel::ARM_NEON>{uint16x8x2_t{
             vaddq_u16(a.data.val[0], a.data.val[1]),
             vaddq_u16(b.data.val[0], b.data.val[1])}};
 }
@@ -651,22 +678,24 @@ inline simd16uint16 combine2x2(const simd16uint16& a, const simd16uint16& b) {
 // compare d0 and d1 to thr, return 32 bits corresponding to the concatenation
 // of d0 and d1 with thr
 inline uint32_t cmp_ge32(
-        const simd16uint16& d0,
-        const simd16uint16& d1,
-        const simd16uint16& thr) {
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& d0,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& d1,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& thr) {
     return detail::simdlib::cmp_xe32<&vcgeq_u16>(d0.data, d1.data, thr.data);
 }
 
 inline uint32_t cmp_le32(
-        const simd16uint16& d0,
-        const simd16uint16& d1,
-        const simd16uint16& thr) {
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& d0,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& d1,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& thr) {
     return detail::simdlib::cmp_xe32<&vcleq_u16>(d0.data, d1.data, thr.data);
 }
 
 // hadd does not cross lanes
-inline simd16uint16 hadd(const simd16uint16& a, const simd16uint16& b) {
-    return simd16uint16{
+inline simd16uint16_tpl<SIMDLevel::ARM_NEON> hadd(
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd16uint16_tpl<SIMDLevel::ARM_NEON>{
             detail::simdlib::binary_func(a.data, b.data).call<&vpaddq_u16>()};
 }
 
@@ -682,14 +711,14 @@ inline simd16uint16 hadd(const simd16uint16& a, const simd16uint16& b) {
 // the last equal value is saved instead of the first one), but this behavior
 // saves instructions.
 inline void cmplt_min_max_fast(
-        const simd16uint16 candidateValues,
-        const simd16uint16 candidateIndices,
-        const simd16uint16 currentValues,
-        const simd16uint16 currentIndices,
-        simd16uint16& minValues,
-        simd16uint16& minIndices,
-        simd16uint16& maxValues,
-        simd16uint16& maxIndices) {
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON> candidateValues,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON> candidateIndices,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON> currentValues,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON> currentIndices,
+        simd16uint16_tpl<SIMDLevel::ARM_NEON>& minValues,
+        simd16uint16_tpl<SIMDLevel::ARM_NEON>& minIndices,
+        simd16uint16_tpl<SIMDLevel::ARM_NEON>& maxValues,
+        simd16uint16_tpl<SIMDLevel::ARM_NEON>& maxIndices) {
     const uint16x8x2_t comparison =
             detail::simdlib::binary_func(
                     candidateValues.data, currentValues.data)
@@ -719,16 +748,17 @@ inline void cmplt_min_max_fast(
 }
 
 // vector of 32 unsigned 8-bit integers
-struct simd32uint8 {
+template <>
+struct simd32uint8_tpl<SIMDLevel::ARM_NEON> {
     uint8x16x2_t data;
 
-    simd32uint8() = default;
+    simd32uint8_tpl() = default;
 
-    explicit simd32uint8(int x) : data{vdupq_n_u8(x), vdupq_n_u8(x)} {}
+    explicit simd32uint8_tpl(int x) : data{vdupq_n_u8(x), vdupq_n_u8(x)} {}
 
-    explicit simd32uint8(uint8_t x) : data{vdupq_n_u8(x), vdupq_n_u8(x)} {}
+    explicit simd32uint8_tpl(uint8_t x) : data{vdupq_n_u8(x), vdupq_n_u8(x)} {}
 
-    explicit simd32uint8(const uint8x16x2_t& v) : data{v} {}
+    explicit simd32uint8_tpl(const uint8x16x2_t& v) : data{v} {}
 
     template <
             uint8_t _0,
@@ -763,12 +793,12 @@ struct simd32uint8 {
             uint8_t _29,
             uint8_t _30,
             uint8_t _31>
-    static simd32uint8 create() {
+    static simd32uint8_tpl create() {
         constexpr uint8_t ds[32] = {_0,  _1,  _2,  _3,  _4,  _5,  _6,  _7,
                                     _8,  _9,  _10, _11, _12, _13, _14, _15,
                                     _16, _17, _18, _19, _20, _21, _22, _23,
                                     _24, _25, _26, _27, _28, _29, _30, _31};
-        return simd32uint8{ds};
+        return simd32uint8_tpl{ds};
     }
 
     template <
@@ -776,10 +806,10 @@ struct simd32uint8 {
             typename std::enable_if<
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
-    explicit simd32uint8(const T& x)
+    explicit simd32uint8_tpl(const T& x)
             : data{detail::simdlib::reinterpret_u8(x.data)} {}
 
-    explicit simd32uint8(const uint8_t* x)
+    explicit simd32uint8_tpl(const uint8_t* x)
             : data{vld1q_u8(x), vld1q_u8(x + 16)} {}
 
     void clear() {
@@ -835,25 +865,25 @@ struct simd32uint8 {
             typename std::enable_if<
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
-    simd32uint8 operator&(const T& other) const {
-        return simd32uint8{
+    simd32uint8_tpl operator&(const T& other) const {
+        return simd32uint8_tpl{
                 detail::simdlib::binary_func(
                         data, detail::simdlib::reinterpret_u8(other.data))
                         .template call<&vandq_u8>()};
     }
 
-    simd32uint8 operator+(const simd32uint8& other) const {
-        return simd32uint8{detail::simdlib::binary_func(data, other.data)
-                                   .call<&vaddq_u8>()};
+    simd32uint8_tpl operator+(const simd32uint8_tpl& other) const {
+        return simd32uint8_tpl{detail::simdlib::binary_func(data, other.data)
+                                       .call<&vaddq_u8>()};
     }
 
     // The very important operation that everything relies on
-    simd32uint8 lookup_2_lanes(const simd32uint8& idx) const {
-        return simd32uint8{detail::simdlib::binary_func(data, idx.data)
-                                   .call<&vqtbl1q_u8>()};
+    simd32uint8_tpl lookup_2_lanes(const simd32uint8_tpl& idx) const {
+        return simd32uint8_tpl{detail::simdlib::binary_func(data, idx.data)
+                                       .call<&vqtbl1q_u8>()};
     }
 
-    simd32uint8 operator+=(const simd32uint8& other) {
+    simd32uint8_tpl& operator+=(const simd32uint8_tpl& other) {
         *this = *this + other;
         return *this;
     }
@@ -879,56 +909,59 @@ struct simd32uint8 {
 
 // convert with saturation
 // careful: this does not cross lanes, so the order is weird
-inline simd32uint8 uint16_to_uint8_saturate(
-        const simd16uint16& a,
-        const simd16uint16& b) {
-    return simd32uint8{uint8x16x2_t{
+inline simd32uint8_tpl<SIMDLevel::ARM_NEON> uint16_to_uint8_saturate(
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd16uint16_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd32uint8_tpl<SIMDLevel::ARM_NEON>{uint8x16x2_t{
             vqmovn_high_u16(vqmovn_u16(a.data.val[0]), b.data.val[0]),
             vqmovn_high_u16(vqmovn_u16(a.data.val[1]), b.data.val[1])}};
 }
 
 /// get most significant bit of each byte
-inline uint32_t get_MSBs(const simd32uint8& a) {
+inline uint32_t get_MSBs(const simd32uint8_tpl<SIMDLevel::ARM_NEON>& a) {
     using detail::simdlib::vmovmask_u8;
     return vmovmask_u8(a.data.val[0]) |
             static_cast<uint32_t>(vmovmask_u8(a.data.val[1])) << 16u;
 }
 
 /// use MSB of each byte of mask to select a byte between a and b
-inline simd32uint8 blendv(
-        const simd32uint8& a,
-        const simd32uint8& b,
-        const simd32uint8& mask) {
+inline simd32uint8_tpl<SIMDLevel::ARM_NEON> blendv(
+        const simd32uint8_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd32uint8_tpl<SIMDLevel::ARM_NEON>& b,
+        const simd32uint8_tpl<SIMDLevel::ARM_NEON>& mask) {
     const auto msb = vdupq_n_u8(0x80);
     const uint8x16x2_t msb_mask = {
             vtstq_u8(mask.data.val[0], msb), vtstq_u8(mask.data.val[1], msb)};
     const uint8x16x2_t selected = {
             vbslq_u8(msb_mask.val[0], b.data.val[0], a.data.val[0]),
             vbslq_u8(msb_mask.val[1], b.data.val[1], a.data.val[1])};
-    return simd32uint8{selected};
+    return simd32uint8_tpl<SIMDLevel::ARM_NEON>{selected};
 }
 
 /// vector of 8 unsigned 32-bit integers
-struct simd8uint32 {
+template <>
+struct simd8uint32_tpl<SIMDLevel::ARM_NEON> {
     uint32x4x2_t data;
 
-    simd8uint32() = default;
+    simd8uint32_tpl() = default;
 
-    explicit simd8uint32(uint32_t x) : data{vdupq_n_u32(x), vdupq_n_u32(x)} {}
+    explicit simd8uint32_tpl(uint32_t x)
+            : data{vdupq_n_u32(x), vdupq_n_u32(x)} {}
 
-    explicit simd8uint32(const uint32x4x2_t& v) : data{v} {}
+    explicit simd8uint32_tpl(const uint32x4x2_t& v) : data{v} {}
 
     template <
             typename T,
             typename std::enable_if<
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
-    explicit simd8uint32(const T& x)
+    explicit simd8uint32_tpl(const T& x)
             : data{detail::simdlib::reinterpret_u32(x.data)} {}
 
-    explicit simd8uint32(const uint8_t* x) : simd8uint32(simd32uint8(x)) {}
+    explicit simd8uint32_tpl(const uint8_t* x)
+            : simd8uint32_tpl(simd32uint8_tpl<SIMDLevel::ARM_NEON>(x)) {}
 
-    explicit simd8uint32(
+    explicit simd8uint32_tpl(
             uint32_t u0,
             uint32_t u1,
             uint32_t u2,
@@ -942,33 +975,33 @@ struct simd8uint32 {
         data.val[1] = vld1q_u32(temp + 4);
     }
 
-    simd8uint32 operator+(simd8uint32 other) const {
-        return simd8uint32{detail::simdlib::binary_func(data, other.data)
-                                   .call<&vaddq_u32>()};
+    simd8uint32_tpl operator+(simd8uint32_tpl other) const {
+        return simd8uint32_tpl{detail::simdlib::binary_func(data, other.data)
+                                       .call<&vaddq_u32>()};
     }
 
-    simd8uint32 operator-(simd8uint32 other) const {
-        return simd8uint32{detail::simdlib::binary_func(data, other.data)
-                                   .call<&vsubq_u32>()};
+    simd8uint32_tpl operator-(simd8uint32_tpl other) const {
+        return simd8uint32_tpl{detail::simdlib::binary_func(data, other.data)
+                                       .call<&vsubq_u32>()};
     }
 
-    simd8uint32& operator+=(const simd8uint32& other) {
+    simd8uint32_tpl& operator+=(const simd8uint32_tpl& other) {
         data.val[0] = vaddq_u32(data.val[0], other.data.val[0]);
         data.val[1] = vaddq_u32(data.val[1], other.data.val[1]);
         return *this;
     }
 
-    simd8uint32 operator==(simd8uint32 other) const {
-        return simd8uint32{detail::simdlib::binary_func(data, other.data)
-                                   .call<&vceqq_u32>()};
+    simd8uint32_tpl operator==(simd8uint32_tpl other) const {
+        return simd8uint32_tpl{detail::simdlib::binary_func(data, other.data)
+                                       .call<&vceqq_u32>()};
     }
 
-    simd8uint32 operator~() const {
-        return simd8uint32{
+    simd8uint32_tpl operator~() const {
+        return simd8uint32_tpl{
                 detail::simdlib::unary_func(data).call<&vmvnq_u32>()};
     }
 
-    simd8uint32 operator!=(simd8uint32 other) const {
+    simd8uint32_tpl operator!=(simd8uint32_tpl other) const {
         return ~(*this == other);
     }
 
@@ -1025,8 +1058,8 @@ struct simd8uint32 {
         detail::simdlib::set1(data, x).call<&vdupq_n_u32>();
     }
 
-    simd8uint32 unzip() const {
-        return simd8uint32{uint32x4x2_t{
+    simd8uint32_tpl unzip() const {
+        return simd8uint32_tpl{uint32x4x2_t{
                 vuzp1q_u32(data.val[0], data.val[1]),
                 vuzp2q_u32(data.val[0], data.val[1])}};
     }
@@ -1044,14 +1077,14 @@ struct simd8uint32 {
 // the last equal value is saved instead of the first one), but this behavior
 // saves instructions.
 inline void cmplt_min_max_fast(
-        const simd8uint32 candidateValues,
-        const simd8uint32 candidateIndices,
-        const simd8uint32 currentValues,
-        const simd8uint32 currentIndices,
-        simd8uint32& minValues,
-        simd8uint32& minIndices,
-        simd8uint32& maxValues,
-        simd8uint32& maxIndices) {
+        const simd8uint32_tpl<SIMDLevel::ARM_NEON> candidateValues,
+        const simd8uint32_tpl<SIMDLevel::ARM_NEON> candidateIndices,
+        const simd8uint32_tpl<SIMDLevel::ARM_NEON> currentValues,
+        const simd8uint32_tpl<SIMDLevel::ARM_NEON> currentIndices,
+        simd8uint32_tpl<SIMDLevel::ARM_NEON>& minValues,
+        simd8uint32_tpl<SIMDLevel::ARM_NEON>& minIndices,
+        simd8uint32_tpl<SIMDLevel::ARM_NEON>& maxValues,
+        simd8uint32_tpl<SIMDLevel::ARM_NEON>& maxIndices) {
     const uint32x4x2_t comparison =
             detail::simdlib::binary_func(
                     candidateValues.data, currentValues.data)
@@ -1084,27 +1117,28 @@ inline void cmplt_min_max_fast(
                     candidateIndices.data.val[1])};
 }
 
-struct simd8float32 {
+template <>
+struct simd8float32_tpl<SIMDLevel::ARM_NEON> {
     float32x4x2_t data;
 
-    simd8float32() = default;
+    simd8float32_tpl() = default;
 
-    explicit simd8float32(float x) : data{vdupq_n_f32(x), vdupq_n_f32(x)} {}
+    explicit simd8float32_tpl(float x) : data{vdupq_n_f32(x), vdupq_n_f32(x)} {}
 
-    explicit simd8float32(const float32x4x2_t& v) : data{v} {}
+    explicit simd8float32_tpl(const float32x4x2_t& v) : data{v} {}
 
     template <
             typename T,
             typename std::enable_if<
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
-    explicit simd8float32(const T& x)
+    explicit simd8float32_tpl(const T& x)
             : data{detail::simdlib::reinterpret_f32(x.data)} {}
 
-    explicit simd8float32(const float* x)
+    explicit simd8float32_tpl(const float* x)
             : data{vld1q_f32(x), vld1q_f32(x + 4)} {}
 
-    explicit simd8float32(
+    explicit simd8float32_tpl(
             float f0,
             float f1,
             float f2,
@@ -1144,22 +1178,22 @@ struct simd8float32 {
         return detail::simdlib::bin(*this);
     }
 
-    simd8float32 operator*(const simd8float32& other) const {
-        return simd8float32{detail::simdlib::binary_func(data, other.data)
-                                    .call<&vmulq_f32>()};
+    simd8float32_tpl operator*(const simd8float32_tpl& other) const {
+        return simd8float32_tpl{detail::simdlib::binary_func(data, other.data)
+                                        .call<&vmulq_f32>()};
     }
 
-    simd8float32 operator+(const simd8float32& other) const {
-        return simd8float32{detail::simdlib::binary_func(data, other.data)
-                                    .call<&vaddq_f32>()};
+    simd8float32_tpl operator+(const simd8float32_tpl& other) const {
+        return simd8float32_tpl{detail::simdlib::binary_func(data, other.data)
+                                        .call<&vaddq_f32>()};
     }
 
-    simd8float32 operator-(const simd8float32& other) const {
-        return simd8float32{detail::simdlib::binary_func(data, other.data)
-                                    .call<&vsubq_f32>()};
+    simd8float32_tpl operator-(const simd8float32_tpl& other) const {
+        return simd8float32_tpl{detail::simdlib::binary_func(data, other.data)
+                                        .call<&vsubq_f32>()};
     }
 
-    simd8float32& operator+=(const simd8float32& other) {
+    simd8float32_tpl& operator+=(const simd8float32_tpl& other) {
         // In this context, it is more compiler friendly to write intrinsics
         // directly instead of using binary_func
         data.val[0] = vaddq_f32(data.val[0], other.data.val[0]);
@@ -1167,13 +1201,15 @@ struct simd8float32 {
         return *this;
     }
 
-    simd8uint32 operator==(simd8float32 other) const {
-        return simd8uint32{
+    simd8uint32_tpl<SIMDLevel::ARM_NEON> operator==(
+            simd8float32_tpl other) const {
+        return simd8uint32_tpl<SIMDLevel::ARM_NEON>{
                 detail::simdlib::binary_func<::uint32x4x2_t>(data, other.data)
                         .call<&vceqq_f32>()};
     }
 
-    simd8uint32 operator!=(simd8float32 other) const {
+    simd8uint32_tpl<SIMDLevel::ARM_NEON> operator!=(
+            simd8float32_tpl other) const {
         return ~(*this == other);
     }
 
@@ -1194,27 +1230,33 @@ struct simd8float32 {
 };
 
 // hadd does not cross lanes
-inline simd8float32 hadd(const simd8float32& a, const simd8float32& b) {
-    return simd8float32{
+inline simd8float32_tpl<SIMDLevel::ARM_NEON> hadd(
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd8float32_tpl<SIMDLevel::ARM_NEON>{
             detail::simdlib::binary_func(a.data, b.data).call<&vpaddq_f32>()};
 }
 
-inline simd8float32 unpacklo(const simd8float32& a, const simd8float32& b) {
-    return simd8float32{
+inline simd8float32_tpl<SIMDLevel::ARM_NEON> unpacklo(
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd8float32_tpl<SIMDLevel::ARM_NEON>{
             detail::simdlib::binary_func(a.data, b.data).call<&vzip1q_f32>()};
 }
 
-inline simd8float32 unpackhi(const simd8float32& a, const simd8float32& b) {
-    return simd8float32{
+inline simd8float32_tpl<SIMDLevel::ARM_NEON> unpackhi(
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd8float32_tpl<SIMDLevel::ARM_NEON>{
             detail::simdlib::binary_func(a.data, b.data).call<&vzip2q_f32>()};
 }
 
 // compute a * b + c
-inline simd8float32 fmadd(
-        const simd8float32& a,
-        const simd8float32& b,
-        const simd8float32& c) {
-    return simd8float32{float32x4x2_t{
+inline simd8float32_tpl<SIMDLevel::ARM_NEON> fmadd(
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& b,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& c) {
+    return simd8float32_tpl<SIMDLevel::ARM_NEON>{float32x4x2_t{
             vfmaq_f32(c.data.val[0], a.data.val[0], b.data.val[0]),
             vfmaq_f32(c.data.val[1], a.data.val[1], b.data.val[1])}};
 }
@@ -1251,10 +1293,10 @@ inline simd8float32 fmadd(
 // confusion for ppl who write in low-level SIMD instructions. Additionally,
 // these two ops (cmp and blend) are very often used together.
 inline void cmplt_and_blend_inplace(
-        const simd8float32 candidateValues,
-        const simd8uint32 candidateIndices,
-        simd8float32& lowestValues,
-        simd8uint32& lowestIndices) {
+        const simd8float32_tpl<SIMDLevel::ARM_NEON> candidateValues,
+        const simd8uint32_tpl<SIMDLevel::ARM_NEON> candidateIndices,
+        simd8float32_tpl<SIMDLevel::ARM_NEON>& lowestValues,
+        simd8uint32_tpl<SIMDLevel::ARM_NEON>& lowestIndices) {
     const auto comparison = detail::simdlib::binary_func<::uint32x4x2_t>(
                                     candidateValues.data, lowestValues.data)
                                     .call<&vcltq_f32>();
@@ -1291,14 +1333,14 @@ inline void cmplt_and_blend_inplace(
 // the last equal value is saved instead of the first one), but this behavior
 // saves instructions.
 inline void cmplt_min_max_fast(
-        const simd8float32 candidateValues,
-        const simd8uint32 candidateIndices,
-        const simd8float32 currentValues,
-        const simd8uint32 currentIndices,
-        simd8float32& minValues,
-        simd8uint32& minIndices,
-        simd8float32& maxValues,
-        simd8uint32& maxIndices) {
+        const simd8float32_tpl<SIMDLevel::ARM_NEON> candidateValues,
+        const simd8uint32_tpl<SIMDLevel::ARM_NEON> candidateIndices,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON> currentValues,
+        const simd8uint32_tpl<SIMDLevel::ARM_NEON> currentIndices,
+        simd8float32_tpl<SIMDLevel::ARM_NEON>& minValues,
+        simd8uint32_tpl<SIMDLevel::ARM_NEON>& minIndices,
+        simd8float32_tpl<SIMDLevel::ARM_NEON>& maxValues,
+        simd8uint32_tpl<SIMDLevel::ARM_NEON>& maxIndices) {
     const uint32x4x2_t comparison =
             detail::simdlib::binary_func<::uint32x4x2_t>(
                     candidateValues.data, currentValues.data)
@@ -1334,29 +1376,39 @@ inline void cmplt_min_max_fast(
 namespace {
 
 // get even float32's of a and b, interleaved
-simd8float32 geteven(const simd8float32& a, const simd8float32& b) {
-    return simd8float32{
+simd8float32_tpl<SIMDLevel::ARM_NEON> geteven(
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd8float32_tpl<SIMDLevel::ARM_NEON>{
             detail::simdlib::binary_func(a.data, b.data).call<&vuzp1q_f32>()};
 }
 
 // get odd float32's of a and b, interleaved
-simd8float32 getodd(const simd8float32& a, const simd8float32& b) {
-    return simd8float32{
+simd8float32_tpl<SIMDLevel::ARM_NEON> getodd(
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd8float32_tpl<SIMDLevel::ARM_NEON>{
             detail::simdlib::binary_func(a.data, b.data).call<&vuzp2q_f32>()};
 }
 
 // 3 cycles
 // if the lanes are a = [a0 a1] and b = [b0 b1], return [a0 b0]
-simd8float32 getlow128(const simd8float32& a, const simd8float32& b) {
-    return simd8float32{float32x4x2_t{a.data.val[0], b.data.val[0]}};
+simd8float32_tpl<SIMDLevel::ARM_NEON> getlow128(
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd8float32_tpl<SIMDLevel::ARM_NEON>{
+            float32x4x2_t{a.data.val[0], b.data.val[0]}};
 }
 
-simd8float32 gethigh128(const simd8float32& a, const simd8float32& b) {
-    return simd8float32{float32x4x2_t{a.data.val[1], b.data.val[1]}};
+simd8float32_tpl<SIMDLevel::ARM_NEON> gethigh128(
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& a,
+        const simd8float32_tpl<SIMDLevel::ARM_NEON>& b) {
+    return simd8float32_tpl<SIMDLevel::ARM_NEON>{
+            float32x4x2_t{a.data.val[1], b.data.val[1]}};
 }
 
 // horizontal add: sum all 8 floats in the register
-inline float horizontal_add(const simd8float32& a) {
+inline float horizontal_add(const simd8float32_tpl<SIMDLevel::ARM_NEON>& a) {
     float32x4_t sum = vaddq_f32(a.data.val[0], a.data.val[1]);
     return vaddvq_f32(sum);
 }