faiss 0.2.6 → 0.2.7

data/vendor/faiss/faiss/utils/simdlib_neon.h

@@ -18,6 +18,8 @@
 
 #include <arm_neon.h>
 
+#include <faiss/impl/FaissAssert.h>
+
 namespace faiss {
 
 namespace detail {
@@ -88,6 +90,23 @@ static inline float32x4x2_t reinterpret_f32(const float32x4x2_t& v) {
     return v;
 }
 
+// Surprisingly, vdupq_n_u16 has the type of
+// uint16x8_t (std::uint32_t) , and vdupq_n_u8 also has
+// uint8x16_t (std::uint32_t) on **some environments**.
+// We want argument type as same as the type of element
+// of result vector type (std::uint16_t for uint16x8_t,
+// and std::uint8_t for uint8x16_t) instead of
+// std::uint32_t due to using set1 function templates,
+// so let's fix the argument type here and use these
+// overload below.
+static inline ::uint16x8_t vdupq_n_u16(std::uint16_t v) {
+    return ::vdupq_n_u16(v);
+}
+
+static inline ::uint8x16_t vdupq_n_u8(std::uint8_t v) {
+    return ::vdupq_n_u8(v);
+}
+
 template <
         typename T,
         typename U = decltype(reinterpret_u8(std::declval<T>().data))>
@@ -119,11 +138,25 @@ static inline std::string bin(const S& simd) {
     return std::string(bits);
 }
 
-template <typename D, typename F, typename T>
-static inline void set1(D& d, F&& f, T t) {
-    const auto v = f(t);
-    d.val[0] = v;
-    d.val[1] = v;
+template <typename T>
+using remove_cv_ref_t =
+        typename std::remove_reference<typename std::remove_cv<T>::type>::type;
+
+template <typename D, typename T>
+struct set1_impl {
+    D& d;
+    T t;
+    template <remove_cv_ref_t<decltype(std::declval<D>().val[0])> (*F)(T)>
+    inline void call() {
+        const auto v = F(t);
+        d.val[0] = v;
+        d.val[1] = v;
+    }
+};
+
+template <typename D, typename T>
+static inline set1_impl<remove_cv_ref_t<D>, T> set1(D& d, T t) {
+    return {d, t};
 }
 
 template <typename T, size_t N, typename S>
@@ -142,20 +175,57 @@ static inline std::string elements_to_string(const char* fmt, const S& simd) {
     return std::string(res);
 }
 
-template <typename T, typename F>
-static inline T unary_func(const T& a, F&& f) {
-    T t;
-    t.val[0] = f(a.val[0]);
-    t.val[1] = f(a.val[1]);
-    return t;
+template <typename T, typename U>
+struct unary_func_impl {
+    const U& a;
+    using Telem = remove_cv_ref_t<decltype(std::declval<T>().val[0])>;
+    using Uelem = remove_cv_ref_t<decltype(std::declval<U>().val[0])>;
+    template <Telem (*F)(Uelem)>
+    inline T call() {
+        T t;
+        t.val[0] = F(a.val[0]);
+        t.val[1] = F(a.val[1]);
+        return t;
+    }
+};
+
+template <typename T>
+static inline unary_func_impl<remove_cv_ref_t<T>, remove_cv_ref_t<T>> unary_func(
+        const T& a) {
+    return {a};
 }
 
-template <typename T, typename F>
-static inline T binary_func(const T& a, const T& b, F&& f) {
-    T t;
-    t.val[0] = f(a.val[0], b.val[0]);
-    t.val[1] = f(a.val[1], b.val[1]);
-    return t;
+template <typename T, typename U>
+static inline unary_func_impl<remove_cv_ref_t<T>, remove_cv_ref_t<U>> unary_func(
+        const U& a) {
+    return {a};
+}
+
+template <typename T, typename U>
+struct binary_func_impl {
+    const U& a;
+    const U& b;
+    using Telem = remove_cv_ref_t<decltype(std::declval<T>().val[0])>;
+    using Uelem = remove_cv_ref_t<decltype(std::declval<U>().val[0])>;
+    template <Telem (*F)(Uelem, Uelem)>
+    inline T call() {
+        T t;
+        t.val[0] = F(a.val[0], b.val[0]);
+        t.val[1] = F(a.val[1], b.val[1]);
+        return t;
+    }
+};
+
+template <typename T>
+static inline binary_func_impl<remove_cv_ref_t<T>, remove_cv_ref_t<T>>
+binary_func(const T& a, const T& b) {
+    return {a, b};
+}
+
+template <typename T, typename U>
+static inline binary_func_impl<remove_cv_ref_t<T>, remove_cv_ref_t<U>>
+binary_func(const U& a, const U& b) {
+    return {a, b};
 }
 
 static inline uint16_t vmovmask_u8(const uint8x16_t& v) {
@@ -172,8 +242,8 @@ static inline uint32_t cmp_xe32(
         const uint16x8x2_t& d0,
         const uint16x8x2_t& d1,
         const uint16x8x2_t& thr) {
-    const auto d0_thr = detail::simdlib::binary_func(d0, thr, F);
-    const auto d1_thr = detail::simdlib::binary_func(d1, thr, F);
+    const auto d0_thr = detail::simdlib::binary_func(d0, thr).call<F>();
+    const auto d1_thr = detail::simdlib::binary_func(d1, thr).call<F>();
     const auto d0_mask = vmovmask_u8(
             vmovn_high_u16(vmovn_u16(d0_thr.val[0]), d0_thr.val[1]));
     const auto d1_mask = vmovmask_u8(
@@ -207,6 +277,44 @@ struct simd16uint16 {
 
     explicit simd16uint16(const uint16x8x2_t& v) : data{v} {}
 
+    explicit simd16uint16(
+            uint16_t u0,
+            uint16_t u1,
+            uint16_t u2,
+            uint16_t u3,
+            uint16_t u4,
+            uint16_t u5,
+            uint16_t u6,
+            uint16_t u7,
+            uint16_t u8,
+            uint16_t u9,
+            uint16_t u10,
+            uint16_t u11,
+            uint16_t u12,
+            uint16_t u13,
+            uint16_t u14,
+            uint16_t u15) {
+        uint16_t temp[16] = {
+                u0,
+                u1,
+                u2,
+                u3,
+                u4,
+                u5,
+                u6,
+                u7,
+                u8,
+                u9,
+                u10,
+                u11,
+                u12,
+                u13,
+                u14,
+                u15};
+        data.val[0] = vld1q_u16(temp);
+        data.val[1] = vld1q_u16(temp + 8);
+    }
+
     template <
             typename T,
             typename std::enable_if<
@@ -219,7 +327,8 @@ struct simd16uint16 {
             : data{vld1q_u16(x), vld1q_u16(x + 8)} {}
 
     void clear() {
-        detail::simdlib::set1(data, &vdupq_n_u16, static_cast<uint16_t>(0));
+        detail::simdlib::set1(data, static_cast<uint16_t>(0))
+                .call<&detail::simdlib::vdupq_n_u16>();
     }
 
     void storeu(uint16_t* ptr) const {
@@ -257,12 +366,12 @@ struct simd16uint16 {
     }
 
     void set1(uint16_t x) {
-        detail::simdlib::set1(data, &vdupq_n_u16, x);
+        detail::simdlib::set1(data, x).call<&detail::simdlib::vdupq_n_u16>();
     }
 
     simd16uint16 operator*(const simd16uint16& other) const {
-        return simd16uint16{
-                detail::simdlib::binary_func(data, other.data, &vmulq_u16)};
+        return simd16uint16{detail::simdlib::binary_func(data, other.data)
+                                    .call<&vmulq_u16>()};
     }
 
     // shift must be known at compile time
@@ -271,50 +380,56 @@ struct simd16uint16 {
             case 0:
                 return *this;
             case 1:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<1>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshrq<1>>()};
             case 2:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<2>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshrq<2>>()};
             case 3:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<3>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshrq<3>>()};
             case 4:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<4>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshrq<4>>()};
             case 5:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<5>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshrq<5>>()};
             case 6:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<6>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshrq<6>>()};
             case 7:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<7>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshrq<7>>()};
             case 8:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<8>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshrq<8>>()};
             case 9:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<9>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshrq<9>>()};
             case 10:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<10>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<10>>()};
             case 11:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<11>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<11>>()};
             case 12:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<12>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<12>>()};
             case 13:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<13>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<13>>()};
             case 14:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<14>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<14>>()};
             case 15:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshrq<15>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshrq<15>>()};
             default:
                 FAISS_THROW_FMT("Invalid shift %d", shift);
         }
@@ -326,50 +441,56 @@ struct simd16uint16 {
             case 0:
                 return *this;
             case 1:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<1>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshlq<1>>()};
             case 2:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<2>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshlq<2>>()};
             case 3:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<3>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshlq<3>>()};
             case 4:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<4>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshlq<4>>()};
             case 5:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<5>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshlq<5>>()};
             case 6:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<6>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshlq<6>>()};
             case 7:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<7>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshlq<7>>()};
             case 8:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<8>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshlq<8>>()};
             case 9:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<9>)};
+                return simd16uint16{detail::simdlib::unary_func(data)
+                                            .call<detail::simdlib::vshlq<9>>()};
             case 10:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<10>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<10>>()};
             case 11:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<11>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<11>>()};
             case 12:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<12>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<12>>()};
             case 13:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<13>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<13>>()};
             case 14:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<14>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<14>>()};
             case 15:
-                return simd16uint16{detail::simdlib::unary_func(
-                        data, detail::simdlib::vshlq<15>)};
+                return simd16uint16{
+                        detail::simdlib::unary_func(data)
+                                .call<detail::simdlib::vshlq<15>>()};
             default:
                 FAISS_THROW_FMT("Invalid shift %d", shift);
         }
@@ -386,13 +507,13 @@ struct simd16uint16 {
     }
 
     simd16uint16 operator+(const simd16uint16& other) const {
-        return simd16uint16{
-                detail::simdlib::binary_func(data, other.data, &vaddq_u16)};
+        return simd16uint16{detail::simdlib::binary_func(data, other.data)
+                                    .call<&vaddq_u16>()};
     }
 
     simd16uint16 operator-(const simd16uint16& other) const {
-        return simd16uint16{
-                detail::simdlib::binary_func(data, other.data, &vsubq_u16)};
+        return simd16uint16{detail::simdlib::binary_func(data, other.data)
+                                    .call<&vsubq_u16>()};
     }
 
     template <
@@ -401,10 +522,10 @@ struct simd16uint16 {
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
     simd16uint16 operator&(const T& other) const {
-        return simd16uint16{detail::simdlib::binary_func(
-                data,
-                detail::simdlib::reinterpret_u16(other.data),
-                &vandq_u16)};
+        return simd16uint16{
+                detail::simdlib::binary_func(
+                        data, detail::simdlib::reinterpret_u16(other.data))
+                        .template call<&vandq_u16>()};
     }
 
     template <
@@ -413,20 +534,45 @@ struct simd16uint16 {
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
     simd16uint16 operator|(const T& other) const {
-        return simd16uint16{detail::simdlib::binary_func(
-                data,
-                detail::simdlib::reinterpret_u16(other.data),
-                &vorrq_u16)};
+        return simd16uint16{
+                detail::simdlib::binary_func(
+                        data, detail::simdlib::reinterpret_u16(other.data))
+                        .template call<&vorrq_u16>()};
+    }
+
+    template <
+            typename T,
+            typename std::enable_if<
+                    detail::simdlib::is_simd256bit<T>::value,
+                    std::nullptr_t>::type = nullptr>
+    simd16uint16 operator^(const T& other) const {
+        return simd16uint16{
+                detail::simdlib::binary_func(
+                        data, detail::simdlib::reinterpret_u16(other.data))
+                        .template call<&veorq_u16>()};
     }
 
     // returns binary masks
     simd16uint16 operator==(const simd16uint16& other) const {
-        return simd16uint16{
-                detail::simdlib::binary_func(data, other.data, &vceqq_u16)};
+        return simd16uint16{detail::simdlib::binary_func(data, other.data)
+                                    .call<&vceqq_u16>()};
+    }
+
+    // Checks whether the other holds exactly the same bytes.
+    bool is_same_as(simd16uint16 other) const {
+        const bool equal0 =
+                (vminvq_u16(vceqq_u16(data.val[0], other.data.val[0])) ==
+                 0xffff);
+        const bool equal1 =
+                (vminvq_u16(vceqq_u16(data.val[1], other.data.val[1])) ==
+                 0xffff);
+
+        return equal0 && equal1;
     }
 
     simd16uint16 operator~() const {
-        return simd16uint16{detail::simdlib::unary_func(data, &vmvnq_u16)};
+        return simd16uint16{
+                detail::simdlib::unary_func(data).call<&vmvnq_u16>()};
     }
 
     // get scalar at index 0
@@ -437,8 +583,8 @@ struct simd16uint16 {
     // mask of elements where this >= thresh
     // 2 bit per component: 16 * 2 = 32 bit
     uint32_t ge_mask(const simd16uint16& thresh) const {
-        const auto input =
-                detail::simdlib::binary_func(data, thresh.data, &vcgeq_u16);
+        const auto input = detail::simdlib::binary_func(data, thresh.data)
+                                   .call<&vcgeq_u16>();
         const auto vmovmask_u16 = [](uint16x8_t v) -> uint16_t {
             uint16_t d[8];
             const auto v2 = vreinterpretq_u32_u16(vshrq_n_u16(v, 14));
@@ -471,23 +617,25 @@ struct simd16uint16 {
     }
 
     void accu_min(const simd16uint16& incoming) {
-        data = detail::simdlib::binary_func(incoming.data, data, &vminq_u16);
+        data = detail::simdlib::binary_func(incoming.data, data)
+                       .call<&vminq_u16>();
     }
 
     void accu_max(const simd16uint16& incoming) {
-        data = detail::simdlib::binary_func(incoming.data, data, &vmaxq_u16);
+        data = detail::simdlib::binary_func(incoming.data, data)
+                       .call<&vmaxq_u16>();
     }
 };
 
 // not really a std::min because it returns an elementwise min
 inline simd16uint16 min(const simd16uint16& av, const simd16uint16& bv) {
     return simd16uint16{
-            detail::simdlib::binary_func(av.data, bv.data, &vminq_u16)};
+            detail::simdlib::binary_func(av.data, bv.data).call<&vminq_u16>()};
 }
 
 inline simd16uint16 max(const simd16uint16& av, const simd16uint16& bv) {
     return simd16uint16{
-            detail::simdlib::binary_func(av.data, bv.data, &vmaxq_u16)};
+            detail::simdlib::binary_func(av.data, bv.data).call<&vmaxq_u16>()};
 }
 
 // decompose in 128-lanes: a = (a0, a1), b = (b0, b1)
@@ -515,6 +663,63 @@ inline uint32_t cmp_le32(
     return detail::simdlib::cmp_xe32<&vcleq_u16>(d0.data, d1.data, thr.data);
 }
 
+// hadd does not cross lanes
+inline simd16uint16 hadd(const simd16uint16& a, const simd16uint16& b) {
+    return simd16uint16{
+            detail::simdlib::binary_func(a.data, b.data).call<&vpaddq_u16>()};
+}
+
+// Vectorized version of the following code:
+//   for (size_t i = 0; i < n; i++) {
+//      bool flag = (candidateValues[i] < currentValues[i]);
+//      minValues[i] = flag ? candidateValues[i] : currentValues[i];
+//      minIndices[i] = flag ? candidateIndices[i] : currentIndices[i];
+//      maxValues[i] = !flag ? candidateValues[i] : currentValues[i];
+//      maxIndices[i] = !flag ? candidateIndices[i] : currentIndices[i];
+//   }
+// Max indices evaluation is inaccurate in case of equal values (the index of
+// the last equal value is saved instead of the first one), but this behavior
+// saves instructions.
+inline void cmplt_min_max_fast(
+        const simd16uint16 candidateValues,
+        const simd16uint16 candidateIndices,
+        const simd16uint16 currentValues,
+        const simd16uint16 currentIndices,
+        simd16uint16& minValues,
+        simd16uint16& minIndices,
+        simd16uint16& maxValues,
+        simd16uint16& maxIndices) {
+    const uint16x8x2_t comparison = uint16x8x2_t{
+            vcltq_u16(candidateValues.data.val[0], currentValues.data.val[0]),
+            vcltq_u16(candidateValues.data.val[1], currentValues.data.val[1])};
+
+    minValues.data = uint16x8x2_t{
+            vminq_u16(candidateValues.data.val[0], currentValues.data.val[0]),
+            vminq_u16(candidateValues.data.val[1], currentValues.data.val[1])};
+    minIndices.data = uint16x8x2_t{
+            vbslq_u16(
+                    comparison.val[0],
+                    candidateIndices.data.val[0],
+                    currentIndices.data.val[0]),
+            vbslq_u16(
+                    comparison.val[1],
+                    candidateIndices.data.val[1],
+                    currentIndices.data.val[1])};
+
+    maxValues.data = uint16x8x2_t{
+            vmaxq_u16(candidateValues.data.val[0], currentValues.data.val[0]),
+            vmaxq_u16(candidateValues.data.val[1], currentValues.data.val[1])};
+    maxIndices.data = uint16x8x2_t{
+            vbslq_u16(
+                    comparison.val[0],
+                    currentIndices.data.val[0],
+                    candidateIndices.data.val[0]),
+            vbslq_u16(
+                    comparison.val[1],
+                    currentIndices.data.val[1],
+                    candidateIndices.data.val[1])};
+}
+
 // vector of 32 unsigned 8-bit integers
 struct simd32uint8 {
     uint8x16x2_t data;
@@ -527,6 +732,47 @@ struct simd32uint8 {
 
     explicit simd32uint8(const uint8x16x2_t& v) : data{v} {}
 
+    template <
+            uint8_t _0,
+            uint8_t _1,
+            uint8_t _2,
+            uint8_t _3,
+            uint8_t _4,
+            uint8_t _5,
+            uint8_t _6,
+            uint8_t _7,
+            uint8_t _8,
+            uint8_t _9,
+            uint8_t _10,
+            uint8_t _11,
+            uint8_t _12,
+            uint8_t _13,
+            uint8_t _14,
+            uint8_t _15,
+            uint8_t _16,
+            uint8_t _17,
+            uint8_t _18,
+            uint8_t _19,
+            uint8_t _20,
+            uint8_t _21,
+            uint8_t _22,
+            uint8_t _23,
+            uint8_t _24,
+            uint8_t _25,
+            uint8_t _26,
+            uint8_t _27,
+            uint8_t _28,
+            uint8_t _29,
+            uint8_t _30,
+            uint8_t _31>
+    static simd32uint8 create() {
+        constexpr uint8_t ds[32] = {_0,  _1,  _2,  _3,  _4,  _5,  _6,  _7,
+                                    _8,  _9,  _10, _11, _12, _13, _14, _15,
+                                    _16, _17, _18, _19, _20, _21, _22, _23,
+                                    _24, _25, _26, _27, _28, _29, _30, _31};
+        return simd32uint8{ds};
+    }
+
     template <
             typename T,
             typename std::enable_if<
@@ -539,7 +785,8 @@ struct simd32uint8 {
             : data{vld1q_u8(x), vld1q_u8(x + 16)} {}
 
     void clear() {
-        detail::simdlib::set1(data, &vdupq_n_u8, static_cast<uint8_t>(0));
+        detail::simdlib::set1(data, static_cast<uint8_t>(0))
+                .call<&detail::simdlib::vdupq_n_u8>();
     }
 
     void storeu(uint8_t* ptr) const {
@@ -582,7 +829,7 @@ struct simd32uint8 {
     }
 
     void set1(uint8_t x) {
-        detail::simdlib::set1(data, &vdupq_n_u8, x);
+        detail::simdlib::set1(data, x).call<&detail::simdlib::vdupq_n_u8>();
     }
 
     template <
@@ -591,19 +838,21 @@ struct simd32uint8 {
                     detail::simdlib::is_simd256bit<T>::value,
                     std::nullptr_t>::type = nullptr>
     simd32uint8 operator&(const T& other) const {
-        return simd32uint8{detail::simdlib::binary_func(
-                data, detail::simdlib::reinterpret_u8(other.data), &vandq_u8)};
+        return simd32uint8{
+                detail::simdlib::binary_func(
+                        data, detail::simdlib::reinterpret_u8(other.data))
+                        .template call<&vandq_u8>()};
     }
 
     simd32uint8 operator+(const simd32uint8& other) const {
-        return simd32uint8{
-                detail::simdlib::binary_func(data, other.data, &vaddq_u8)};
+        return simd32uint8{detail::simdlib::binary_func(data, other.data)
+                                   .call<&vaddq_u8>()};
     }
 
     // The very important operation that everything relies on
     simd32uint8 lookup_2_lanes(const simd32uint8& idx) const {
-        return simd32uint8{
-                detail::simdlib::binary_func(data, idx.data, &vqtbl1q_u8)};
+        return simd32uint8{detail::simdlib::binary_func(data, idx.data)
+                                   .call<&vqtbl1q_u8>()};
     }
 
     simd32uint8 operator+=(const simd32uint8& other) {
@@ -618,6 +867,16 @@ struct simd32uint8 {
         vst1q_u8(tab, data.val[high]);
         return tab[i - high * 16];
     }
+
+    // Checks whether the other holds exactly the same bytes.
+    bool is_same_as(simd32uint8 other) const {
+        const bool equal0 =
+                (vminvq_u8(vceqq_u8(data.val[0], other.data.val[0])) == 0xff);
+        const bool equal1 =
+                (vminvq_u8(vceqq_u8(data.val[1], other.data.val[1])) == 0xff);
+
+        return equal0 && equal1;
+    }
 };
 
 // convert with saturation
@@ -671,8 +930,62 @@ struct simd8uint32 {
 
     explicit simd8uint32(const uint8_t* x) : simd8uint32(simd32uint8(x)) {}
 
+    explicit simd8uint32(
+            uint32_t u0,
+            uint32_t u1,
+            uint32_t u2,
+            uint32_t u3,
+            uint32_t u4,
+            uint32_t u5,
+            uint32_t u6,
+            uint32_t u7) {
+        uint32_t temp[8] = {u0, u1, u2, u3, u4, u5, u6, u7};
+        data.val[0] = vld1q_u32(temp);
+        data.val[1] = vld1q_u32(temp + 4);
+    }
+
+    simd8uint32 operator+(simd8uint32 other) const {
+        return simd8uint32{detail::simdlib::binary_func(data, other.data)
+                                   .call<&vaddq_u32>()};
+    }
+
+    simd8uint32 operator-(simd8uint32 other) const {
+        return simd8uint32{detail::simdlib::binary_func(data, other.data)
+                                   .call<&vsubq_u32>()};
+    }
+
+    simd8uint32& operator+=(const simd8uint32& other) {
+        data.val[0] = vaddq_u32(data.val[0], other.data.val[0]);
+        data.val[1] = vaddq_u32(data.val[1], other.data.val[1]);
+        return *this;
+    }
+
+    bool operator==(simd8uint32 other) const {
+        const auto equals = detail::simdlib::binary_func(data, other.data)
+                                    .call<&vceqq_u32>();
+        const auto equal = vandq_u32(equals.val[0], equals.val[1]);
+        return vminvq_u32(equal) == 0xffffffff;
+    }
+
+    bool operator!=(simd8uint32 other) const {
+        return !(*this == other);
+    }
+
+    // Checks whether the other holds exactly the same bytes.
+    bool is_same_as(simd8uint32 other) const {
+        const bool equal0 =
+                (vminvq_u32(vceqq_u32(data.val[0], other.data.val[0])) ==
+                 0xffffffff);
+        const bool equal1 =
+                (vminvq_u32(vceqq_u32(data.val[1], other.data.val[1])) ==
+                 0xffffffff);
+
+        return equal0 && equal1;
+    }
+
     void clear() {
-        detail::simdlib::set1(data, &vdupq_n_u32, static_cast<uint32_t>(0));
+        detail::simdlib::set1(data, static_cast<uint32_t>(0))
+                .call<&vdupq_n_u32>();
     }
 
     void storeu(uint32_t* ptr) const {
@@ -710,10 +1023,67 @@ struct simd8uint32 {
     }
 
     void set1(uint32_t x) {
-        detail::simdlib::set1(data, &vdupq_n_u32, x);
+        detail::simdlib::set1(data, x).call<&vdupq_n_u32>();
+    }
+
+    simd8uint32 unzip() const {
+        return simd8uint32{uint32x4x2_t{
+                vuzp1q_u32(data.val[0], data.val[1]),
+                vuzp2q_u32(data.val[0], data.val[1])}};
     }
 };
 
+// Vectorized version of the following code:
+//   for (size_t i = 0; i < n; i++) {
+//      bool flag = (candidateValues[i] < currentValues[i]);
+//      minValues[i] = flag ? candidateValues[i] : currentValues[i];
+//      minIndices[i] = flag ? candidateIndices[i] : currentIndices[i];
+//      maxValues[i] = !flag ? candidateValues[i] : currentValues[i];
+//      maxIndices[i] = !flag ? candidateIndices[i] : currentIndices[i];
+//   }
+// Max indices evaluation is inaccurate in case of equal values (the index of
+// the last equal value is saved instead of the first one), but this behavior
+// saves instructions.
+inline void cmplt_min_max_fast(
+        const simd8uint32 candidateValues,
+        const simd8uint32 candidateIndices,
+        const simd8uint32 currentValues,
+        const simd8uint32 currentIndices,
+        simd8uint32& minValues,
+        simd8uint32& minIndices,
+        simd8uint32& maxValues,
+        simd8uint32& maxIndices) {
+    const uint32x4x2_t comparison = uint32x4x2_t{
+            vcltq_u32(candidateValues.data.val[0], currentValues.data.val[0]),
+            vcltq_u32(candidateValues.data.val[1], currentValues.data.val[1])};
+
+    minValues.data = uint32x4x2_t{
+            vminq_u32(candidateValues.data.val[0], currentValues.data.val[0]),
+            vminq_u32(candidateValues.data.val[1], currentValues.data.val[1])};
+    minIndices.data = uint32x4x2_t{
+            vbslq_u32(
+                    comparison.val[0],
+                    candidateIndices.data.val[0],
+                    currentIndices.data.val[0]),
+            vbslq_u32(
+                    comparison.val[1],
+                    candidateIndices.data.val[1],
+                    currentIndices.data.val[1])};
+
+    maxValues.data = uint32x4x2_t{
+            vmaxq_u32(candidateValues.data.val[0], currentValues.data.val[0]),
+            vmaxq_u32(candidateValues.data.val[1], currentValues.data.val[1])};
+    maxIndices.data = uint32x4x2_t{
+            vbslq_u32(
+                    comparison.val[0],
+                    currentIndices.data.val[0],
+                    candidateIndices.data.val[0]),
+            vbslq_u32(
+                    comparison.val[1],
+                    currentIndices.data.val[1],
+                    candidateIndices.data.val[1])};
+}
+
 struct simd8float32 {
     float32x4x2_t data;
 
@@ -734,8 +1104,22 @@ struct simd8float32 {
     explicit simd8float32(const float* x)
             : data{vld1q_f32(x), vld1q_f32(x + 4)} {}
 
+    explicit simd8float32(
+            float f0,
+            float f1,
+            float f2,
+            float f3,
+            float f4,
+            float f5,
+            float f6,
+            float f7) {
+        float temp[8] = {f0, f1, f2, f3, f4, f5, f6, f7};
+        data.val[0] = vld1q_f32(temp);
+        data.val[1] = vld1q_f32(temp + 4);
+    }
+
     void clear() {
-        detail::simdlib::set1(data, &vdupq_n_f32, 0.f);
+        detail::simdlib::set1(data, 0.f).call<&vdupq_n_f32>();
     }
 
     void storeu(float* ptr) const {
@@ -761,18 +1145,50 @@ struct simd8float32 {
     }
 
     simd8float32 operator*(const simd8float32& other) const {
-        return simd8float32{
-                detail::simdlib::binary_func(data, other.data, &vmulq_f32)};
+        return simd8float32{detail::simdlib::binary_func(data, other.data)
+                                    .call<&vmulq_f32>()};
     }
 
     simd8float32 operator+(const simd8float32& other) const {
-        return simd8float32{
-                detail::simdlib::binary_func(data, other.data, &vaddq_f32)};
+        return simd8float32{detail::simdlib::binary_func(data, other.data)
+                                    .call<&vaddq_f32>()};
     }
 
     simd8float32 operator-(const simd8float32& other) const {
-        return simd8float32{
-                detail::simdlib::binary_func(data, other.data, &vsubq_f32)};
+        return simd8float32{detail::simdlib::binary_func(data, other.data)
+                                    .call<&vsubq_f32>()};
+    }
+
+    simd8float32& operator+=(const simd8float32& other) {
+        // In this context, it is more compiler friendly to write intrinsics
+        // directly instead of using binary_func
+        data.val[0] = vaddq_f32(data.val[0], other.data.val[0]);
+        data.val[1] = vaddq_f32(data.val[1], other.data.val[1]);
+        return *this;
+    }
+
+    bool operator==(simd8float32 other) const {
+        const auto equals =
+                detail::simdlib::binary_func<::uint32x4x2_t>(data, other.data)
+                        .call<&vceqq_f32>();
+        const auto equal = vandq_u32(equals.val[0], equals.val[1]);
+        return vminvq_u32(equal) == 0xffffffff;
+    }
+
+    bool operator!=(simd8float32 other) const {
+        return !(*this == other);
+    }
+
+    // Checks whether the other holds exactly the same bytes.
+    bool is_same_as(simd8float32 other) const {
+        const bool equal0 =
+                (vminvq_u32(vceqq_f32(data.val[0], other.data.val[0])) ==
+                 0xffffffff);
+        const bool equal1 =
+                (vminvq_u32(vceqq_f32(data.val[1], other.data.val[1])) ==
+                 0xffffffff);
+
+        return equal0 && equal1;
     }
 
     std::string tostring() const {
@@ -783,17 +1199,17 @@ struct simd8float32 {
 // hadd does not cross lanes
 inline simd8float32 hadd(const simd8float32& a, const simd8float32& b) {
     return simd8float32{
-            detail::simdlib::binary_func(a.data, b.data, &vpaddq_f32)};
+            detail::simdlib::binary_func(a.data, b.data).call<&vpaddq_f32>()};
 }
 
 inline simd8float32 unpacklo(const simd8float32& a, const simd8float32& b) {
     return simd8float32{
-            detail::simdlib::binary_func(a.data, b.data, &vzip1q_f32)};
+            detail::simdlib::binary_func(a.data, b.data).call<&vzip1q_f32>()};
 }
 
 inline simd8float32 unpackhi(const simd8float32& a, const simd8float32& b) {
     return simd8float32{
-            detail::simdlib::binary_func(a.data, b.data, &vzip2q_f32)};
+            detail::simdlib::binary_func(a.data, b.data).call<&vzip2q_f32>()};
 }
 
 // compute a * b + c
@@ -806,20 +1222,129 @@ inline simd8float32 fmadd(
             vfmaq_f32(c.data.val[1], a.data.val[1], b.data.val[1])}};
 }
 
+// The following primitive is a vectorized version of the following code
+// snippet:
+//   float lowestValue = HUGE_VAL;
+//   uint lowestIndex = 0;
+//   for (size_t i = 0; i < n; i++) {
+//     if (values[i] < lowestValue) {
+//       lowestValue = values[i];
+//       lowestIndex = i;
+//     }
+//   }
+// Vectorized version can be implemented via two operations: cmp and blend
+// with something like this:
+//   lowestValues = [HUGE_VAL; 8];
+//   lowestIndices = {0, 1, 2, 3, 4, 5, 6, 7};
+//   for (size_t i = 0; i < n; i += 8) {
+//     auto comparison = cmp(values + i, lowestValues);
+//     lowestValues = blend(
+//         comparison,
+//         values + i,
+//         lowestValues);
+//     lowestIndices = blend(
+//         comparison,
+//         i + {0, 1, 2, 3, 4, 5, 6, 7},
+//         lowestIndices);
+//     lowestIndices += {8, 8, 8, 8, 8, 8, 8, 8};
+//   }
+// The problem is that blend primitive needs very different instruction
+// order for AVX and ARM.
+// So, let's introduce a combination of these two in order to avoid
+// confusion for ppl who write in low-level SIMD instructions. Additionally,
+// these two ops (cmp and blend) are very often used together.
+inline void cmplt_and_blend_inplace(
+        const simd8float32 candidateValues,
+        const simd8uint32 candidateIndices,
+        simd8float32& lowestValues,
+        simd8uint32& lowestIndices) {
+    const auto comparison = detail::simdlib::binary_func<::uint32x4x2_t>(
+                                    candidateValues.data, lowestValues.data)
+                                    .call<&vcltq_f32>();
+
+    lowestValues.data = float32x4x2_t{
+            vbslq_f32(
+                    comparison.val[0],
+                    candidateValues.data.val[0],
+                    lowestValues.data.val[0]),
+            vbslq_f32(
+                    comparison.val[1],
+                    candidateValues.data.val[1],
+                    lowestValues.data.val[1])};
+    lowestIndices.data = uint32x4x2_t{
+            vbslq_u32(
+                    comparison.val[0],
+                    candidateIndices.data.val[0],
+                    lowestIndices.data.val[0]),
+            vbslq_u32(
+                    comparison.val[1],
+                    candidateIndices.data.val[1],
+                    lowestIndices.data.val[1])};
+}
+
+// Vectorized version of the following code:
+//   for (size_t i = 0; i < n; i++) {
+//      bool flag = (candidateValues[i] < currentValues[i]);
+//      minValues[i] = flag ? candidateValues[i] : currentValues[i];
+//      minIndices[i] = flag ? candidateIndices[i] : currentIndices[i];
+//      maxValues[i] = !flag ? candidateValues[i] : currentValues[i];
+//      maxIndices[i] = !flag ? candidateIndices[i] : currentIndices[i];
+//   }
+// Max indices evaluation is inaccurate in case of equal values (the index of
+// the last equal value is saved instead of the first one), but this behavior
+// saves instructions.
+inline void cmplt_min_max_fast(
+        const simd8float32 candidateValues,
+        const simd8uint32 candidateIndices,
+        const simd8float32 currentValues,
+        const simd8uint32 currentIndices,
+        simd8float32& minValues,
+        simd8uint32& minIndices,
+        simd8float32& maxValues,
+        simd8uint32& maxIndices) {
+    const uint32x4x2_t comparison = uint32x4x2_t{
+            vcltq_f32(candidateValues.data.val[0], currentValues.data.val[0]),
+            vcltq_f32(candidateValues.data.val[1], currentValues.data.val[1])};
+
+    minValues.data = float32x4x2_t{
+            vminq_f32(candidateValues.data.val[0], currentValues.data.val[0]),
+            vminq_f32(candidateValues.data.val[1], currentValues.data.val[1])};
+    minIndices.data = uint32x4x2_t{
+            vbslq_u32(
+                    comparison.val[0],
+                    candidateIndices.data.val[0],
+                    currentIndices.data.val[0]),
+            vbslq_u32(
+                    comparison.val[1],
+                    candidateIndices.data.val[1],
+                    currentIndices.data.val[1])};
+
+    maxValues.data = float32x4x2_t{
+            vmaxq_f32(candidateValues.data.val[0], currentValues.data.val[0]),
+            vmaxq_f32(candidateValues.data.val[1], currentValues.data.val[1])};
+    maxIndices.data = uint32x4x2_t{
+            vbslq_u32(
+                    comparison.val[0],
+                    currentIndices.data.val[0],
+                    candidateIndices.data.val[0]),
+            vbslq_u32(
+                    comparison.val[1],
+                    currentIndices.data.val[1],
+                    candidateIndices.data.val[1])};
+}
+
 namespace {
 
 // get even float32's of a and b, interleaved
 simd8float32 geteven(const simd8float32& a, const simd8float32& b) {
-    return simd8float32{float32x4x2_t{
-            vuzp1q_f32(a.data.val[0], b.data.val[0]),
-            vuzp1q_f32(a.data.val[1], b.data.val[1])}};
+    return simd8float32{
+            detail::simdlib::binary_func(a.data, b.data).call<&vuzp1q_f32>()};
 }
 
 // get odd float32's of a and b, interleaved
 simd8float32 getodd(const simd8float32& a, const simd8float32& b) {
-    return simd8float32{float32x4x2_t{
-            vuzp2q_f32(a.data.val[0], b.data.val[0]),
-            vuzp2q_f32(a.data.val[1], b.data.val[1])}};
+    return simd8float32{
+            detail::simdlib::binary_func(a.data, b.data).call<&vuzp2q_f32>()};
 }
 
 // 3 cycles