numkong 7.0.0 → 7.4.2

package/include/numkong/sets.h

@@ -73,17 +73,17 @@ NK_DYNAMIC void nk_hammings_packed_u1(nk_u1x8_t const *v, void const *q_packed,
 /**
  *  @brief Computes C = A × Aᵀ symmetric Gram matrix of Hamming distances.
  *  @param[in] vectors Input matrix of row vectors in row-major order.
- *  @param[in] n_vectors Number of vectors (rows) in the input matrix.
+ *  @param[in] vectors_count Number of vectors (rows) in the input matrix.
  *  @param[in] d Dimension of each vector (columns).
  *  @param[in] stride Row stride in bytes for the input matrix.
- *  @param[out] result Output symmetric matrix (n_vectors × n_vectors).
+ *  @param[out] result Output symmetric matrix (vectors_count × vectors_count).
  *  @param[in] result_stride Row stride in bytes for the result matrix.
  *  @param[in] row_start Starting row offset of results to compute (needed for parallelism).
  *  @param[in] row_count Number of rows of results to compute (needed for parallelism).
  */
-NK_DYNAMIC void nk_hammings_symmetric_u1(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d, nk_size_t stride,
-                                         nk_u32_t *result, nk_size_t result_stride, nk_size_t row_start,
-                                         nk_size_t row_count);
+NK_DYNAMIC void nk_hammings_symmetric_u1(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
+                                         nk_size_t stride, nk_u32_t *result, nk_size_t result_stride,
+                                         nk_size_t row_start, nk_size_t row_count);
 
 /**
  *  @brief Compute Jaccard distances between V rows and packed Q rows.
@@ -103,24 +103,24 @@ NK_DYNAMIC void nk_jaccards_packed_u1(nk_u1x8_t const *v, void const *q_packed,
 /**
  *  @brief Computes C = f(A, Aᵀ) symmetric Gram matrix of Jaccard distances.
  *  @param[in] vectors Input matrix of row vectors in row-major order.
- *  @param[in] n_vectors Number of vectors (rows).
+ *  @param[in] vectors_count Number of vectors (rows).
  *  @param[in] d Dimension of each vector (columns).
  *  @param[in] stride Row stride in bytes.
- *  @param[out] result Output symmetric f32 matrix (n_vectors × n_vectors).
+ *  @param[out] result Output symmetric f32 matrix (vectors_count × vectors_count).
  *  @param[in] result_stride Row stride in bytes for the result matrix.
  *  @param[in] row_start Starting row offset (for parallelism).
  *  @param[in] row_count Number of rows to compute (for parallelism).
  */
-NK_DYNAMIC void nk_jaccards_symmetric_u1(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d, nk_size_t stride,
-                                         nk_f32_t *result, nk_size_t result_stride, nk_size_t row_start,
-                                         nk_size_t row_count);
+NK_DYNAMIC void nk_jaccards_symmetric_u1(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
+                                         nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
+                                         nk_size_t row_start, nk_size_t row_count);
 
 /** @copydoc nk_hammings_packed_u1 */
 NK_PUBLIC void nk_hammings_packed_u1_serial(nk_u1x8_t const *v, void const *q_packed, nk_u32_t *result, nk_size_t rows,
                                             nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                             nk_size_t r_stride_in_bytes);
 /** @copydoc nk_hammings_symmetric_u1 */
-NK_PUBLIC void nk_hammings_symmetric_u1_serial(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_hammings_symmetric_u1_serial(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                nk_size_t stride, nk_u32_t *result, nk_size_t result_stride,
                                                nk_size_t row_start, nk_size_t row_count);
 /** @copydoc nk_jaccards_packed_u1 */
@@ -128,7 +128,7 @@ NK_PUBLIC void nk_jaccards_packed_u1_serial(nk_u1x8_t const *v, void const *q_pa
                                             nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                             nk_size_t r_stride_in_bytes);
 /** @copydoc nk_jaccards_symmetric_u1 */
-NK_PUBLIC void nk_jaccards_symmetric_u1_serial(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_jaccards_symmetric_u1_serial(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
                                                nk_size_t row_start, nk_size_t row_count);
 
@@ -141,7 +141,7 @@ NK_PUBLIC void nk_hammings_packed_u1_smebi32(nk_u1x8_t const *v, void const *q_p
                                              nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                              nk_size_t r_stride_in_bytes);
 /** @copydoc nk_hammings_symmetric_u1 */
-NK_PUBLIC void nk_hammings_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_hammings_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                 nk_size_t stride, nk_u32_t *result, nk_size_t result_stride,
                                                 nk_size_t row_start, nk_size_t row_count);
 /** @copydoc nk_jaccards_packed_u1 */
@@ -149,7 +149,7 @@ NK_PUBLIC void nk_jaccards_packed_u1_smebi32(nk_u1x8_t const *v, void const *q_p
                                              nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                              nk_size_t r_stride_in_bytes);
 /** @copydoc nk_jaccards_symmetric_u1 */
-NK_PUBLIC void nk_jaccards_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_jaccards_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                 nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
                                                 nk_size_t row_start, nk_size_t row_count);
 #endif // NK_TARGET_SMEBI32
@@ -163,7 +163,7 @@ NK_PUBLIC void nk_hammings_packed_u1_haswell(nk_u1x8_t const *v, void const *q_p
                                              nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                              nk_size_t r_stride_in_bytes);
 /** @copydoc nk_hammings_symmetric_u1 */
-NK_PUBLIC void nk_hammings_symmetric_u1_haswell(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_hammings_symmetric_u1_haswell(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                 nk_size_t stride, nk_u32_t *result, nk_size_t result_stride,
                                                 nk_size_t row_start, nk_size_t row_count);
 /** @copydoc nk_jaccards_packed_u1 */
@@ -171,7 +171,7 @@ NK_PUBLIC void nk_jaccards_packed_u1_haswell(nk_u1x8_t const *v, void const *q_p
                                              nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                              nk_size_t r_stride_in_bytes);
 /** @copydoc nk_jaccards_symmetric_u1 */
-NK_PUBLIC void nk_jaccards_symmetric_u1_haswell(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_jaccards_symmetric_u1_haswell(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                 nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
                                                 nk_size_t row_start, nk_size_t row_count);
 #endif // NK_TARGET_HASWELL
@@ -185,7 +185,7 @@ NK_PUBLIC void nk_hammings_packed_u1_icelake(nk_u1x8_t const *v, void const *q_p
                                              nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                              nk_size_t r_stride_in_bytes);
 /** @copydoc nk_hammings_symmetric_u1 */
-NK_PUBLIC void nk_hammings_symmetric_u1_icelake(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_hammings_symmetric_u1_icelake(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                 nk_size_t stride, nk_u32_t *result, nk_size_t result_stride,
                                                 nk_size_t row_start, nk_size_t row_count);
 /** @copydoc nk_jaccards_packed_u1 */
@@ -193,7 +193,7 @@ NK_PUBLIC void nk_jaccards_packed_u1_icelake(nk_u1x8_t const *v, void const *q_p
                                              nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                              nk_size_t r_stride_in_bytes);
 /** @copydoc nk_jaccards_symmetric_u1 */
-NK_PUBLIC void nk_jaccards_symmetric_u1_icelake(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_jaccards_symmetric_u1_icelake(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                 nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
                                                 nk_size_t row_start, nk_size_t row_count);
 #endif // NK_TARGET_ICELAKE
@@ -207,7 +207,7 @@ NK_PUBLIC void nk_hammings_packed_u1_neon(nk_u1x8_t const *v, void const *q_pack
                                           nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                           nk_size_t r_stride_in_bytes);
 /** @copydoc nk_hammings_symmetric_u1 */
-NK_PUBLIC void nk_hammings_symmetric_u1_neon(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_hammings_symmetric_u1_neon(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                              nk_size_t stride, nk_u32_t *result, nk_size_t result_stride,
                                              nk_size_t row_start, nk_size_t row_count);
 /** @copydoc nk_jaccards_packed_u1 */
@@ -215,7 +215,7 @@ NK_PUBLIC void nk_jaccards_packed_u1_neon(nk_u1x8_t const *v, void const *q_pack
                                           nk_size_t cols, nk_size_t d, nk_size_t v_stride_in_bytes,
                                           nk_size_t r_stride_in_bytes);
 /** @copydoc nk_jaccards_symmetric_u1 */
-NK_PUBLIC void nk_jaccards_symmetric_u1_neon(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_jaccards_symmetric_u1_neon(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                              nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
                                              nk_size_t row_start, nk_size_t row_count);
 #endif // NK_TARGET_NEON
@@ -228,7 +228,7 @@ NK_PUBLIC void nk_hammings_packed_u1_v128relaxed(nk_u1x8_t const *v, void const
                                                  nk_size_t rows, nk_size_t cols, nk_size_t d,
                                                  nk_size_t v_stride_in_bytes, nk_size_t r_stride_in_bytes);
 /** @copydoc nk_hammings_symmetric_u1 */
-NK_PUBLIC void nk_hammings_symmetric_u1_v128relaxed(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_hammings_symmetric_u1_v128relaxed(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                     nk_size_t stride, nk_u32_t *result, nk_size_t result_stride,
                                                     nk_size_t row_start, nk_size_t row_count);
 /** @copydoc nk_jaccards_packed_u1 */
@@ -236,11 +236,32 @@ NK_PUBLIC void nk_jaccards_packed_u1_v128relaxed(nk_u1x8_t const *v, void const
                                                  nk_size_t rows, nk_size_t cols, nk_size_t d,
                                                  nk_size_t v_stride_in_bytes, nk_size_t r_stride_in_bytes);
 /** @copydoc nk_jaccards_symmetric_u1 */
-NK_PUBLIC void nk_jaccards_symmetric_u1_v128relaxed(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d,
+NK_PUBLIC void nk_jaccards_symmetric_u1_v128relaxed(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
                                                     nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
                                                     nk_size_t row_start, nk_size_t row_count);
 #endif // NK_TARGET_V128RELAXED
 
+/*  Loongson LASX backends using 256-bit SIMD with XVPCNT.W for popcount-based set distances.
+ */
+#if NK_TARGET_LOONGSONASX
+/** @copydoc nk_hammings_packed_u1 */
+NK_PUBLIC void nk_hammings_packed_u1_loongsonasx(nk_u1x8_t const *v, void const *q_packed, nk_u32_t *result,
+                                                 nk_size_t rows, nk_size_t cols, nk_size_t d,
+                                                 nk_size_t v_stride_in_bytes, nk_size_t r_stride_in_bytes);
+/** @copydoc nk_hammings_symmetric_u1 */
+NK_PUBLIC void nk_hammings_symmetric_u1_loongsonasx(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
+                                                    nk_size_t stride, nk_u32_t *result, nk_size_t result_stride,
+                                                    nk_size_t row_start, nk_size_t row_count);
+/** @copydoc nk_jaccards_packed_u1 */
+NK_PUBLIC void nk_jaccards_packed_u1_loongsonasx(nk_u1x8_t const *v, void const *q_packed, nk_f32_t *result,
+                                                 nk_size_t rows, nk_size_t cols, nk_size_t d,
+                                                 nk_size_t v_stride_in_bytes, nk_size_t r_stride_in_bytes);
+/** @copydoc nk_jaccards_symmetric_u1 */
+NK_PUBLIC void nk_jaccards_symmetric_u1_loongsonasx(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
+                                                    nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
+                                                    nk_size_t row_start, nk_size_t row_count);
+#endif // NK_TARGET_LOONGSONASX
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif
@@ -251,6 +272,8 @@ NK_PUBLIC void nk_jaccards_symmetric_u1_v128relaxed(nk_u1x8_t const *vectors, nk
 #include "numkong/sets/haswell.h"
 #include "numkong/sets/smebi32.h"
 #include "numkong/sets/v128relaxed.h"
+#include "numkong/sets/powervsx.h"
+#include "numkong/sets/loongsonasx.h"
 
 #if defined(__cplusplus)
 extern "C" {
@@ -269,6 +292,10 @@ NK_PUBLIC void nk_hammings_packed_u1(nk_u1x8_t const *v, void const *q_packed, n
     nk_hammings_packed_u1_icelake(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
 #elif NK_TARGET_HASWELL
     nk_hammings_packed_u1_haswell(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
+#elif NK_TARGET_POWERVSX
+    nk_hammings_packed_u1_powervsx(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
+#elif NK_TARGET_LOONGSONASX
+    nk_hammings_packed_u1_loongsonasx(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
 #elif NK_TARGET_V128RELAXED
     nk_hammings_packed_u1_v128relaxed(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
 #else
@@ -276,21 +303,27 @@ NK_PUBLIC void nk_hammings_packed_u1(nk_u1x8_t const *v, void const *q_packed, n
 #endif
 }
 
-NK_PUBLIC void nk_hammings_symmetric_u1(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d, nk_size_t stride,
-                                        nk_u32_t *result, nk_size_t result_stride, nk_size_t row_start,
-                                        nk_size_t row_count) {
+NK_PUBLIC void nk_hammings_symmetric_u1(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
+                                        nk_size_t stride, nk_u32_t *result, nk_size_t result_stride,
+                                        nk_size_t row_start, nk_size_t row_count) {
 #if NK_TARGET_SMEBI32
-    nk_hammings_symmetric_u1_smebi32(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_hammings_symmetric_u1_smebi32(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
 #elif NK_TARGET_NEON
-    nk_hammings_symmetric_u1_neon(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_hammings_symmetric_u1_neon(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
 #elif NK_TARGET_ICELAKE
-    nk_hammings_symmetric_u1_icelake(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_hammings_symmetric_u1_icelake(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
 #elif NK_TARGET_HASWELL
-    nk_hammings_symmetric_u1_haswell(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_hammings_symmetric_u1_haswell(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
+#elif NK_TARGET_POWERVSX
+    nk_hammings_symmetric_u1_powervsx(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
+#elif NK_TARGET_LOONGSONASX
+    nk_hammings_symmetric_u1_loongsonasx(vectors, vectors_count, d, stride, result, result_stride, row_start,
+                                         row_count);
 #elif NK_TARGET_V128RELAXED
-    nk_hammings_symmetric_u1_v128relaxed(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_hammings_symmetric_u1_v128relaxed(vectors, vectors_count, d, stride, result, result_stride, row_start,
+                                         row_count);
 #else
-    nk_hammings_symmetric_u1_serial(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_hammings_symmetric_u1_serial(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
 #endif
 }
 
@@ -305,6 +338,10 @@ NK_PUBLIC void nk_jaccards_packed_u1(nk_u1x8_t const *v, void const *q_packed, n
     nk_jaccards_packed_u1_icelake(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
 #elif NK_TARGET_HASWELL
     nk_jaccards_packed_u1_haswell(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
+#elif NK_TARGET_POWERVSX
+    nk_jaccards_packed_u1_powervsx(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
+#elif NK_TARGET_LOONGSONASX
+    nk_jaccards_packed_u1_loongsonasx(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
 #elif NK_TARGET_V128RELAXED
     nk_jaccards_packed_u1_v128relaxed(v, q_packed, result, rows, cols, d, v_stride_in_bytes, r_stride_in_bytes);
 #else
@@ -312,21 +349,27 @@ NK_PUBLIC void nk_jaccards_packed_u1(nk_u1x8_t const *v, void const *q_packed, n
 #endif
 }
 
-NK_PUBLIC void nk_jaccards_symmetric_u1(nk_u1x8_t const *vectors, nk_size_t n_vectors, nk_size_t d, nk_size_t stride,
-                                        nk_f32_t *result, nk_size_t result_stride, nk_size_t row_start,
-                                        nk_size_t row_count) {
+NK_PUBLIC void nk_jaccards_symmetric_u1(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t d,
+                                        nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
+                                        nk_size_t row_start, nk_size_t row_count) {
 #if NK_TARGET_SMEBI32
-    nk_jaccards_symmetric_u1_smebi32(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_jaccards_symmetric_u1_smebi32(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
 #elif NK_TARGET_NEON
-    nk_jaccards_symmetric_u1_neon(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_jaccards_symmetric_u1_neon(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
 #elif NK_TARGET_ICELAKE
-    nk_jaccards_symmetric_u1_icelake(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_jaccards_symmetric_u1_icelake(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
 #elif NK_TARGET_HASWELL
-    nk_jaccards_symmetric_u1_haswell(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_jaccards_symmetric_u1_haswell(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
+#elif NK_TARGET_POWERVSX
+    nk_jaccards_symmetric_u1_powervsx(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
+#elif NK_TARGET_LOONGSONASX
+    nk_jaccards_symmetric_u1_loongsonasx(vectors, vectors_count, d, stride, result, result_stride, row_start,
+                                         row_count);
 #elif NK_TARGET_V128RELAXED
-    nk_jaccards_symmetric_u1_v128relaxed(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_jaccards_symmetric_u1_v128relaxed(vectors, vectors_count, d, stride, result, result_stride, row_start,
+                                         row_count);
 #else
-    nk_jaccards_symmetric_u1_serial(vectors, n_vectors, d, stride, result, result_stride, row_start, row_count);
+    nk_jaccards_symmetric_u1_serial(vectors, vectors_count, d, stride, result, result_stride, row_start, row_count);
 #endif
 }
 

package/include/numkong/sparse/README.md

@@ -7,15 +7,15 @@ The separate index/weight stream design makes these primitives composable into b
 
 Set intersection:
 
-```math
+$$
 |A \cap B| = |\{i : i \in A \land i \in B\}|
-```
+$$
 
 Sparse dot product:
 
-```math
+$$
 \text{dot}(a, b) = \sum_{i \in A \cap B} w_a(i) \cdot w_b(i)
-```
+$$
 
 Reformulating as Python pseudocode:
 

package/include/numkong/sparse/icelake.h

@@ -45,58 +45,58 @@ extern "C" {
  *          slightly faster than the native Tiger Lake implementation, but returns only one mask.
  */
 NK_INTERNAL nk_u32_t nk_intersect_u16x32_icelake_(__m512i a, __m512i b) {
-    __m512i a1 = _mm512_alignr_epi32(a, a, 4);
-    __m512i a2 = _mm512_alignr_epi32(a, a, 8);
-    __m512i a3 = _mm512_alignr_epi32(a, a, 12);
+    __m512i a1_u16x32 = _mm512_alignr_epi32(a, a, 4);
+    __m512i a2_u16x32 = _mm512_alignr_epi32(a, a, 8);
+    __m512i a3_u16x32 = _mm512_alignr_epi32(a, a, 12);
 
-    __m512i b1 = _mm512_shuffle_epi32(b, _MM_PERM_ADCB);
-    __m512i b2 = _mm512_shuffle_epi32(b, _MM_PERM_BADC);
-    __m512i b3 = _mm512_shuffle_epi32(b, _MM_PERM_CBAD);
+    __m512i b1_u16x32 = _mm512_shuffle_epi32(b, _MM_PERM_ADCB);
+    __m512i b2_u16x32 = _mm512_shuffle_epi32(b, _MM_PERM_BADC);
+    __m512i b3_u16x32 = _mm512_shuffle_epi32(b, _MM_PERM_CBAD);
 
-    __m512i b01 = _mm512_shrdi_epi32(b, b, 16);
-    __m512i b11 = _mm512_shrdi_epi32(b1, b1, 16);
-    __m512i b21 = _mm512_shrdi_epi32(b2, b2, 16);
-    __m512i b31 = _mm512_shrdi_epi32(b3, b3, 16);
+    __m512i b01_u16x32 = _mm512_shrdi_epi32(b, b, 16);
+    __m512i b11_u16x32 = _mm512_shrdi_epi32(b1_u16x32, b1_u16x32, 16);
+    __m512i b21_u16x32 = _mm512_shrdi_epi32(b2_u16x32, b2_u16x32, 16);
+    __m512i b31_u16x32 = _mm512_shrdi_epi32(b3_u16x32, b3_u16x32, 16);
 
     __mmask32 nm00 = _mm512_cmpneq_epi16_mask(a, b);
-    __mmask32 nm01 = _mm512_cmpneq_epi16_mask(a1, b);
-    __mmask32 nm02 = _mm512_cmpneq_epi16_mask(a2, b);
-    __mmask32 nm03 = _mm512_cmpneq_epi16_mask(a3, b);
-
-    __mmask32 nm10 = _mm512_mask_cmpneq_epi16_mask(nm00, a, b01);
-    __mmask32 nm11 = _mm512_mask_cmpneq_epi16_mask(nm01, a1, b01);
-    __mmask32 nm12 = _mm512_mask_cmpneq_epi16_mask(nm02, a2, b01);
-    __mmask32 nm13 = _mm512_mask_cmpneq_epi16_mask(nm03, a3, b01);
-
-    __mmask32 nm20 = _mm512_mask_cmpneq_epi16_mask(nm10, a, b1);
-    __mmask32 nm21 = _mm512_mask_cmpneq_epi16_mask(nm11, a1, b1);
-    __mmask32 nm22 = _mm512_mask_cmpneq_epi16_mask(nm12, a2, b1);
-    __mmask32 nm23 = _mm512_mask_cmpneq_epi16_mask(nm13, a3, b1);
-
-    __mmask32 nm30 = _mm512_mask_cmpneq_epi16_mask(nm20, a, b11);
-    __mmask32 nm31 = _mm512_mask_cmpneq_epi16_mask(nm21, a1, b11);
-    __mmask32 nm32 = _mm512_mask_cmpneq_epi16_mask(nm22, a2, b11);
-    __mmask32 nm33 = _mm512_mask_cmpneq_epi16_mask(nm23, a3, b11);
-
-    __mmask32 nm40 = _mm512_mask_cmpneq_epi16_mask(nm30, a, b2);
-    __mmask32 nm41 = _mm512_mask_cmpneq_epi16_mask(nm31, a1, b2);
-    __mmask32 nm42 = _mm512_mask_cmpneq_epi16_mask(nm32, a2, b2);
-    __mmask32 nm43 = _mm512_mask_cmpneq_epi16_mask(nm33, a3, b2);
-
-    __mmask32 nm50 = _mm512_mask_cmpneq_epi16_mask(nm40, a, b21);
-    __mmask32 nm51 = _mm512_mask_cmpneq_epi16_mask(nm41, a1, b21);
-    __mmask32 nm52 = _mm512_mask_cmpneq_epi16_mask(nm42, a2, b21);
-    __mmask32 nm53 = _mm512_mask_cmpneq_epi16_mask(nm43, a3, b21);
-
-    __mmask32 nm60 = _mm512_mask_cmpneq_epi16_mask(nm50, a, b3);
-    __mmask32 nm61 = _mm512_mask_cmpneq_epi16_mask(nm51, a1, b3);
-    __mmask32 nm62 = _mm512_mask_cmpneq_epi16_mask(nm52, a2, b3);
-    __mmask32 nm63 = _mm512_mask_cmpneq_epi16_mask(nm53, a3, b3);
-
-    __mmask32 nm70 = _mm512_mask_cmpneq_epi16_mask(nm60, a, b31);
-    __mmask32 nm71 = _mm512_mask_cmpneq_epi16_mask(nm61, a1, b31);
-    __mmask32 nm72 = _mm512_mask_cmpneq_epi16_mask(nm62, a2, b31);
-    __mmask32 nm73 = _mm512_mask_cmpneq_epi16_mask(nm63, a3, b31);
+    __mmask32 nm01 = _mm512_cmpneq_epi16_mask(a1_u16x32, b);
+    __mmask32 nm02 = _mm512_cmpneq_epi16_mask(a2_u16x32, b);
+    __mmask32 nm03 = _mm512_cmpneq_epi16_mask(a3_u16x32, b);
+
+    __mmask32 nm10 = _mm512_mask_cmpneq_epi16_mask(nm00, a, b01_u16x32);
+    __mmask32 nm11 = _mm512_mask_cmpneq_epi16_mask(nm01, a1_u16x32, b01_u16x32);
+    __mmask32 nm12 = _mm512_mask_cmpneq_epi16_mask(nm02, a2_u16x32, b01_u16x32);
+    __mmask32 nm13 = _mm512_mask_cmpneq_epi16_mask(nm03, a3_u16x32, b01_u16x32);
+
+    __mmask32 nm20 = _mm512_mask_cmpneq_epi16_mask(nm10, a, b1_u16x32);
+    __mmask32 nm21 = _mm512_mask_cmpneq_epi16_mask(nm11, a1_u16x32, b1_u16x32);
+    __mmask32 nm22 = _mm512_mask_cmpneq_epi16_mask(nm12, a2_u16x32, b1_u16x32);
+    __mmask32 nm23 = _mm512_mask_cmpneq_epi16_mask(nm13, a3_u16x32, b1_u16x32);
+
+    __mmask32 nm30 = _mm512_mask_cmpneq_epi16_mask(nm20, a, b11_u16x32);
+    __mmask32 nm31 = _mm512_mask_cmpneq_epi16_mask(nm21, a1_u16x32, b11_u16x32);
+    __mmask32 nm32 = _mm512_mask_cmpneq_epi16_mask(nm22, a2_u16x32, b11_u16x32);
+    __mmask32 nm33 = _mm512_mask_cmpneq_epi16_mask(nm23, a3_u16x32, b11_u16x32);
+
+    __mmask32 nm40 = _mm512_mask_cmpneq_epi16_mask(nm30, a, b2_u16x32);
+    __mmask32 nm41 = _mm512_mask_cmpneq_epi16_mask(nm31, a1_u16x32, b2_u16x32);
+    __mmask32 nm42 = _mm512_mask_cmpneq_epi16_mask(nm32, a2_u16x32, b2_u16x32);
+    __mmask32 nm43 = _mm512_mask_cmpneq_epi16_mask(nm33, a3_u16x32, b2_u16x32);
+
+    __mmask32 nm50 = _mm512_mask_cmpneq_epi16_mask(nm40, a, b21_u16x32);
+    __mmask32 nm51 = _mm512_mask_cmpneq_epi16_mask(nm41, a1_u16x32, b21_u16x32);
+    __mmask32 nm52 = _mm512_mask_cmpneq_epi16_mask(nm42, a2_u16x32, b21_u16x32);
+    __mmask32 nm53 = _mm512_mask_cmpneq_epi16_mask(nm43, a3_u16x32, b21_u16x32);
+
+    __mmask32 nm60 = _mm512_mask_cmpneq_epi16_mask(nm50, a, b3_u16x32);
+    __mmask32 nm61 = _mm512_mask_cmpneq_epi16_mask(nm51, a1_u16x32, b3_u16x32);
+    __mmask32 nm62 = _mm512_mask_cmpneq_epi16_mask(nm52, a2_u16x32, b3_u16x32);
+    __mmask32 nm63 = _mm512_mask_cmpneq_epi16_mask(nm53, a3_u16x32, b3_u16x32);
+
+    __mmask32 nm70 = _mm512_mask_cmpneq_epi16_mask(nm60, a, b31_u16x32);
+    __mmask32 nm71 = _mm512_mask_cmpneq_epi16_mask(nm61, a1_u16x32, b31_u16x32);
+    __mmask32 nm72 = _mm512_mask_cmpneq_epi16_mask(nm62, a2_u16x32, b31_u16x32);
+    __mmask32 nm73 = _mm512_mask_cmpneq_epi16_mask(nm63, a3_u16x32, b31_u16x32);
 
     return ~(nk_u32_t)(nm70 & nk_u32_rol(nm71, 8) & nk_u32_rol(nm72, 16) & nk_u32_ror(nm73, 8));
 }
@@ -106,33 +106,33 @@ NK_INTERNAL nk_u32_t nk_intersect_u16x32_icelake_(__m512i a, __m512i b) {
  *          slightly faster than the native Tiger Lake implementation, but returns only one mask.
  */
 NK_INTERNAL nk_u16_t nk_intersect_u32x16_icelake_(__m512i a, __m512i b) {
-    __m512i a1 = _mm512_alignr_epi32(a, a, 4);
-    __m512i b1 = _mm512_shuffle_epi32(b, _MM_PERM_ADCB);
+    __m512i a1_u32x16 = _mm512_alignr_epi32(a, a, 4);
+    __m512i b1_u32x16 = _mm512_shuffle_epi32(b, _MM_PERM_ADCB);
     __mmask16 nm00 = _mm512_cmpneq_epi32_mask(a, b);
 
-    __m512i a2 = _mm512_alignr_epi32(a, a, 8);
-    __m512i a3 = _mm512_alignr_epi32(a, a, 12);
-    __mmask16 nm01 = _mm512_cmpneq_epi32_mask(a1, b);
-    __mmask16 nm02 = _mm512_cmpneq_epi32_mask(a2, b);
+    __m512i a2_u32x16 = _mm512_alignr_epi32(a, a, 8);
+    __m512i a3_u32x16 = _mm512_alignr_epi32(a, a, 12);
+    __mmask16 nm01 = _mm512_cmpneq_epi32_mask(a1_u32x16, b);
+    __mmask16 nm02 = _mm512_cmpneq_epi32_mask(a2_u32x16, b);
 
-    __mmask16 nm03 = _mm512_cmpneq_epi32_mask(a3, b);
-    __mmask16 nm10 = _mm512_mask_cmpneq_epi32_mask(nm00, a, b1);
-    __mmask16 nm11 = _mm512_mask_cmpneq_epi32_mask(nm01, a1, b1);
+    __mmask16 nm03 = _mm512_cmpneq_epi32_mask(a3_u32x16, b);
+    __mmask16 nm10 = _mm512_mask_cmpneq_epi32_mask(nm00, a, b1_u32x16);
+    __mmask16 nm11 = _mm512_mask_cmpneq_epi32_mask(nm01, a1_u32x16, b1_u32x16);
 
-    __m512i b2 = _mm512_shuffle_epi32(b, _MM_PERM_BADC);
-    __mmask16 nm12 = _mm512_mask_cmpneq_epi32_mask(nm02, a2, b1);
-    __mmask16 nm13 = _mm512_mask_cmpneq_epi32_mask(nm03, a3, b1);
-    __mmask16 nm20 = _mm512_mask_cmpneq_epi32_mask(nm10, a, b2);
+    __m512i b2_u32x16 = _mm512_shuffle_epi32(b, _MM_PERM_BADC);
+    __mmask16 nm12 = _mm512_mask_cmpneq_epi32_mask(nm02, a2_u32x16, b1_u32x16);
+    __mmask16 nm13 = _mm512_mask_cmpneq_epi32_mask(nm03, a3_u32x16, b1_u32x16);
+    __mmask16 nm20 = _mm512_mask_cmpneq_epi32_mask(nm10, a, b2_u32x16);
 
-    __m512i b3 = _mm512_shuffle_epi32(b, _MM_PERM_CBAD);
-    __mmask16 nm21 = _mm512_mask_cmpneq_epi32_mask(nm11, a1, b2);
-    __mmask16 nm22 = _mm512_mask_cmpneq_epi32_mask(nm12, a2, b2);
-    __mmask16 nm23 = _mm512_mask_cmpneq_epi32_mask(nm13, a3, b2);
+    __m512i b3_u32x16 = _mm512_shuffle_epi32(b, _MM_PERM_CBAD);
+    __mmask16 nm21 = _mm512_mask_cmpneq_epi32_mask(nm11, a1_u32x16, b2_u32x16);
+    __mmask16 nm22 = _mm512_mask_cmpneq_epi32_mask(nm12, a2_u32x16, b2_u32x16);
+    __mmask16 nm23 = _mm512_mask_cmpneq_epi32_mask(nm13, a3_u32x16, b2_u32x16);
 
-    __mmask16 nm0 = _mm512_mask_cmpneq_epi32_mask(nm20, a, b3);
-    __mmask16 nm1 = _mm512_mask_cmpneq_epi32_mask(nm21, a1, b3);
-    __mmask16 nm2 = _mm512_mask_cmpneq_epi32_mask(nm22, a2, b3);
-    __mmask16 nm3 = _mm512_mask_cmpneq_epi32_mask(nm23, a3, b3);
+    __mmask16 nm0 = _mm512_mask_cmpneq_epi32_mask(nm20, a, b3_u32x16);
+    __mmask16 nm1 = _mm512_mask_cmpneq_epi32_mask(nm21, a1_u32x16, b3_u32x16);
+    __mmask16 nm2 = _mm512_mask_cmpneq_epi32_mask(nm22, a2_u32x16, b3_u32x16);
+    __mmask16 nm3 = _mm512_mask_cmpneq_epi32_mask(nm23, a3_u32x16, b3_u32x16);
 
     return ~(nk_u16_t)(nm0 & nk_u16_rol(nm1, 4) & nk_u16_rol(nm2, 8) & nk_u16_ror(nm3, 4));
 }
@@ -268,33 +268,33 @@ NK_PUBLIC void nk_sparse_intersect_u32_icelake( //
  *          returns only one mask indicating which elements in `a` have a match in `b`.
  */
 NK_INTERNAL nk_u8_t nk_intersect_u64x8_icelake_(__m512i a, __m512i b) {
-    __m512i a1 = _mm512_alignr_epi64(a, a, 2);
-    __m512i b1 = _mm512_permutex_epi64(b, _MM_PERM_ADCB);
+    __m512i a1_u64x8 = _mm512_alignr_epi64(a, a, 2);
+    __m512i b1_u64x8 = _mm512_permutex_epi64(b, _MM_PERM_ADCB);
     __mmask8 nm00 = _mm512_cmpneq_epi64_mask(a, b);
 
-    __m512i a2 = _mm512_alignr_epi64(a, a, 4);
-    __m512i a3 = _mm512_alignr_epi64(a, a, 6);
-    __mmask8 nm01 = _mm512_cmpneq_epi64_mask(a1, b);
-    __mmask8 nm02 = _mm512_cmpneq_epi64_mask(a2, b);
+    __m512i a2_u64x8 = _mm512_alignr_epi64(a, a, 4);
+    __m512i a3_u64x8 = _mm512_alignr_epi64(a, a, 6);
+    __mmask8 nm01 = _mm512_cmpneq_epi64_mask(a1_u64x8, b);
+    __mmask8 nm02 = _mm512_cmpneq_epi64_mask(a2_u64x8, b);
 
-    __m512i b2 = _mm512_permutex_epi64(b, _MM_PERM_BADC);
-    __mmask8 nm03 = _mm512_cmpneq_epi64_mask(a3, b);
-    __mmask8 nm10 = _mm512_mask_cmpneq_epi64_mask(nm00, a, b1);
-    __mmask8 nm11 = _mm512_mask_cmpneq_epi64_mask(nm01, a1, b1);
+    __m512i b2_u64x8 = _mm512_permutex_epi64(b, _MM_PERM_BADC);
+    __mmask8 nm03 = _mm512_cmpneq_epi64_mask(a3_u64x8, b);
+    __mmask8 nm10 = _mm512_mask_cmpneq_epi64_mask(nm00, a, b1_u64x8);
+    __mmask8 nm11 = _mm512_mask_cmpneq_epi64_mask(nm01, a1_u64x8, b1_u64x8);
 
-    __m512i b3 = _mm512_permutex_epi64(b, _MM_PERM_CBAD);
-    __mmask8 nm12 = _mm512_mask_cmpneq_epi64_mask(nm02, a2, b1);
-    __mmask8 nm13 = _mm512_mask_cmpneq_epi64_mask(nm03, a3, b1);
-    __mmask8 nm20 = _mm512_mask_cmpneq_epi64_mask(nm10, a, b2);
+    __m512i b3_u64x8 = _mm512_permutex_epi64(b, _MM_PERM_CBAD);
+    __mmask8 nm12 = _mm512_mask_cmpneq_epi64_mask(nm02, a2_u64x8, b1_u64x8);
+    __mmask8 nm13 = _mm512_mask_cmpneq_epi64_mask(nm03, a3_u64x8, b1_u64x8);
+    __mmask8 nm20 = _mm512_mask_cmpneq_epi64_mask(nm10, a, b2_u64x8);
 
-    __mmask8 nm21 = _mm512_mask_cmpneq_epi64_mask(nm11, a1, b2);
-    __mmask8 nm22 = _mm512_mask_cmpneq_epi64_mask(nm12, a2, b2);
-    __mmask8 nm23 = _mm512_mask_cmpneq_epi64_mask(nm13, a3, b2);
+    __mmask8 nm21 = _mm512_mask_cmpneq_epi64_mask(nm11, a1_u64x8, b2_u64x8);
+    __mmask8 nm22 = _mm512_mask_cmpneq_epi64_mask(nm12, a2_u64x8, b2_u64x8);
+    __mmask8 nm23 = _mm512_mask_cmpneq_epi64_mask(nm13, a3_u64x8, b2_u64x8);
 
-    __mmask8 nm0 = _mm512_mask_cmpneq_epi64_mask(nm20, a, b3);
-    __mmask8 nm1 = _mm512_mask_cmpneq_epi64_mask(nm21, a1, b3);
-    __mmask8 nm2 = _mm512_mask_cmpneq_epi64_mask(nm22, a2, b3);
-    __mmask8 nm3 = _mm512_mask_cmpneq_epi64_mask(nm23, a3, b3);
+    __mmask8 nm0 = _mm512_mask_cmpneq_epi64_mask(nm20, a, b3_u64x8);
+    __mmask8 nm1 = _mm512_mask_cmpneq_epi64_mask(nm21, a1_u64x8, b3_u64x8);
+    __mmask8 nm2 = _mm512_mask_cmpneq_epi64_mask(nm22, a2_u64x8, b3_u64x8);
+    __mmask8 nm3 = _mm512_mask_cmpneq_epi64_mask(nm23, a3_u64x8, b3_u64x8);
 
     return ~(nk_u8_t)(nm0 & nk_u8_rol(nm1, 2) & nk_u8_rol(nm2, 4) & nk_u8_ror(nm3, 2));
 }
@@ -377,8 +377,8 @@ NK_PUBLIC void nk_sparse_dot_u32f32_icelake(              //
 
     nk_u32_t const *const a_end = a + a_length;
     nk_u32_t const *const b_end = b + b_length;
-    __m512d product_lower_f64x8 = _mm512_setzero_pd();
-    __m512d product_upper_f64x8 = _mm512_setzero_pd();
+    __m512d product_low_f64x8 = _mm512_setzero_pd();
+    __m512d product_high_f64x8 = _mm512_setzero_pd();
     nk_b512_vec_t a_vec, b_vec;
 
     while (a + 16 <= a_end && b + 16 <= b_end) {
@@ -425,15 +425,15 @@ NK_PUBLIC void nk_sparse_dot_u32f32_icelake(              //
             __m512 a_matched_f32x16 = _mm512_maskz_compress_ps(a_matches, a_weights_f32x16);
             __m512 b_matched_f32x16 = _mm512_maskz_compress_ps(b_matches, b_weights_f32x16);
 
-            __m256 a_matched_lower_f32x8 = _mm512_castps512_ps256(a_matched_f32x16);
-            __m256 a_matched_upper_f32x8 = _mm512_extractf32x8_ps(a_matched_f32x16, 1);
-            __m256 b_matched_lower_f32x8 = _mm512_castps512_ps256(b_matched_f32x16);
-            __m256 b_matched_upper_f32x8 = _mm512_extractf32x8_ps(b_matched_f32x16, 1);
+            __m256 a_matched_low_f32x8 = _mm512_castps512_ps256(a_matched_f32x16);
+            __m256 a_matched_high_f32x8 = _mm512_extractf32x8_ps(a_matched_f32x16, 1);
+            __m256 b_matched_low_f32x8 = _mm512_castps512_ps256(b_matched_f32x16);
+            __m256 b_matched_high_f32x8 = _mm512_extractf32x8_ps(b_matched_f32x16, 1);
 
-            product_lower_f64x8 = _mm512_fmadd_pd(_mm512_cvtps_pd(a_matched_lower_f32x8),
-                                                  _mm512_cvtps_pd(b_matched_lower_f32x8), product_lower_f64x8);
-            product_upper_f64x8 = _mm512_fmadd_pd(_mm512_cvtps_pd(a_matched_upper_f32x8),
-                                                  _mm512_cvtps_pd(b_matched_upper_f32x8), product_upper_f64x8);
+            product_low_f64x8 = _mm512_fmadd_pd(_mm512_cvtps_pd(a_matched_low_f32x8),
+                                                _mm512_cvtps_pd(b_matched_low_f32x8), product_low_f64x8);
+            product_high_f64x8 = _mm512_fmadd_pd(_mm512_cvtps_pd(a_matched_high_f32x8),
+                                                 _mm512_cvtps_pd(b_matched_high_f32x8), product_high_f64x8);
         }
 
         // Advance pointers after processing
@@ -445,7 +445,7 @@ NK_PUBLIC void nk_sparse_dot_u32f32_icelake(              //
 
     nk_f64_t tail_product = 0;
     nk_sparse_dot_u32f32_serial(a, b, a_weights, b_weights, a_end - a, b_end - b, &tail_product);
-    *product = _mm512_reduce_add_pd(product_lower_f64x8) + _mm512_reduce_add_pd(product_upper_f64x8) + tail_product;
+    *product = _mm512_reduce_add_pd(product_low_f64x8) + _mm512_reduce_add_pd(product_high_f64x8) + tail_product;
 }
 
 #if defined(__clang__)

package/include/numkong/sparse/serial.h

@@ -96,7 +96,7 @@ extern "C" {
             int matches = ai == bj;                                                                        \
             load_and_convert(a_weights + i, &awi);                                                         \
             load_and_convert(b_weights + j, &bwi);                                                         \
-            weights_product += matches * awi * bwi;                                                        \
+            weights_product += (nk_##accumulator_type##_t)matches * awi * bwi;                             \
             i += ai < bj;                                                                                  \
             j += ai >= bj;                                                                                 \
         }                                                                                                  \