numkong 7.0.0 → 7.4.2

package/include/numkong/spatials/smef64.h

@@ -20,69 +20,70 @@ extern "C" {
 #endif
 
 #if defined(__clang__)
-#pragma clang attribute push(__attribute__((target("sme,sve,sme-f64f64"))), apply_to = function)
+#pragma clang attribute push(__attribute__((target("sme,sme-f64f64"))), apply_to = function)
 #elif defined(__GNUC__)
 #pragma GCC push_options
 #pragma GCC target("+sme+sme-f64f64")
 #endif
 
 NK_PUBLIC nk_f64_t nk_dots_reduce_sumsq_f32_ssve_(nk_f32_t const *data, nk_size_t count) NK_STREAMING_ {
-    svfloat64_t accumulator_lo_f64x = svdup_f64(0.0);
-    svfloat64_t accumulator_hi_f64x = svdup_f64(0.0);
+    svfloat64_t accumulator_even_f64x = svdup_f64(0.0);
+    svfloat64_t accumulator_odd_f64x = svdup_f64(0.0);
     nk_size_t const vector_length = svcntw();
     nk_size_t const half_vector_length = svcntd();
     for (nk_size_t i = 0; i < count; i += vector_length) {
-        svbool_t predicate_f32x = svwhilelt_b32_u64(i, count);
-        svfloat32_t values_f32x = svld1_f32(predicate_f32x, data + i);
+        svbool_t predicate_b32x = svwhilelt_b32_u64(i, count);
+        svfloat32_t values_f32x = svld1_f32(predicate_b32x, data + i);
 
-        svbool_t predicate_lo_f64x = svwhilelt_b64_u64(i, count);
-        svfloat64_t values_lo_f64x = svcvt_f64_f32_x(predicate_lo_f64x, values_f32x);
-        accumulator_lo_f64x = svmla_f64_x(predicate_lo_f64x, accumulator_lo_f64x, values_lo_f64x, values_lo_f64x);
+        svbool_t predicate_even_b64x = svwhilelt_b64_u64(i, count);
+        svfloat64_t values_even_f64x = svcvt_f64_f32_x(predicate_even_b64x, values_f32x);
+        accumulator_even_f64x = svmla_f64_m(predicate_even_b64x, accumulator_even_f64x, values_even_f64x,
+                                            values_even_f64x);
 
-        svbool_t predicate_hi_f64x = svwhilelt_b64_u64(i + half_vector_length, count);
-        svfloat64_t values_hi_f64x = svcvtlt_f64_f32_x(predicate_hi_f64x, values_f32x);
-        accumulator_hi_f64x = svmla_f64_x(predicate_hi_f64x, accumulator_hi_f64x, values_hi_f64x, values_hi_f64x);
+        svbool_t predicate_odd_b64x = svwhilelt_b64_u64(i + half_vector_length, count);
+        svfloat64_t values_odd_f64x = svcvtlt_f64_f32_x(predicate_odd_b64x, values_f32x);
+        accumulator_odd_f64x = svmla_f64_m(predicate_odd_b64x, accumulator_odd_f64x, values_odd_f64x, values_odd_f64x);
     }
-    return svaddv_f64(svptrue_b64(), accumulator_lo_f64x) + svaddv_f64(svptrue_b64(), accumulator_hi_f64x);
+    return svaddv_f64(svptrue_b64(), accumulator_even_f64x) + svaddv_f64(svptrue_b64(), accumulator_odd_f64x);
 }
 
-NK_PUBLIC nk_f64_t nk_dots_reduce_sumsq_f64_ssve_(nk_f64_t const *data, nk_size_t count) NK_STREAMING_COMPATIBLE_ {
+NK_PUBLIC nk_f64_t nk_dots_reduce_sumsq_f64_ssve_(nk_f64_t const *data, nk_size_t count) NK_STREAMING_ {
     svfloat64_t accumulator_f64x = svdup_f64(0.0);
     nk_size_t const vector_length = svcntd();
     for (nk_size_t i = 0; i < count; i += vector_length) {
-        svbool_t predicate_f64x = svwhilelt_b64_u64(i, count);
-        svfloat64_t values_f64x = svld1_f64(predicate_f64x, data + i);
-        accumulator_f64x = svmla_f64_x(predicate_f64x, accumulator_f64x, values_f64x, values_f64x);
+        svbool_t predicate_b64x = svwhilelt_b64_u64(i, count);
+        svfloat64_t values_f64x = svld1_f64(predicate_b64x, data + i);
+        accumulator_f64x = svmla_f64_m(predicate_b64x, accumulator_f64x, values_f64x, values_f64x);
     }
     return svaddv_f64(svptrue_b64(), accumulator_f64x);
 }
 
-NK_PUBLIC svfloat64_t nk_angulars_from_dot_f64x_ssvef64_(svbool_t predicate_f64x, svfloat64_t dots_f64x,
+NK_PUBLIC svfloat64_t nk_angulars_from_dot_f64x_ssvef64_(svbool_t predicate_b64x, svfloat64_t dots_f64x,
                                                          svfloat64_t query_norm_sq_f64x,
-                                                         svfloat64_t target_norms_sq_f64x) NK_STREAMING_COMPATIBLE_ {
-    svfloat64_t norms_product_f64x = svmul_f64_x(predicate_f64x, query_norm_sq_f64x, target_norms_sq_f64x);
-    svbool_t positive_norms_f64x = svcmpgt_n_f64(predicate_f64x, norms_product_f64x, 0.0);
-    svfloat64_t denom_f64x = svsqrt_f64_x(positive_norms_f64x, norms_product_f64x);
-    svfloat64_t safe_denom_f64x = svsel_f64(positive_norms_f64x, denom_f64x, svdup_n_f64(1.0));
-    svfloat64_t normalized_f64x = svdiv_f64_x(predicate_f64x, dots_f64x, safe_denom_f64x);
-    svfloat64_t angular_f64x = svsub_f64_x(predicate_f64x, svdup_n_f64(1.0), normalized_f64x);
+                                                         svfloat64_t target_norms_sq_f64x) NK_STREAMING_ {
+    svfloat64_t norms_product_f64x = svmul_f64_x(predicate_b64x, query_norm_sq_f64x, target_norms_sq_f64x);
+    svbool_t positive_norms_b64x = svcmpgt_n_f64(predicate_b64x, norms_product_f64x, 0.0);
+    svfloat64_t denom_f64x = svsqrt_f64_x(positive_norms_b64x, norms_product_f64x);
+    svfloat64_t safe_denom_f64x = svsel_f64(positive_norms_b64x, denom_f64x, svdup_n_f64(1.0));
+    svfloat64_t normalized_f64x = svdiv_f64_x(predicate_b64x, dots_f64x, safe_denom_f64x);
+    svfloat64_t angular_f64x = svsub_f64_x(predicate_b64x, svdup_n_f64(1.0), normalized_f64x);
     angular_f64x = svsel_f64(
-        positive_norms_f64x, angular_f64x,
-        svsel_f64(svcmpeq_n_f64(predicate_f64x, dots_f64x, 0.0), svdup_n_f64(0.0), svdup_n_f64(1.0)));
-    return svmax_f64_x(predicate_f64x, angular_f64x, svdup_n_f64(0.0));
+        positive_norms_b64x, angular_f64x,
+        svsel_f64(svcmpeq_n_f64(predicate_b64x, dots_f64x, 0.0), svdup_n_f64(0.0), svdup_n_f64(1.0)));
+    return svmax_f64_x(predicate_b64x, angular_f64x, svdup_n_f64(0.0));
 }
 
-NK_PUBLIC svfloat64_t nk_euclideans_from_dot_f64x_ssvef64_(svbool_t predicate_f64x, svfloat64_t dots_f64x,
+NK_PUBLIC svfloat64_t nk_euclideans_from_dot_f64x_ssvef64_(svbool_t predicate_b64x, svfloat64_t dots_f64x,
                                                            svfloat64_t query_norm_sq_f64x,
-                                                           svfloat64_t target_norms_sq_f64x) NK_STREAMING_COMPATIBLE_ {
-    svfloat64_t sum_sq_f64x = svadd_f64_x(predicate_f64x, query_norm_sq_f64x, target_norms_sq_f64x);
-    svfloat64_t dist_sq_f64x = svsub_f64_x(predicate_f64x, sum_sq_f64x,
-                                           svmul_f64_x(predicate_f64x, svdup_n_f64(2.0), dots_f64x));
-    dist_sq_f64x = svmax_f64_x(predicate_f64x, dist_sq_f64x, svdup_n_f64(0.0));
-    return svsqrt_f64_x(predicate_f64x, dist_sq_f64x);
+                                                           svfloat64_t target_norms_sq_f64x) NK_STREAMING_ {
+    svfloat64_t sum_sq_f64x = svadd_f64_x(predicate_b64x, query_norm_sq_f64x, target_norms_sq_f64x);
+    svfloat64_t dist_sq_f64x = svsub_f64_x(predicate_b64x, sum_sq_f64x,
+                                           svmul_f64_x(predicate_b64x, svdup_n_f64(2.0), dots_f64x));
+    dist_sq_f64x = svmax_f64_x(predicate_b64x, dist_sq_f64x, svdup_n_f64(0.0));
+    return svsqrt_f64_x(predicate_b64x, dist_sq_f64x);
 }
 
-#pragma region Single Precision Packed Angular
+#pragma region F32 Packed Angular
 
 __arm_locally_streaming static void nk_angulars_packed_f32_smef64_finalize_streaming_( //
     nk_f32_t const *a, void const *b_packed, nk_f64_t *c,                              //
@@ -99,11 +100,11 @@ __arm_locally_streaming static void nk_angulars_packed_f32_smef64_finalize_strea
         svfloat64_t query_norm_sq_f64x = svdup_n_f64(query_norm_sq_f64);
 
         for (nk_size_t col_index = 0; col_index < columns; col_index += svcntd()) {
-            svbool_t predicate_f64x = svwhilelt_b64_u64(col_index, columns);
-            svfloat64_t dots_f64x = svld1_f64(predicate_f64x, c_row + col_index);
-            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_f64x, b_norms + col_index);
-            svst1_f64(predicate_f64x, c_row + col_index,
-                      nk_angulars_from_dot_f64x_ssvef64_(predicate_f64x, dots_f64x, query_norm_sq_f64x,
+            svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, columns);
+            svfloat64_t dots_f64x = svld1_f64(predicate_b64x, c_row + col_index);
+            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, b_norms + col_index);
+            svst1_f64(predicate_b64x, c_row + col_index,
+                      nk_angulars_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
                                                          target_norms_sq_f64x));
         }
     }
@@ -122,7 +123,8 @@ NK_PUBLIC void nk_angulars_packed_f32_smef64(             //
                                                       c_stride_elements);
 }
 
-#pragma region Single Precision Packed Euclidean
+#pragma endregion F32 Packed Angular
+#pragma region F32 Packed Euclidean
 
 __arm_locally_streaming static void nk_euclideans_packed_f32_smef64_finalize_streaming_( //
     nk_f32_t const *a, void const *b_packed, nk_f64_t *c,                                //
@@ -139,11 +141,11 @@ __arm_locally_streaming static void nk_euclideans_packed_f32_smef64_finalize_str
         svfloat64_t query_norm_sq_f64x = svdup_n_f64(query_norm_sq_f64);
 
         for (nk_size_t col_index = 0; col_index < columns; col_index += svcntd()) {
-            svbool_t predicate_f64x = svwhilelt_b64_u64(col_index, columns);
-            svfloat64_t dots_f64x = svld1_f64(predicate_f64x, c_row + col_index);
-            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_f64x, b_norms + col_index);
-            svst1_f64(predicate_f64x, c_row + col_index,
-                      nk_euclideans_from_dot_f64x_ssvef64_(predicate_f64x, dots_f64x, query_norm_sq_f64x,
+            svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, columns);
+            svfloat64_t dots_f64x = svld1_f64(predicate_b64x, c_row + col_index);
+            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, b_norms + col_index);
+            svst1_f64(predicate_b64x, c_row + col_index,
+                      nk_euclideans_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
                                                            target_norms_sq_f64x));
         }
     }
@@ -162,10 +164,11 @@ NK_PUBLIC void nk_euclideans_packed_f32_smef64(           //
                                                         c_stride_elements);
 }
 
-#pragma region Single Precision Symmetric Angular
+#pragma endregion F32 Packed Euclidean
+#pragma region F32 Symmetric Angular
 
-__arm_locally_streaming static void nk_angulars_symmetric_f32_smef64_finalize_streaming_(     //
-    nk_f32_t const *vectors, nk_size_t n_vectors, nk_size_t depth, nk_size_t stride_elements, //
+__arm_locally_streaming static void nk_angulars_symmetric_f32_smef64_finalize_streaming_(         //
+    nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, //
     nk_f64_t *result, nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) {
     // Phase 1: cache row norms on diagonal
     for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
@@ -175,8 +178,8 @@ __arm_locally_streaming static void nk_angulars_symmetric_f32_smef64_finalize_st
     }
     // Phase 2: column-chunked post-processing
     nk_f64_t column_norms[256];
-    for (nk_size_t chunk_start = 0; chunk_start < n_vectors; chunk_start += 256) {
-        nk_size_t chunk_end = chunk_start + 256 < n_vectors ? chunk_start + 256 : n_vectors;
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
         for (nk_size_t col = chunk_start; col < chunk_end; ++col) {
             nk_f32_t const *col_vector = vectors + col * stride_elements;
             column_norms[col - chunk_start] = nk_dots_reduce_sumsq_f32_ssve_(col_vector, depth);
@@ -187,11 +190,11 @@ __arm_locally_streaming static void nk_angulars_symmetric_f32_smef64_finalize_st
             nk_f64_t *result_row = result + row_index * result_stride_elements;
             svfloat64_t query_norm_sq_f64x = svdup_n_f64(result_row[row_index]);
             for (nk_size_t col_index = col_start; col_index < chunk_end; col_index += svcntd()) {
-                svbool_t predicate_f64x = svwhilelt_b64_u64(col_index, chunk_end);
-                svfloat64_t dots_f64x = svld1_f64(predicate_f64x, result_row + col_index);
-                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_f64x, column_norms + (col_index - chunk_start));
-                svst1_f64(predicate_f64x, result_row + col_index,
-                          nk_angulars_from_dot_f64x_ssvef64_(predicate_f64x, dots_f64x, query_norm_sq_f64x,
+                svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, chunk_end);
+                svfloat64_t dots_f64x = svld1_f64(predicate_b64x, result_row + col_index);
+                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, column_norms + (col_index - chunk_start));
+                svst1_f64(predicate_b64x, result_row + col_index,
+                          nk_angulars_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
                                                              target_norms_sq_f64x));
             }
         }
@@ -201,23 +204,24 @@ __arm_locally_streaming static void nk_angulars_symmetric_f32_smef64_finalize_st
         result[row_index * result_stride_elements + row_index] = 0;
 }
 
-NK_PUBLIC void nk_angulars_symmetric_f32_smef64(                                     //
-    nk_f32_t const *vectors, nk_size_t n_vectors, nk_size_t depth, nk_size_t stride, //
-    nk_f64_t *result, nk_size_t result_stride, nk_size_t row_start, nk_size_t row_count) {
+NK_PUBLIC void nk_angulars_symmetric_f32_smef64(                                                  //
+    nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, //
+    nk_f64_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
 
-    nk_size_t const stride_elements = stride / sizeof(nk_f32_t);
-    nk_size_t const result_stride_elements = result_stride / sizeof(nk_f64_t);
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f32_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
 
-    nk_dots_symmetric_f32_smef64_streaming_(vectors, n_vectors, depth, stride_elements, result, result_stride_elements,
-                                            row_start, row_count);
-    nk_angulars_symmetric_f32_smef64_finalize_streaming_(vectors, n_vectors, depth, stride_elements, result,
+    nk_dots_symmetric_f32_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
+                                            result_stride_elements, row_start, row_count);
+    nk_angulars_symmetric_f32_smef64_finalize_streaming_(vectors, vectors_count, depth, stride_elements, result,
                                                          result_stride_elements, row_start, row_count);
 }
 
-#pragma region Single Precision Symmetric Euclidean
+#pragma endregion F32 Symmetric Angular
+#pragma region F32 Symmetric Euclidean
 
-__arm_locally_streaming static void nk_euclideans_symmetric_f32_smef64_finalize_streaming_(   //
-    nk_f32_t const *vectors, nk_size_t n_vectors, nk_size_t depth, nk_size_t stride_elements, //
+__arm_locally_streaming static void nk_euclideans_symmetric_f32_smef64_finalize_streaming_(       //
+    nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, //
     nk_f64_t *result, nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) {
     // Phase 1: cache row norms on diagonal
     for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
@@ -227,8 +231,8 @@ __arm_locally_streaming static void nk_euclideans_symmetric_f32_smef64_finalize_
     }
     // Phase 2: column-chunked post-processing
     nk_f64_t column_norms[256];
-    for (nk_size_t chunk_start = 0; chunk_start < n_vectors; chunk_start += 256) {
-        nk_size_t chunk_end = chunk_start + 256 < n_vectors ? chunk_start + 256 : n_vectors;
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
         for (nk_size_t col = chunk_start; col < chunk_end; ++col) {
             nk_f32_t const *col_vector = vectors + col * stride_elements;
             column_norms[col - chunk_start] = nk_dots_reduce_sumsq_f32_ssve_(col_vector, depth);
@@ -239,11 +243,11 @@ __arm_locally_streaming static void nk_euclideans_symmetric_f32_smef64_finalize_
             nk_f64_t *result_row = result + row_index * result_stride_elements;
             svfloat64_t query_norm_sq_f64x = svdup_n_f64(result_row[row_index]);
             for (nk_size_t col_index = col_start; col_index < chunk_end; col_index += svcntd()) {
-                svbool_t predicate_f64x = svwhilelt_b64_u64(col_index, chunk_end);
-                svfloat64_t dots_f64x = svld1_f64(predicate_f64x, result_row + col_index);
-                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_f64x, column_norms + (col_index - chunk_start));
-                svst1_f64(predicate_f64x, result_row + col_index,
-                          nk_euclideans_from_dot_f64x_ssvef64_(predicate_f64x, dots_f64x, query_norm_sq_f64x,
+                svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, chunk_end);
+                svfloat64_t dots_f64x = svld1_f64(predicate_b64x, result_row + col_index);
+                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, column_norms + (col_index - chunk_start));
+                svst1_f64(predicate_b64x, result_row + col_index,
+                          nk_euclideans_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
                                                                target_norms_sq_f64x));
             }
         }
@@ -253,20 +257,21 @@ __arm_locally_streaming static void nk_euclideans_symmetric_f32_smef64_finalize_
         result[row_index * result_stride_elements + row_index] = 0;
 }
 
-NK_PUBLIC void nk_euclideans_symmetric_f32_smef64(                                   //
-    nk_f32_t const *vectors, nk_size_t n_vectors, nk_size_t depth, nk_size_t stride, //
-    nk_f64_t *result, nk_size_t result_stride, nk_size_t row_start, nk_size_t row_count) {
+NK_PUBLIC void nk_euclideans_symmetric_f32_smef64(                                                //
+    nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, //
+    nk_f64_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
 
-    nk_size_t const stride_elements = stride / sizeof(nk_f32_t);
-    nk_size_t const result_stride_elements = result_stride / sizeof(nk_f64_t);
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f32_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
 
-    nk_dots_symmetric_f32_smef64_streaming_(vectors, n_vectors, depth, stride_elements, result, result_stride_elements,
-                                            row_start, row_count);
-    nk_euclideans_symmetric_f32_smef64_finalize_streaming_(vectors, n_vectors, depth, stride_elements, result,
+    nk_dots_symmetric_f32_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
+                                            result_stride_elements, row_start, row_count);
+    nk_euclideans_symmetric_f32_smef64_finalize_streaming_(vectors, vectors_count, depth, stride_elements, result,
                                                            result_stride_elements, row_start, row_count);
 }
 
-#pragma region Double Precision Packed Angular
+#pragma endregion F32 Symmetric Euclidean
+#pragma region F64 Packed Angular
 
 __arm_locally_streaming static void nk_angulars_packed_f64_smef64_finalize_streaming_( //
     nk_f64_t const *a, void const *b_packed, nk_f64_t *c,                              //
@@ -283,11 +288,11 @@ __arm_locally_streaming static void nk_angulars_packed_f64_smef64_finalize_strea
         svfloat64_t query_norm_sq_f64x = svdup_n_f64(query_norm_sq_f64);
 
         for (nk_size_t col_index = 0; col_index < columns; col_index += svcntd()) {
-            svbool_t predicate_f64x = svwhilelt_b64_u64(col_index, columns);
-            svfloat64_t dots_f64x = svld1_f64(predicate_f64x, c_row + col_index);
-            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_f64x, b_norms + col_index);
-            svst1_f64(predicate_f64x, c_row + col_index,
-                      nk_angulars_from_dot_f64x_ssvef64_(predicate_f64x, dots_f64x, query_norm_sq_f64x,
+            svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, columns);
+            svfloat64_t dots_f64x = svld1_f64(predicate_b64x, c_row + col_index);
+            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, b_norms + col_index);
+            svst1_f64(predicate_b64x, c_row + col_index,
+                      nk_angulars_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
                                                          target_norms_sq_f64x));
         }
     }
@@ -306,7 +311,8 @@ NK_PUBLIC void nk_angulars_packed_f64_smef64(             //
                                                       c_stride_elements);
 }
 
-#pragma region Double Precision Packed Euclidean
+#pragma endregion F64 Packed Angular
+#pragma region F64 Packed Euclidean
 
 __arm_locally_streaming static void nk_euclideans_packed_f64_smef64_finalize_streaming_( //
     nk_f64_t const *a, void const *b_packed, nk_f64_t *c,                                //
@@ -323,11 +329,11 @@ __arm_locally_streaming static void nk_euclideans_packed_f64_smef64_finalize_str
         svfloat64_t query_norm_sq_f64x = svdup_n_f64(query_norm_sq_f64);
 
         for (nk_size_t col_index = 0; col_index < columns; col_index += svcntd()) {
-            svbool_t predicate_f64x = svwhilelt_b64_u64(col_index, columns);
-            svfloat64_t dots_f64x = svld1_f64(predicate_f64x, c_row + col_index);
-            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_f64x, b_norms + col_index);
-            svst1_f64(predicate_f64x, c_row + col_index,
-                      nk_euclideans_from_dot_f64x_ssvef64_(predicate_f64x, dots_f64x, query_norm_sq_f64x,
+            svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, columns);
+            svfloat64_t dots_f64x = svld1_f64(predicate_b64x, c_row + col_index);
+            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, b_norms + col_index);
+            svst1_f64(predicate_b64x, c_row + col_index,
+                      nk_euclideans_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
                                                            target_norms_sq_f64x));
         }
     }
@@ -346,10 +352,11 @@ NK_PUBLIC void nk_euclideans_packed_f64_smef64(           //
                                                         c_stride_elements);
 }
 
-#pragma region Double Precision Symmetric Angular
+#pragma endregion F64 Packed Euclidean
+#pragma region F64 Symmetric Angular
 
-__arm_locally_streaming static void nk_angulars_symmetric_f64_smef64_finalize_streaming_(     //
-    nk_f64_t const *vectors, nk_size_t n_vectors, nk_size_t depth, nk_size_t stride_elements, //
+__arm_locally_streaming static void nk_angulars_symmetric_f64_smef64_finalize_streaming_(         //
+    nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, //
     nk_f64_t *result, nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) {
     // Phase 1: cache row norms on diagonal
     for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
@@ -359,8 +366,8 @@ __arm_locally_streaming static void nk_angulars_symmetric_f64_smef64_finalize_st
     }
     // Phase 2: column-chunked post-processing
     nk_f64_t column_norms[256];
-    for (nk_size_t chunk_start = 0; chunk_start < n_vectors; chunk_start += 256) {
-        nk_size_t chunk_end = chunk_start + 256 < n_vectors ? chunk_start + 256 : n_vectors;
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
         for (nk_size_t col = chunk_start; col < chunk_end; ++col) {
             nk_f64_t const *col_vector = vectors + col * stride_elements;
             column_norms[col - chunk_start] = nk_dots_reduce_sumsq_f64_ssve_(col_vector, depth);
@@ -371,11 +378,11 @@ __arm_locally_streaming static void nk_angulars_symmetric_f64_smef64_finalize_st
             nk_f64_t *result_row = result + row_index * result_stride_elements;
             svfloat64_t query_norm_sq_f64x = svdup_n_f64(result_row[row_index]);
             for (nk_size_t col_index = col_start; col_index < chunk_end; col_index += svcntd()) {
-                svbool_t predicate_f64x = svwhilelt_b64_u64(col_index, chunk_end);
-                svfloat64_t dots_f64x = svld1_f64(predicate_f64x, result_row + col_index);
-                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_f64x, column_norms + (col_index - chunk_start));
-                svst1_f64(predicate_f64x, result_row + col_index,
-                          nk_angulars_from_dot_f64x_ssvef64_(predicate_f64x, dots_f64x, query_norm_sq_f64x,
+                svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, chunk_end);
+                svfloat64_t dots_f64x = svld1_f64(predicate_b64x, result_row + col_index);
+                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, column_norms + (col_index - chunk_start));
+                svst1_f64(predicate_b64x, result_row + col_index,
+                          nk_angulars_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
                                                              target_norms_sq_f64x));
             }
         }
@@ -385,23 +392,24 @@ __arm_locally_streaming static void nk_angulars_symmetric_f64_smef64_finalize_st
         result[row_index * result_stride_elements + row_index] = 0;
 }
 
-NK_PUBLIC void nk_angulars_symmetric_f64_smef64(                                     //
-    nk_f64_t const *vectors, nk_size_t n_vectors, nk_size_t depth, nk_size_t stride, //
-    nk_f64_t *result, nk_size_t result_stride, nk_size_t row_start, nk_size_t row_count) {
+NK_PUBLIC void nk_angulars_symmetric_f64_smef64(                                                  //
+    nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, //
+    nk_f64_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
 
-    nk_size_t const stride_elements = stride / sizeof(nk_f64_t);
-    nk_size_t const result_stride_elements = result_stride / sizeof(nk_f64_t);
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f64_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
 
-    nk_dots_symmetric_f64_smef64_streaming_(vectors, n_vectors, depth, stride_elements, result, result_stride_elements,
-                                            row_start, row_count);
-    nk_angulars_symmetric_f64_smef64_finalize_streaming_(vectors, n_vectors, depth, stride_elements, result,
+    nk_dots_symmetric_f64_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
+                                            result_stride_elements, row_start, row_count);
+    nk_angulars_symmetric_f64_smef64_finalize_streaming_(vectors, vectors_count, depth, stride_elements, result,
                                                          result_stride_elements, row_start, row_count);
 }
 
-#pragma region Double Precision Symmetric Euclidean
+#pragma endregion F64 Symmetric Angular
+#pragma region F64 Symmetric Euclidean
 
-__arm_locally_streaming static void nk_euclideans_symmetric_f64_smef64_finalize_streaming_(   //
-    nk_f64_t const *vectors, nk_size_t n_vectors, nk_size_t depth, nk_size_t stride_elements, //
+__arm_locally_streaming static void nk_euclideans_symmetric_f64_smef64_finalize_streaming_(       //
+    nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, //
     nk_f64_t *result, nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) {
     // Phase 1: cache row norms on diagonal
     for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
@@ -411,8 +419,8 @@ __arm_locally_streaming static void nk_euclideans_symmetric_f64_smef64_finalize_
     }
     // Phase 2: column-chunked post-processing
     nk_f64_t column_norms[256];
-    for (nk_size_t chunk_start = 0; chunk_start < n_vectors; chunk_start += 256) {
-        nk_size_t chunk_end = chunk_start + 256 < n_vectors ? chunk_start + 256 : n_vectors;
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
         for (nk_size_t col = chunk_start; col < chunk_end; ++col) {
             nk_f64_t const *col_vector = vectors + col * stride_elements;
             column_norms[col - chunk_start] = nk_dots_reduce_sumsq_f64_ssve_(col_vector, depth);
@@ -423,11 +431,11 @@ __arm_locally_streaming static void nk_euclideans_symmetric_f64_smef64_finalize_
             nk_f64_t *result_row = result + row_index * result_stride_elements;
             svfloat64_t query_norm_sq_f64x = svdup_n_f64(result_row[row_index]);
             for (nk_size_t col_index = col_start; col_index < chunk_end; col_index += svcntd()) {
-                svbool_t predicate_f64x = svwhilelt_b64_u64(col_index, chunk_end);
-                svfloat64_t dots_f64x = svld1_f64(predicate_f64x, result_row + col_index);
-                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_f64x, column_norms + (col_index - chunk_start));
-                svst1_f64(predicate_f64x, result_row + col_index,
-                          nk_euclideans_from_dot_f64x_ssvef64_(predicate_f64x, dots_f64x, query_norm_sq_f64x,
+                svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, chunk_end);
+                svfloat64_t dots_f64x = svld1_f64(predicate_b64x, result_row + col_index);
+                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, column_norms + (col_index - chunk_start));
+                svst1_f64(predicate_b64x, result_row + col_index,
+                          nk_euclideans_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
                                                                target_norms_sq_f64x));
             }
         }
@@ -437,19 +445,20 @@ __arm_locally_streaming static void nk_euclideans_symmetric_f64_smef64_finalize_
         result[row_index * result_stride_elements + row_index] = 0;
 }
 
-NK_PUBLIC void nk_euclideans_symmetric_f64_smef64(                                   //
-    nk_f64_t const *vectors, nk_size_t n_vectors, nk_size_t depth, nk_size_t stride, //
-    nk_f64_t *result, nk_size_t result_stride, nk_size_t row_start, nk_size_t row_count) {
+NK_PUBLIC void nk_euclideans_symmetric_f64_smef64(                                                //
+    nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, //
+    nk_f64_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
 
-    nk_size_t const stride_elements = stride / sizeof(nk_f64_t);
-    nk_size_t const result_stride_elements = result_stride / sizeof(nk_f64_t);
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f64_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
 
-    nk_dots_symmetric_f64_smef64_streaming_(vectors, n_vectors, depth, stride_elements, result, result_stride_elements,
-                                            row_start, row_count);
-    nk_euclideans_symmetric_f64_smef64_finalize_streaming_(vectors, n_vectors, depth, stride_elements, result,
+    nk_dots_symmetric_f64_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
+                                            result_stride_elements, row_start, row_count);
+    nk_euclideans_symmetric_f64_smef64_finalize_streaming_(vectors, vectors_count, depth, stride_elements, result,
                                                            result_stride_elements, row_start, row_count);
 }
 
+#pragma endregion F64 Symmetric Euclidean
 #if defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)

package/include/numkong/spatials/v128relaxed.h

@@ -144,7 +144,7 @@ nk_define_cross_normalized_symmetric_(euclidean, e5m2, v128relaxed, e5m2, f32, /
                                       nk_load_b128_v128relaxed_, nk_partial_load_b32x4_serial_,
                                       nk_store_b128_v128relaxed_, nk_partial_store_b32x4_serial_, 1)
 
-nk_define_cross_normalized_packed_(angular, bf16, v128relaxed, bf16, f32, f32, /*norm_value_type=*/f32, f32,
+nk_define_cross_normalized_packed_(angular, bf16, v128relaxed, bf16, bf16, f32, /*norm_value_type=*/f32, f32,
                                    nk_b128_vec_t, nk_dots_packed_bf16_v128relaxed,
                                    nk_angular_through_f32_from_dot_v128relaxed_, nk_dots_reduce_sumsq_bf16_,
                                    nk_load_b128_v128relaxed_, nk_partial_load_b32x4_serial_, nk_store_b128_v128relaxed_,
@@ -154,7 +154,7 @@ nk_define_cross_normalized_symmetric_(angular, bf16, v128relaxed, bf16, f32, /*n
                                       nk_angular_through_f32_from_dot_v128relaxed_, nk_dots_reduce_sumsq_bf16_,
                                       nk_load_b128_v128relaxed_, nk_partial_load_b32x4_serial_,
                                       nk_store_b128_v128relaxed_, nk_partial_store_b32x4_serial_, 1)
-nk_define_cross_normalized_packed_(euclidean, bf16, v128relaxed, bf16, f32, f32, /*norm_value_type=*/f32, f32,
+nk_define_cross_normalized_packed_(euclidean, bf16, v128relaxed, bf16, bf16, f32, /*norm_value_type=*/f32, f32,
                                    nk_b128_vec_t, nk_dots_packed_bf16_v128relaxed,
                                    nk_euclidean_through_f32_from_dot_v128relaxed_, nk_dots_reduce_sumsq_bf16_,
                                    nk_load_b128_v128relaxed_, nk_partial_load_b32x4_serial_, nk_store_b128_v128relaxed_,