numkong 7.0.0 → 7.4.1

package/include/numkong/reduce/serial.h

@@ -140,22 +140,22 @@ NK_PUBLIC void nk_reduce_moments_u16_serial(                       //
 NK_PUBLIC void nk_reduce_moments_i32_serial(                       //
     nk_i32_t const *data, nk_size_t count, nk_size_t stride_bytes, //
     nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
-    nk_u64_t sum_lower = 0;
-    nk_i64_t sum_upper = 0;
+    nk_u64_t sum_low = 0;
+    nk_i64_t sum_high = 0;
     nk_u64_t sumsq = 0;
     unsigned char const *ptr = (unsigned char const *)data;
     for (nk_size_t i = 0; i < count; ++i, ptr += stride_bytes) {
         nk_i64_t val = (nk_i64_t)(*(nk_i32_t const *)ptr);
         nk_u64_t product = (nk_u64_t)(val * val);
-        nk_u64_t sum_before = sum_lower;
-        sum_lower += (nk_u64_t)val;
-        if (sum_lower < sum_before) sum_upper++;
-        sum_upper += (val >> 63);
+        nk_u64_t sum_before = sum_low;
+        sum_low += (nk_u64_t)val;
+        if (sum_low < sum_before) sum_high++;
+        sum_high += (val >> 63);
         sumsq = nk_u64_saturating_add_serial(sumsq, product);
     }
-    nk_i64_t sum_lower_signed = (nk_i64_t)sum_lower;
-    if (sum_upper == (sum_lower_signed >> 63)) *sum_ptr = sum_lower_signed;
-    else if (sum_upper >= 0) *sum_ptr = NK_I64_MAX;
+    nk_i64_t sum_low_signed = (nk_i64_t)sum_low;
+    if (sum_high == (sum_low_signed >> 63)) *sum_ptr = sum_low_signed;
+    else if (sum_high >= 0) *sum_ptr = NK_I64_MAX;
     else *sum_ptr = NK_I64_MIN;
     *sumsq_ptr = sumsq;
 }
@@ -177,8 +177,8 @@ NK_PUBLIC void nk_reduce_moments_u32_serial(                       //
 NK_PUBLIC void nk_reduce_moments_i64_serial(                       //
     nk_i64_t const *data, nk_size_t count, nk_size_t stride_bytes, //
     nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
-    nk_u64_t sum_lower = 0;
-    nk_i64_t sum_upper = 0;
+    nk_u64_t sum_low = 0;
+    nk_i64_t sum_high = 0;
     nk_u64_t sumsq = 0;
     unsigned char const *ptr = (unsigned char const *)data;
     for (nk_size_t i = 0; i < count; ++i, ptr += stride_bytes) {
@@ -186,14 +186,14 @@ NK_PUBLIC void nk_reduce_moments_i64_serial(                       //
         nk_i64_t product = nk_i64_saturating_mul_serial(val, val);
         nk_u64_t unsigned_product = (nk_u64_t)product;
         sumsq = nk_u64_saturating_add_serial(sumsq, unsigned_product);
-        nk_u64_t sum_before = sum_lower;
-        sum_lower += (nk_u64_t)val;
-        if (sum_lower < sum_before) sum_upper++;
-        sum_upper += (val >> 63);
-    }
-    nk_i64_t sum_lower_signed = (nk_i64_t)sum_lower;
-    if (sum_upper == (sum_lower_signed >> 63)) *sum_ptr = sum_lower_signed;
-    else if (sum_upper >= 0) *sum_ptr = NK_I64_MAX;
+        nk_u64_t sum_before = sum_low;
+        sum_low += (nk_u64_t)val;
+        if (sum_low < sum_before) sum_high++;
+        sum_high += (val >> 63);
+    }
+    nk_i64_t sum_low_signed = (nk_i64_t)sum_low;
+    if (sum_high == (sum_low_signed >> 63)) *sum_ptr = sum_low_signed;
+    else if (sum_high >= 0) *sum_ptr = NK_I64_MAX;
     else *sum_ptr = NK_I64_MIN;
     *sumsq_ptr = sumsq;
 }
@@ -572,13 +572,11 @@ NK_PUBLIC void nk_reduce_minmax_f16_serial(                        //
     nk_f16_t *min_value_ptr, nk_size_t *min_index_ptr,             //
     nk_f16_t *max_value_ptr, nk_size_t *max_index_ptr) {
     unsigned char const *ptr = (unsigned char const *)data;
-    nk_f16_t min_value = nk_f16_from_u16_(NK_F16_MAX), max_value = nk_f16_from_u16_(NK_F16_MIN);
+    nk_f16_t min_value = NK_F16_MAX, max_value = NK_F16_MIN;
     nk_size_t min_idx = NK_SIZE_MAX, max_idx = NK_SIZE_MAX;
     for (nk_size_t i = 0; i < count; ++i, ptr += stride_bytes) {
         nk_f16_t raw_value = *(nk_f16_t const *)ptr;
-        nk_fui16_t raw_fui;
-        raw_fui.f = raw_value;
-        if (nk_f16_is_nan_(raw_fui.u)) continue;
+        if (nk_f16_is_nan_(raw_value)) continue;
         if (min_idx == NK_SIZE_MAX || nk_f16_order_serial(raw_value, min_value) < 0) min_value = raw_value, min_idx = i;
         if (max_idx == NK_SIZE_MAX || nk_f16_order_serial(raw_value, max_value) > 0) max_value = raw_value, max_idx = i;
     }
@@ -591,13 +589,11 @@ NK_PUBLIC void nk_reduce_minmax_bf16_serial(                        //
     nk_bf16_t *min_value_ptr, nk_size_t *min_index_ptr,             //
     nk_bf16_t *max_value_ptr, nk_size_t *max_index_ptr) {
     unsigned char const *ptr = (unsigned char const *)data;
-    nk_bf16_t min_value = nk_bf16_from_u16_(NK_BF16_MAX), max_value = nk_bf16_from_u16_(NK_BF16_MIN);
+    nk_bf16_t min_value = NK_BF16_MAX, max_value = NK_BF16_MIN;
     nk_size_t min_idx = NK_SIZE_MAX, max_idx = NK_SIZE_MAX;
     for (nk_size_t i = 0; i < count; ++i, ptr += stride_bytes) {
         nk_bf16_t raw_value = *(nk_bf16_t const *)ptr;
-        nk_fui16_t raw_fui;
-        raw_fui.bf = raw_value;
-        if (nk_bf16_is_nan_(raw_fui.u)) continue;
+        if (nk_bf16_is_nan_(raw_value)) continue;
         if (min_idx == NK_SIZE_MAX || nk_bf16_order_serial(raw_value, min_value) < 0)
             min_value = raw_value, min_idx = i;
         if (max_idx == NK_SIZE_MAX || nk_bf16_order_serial(raw_value, max_value) > 0)

package/include/numkong/reduce/sierra.h

@@ -7,8 +7,8 @@
  *  @sa include/numkong/reduce.h
  *
  *  Uses AVX-VNNI-INT8 (256-bit) for efficient widening dot-products on i8, u8, and e2m3:
- *  - `_mm256_dpbssd_epi32`: i8 x i8 -> i32 signed dot product (AVXVNNIINT8)
- *  - `_mm256_dpbuud_epi32`: u8 x u8 -> u32 unsigned dot product (AVXVNNIINT8)
+ *  - `_mm256_dpbssd_epi32`: i8 × i8 → i32 signed dot product (AVXVNNIINT8)
+ *  - `_mm256_dpbuud_epi32`: u8 × u8 → u32 unsigned dot product (AVXVNNIINT8)
  */
 #ifndef NK_REDUCE_SIERRA_H
 #define NK_REDUCE_SIERRA_H
@@ -68,7 +68,7 @@ NK_INTERNAL void nk_reduce_moments_i8_sierra_strided_(               //
     nk_size_t idx_scalars = 0;
     nk_size_t total_scalars = count * stride_elements;
     nk_size_t step = nk_size_round_up_to_multiple_(32, stride_elements);
-    for (; idx_scalars + step <= total_scalars; idx_scalars += step) {
+    for (; idx_scalars + stride_elements + 31 <= total_scalars; idx_scalars += step) {
         __m256i data_i8x32 = _mm256_loadu_si256((__m256i const *)(data + idx_scalars));
         data_i8x32 = _mm256_and_si256(data_i8x32, stride_mask_i8x32);
         sum_i32x8 = _mm256_dpbssd_epi32(sum_i32x8, data_i8x32, ones_i8x32);
@@ -109,7 +109,7 @@ NK_PUBLIC void nk_reduce_moments_i8_sierra(                       //
 }
 
 /**
- *  @section u8 moments via VPDPBUUD (unsigned u8 x u8 -> u32)
+ *  @section u8 moments via VPDPBUUD (unsigned u8 × u8 → u32)
  *
  *  Sierra's `_mm256_dpbuud_epi32` provides native u8×u8→u32 dot product, replacing
  *  Haswell's 8-instruction SAD+widen+MADD sequence with 3 instructions per 32 elements.
@@ -153,7 +153,7 @@ NK_INTERNAL void nk_reduce_moments_u8_sierra_strided_(               //
     nk_size_t idx_scalars = 0;
     nk_size_t total_scalars = count * stride_elements;
     nk_size_t step = nk_size_round_up_to_multiple_(32, stride_elements);
-    for (; idx_scalars + step <= total_scalars; idx_scalars += step) {
+    for (; idx_scalars + stride_elements + 31 <= total_scalars; idx_scalars += step) {
         __m256i data_u8x32 = _mm256_loadu_si256((__m256i const *)(data + idx_scalars));
         data_u8x32 = _mm256_and_si256(data_u8x32, stride_mask_u8x32);
         sum_i32x8 = _mm256_dpbuud_epi32(sum_i32x8, data_u8x32, ones_u8x32);
@@ -203,10 +203,10 @@ NK_PUBLIC void nk_reduce_moments_u8_sierra(                       //
 NK_INTERNAL void nk_reduce_moments_e2m3_sierra_contiguous_( //
     nk_e2m3_t const *data, nk_size_t count,                 //
     nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
-    __m256i const lut_lower_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, //
-                                                    30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
-    __m256i const lut_upper_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
-                                                    120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
+    __m256i const lut_low_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, //
+                                                  30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
+    __m256i const lut_high_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
+                                                   120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
     __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
     __m256i const magnitude_mask_u8x32 = _mm256_set1_epi8(0x1F);
     __m256i const half_select_u8x32 = _mm256_set1_epi8(0x10);
@@ -221,8 +221,8 @@ NK_INTERNAL void nk_reduce_moments_e2m3_sierra_contiguous_( //
         __m256i shuffle_idx_u8x32 = _mm256_and_si256(magnitude_u8x32, nibble_mask_u8x32);
         __m256i upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(magnitude_u8x32, half_select_u8x32),
                                                        half_select_u8x32);
-        __m256i unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, shuffle_idx_u8x32),
-                                                    _mm256_shuffle_epi8(lut_upper_u8x32, shuffle_idx_u8x32),
+        __m256i unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, shuffle_idx_u8x32),
+                                                    _mm256_shuffle_epi8(lut_high_u8x32, shuffle_idx_u8x32),
                                                     upper_select_u8x32);
         __m256i negate_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(data_u8x32, sign_mask_u8x32), sign_mask_u8x32);
         __m256i signed_i8x32 = _mm256_blendv_epi8(
@@ -241,8 +241,8 @@ NK_INTERNAL void nk_reduce_moments_e2m3_sierra_contiguous_( //
         __m256i shuffle_idx_u8x32 = _mm256_and_si256(magnitude_u8x32, nibble_mask_u8x32);
         __m256i upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(magnitude_u8x32, half_select_u8x32),
                                                        half_select_u8x32);
-        __m256i unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, shuffle_idx_u8x32),
-                                                    _mm256_shuffle_epi8(lut_upper_u8x32, shuffle_idx_u8x32),
+        __m256i unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, shuffle_idx_u8x32),
+                                                    _mm256_shuffle_epi8(lut_high_u8x32, shuffle_idx_u8x32),
                                                     upper_select_u8x32);
         __m256i negate_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(data_u8x32, sign_mask_u8x32), sign_mask_u8x32);
         __m256i signed_i8x32 = _mm256_blendv_epi8(
@@ -258,10 +258,10 @@ NK_INTERNAL void nk_reduce_moments_e2m3_sierra_strided_(               //
     nk_e2m3_t const *data, nk_size_t count, nk_size_t stride_elements, //
     nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
     __m256i stride_mask_u8x32 = nk_stride_blend_u1x32_(stride_elements);
-    __m256i const lut_lower_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, //
-                                                    30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
-    __m256i const lut_upper_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
-                                                    120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
+    __m256i const lut_low_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, //
+                                                  30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
+    __m256i const lut_high_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
+                                                   120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
     __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
     __m256i const magnitude_mask_u8x32 = _mm256_set1_epi8(0x1F);
     __m256i const half_select_u8x32 = _mm256_set1_epi8(0x10);
@@ -272,15 +272,15 @@ NK_INTERNAL void nk_reduce_moments_e2m3_sierra_strided_(               //
     nk_size_t idx_scalars = 0;
     nk_size_t total_scalars = count * stride_elements;
     nk_size_t step = nk_size_round_up_to_multiple_(32, stride_elements);
-    for (; idx_scalars + step <= total_scalars; idx_scalars += step) {
+    for (; idx_scalars + stride_elements + 31 <= total_scalars; idx_scalars += step) {
         __m256i data_u8x32 = _mm256_loadu_si256((__m256i const *)(data + idx_scalars));
         data_u8x32 = _mm256_and_si256(data_u8x32, stride_mask_u8x32);
         __m256i magnitude_u8x32 = _mm256_and_si256(data_u8x32, magnitude_mask_u8x32);
         __m256i shuffle_idx_u8x32 = _mm256_and_si256(magnitude_u8x32, nibble_mask_u8x32);
         __m256i upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(magnitude_u8x32, half_select_u8x32),
                                                        half_select_u8x32);
-        __m256i unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, shuffle_idx_u8x32),
-                                                    _mm256_shuffle_epi8(lut_upper_u8x32, shuffle_idx_u8x32),
+        __m256i unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, shuffle_idx_u8x32),
+                                                    _mm256_shuffle_epi8(lut_high_u8x32, shuffle_idx_u8x32),
                                                     upper_select_u8x32);
         __m256i negate_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(data_u8x32, sign_mask_u8x32), sign_mask_u8x32);
         __m256i signed_i8x32 = _mm256_blendv_epi8(