numkong 7.0.0 → 7.4.1

package/include/numkong/dot/serial.h

@@ -94,11 +94,11 @@ extern "C" {
 #define nk_define_dot_(input_type, accumulator_type, output_type, load_and_convert)                         \
     NK_PUBLIC void nk_dot_##input_type##_serial(nk_##input_type##_t const *a, nk_##input_type##_t const *b, \
                                                 nk_size_t n, nk_##output_type##_t *result) {                \
-        nk_##accumulator_type##_t sum = 0, a_val, b_val;                                                    \
+        nk_##accumulator_type##_t sum = 0, a_value, b_value;                                                \
         for (nk_size_t i = 0; i != n; ++i) {                                                                \
-            load_and_convert(a + i, &a_val);                                                                \
-            load_and_convert(b + i, &b_val);                                                                \
-            sum += a_val * b_val;                                                                           \
+            load_and_convert(a + i, &a_value);                                                              \
+            load_and_convert(b + i, &b_value);                                                              \
+            sum += a_value * b_value;                                                                       \
         }                                                                                                   \
         *result = (nk_##output_type##_t)sum;                                                                \
     }
@@ -139,15 +139,15 @@ extern "C" {
         result->imag = sum_imag;                                                                            \
     }
 
-#pragma region - Traditional Floats
+#pragma region F32 and F64 Floats
 
 nk_define_dot_(f32, f64, f64, nk_assign_from_to_)            // nk_dot_f32_serial
 nk_define_dot_complex_(f32c, f64, f64c, nk_assign_from_to_)  // nk_dot_f32c_serial
 nk_define_vdot_complex_(f32c, f64, f64c, nk_assign_from_to_) // nk_vdot_f32c_serial
 
-#pragma endregion - Traditional Floats
+#pragma endregion F32 and F64 Floats
 
-#pragma region - Smaller Floats
+#pragma region F16 and BF16 Floats
 
 nk_define_dot_(f16, f32, f32, nk_f16_to_f32_serial)            // nk_dot_f16_serial
 nk_define_dot_complex_(f16c, f32, f32c, nk_f16_to_f32_serial)  // nk_dot_f16c_serial
@@ -162,9 +162,9 @@ nk_define_dot_(e5m2, f32, f32, nk_e5m2_to_f32_serial) // nk_dot_e5m2_serial
 nk_define_dot_(e2m3, f32, f32, nk_e2m3_to_f32_serial) // nk_dot_e2m3_serial
 nk_define_dot_(e3m2, f32, f32, nk_e3m2_to_f32_serial) // nk_dot_e3m2_serial
 
-#pragma endregion - Smaller Floats
+#pragma endregion F16 and BF16 Floats
 
-#pragma region - Small Integers
+#pragma region I8 and U8 Integers
 
 nk_define_dot_(i8, i32, i32, nk_assign_from_to_) // nk_dot_i8_serial
 nk_define_dot_(u8, u32, u32, nk_assign_from_to_) // nk_dot_u8_serial
@@ -207,9 +207,9 @@ NK_PUBLIC void nk_dot_u4_serial(nk_u4x2_t const *a, nk_u4x2_t const *b, nk_size_
     *result = sum;
 }
 
-#pragma endregion - Small Integers
+#pragma endregion I8 and U8 Integers
 
-#pragma region - Traditional Floats
+#pragma region F32 and F64 Floats
 
 /*  Double-precision dot-produce variants
  *
@@ -325,9 +325,9 @@ NK_INTERNAL void nk_dot_f32x4_finalize_serial(
     result->f64s[3] = state_d->sums[0] + state_d->sums[1] + state_d->sums[2] + state_d->sums[3];
 }
 
-#pragma endregion - Traditional Floats
+#pragma endregion F32 and F64 Floats
 
-#pragma region - Smaller Floats
+#pragma region F16 and BF16 Floats
 
 typedef struct nk_dot_f16x8_state_serial_t {
     nk_f32_t sums[4];
@@ -364,6 +364,36 @@ NK_INTERNAL void nk_dot_f16x8_finalize_serial(
     result->f32s[3] = state_d->sums[0] + state_d->sums[1] + state_d->sums[2] + state_d->sums[3];
 }
 
+typedef struct nk_dot_through_f32x4_state_serial_t {
+    nk_f32_t sums[4];
+} nk_dot_through_f32x4_state_serial_t;
+
+NK_INTERNAL void nk_dot_through_f32x4_init_serial(nk_dot_through_f32x4_state_serial_t *state) {
+    state->sums[0] = 0, state->sums[1] = 0, state->sums[2] = 0, state->sums[3] = 0;
+}
+
+NK_INTERNAL void nk_dot_through_f32x4_update_serial(nk_dot_through_f32x4_state_serial_t *state, nk_b128_vec_t a,
+                                                    nk_b128_vec_t b, nk_size_t depth_offset,
+                                                    nk_size_t active_dimensions) {
+    nk_unused_(depth_offset);
+    nk_unused_(active_dimensions);
+    state->sums[0] += a.f32s[0] * b.f32s[0];
+    state->sums[1] += a.f32s[1] * b.f32s[1];
+    state->sums[2] += a.f32s[2] * b.f32s[2];
+    state->sums[3] += a.f32s[3] * b.f32s[3];
+}
+
+NK_INTERNAL void nk_dot_through_f32x4_finalize_serial(                                                      //
+    nk_dot_through_f32x4_state_serial_t const *state_a, nk_dot_through_f32x4_state_serial_t const *state_b, //
+    nk_dot_through_f32x4_state_serial_t const *state_c, nk_dot_through_f32x4_state_serial_t const *state_d, //
+    nk_size_t total_dimensions, nk_b128_vec_t *result) {
+    nk_unused_(total_dimensions);
+    result->f32s[0] = state_a->sums[0] + state_a->sums[1] + state_a->sums[2] + state_a->sums[3];
+    result->f32s[1] = state_b->sums[0] + state_b->sums[1] + state_b->sums[2] + state_b->sums[3];
+    result->f32s[2] = state_c->sums[0] + state_c->sums[1] + state_c->sums[2] + state_c->sums[3];
+    result->f32s[3] = state_d->sums[0] + state_d->sums[1] + state_d->sums[2] + state_d->sums[3];
+}
+
 typedef struct nk_dot_bf16x8_state_serial_t {
     nk_f32_t sums[4];
 } nk_dot_bf16x8_state_serial_t;
@@ -399,9 +429,9 @@ NK_INTERNAL void nk_dot_bf16x8_finalize_serial(
     result->f32s[3] = state_d->sums[0] + state_d->sums[1] + state_d->sums[2] + state_d->sums[3];
 }
 
-#pragma endregion - Smaller Floats
+#pragma endregion F16 and BF16 Floats
 
-#pragma region - Small Integers
+#pragma region I8 and U8 Integers
 
 typedef struct nk_dot_i8x16_state_serial_t {
     nk_i64_t sums[2];
@@ -476,9 +506,9 @@ NK_INTERNAL void nk_dot_u8x16_finalize_serial(
     result->u32s[3] = (nk_u32_t)(state_d->sums[0] + state_d->sums[1]);
 }
 
-#pragma endregion - Small Integers
+#pragma endregion I8 and U8 Integers
 
-#pragma region - Smaller Floats
+#pragma region F16 and BF16 Floats
 
 typedef struct nk_dot_e4m3x16_state_serial_t {
     nk_f32_t sums[4];
@@ -640,9 +670,9 @@ NK_INTERNAL void nk_dot_e3m2x16_finalize_serial(
     result->f32s[3] = state_d->sums[0] + state_d->sums[1] + state_d->sums[2] + state_d->sums[3];
 }
 
-#pragma endregion - Smaller Floats
+#pragma endregion F16 and BF16 Floats
 
-#pragma region - Small Integers
+#pragma region I8 and U8 Integers
 
 // U4x2 state: processes 16 nibbles (8 bytes = 64 bits) per update
 typedef struct nk_dot_u4x16_state_serial_t {
@@ -694,20 +724,26 @@ NK_INTERNAL void nk_dot_u4x16_finalize_serial(nk_dot_u4x16_state_serial_t const
 }
 
 NK_INTERNAL void nk_load_i4x16_to_i8x16_serial_(void const *src, nk_b128_vec_t *dst) {
-    nk_i4_to_i8_serial_((nk_i4x2_t const *)src, dst->i8s, 16);
+    nk_i4x2_t const *pairs = (nk_i4x2_t const *)src;
+    for (nk_size_t i = 0; i < 8; ++i) nk_i4x2_to_i8x2_serial(&pairs[i], &dst->i8s[i * 2]);
 }
 
 NK_INTERNAL void nk_partial_load_i4x16_to_i8x16_serial_(void const *src, nk_b128_vec_t *dst, nk_size_t n) {
-    nk_i4_to_i8_serial_((nk_i4x2_t const *)src, dst->i8s, n);
+    nk_i4x2_t const *pairs = (nk_i4x2_t const *)src;
+    nk_size_t count_pairs = n / 2;
+    for (nk_size_t i = 0; i < count_pairs; ++i) nk_i4x2_to_i8x2_serial(&pairs[i], &dst->i8s[i * 2]);
     for (nk_size_t i = n; i < 16; ++i) dst->i8s[i] = 0;
 }
 
 NK_INTERNAL void nk_load_u4x16_to_u8x16_serial_(void const *src, nk_b128_vec_t *dst) {
-    nk_u4_to_u8_serial_((nk_u4x2_t const *)src, dst->u8s, 16);
+    nk_u4x2_t const *pairs = (nk_u4x2_t const *)src;
+    for (nk_size_t i = 0; i < 8; ++i) nk_u4x2_to_u8x2_serial(&pairs[i], &dst->u8s[i * 2]);
 }
 
 NK_INTERNAL void nk_partial_load_u4x16_to_u8x16_serial_(void const *src, nk_b128_vec_t *dst, nk_size_t n) {
-    nk_u4_to_u8_serial_((nk_u4x2_t const *)src, dst->u8s, n);
+    nk_u4x2_t const *pairs = (nk_u4x2_t const *)src;
+    nk_size_t count_pairs = n / 2;
+    for (nk_size_t i = 0; i < count_pairs; ++i) nk_u4x2_to_u8x2_serial(&pairs[i], &dst->u8s[i * 2]);
     for (nk_size_t i = n; i < 16; ++i) dst->u8s[i] = 0;
 }
 
@@ -759,9 +795,9 @@ NK_INTERNAL void nk_dot_i4x16_finalize_serial(nk_dot_i4x16_state_serial_t const
     result->i32s[3] = (nk_i32_t)(state_d->sums[0] + state_d->sums[1]);
 }
 
-#pragma endregion - Small Integers
+#pragma endregion I8 and U8 Integers
 
-#pragma region - Binary
+#pragma region Binary
 
 NK_PUBLIC void nk_dot_u1_serial(nk_u1x8_t const *a, nk_u1x8_t const *b, nk_size_t n_bits, nk_u32_t *result) {
     nk_u32_t dot = 0;
@@ -798,7 +834,7 @@ NK_INTERNAL void nk_dot_u1x128_finalize_serial(nk_dot_u1x128_state_serial_t cons
     result->u32s[3] = state_d->dot_count;
 }
 
-#pragma endregion - Binary
+#pragma endregion Binary
 
 /**
  *  Serial fallback sum helpers for progressive element-sum accumulation.
@@ -806,7 +842,7 @@ NK_INTERNAL void nk_dot_u1x128_finalize_serial(nk_dot_u1x128_state_serial_t cons
  *  on the depth loop's already-loaded vectors, avoiding a separate sum pass.
  */
 
-#pragma region - Stateful Element Sum Helpers (for compensated GEMM)
+#pragma region Stateful Element Sum Helpers (for compensated GEMM)
 
 /* i4x32: Haswell i4 (nk_b128_vec_t containing 32 nibbles in 16 bytes) */
 typedef struct nk_sum_i4x32_state_serial_t {
@@ -818,8 +854,8 @@ NK_INTERNAL void nk_sum_i4x32_init_serial(nk_sum_i4x32_state_serial_t *state) {
 NK_INTERNAL void nk_sum_i4x32_update_serial(nk_sum_i4x32_state_serial_t *state, nk_b128_vec_t v) {
     nk_u8_t const *d = (nk_u8_t const *)&v;
     for (int i = 0; i < 16; i++) {
-        nk_i8_t low = (nk_i8_t)((d[i] & 0x0F) ^ 0x08) - 8; /* sign-extend low nibble */
-        nk_i8_t high = (nk_i8_t)((d[i] >> 4) ^ 0x08) - 8;  /* sign-extend high nibble */
+        nk_i8_t low = (nk_i8_t)((d[i] & 0x0F) ^ 0x08) - 8; // sign-extend low nibble
+        nk_i8_t high = (nk_i8_t)((d[i] >> 4) ^ 0x08) - 8;  // sign-extend high nibble
         state->sum += low + high;
     }
 }
@@ -829,7 +865,7 @@ NK_INTERNAL nk_i32_t nk_sum_i4x32_finalize_serial(nk_sum_i4x32_state_serial_t co
     return (nk_i32_t)state->sum;
 }
 
-#pragma endregion - Stateful Element Sum Helpers
+#pragma endregion Stateful Element Sum Helpers
 
 #if defined(__cplusplus)
 } // extern "C"

package/include/numkong/dot/sierra.h

@@ -8,9 +8,9 @@
  *
  *  @section dot_sierra_instructions AVX-VNNI-INT8 Instructions
  *
- *      Intrinsic                   Instruction
- *      _mm256_dpbssd_epi32         VPDPBSSD (YMM, YMM, YMM)   i8 x i8 -> i32
- *      _mm256_dpbuud_epi32         VPDPBUUD (YMM, YMM, YMM)   u8 x u8 -> u32
+ *      Intrinsic            Instruction
+ *      _mm256_dpbssd_epi32  VPDPBSSD (YMM, YMM, YMM)  i8 × i8 → i32
+ *      _mm256_dpbuud_epi32  VPDPBUUD (YMM, YMM, YMM)  u8 × u8 → u32
  *
  *  Sierra Forest CPUs support AVX-VNNI-INT8, adding native signed*signed and
  *  unsigned*unsigned 8-bit dot products. This eliminates the algebraic sign
@@ -248,10 +248,10 @@ NK_PUBLIC void nk_dot_e2m3_sierra(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b
     // Uses dpbssd instead of dpbusd — both operands are already signed i8 after
     // LUT + sign application, so no unsigned conversion is needed.
     //
-    __m256i const lut_lower_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, 30, 28,
-                                                    26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
-    __m256i const lut_upper_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
-                                                    120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
+    __m256i const lut_low_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, 30, 28, 26,
+                                                  24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
+    __m256i const lut_high_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
+                                                   120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
     __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
     __m256i const magnitude_mask_u8x32 = _mm256_set1_epi8(0x1F);
     __m256i const half_select_u8x32 = _mm256_set1_epi8(0x10);
@@ -277,11 +277,11 @@ nk_dot_e2m3_sierra_cycle:
     // Decode a: extract magnitude, dual-VPSHUFB LUT, apply sign
     __m256i a_magnitude_u8x32 = _mm256_and_si256(a_e2m3_u8x32, magnitude_mask_u8x32);
     __m256i a_shuffle_index_u8x32 = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i a_upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
-                                                     half_select_u8x32);
-    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, a_shuffle_index_u8x32),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, a_shuffle_index_u8x32),
-                                                  a_upper_select_u8x32);
+    __m256i a_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, a_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, a_shuffle_index_u8x32),
+                                                  a_high_select_u8x32);
     __m256i a_negate_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_e2m3_u8x32, sign_mask_u8x32), sign_mask_u8x32);
     __m256i a_signed_i8x32 = _mm256_blendv_epi8(
         a_unsigned_u8x32, _mm256_sub_epi8(_mm256_setzero_si256(), a_unsigned_u8x32), a_negate_mask_u8x32);
@@ -289,11 +289,11 @@ nk_dot_e2m3_sierra_cycle:
     // Decode b: same LUT decode + sign
     __m256i b_magnitude_u8x32 = _mm256_and_si256(b_e2m3_u8x32, magnitude_mask_u8x32);
     __m256i b_shuffle_index_u8x32 = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i b_upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
-                                                     half_select_u8x32);
-    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, b_shuffle_index_u8x32),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, b_shuffle_index_u8x32),
-                                                  b_upper_select_u8x32);
+    __m256i b_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, b_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, b_shuffle_index_u8x32),
+                                                  b_high_select_u8x32);
     __m256i b_negate_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_e2m3_u8x32, sign_mask_u8x32), sign_mask_u8x32);
     __m256i b_signed_i8x32 = _mm256_blendv_epi8(
         b_unsigned_u8x32, _mm256_sub_epi8(_mm256_setzero_si256(), b_unsigned_u8x32), b_negate_mask_u8x32);
@@ -318,10 +318,10 @@ NK_INTERNAL void nk_dot_e2m3x32_update_sierra(nk_dot_e2m3x32_state_sierra_t *sta
     nk_unused_(depth_offset);
     nk_unused_(active_dimensions);
     // Same LUT constants...
-    __m256i const lut_lower_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, 30, 28,
-                                                    26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
-    __m256i const lut_upper_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
-                                                    120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
+    __m256i const lut_low_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, 30, 28, 26,
+                                                  24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
+    __m256i const lut_high_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
+                                                   120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
     __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
     __m256i const magnitude_mask_u8x32 = _mm256_set1_epi8(0x1F);
     __m256i const half_select_u8x32 = _mm256_set1_epi8(0x10);
@@ -333,11 +333,11 @@ NK_INTERNAL void nk_dot_e2m3x32_update_sierra(nk_dot_e2m3x32_state_sierra_t *sta
     // Decode a
     __m256i a_magnitude_u8x32 = _mm256_and_si256(a_e2m3_u8x32, magnitude_mask_u8x32);
     __m256i a_shuffle_index_u8x32 = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i a_upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
-                                                     half_select_u8x32);
-    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, a_shuffle_index_u8x32),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, a_shuffle_index_u8x32),
-                                                  a_upper_select_u8x32);
+    __m256i a_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, a_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, a_shuffle_index_u8x32),
+                                                  a_high_select_u8x32);
     __m256i a_negate_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_e2m3_u8x32, sign_mask_u8x32), sign_mask_u8x32);
     __m256i a_signed_i8x32 = _mm256_blendv_epi8(
         a_unsigned_u8x32, _mm256_sub_epi8(_mm256_setzero_si256(), a_unsigned_u8x32), a_negate_mask_u8x32);
@@ -345,11 +345,11 @@ NK_INTERNAL void nk_dot_e2m3x32_update_sierra(nk_dot_e2m3x32_state_sierra_t *sta
     // Decode b
     __m256i b_magnitude_u8x32 = _mm256_and_si256(b_e2m3_u8x32, magnitude_mask_u8x32);
     __m256i b_shuffle_index_u8x32 = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i b_upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
-                                                     half_select_u8x32);
-    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, b_shuffle_index_u8x32),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, b_shuffle_index_u8x32),
-                                                  b_upper_select_u8x32);
+    __m256i b_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, b_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, b_shuffle_index_u8x32),
+                                                  b_high_select_u8x32);
     __m256i b_negate_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_e2m3_u8x32, sign_mask_u8x32), sign_mask_u8x32);
     __m256i b_signed_i8x32 = _mm256_blendv_epi8(
         b_unsigned_u8x32, _mm256_sub_epi8(_mm256_setzero_si256(), b_unsigned_u8x32), b_negate_mask_u8x32);