numkong 7.5.0 → 7.6.0

package/include/numkong/spatial/skylake.h

@@ -346,28 +346,36 @@ nk_angular_f16_skylake_cycle:
 }
 
 NK_PUBLIC void nk_sqeuclidean_e4m3_skylake(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t *result) {
-    __m512 sum_f32x16 = _mm512_setzero_ps();
-    __m128i a_e4m3_u8x16, b_e4m3_u8x16;
+    // E4M3 has no free widen shift (its 4-bit exponent doesn't line up with F16's 5-bit
+    // at bit 10), so we call the Giesen-based 16-lane cast helper twice per iter and
+    // run with two F32 accumulators to break the FMA dependency chain.
+    __m512 first_acc_f32x16 = _mm512_setzero_ps();
+    __m512 second_acc_f32x16 = _mm512_setzero_ps();
+    __m256i a_u8x32, b_u8x32;
 
 nk_sqeuclidean_e4m3_skylake_cycle:
-    if (n < 16) {
-        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, b);
+    if (n < 32) {
+        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, (unsigned int)n);
+        a_u8x32 = _mm256_maskz_loadu_epi8(mask, a);
+        b_u8x32 = _mm256_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e4m3_u8x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e4m3_u8x16 = _mm_loadu_si128((__m128i const *)b);
-        a += 16, b += 16, n -= 16;
+        a_u8x32 = _mm256_loadu_si256((__m256i const *)a);
+        b_u8x32 = _mm256_loadu_si256((__m256i const *)b);
+        a += 32, b += 32, n -= 32;
     }
-    __m512 a_f32x16 = nk_e4m3x16_to_f32x16_skylake_(a_e4m3_u8x16);
-    __m512 b_f32x16 = nk_e4m3x16_to_f32x16_skylake_(b_e4m3_u8x16);
-    __m512 diff_f32x16 = _mm512_sub_ps(a_f32x16, b_f32x16);
-    sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
+    __m512 a_low_f32x16 = nk_e4m3x16_to_f32x16_skylake_(_mm256_castsi256_si128(a_u8x32));
+    __m512 a_high_f32x16 = nk_e4m3x16_to_f32x16_skylake_(_mm256_extracti128_si256(a_u8x32, 1));
+    __m512 b_low_f32x16 = nk_e4m3x16_to_f32x16_skylake_(_mm256_castsi256_si128(b_u8x32));
+    __m512 b_high_f32x16 = nk_e4m3x16_to_f32x16_skylake_(_mm256_extracti128_si256(b_u8x32, 1));
+    __m512 diff_low_f32x16 = _mm512_sub_ps(a_low_f32x16, b_low_f32x16);
+    __m512 diff_high_f32x16 = _mm512_sub_ps(a_high_f32x16, b_high_f32x16);
+    first_acc_f32x16 = _mm512_fmadd_ps(diff_low_f32x16, diff_low_f32x16, first_acc_f32x16);
+    second_acc_f32x16 = _mm512_fmadd_ps(diff_high_f32x16, diff_high_f32x16, second_acc_f32x16);
     if (n) goto nk_sqeuclidean_e4m3_skylake_cycle;
 
-    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+    *result = nk_reduce_add_f32x16_skylake_(_mm512_add_ps(first_acc_f32x16, second_acc_f32x16));
 }
 
 NK_PUBLIC void nk_euclidean_e4m3_skylake(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t *result) {
@@ -379,25 +387,30 @@ NK_PUBLIC void nk_angular_e4m3_skylake(nk_e4m3_t const *a, nk_e4m3_t const *b, n
     __m512 dot_f32x16 = _mm512_setzero_ps();
     __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
     __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
-    __m128i a_e4m3_u8x16, b_e4m3_u8x16;
+    __m256i a_u8x32, b_u8x32;
 
 nk_angular_e4m3_skylake_cycle:
-    if (n < 16) {
-        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, b);
+    if (n < 32) {
+        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, (unsigned int)n);
+        a_u8x32 = _mm256_maskz_loadu_epi8(mask, a);
+        b_u8x32 = _mm256_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e4m3_u8x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e4m3_u8x16 = _mm_loadu_si128((__m128i const *)b);
-        a += 16, b += 16, n -= 16;
+        a_u8x32 = _mm256_loadu_si256((__m256i const *)a);
+        b_u8x32 = _mm256_loadu_si256((__m256i const *)b);
+        a += 32, b += 32, n -= 32;
     }
-    __m512 a_f32x16 = nk_e4m3x16_to_f32x16_skylake_(a_e4m3_u8x16);
-    __m512 b_f32x16 = nk_e4m3x16_to_f32x16_skylake_(b_e4m3_u8x16);
-    dot_f32x16 = _mm512_fmadd_ps(a_f32x16, b_f32x16, dot_f32x16);
-    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_f32x16, a_f32x16, a_norm_sq_f32x16);
-    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_f32x16, b_f32x16, b_norm_sq_f32x16);
+    __m512 a_low_f32x16 = nk_e4m3x16_to_f32x16_skylake_(_mm256_castsi256_si128(a_u8x32));
+    __m512 a_high_f32x16 = nk_e4m3x16_to_f32x16_skylake_(_mm256_extracti128_si256(a_u8x32, 1));
+    __m512 b_low_f32x16 = nk_e4m3x16_to_f32x16_skylake_(_mm256_castsi256_si128(b_u8x32));
+    __m512 b_high_f32x16 = nk_e4m3x16_to_f32x16_skylake_(_mm256_extracti128_si256(b_u8x32, 1));
+    dot_f32x16 = _mm512_fmadd_ps(a_low_f32x16, b_low_f32x16, dot_f32x16);
+    dot_f32x16 = _mm512_fmadd_ps(a_high_f32x16, b_high_f32x16, dot_f32x16);
+    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_low_f32x16, a_low_f32x16, a_norm_sq_f32x16);
+    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_high_f32x16, a_high_f32x16, a_norm_sq_f32x16);
+    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_low_f32x16, b_low_f32x16, b_norm_sq_f32x16);
+    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_high_f32x16, b_high_f32x16, b_norm_sq_f32x16);
     if (n) goto nk_angular_e4m3_skylake_cycle;
 
     nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(dot_f32x16);
@@ -407,28 +420,53 @@ nk_angular_e4m3_skylake_cycle:
 }
 
 NK_PUBLIC void nk_sqeuclidean_e5m2_skylake(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
-    __m512 sum_f32x16 = _mm512_setzero_ps();
-    __m128i a_e5m2_u8x16, b_e5m2_u8x16;
+    // E5M2 shares F16's exponent bias (15): `byte << 8` equals the matching F16 bit-pattern
+    // for normals, subnormals, zero, Inf, and NaN. We expose that shift for free by unpacking
+    // against zero — the zero byte lands in the low half of each 16-bit lane, the E5M2 byte
+    // in the high half. `vpunpck*bw` is per-128-bit-lane so the F32 outputs are lane-scrambled
+    // across 512 bits, but the commutative sum reduction is invariant under that.
+    __m512 first_acc_f32x16 = _mm512_setzero_ps();
+    __m512 second_acc_f32x16 = _mm512_setzero_ps();
+    __m512i const zero_u8x64 = _mm512_setzero_si512();
+    __m512i a_u8x64, b_u8x64;
 
 nk_sqeuclidean_e5m2_skylake_cycle:
-    if (n < 16) {
-        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, b);
+    if (n < 64) {
+        __mmask64 mask = _bzhi_u64(0xFFFFFFFFFFFFFFFFULL, (unsigned int)n);
+        a_u8x64 = _mm512_maskz_loadu_epi8(mask, a);
+        b_u8x64 = _mm512_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)b);
-        a += 16, b += 16, n -= 16;
+        a_u8x64 = _mm512_loadu_si512((__m512i const *)a);
+        b_u8x64 = _mm512_loadu_si512((__m512i const *)b);
+        a += 64, b += 64, n -= 64;
     }
-    __m512 a_f32x16 = nk_e5m2x16_to_f32x16_skylake_(a_e5m2_u8x16);
-    __m512 b_f32x16 = nk_e5m2x16_to_f32x16_skylake_(b_e5m2_u8x16);
-    __m512 diff_f32x16 = _mm512_sub_ps(a_f32x16, b_f32x16);
-    sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
+    __m512i a_even_f16x32 = _mm512_unpacklo_epi8(zero_u8x64, a_u8x64);
+    __m512i a_odd_f16x32 = _mm512_unpackhi_epi8(zero_u8x64, a_u8x64);
+    __m512i b_even_f16x32 = _mm512_unpacklo_epi8(zero_u8x64, b_u8x64);
+    __m512i b_odd_f16x32 = _mm512_unpackhi_epi8(zero_u8x64, b_u8x64);
+
+    __m512 a_first_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(a_even_f16x32));
+    __m512 a_second_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(a_even_f16x32, 1));
+    __m512 a_third_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(a_odd_f16x32));
+    __m512 a_fourth_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(a_odd_f16x32, 1));
+    __m512 b_first_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(b_even_f16x32));
+    __m512 b_second_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(b_even_f16x32, 1));
+    __m512 b_third_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(b_odd_f16x32));
+    __m512 b_fourth_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(b_odd_f16x32, 1));
+
+    __m512 diff_first_f32x16 = _mm512_sub_ps(a_first_f32x16, b_first_f32x16);
+    __m512 diff_second_f32x16 = _mm512_sub_ps(a_second_f32x16, b_second_f32x16);
+    __m512 diff_third_f32x16 = _mm512_sub_ps(a_third_f32x16, b_third_f32x16);
+    __m512 diff_fourth_f32x16 = _mm512_sub_ps(a_fourth_f32x16, b_fourth_f32x16);
+    first_acc_f32x16 = _mm512_fmadd_ps(diff_first_f32x16, diff_first_f32x16, first_acc_f32x16);
+    second_acc_f32x16 = _mm512_fmadd_ps(diff_second_f32x16, diff_second_f32x16, second_acc_f32x16);
+    first_acc_f32x16 = _mm512_fmadd_ps(diff_third_f32x16, diff_third_f32x16, first_acc_f32x16);
+    second_acc_f32x16 = _mm512_fmadd_ps(diff_fourth_f32x16, diff_fourth_f32x16, second_acc_f32x16);
     if (n) goto nk_sqeuclidean_e5m2_skylake_cycle;
 
-    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+    *result = nk_reduce_add_f32x16_skylake_(_mm512_add_ps(first_acc_f32x16, second_acc_f32x16));
 }
 
 NK_PUBLIC void nk_euclidean_e5m2_skylake(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
@@ -440,25 +478,47 @@ NK_PUBLIC void nk_angular_e5m2_skylake(nk_e5m2_t const *a, nk_e5m2_t const *b, n
     __m512 dot_f32x16 = _mm512_setzero_ps();
     __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
     __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
-    __m128i a_e5m2_u8x16, b_e5m2_u8x16;
+    __m512i const zero_u8x64 = _mm512_setzero_si512();
+    __m512i a_u8x64, b_u8x64;
 
 nk_angular_e5m2_skylake_cycle:
-    if (n < 16) {
-        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, b);
+    if (n < 64) {
+        __mmask64 mask = _bzhi_u64(0xFFFFFFFFFFFFFFFFULL, (unsigned int)n);
+        a_u8x64 = _mm512_maskz_loadu_epi8(mask, a);
+        b_u8x64 = _mm512_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)b);
-        a += 16, b += 16, n -= 16;
+        a_u8x64 = _mm512_loadu_si512((__m512i const *)a);
+        b_u8x64 = _mm512_loadu_si512((__m512i const *)b);
+        a += 64, b += 64, n -= 64;
     }
-    __m512 a_f32x16 = nk_e5m2x16_to_f32x16_skylake_(a_e5m2_u8x16);
-    __m512 b_f32x16 = nk_e5m2x16_to_f32x16_skylake_(b_e5m2_u8x16);
-    dot_f32x16 = _mm512_fmadd_ps(a_f32x16, b_f32x16, dot_f32x16);
-    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_f32x16, a_f32x16, a_norm_sq_f32x16);
-    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_f32x16, b_f32x16, b_norm_sq_f32x16);
+    __m512i a_even_f16x32 = _mm512_unpacklo_epi8(zero_u8x64, a_u8x64);
+    __m512i a_odd_f16x32 = _mm512_unpackhi_epi8(zero_u8x64, a_u8x64);
+    __m512i b_even_f16x32 = _mm512_unpacklo_epi8(zero_u8x64, b_u8x64);
+    __m512i b_odd_f16x32 = _mm512_unpackhi_epi8(zero_u8x64, b_u8x64);
+
+    __m512 a_first_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(a_even_f16x32));
+    __m512 a_second_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(a_even_f16x32, 1));
+    __m512 a_third_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(a_odd_f16x32));
+    __m512 a_fourth_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(a_odd_f16x32, 1));
+    __m512 b_first_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(b_even_f16x32));
+    __m512 b_second_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(b_even_f16x32, 1));
+    __m512 b_third_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(b_odd_f16x32));
+    __m512 b_fourth_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(b_odd_f16x32, 1));
+
+    dot_f32x16 = _mm512_fmadd_ps(a_first_f32x16, b_first_f32x16, dot_f32x16);
+    dot_f32x16 = _mm512_fmadd_ps(a_second_f32x16, b_second_f32x16, dot_f32x16);
+    dot_f32x16 = _mm512_fmadd_ps(a_third_f32x16, b_third_f32x16, dot_f32x16);
+    dot_f32x16 = _mm512_fmadd_ps(a_fourth_f32x16, b_fourth_f32x16, dot_f32x16);
+    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_first_f32x16, a_first_f32x16, a_norm_sq_f32x16);
+    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_second_f32x16, a_second_f32x16, a_norm_sq_f32x16);
+    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_third_f32x16, a_third_f32x16, a_norm_sq_f32x16);
+    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_fourth_f32x16, a_fourth_f32x16, a_norm_sq_f32x16);
+    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_first_f32x16, b_first_f32x16, b_norm_sq_f32x16);
+    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_second_f32x16, b_second_f32x16, b_norm_sq_f32x16);
+    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_third_f32x16, b_third_f32x16, b_norm_sq_f32x16);
+    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_fourth_f32x16, b_fourth_f32x16, b_norm_sq_f32x16);
     if (n) goto nk_angular_e5m2_skylake_cycle;
 
     nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(dot_f32x16);

package/include/numkong/spatial.h

@@ -604,12 +604,6 @@ NK_PUBLIC void nk_euclidean_bf16_genoa(nk_bf16_t const *a, nk_bf16_t const *b, n
 NK_PUBLIC void nk_sqeuclidean_bf16_genoa(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *result);
 /** @copydoc nk_angular_f64 */
 NK_PUBLIC void nk_angular_bf16_genoa(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *result);
-/** @copydoc nk_euclidean_f64 */
-NK_PUBLIC void nk_euclidean_e5m2_genoa(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result);
-/** @copydoc nk_sqeuclidean_f64 */
-NK_PUBLIC void nk_sqeuclidean_e5m2_genoa(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result);
-/** @copydoc nk_angular_f64 */
-NK_PUBLIC void nk_angular_e5m2_genoa(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result);
 #endif // NK_TARGET_GENOA
 
 #if NK_TARGET_DIAMOND
@@ -1263,8 +1257,6 @@ NK_PUBLIC void nk_euclidean_e5m2(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size
     nk_euclidean_e5m2_neonfp8(a, b, n, result);
 #elif NK_TARGET_DIAMOND
     nk_euclidean_e5m2_diamond(a, b, n, result);
-#elif NK_TARGET_GENOA
-    nk_euclidean_e5m2_genoa(a, b, n, result);
 #elif NK_TARGET_SKYLAKE
     nk_euclidean_e5m2_skylake(a, b, n, result);
 #elif NK_TARGET_RVV
@@ -1281,8 +1273,6 @@ NK_PUBLIC void nk_sqeuclidean_e5m2(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_si
     nk_sqeuclidean_e5m2_neonfp8(a, b, n, result);
 #elif NK_TARGET_DIAMOND
     nk_sqeuclidean_e5m2_diamond(a, b, n, result);
-#elif NK_TARGET_GENOA
-    nk_sqeuclidean_e5m2_genoa(a, b, n, result);
 #elif NK_TARGET_SKYLAKE
     nk_sqeuclidean_e5m2_skylake(a, b, n, result);
 #elif NK_TARGET_RVV
@@ -1299,8 +1289,6 @@ NK_PUBLIC void nk_angular_e5m2(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t
     nk_angular_e5m2_neonfp8(a, b, n, result);
 #elif NK_TARGET_DIAMOND
     nk_angular_e5m2_diamond(a, b, n, result);
-#elif NK_TARGET_GENOA
-    nk_angular_e5m2_genoa(a, b, n, result);
 #elif NK_TARGET_SKYLAKE
     nk_angular_e5m2_skylake(a, b, n, result);
 #elif NK_TARGET_RVV

package/include/numkong/spatials/graniteamx.h

@@ -158,6 +158,134 @@ NK_PUBLIC void nk_euclideans_symmetric_f16_graniteamx(
 
 #pragma endregion F16 Symmetric
 
+#pragma region E5M2 Packed
+
+NK_INTERNAL void nk_angulars_packed_e5m2_graniteamx_finalize_(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *c,
+                                                              nk_size_t rows, nk_size_t columns, nk_size_t depth,
+                                                              nk_size_t a_stride_elements,
+                                                              nk_size_t c_stride_elements) {
+    nk_dots_amx_packed_header_t const *header = (nk_dots_amx_packed_header_t const *)b_packed;
+    nk_f32_t const *b_norms = (nk_f32_t const *)((char const *)b_packed + header->norms_byte_offset);
+    for (nk_size_t row = 0; row < rows; row++) {
+        nk_f32_t query_norm_sq = nk_dots_reduce_sumsq_e5m2_(a + row * a_stride_elements, depth);
+        nk_angulars_row_f32dots_sapphireamx_(c + row * c_stride_elements, b_norms, query_norm_sq, columns);
+    }
+}
+
+NK_PUBLIC void nk_angulars_packed_e5m2_graniteamx(         //
+    nk_e5m2_t const *a, void const *b_packed, nk_f32_t *c, //
+    nk_size_t rows, nk_size_t columns, nk_size_t depth,    //
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_size_t const a_stride_elements = a_stride_in_bytes;
+    nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f32_t);
+    nk_dots_packed_e5m2_graniteamx(a, b_packed, c, rows, columns, depth, a_stride_in_bytes, c_stride_in_bytes);
+    nk_angulars_packed_e5m2_graniteamx_finalize_(a, b_packed, c, rows, columns, depth, a_stride_elements,
+                                                 c_stride_elements);
+}
+
+NK_INTERNAL void nk_euclideans_packed_e5m2_graniteamx_finalize_(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *c,
+                                                                nk_size_t rows, nk_size_t columns, nk_size_t depth,
+                                                                nk_size_t a_stride_elements,
+                                                                nk_size_t c_stride_elements) {
+    nk_dots_amx_packed_header_t const *header = (nk_dots_amx_packed_header_t const *)b_packed;
+    nk_f32_t const *b_norms = (nk_f32_t const *)((char const *)b_packed + header->norms_byte_offset);
+    for (nk_size_t row = 0; row < rows; row++) {
+        nk_f32_t query_norm_sq = nk_dots_reduce_sumsq_e5m2_(a + row * a_stride_elements, depth);
+        nk_euclideans_row_f32dots_sapphireamx_(c + row * c_stride_elements, b_norms, query_norm_sq, columns);
+    }
+}
+
+NK_PUBLIC void nk_euclideans_packed_e5m2_graniteamx(       //
+    nk_e5m2_t const *a, void const *b_packed, nk_f32_t *c, //
+    nk_size_t rows, nk_size_t columns, nk_size_t depth,    //
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_size_t const a_stride_elements = a_stride_in_bytes;
+    nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f32_t);
+    nk_dots_packed_e5m2_graniteamx(a, b_packed, c, rows, columns, depth, a_stride_in_bytes, c_stride_in_bytes);
+    nk_euclideans_packed_e5m2_graniteamx_finalize_(a, b_packed, c, rows, columns, depth, a_stride_elements,
+                                                   c_stride_elements);
+}
+
+#pragma endregion E5M2 Packed
+
+#pragma region E5M2 Symmetric
+
+NK_INTERNAL void nk_angulars_symmetric_e5m2_graniteamx_finalize_(nk_e5m2_t const *vectors, nk_size_t vectors_count,
+                                                                 nk_size_t depth, nk_size_t stride_elements,
+                                                                 nk_f32_t *result, nk_size_t result_stride_elements,
+                                                                 nk_size_t row_start, nk_size_t row_count) {
+
+    for (nk_size_t row = row_start; row < row_start + row_count; row++)
+        result[row * result_stride_elements + row] = nk_dots_reduce_sumsq_e5m2_(vectors + row * stride_elements, depth);
+
+    nk_f32_t column_norms_cache[256];
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
+        for (nk_size_t col = chunk_start; col < chunk_end; col++)
+            column_norms_cache[col - chunk_start] = nk_dots_reduce_sumsq_e5m2_(vectors + col * stride_elements, depth);
+
+        for (nk_size_t row = row_start; row < row_start + row_count; row++) {
+            nk_f32_t *r_row = result + row * result_stride_elements;
+            nk_size_t col_start = chunk_start > row + 1 ? chunk_start : row + 1;
+            if (col_start >= chunk_end) continue;
+            nk_angulars_row_f32dots_sapphireamx_(r_row + col_start, column_norms_cache + col_start - chunk_start,
+                                                 r_row[row], chunk_end - col_start);
+        }
+    }
+
+    for (nk_size_t row = row_start; row < row_start + row_count; row++) result[row * result_stride_elements + row] = 0;
+}
+
+NK_PUBLIC void nk_angulars_symmetric_e5m2_graniteamx(                                              //
+    nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, //
+    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_size_t const stride_elements = stride_in_bytes;
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f32_t);
+    nk_dots_symmetric_e5m2_graniteamx(vectors, vectors_count, depth, stride_in_bytes, result, result_stride_in_bytes,
+                                      row_start, row_count);
+    nk_angulars_symmetric_e5m2_graniteamx_finalize_(vectors, vectors_count, depth, stride_elements, result,
+                                                    result_stride_elements, row_start, row_count);
+}
+
+NK_INTERNAL void nk_euclideans_symmetric_e5m2_graniteamx_finalize_(nk_e5m2_t const *vectors, nk_size_t vectors_count,
+                                                                   nk_size_t depth, nk_size_t stride_elements,
+                                                                   nk_f32_t *result, nk_size_t result_stride_elements,
+                                                                   nk_size_t row_start, nk_size_t row_count) {
+
+    for (nk_size_t row = row_start; row < row_start + row_count; row++)
+        result[row * result_stride_elements + row] = nk_dots_reduce_sumsq_e5m2_(vectors + row * stride_elements, depth);
+
+    nk_f32_t column_norms_cache[256];
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
+        for (nk_size_t col = chunk_start; col < chunk_end; col++)
+            column_norms_cache[col - chunk_start] = nk_dots_reduce_sumsq_e5m2_(vectors + col * stride_elements, depth);
+
+        for (nk_size_t row = row_start; row < row_start + row_count; row++) {
+            nk_f32_t *r_row = result + row * result_stride_elements;
+            nk_size_t col_start = chunk_start > row + 1 ? chunk_start : row + 1;
+            if (col_start >= chunk_end) continue;
+            nk_euclideans_row_f32dots_sapphireamx_(r_row + col_start, column_norms_cache + col_start - chunk_start,
+                                                   r_row[row], chunk_end - col_start);
+        }
+    }
+
+    for (nk_size_t row = row_start; row < row_start + row_count; row++) result[row * result_stride_elements + row] = 0;
+}
+
+NK_PUBLIC void nk_euclideans_symmetric_e5m2_graniteamx(                                            //
+    nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, //
+    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_size_t const stride_elements = stride_in_bytes;
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f32_t);
+    nk_dots_symmetric_e5m2_graniteamx(vectors, vectors_count, depth, stride_in_bytes, result, result_stride_in_bytes,
+                                      row_start, row_count);
+    nk_euclideans_symmetric_e5m2_graniteamx_finalize_(vectors, vectors_count, depth, stride_elements, result,
+                                                      result_stride_elements, row_start, row_count);
+}
+
+#pragma endregion E5M2 Symmetric
+
 #if defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)

package/include/numkong/spatials/serial.h

@@ -15,7 +15,18 @@
 extern "C" {
 #endif
 
-/* Optimize serial fallbacks for size — see dots/serial.h for rationale. */
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  Without this, -O3 + LTO can vectorize or clone the serial kernels under AVX-512
+ *  callers in dispatch_*.c, which wastes binary and breaks the nk_*_serial-as-scalar-oracle
+ *  contract that tests and numerical-stability docs rely on. See dots/serial.h. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
+/* Size bias for release. Gated on NDEBUG so Debug builds keep -O0 for stepping. */
 #if defined(NDEBUG)
 #if defined(_MSC_VER)
 #pragma optimize("s", on)
@@ -241,6 +252,12 @@ nk_define_cross_normalized_symmetric_(euclidean, u4, serial, u4x2, u32, /*norm_v
 #endif
 #endif
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif

package/include/numkong/spatials/skylake.h

@@ -97,11 +97,11 @@ nk_define_cross_normalized_symmetric_(euclidean, bf16, skylake, bf16, f32, /*nor
                                       nk_dots_reduce_sumsq_bf16_, nk_load_b128_haswell_, nk_partial_load_b32x4_skylake_,
                                       nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_, 1)
 
-nk_define_cross_normalized_packed_(angular, e4m3, skylake, e4m3, f32, f32, /*norm_value_type=*/f32, f32, nk_b128_vec_t,
+nk_define_cross_normalized_packed_(angular, e4m3, skylake, e4m3, f16, f32, /*norm_value_type=*/f32, f32, nk_b128_vec_t,
                                    nk_dots_packed_e4m3_skylake, nk_angular_through_f32_from_dot_haswell_,
                                    nk_dots_reduce_sumsq_e4m3_, nk_load_b128_haswell_, nk_partial_load_b32x4_skylake_,
                                    nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_, 1)
-nk_define_cross_normalized_packed_(euclidean, e4m3, skylake, e4m3, f32, f32, /*norm_value_type=*/f32, f32,
+nk_define_cross_normalized_packed_(euclidean, e4m3, skylake, e4m3, f16, f32, /*norm_value_type=*/f32, f32,
                                    nk_b128_vec_t, nk_dots_packed_e4m3_skylake,
                                    nk_euclidean_through_f32_from_dot_haswell_, nk_dots_reduce_sumsq_e4m3_,
                                    nk_load_b128_haswell_, nk_partial_load_b32x4_skylake_, nk_store_b128_haswell_,

package/include/numkong/spatials.h

@@ -759,6 +759,23 @@ NK_PUBLIC void nk_euclideans_packed_f16_graniteamx(nk_f16_t const *a, void const
 NK_PUBLIC void nk_euclideans_symmetric_f16_graniteamx(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
                                                       nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
                                                       nk_size_t row_start, nk_size_t row_count);
+/** @copydoc nk_angulars_packed_f16 */
+NK_PUBLIC void nk_angulars_packed_e5m2_graniteamx(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
+                                                  nk_size_t rows, nk_size_t cols, nk_size_t depth,
+                                                  nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
+/** @copydoc nk_angulars_symmetric_f16 */
+NK_PUBLIC void nk_angulars_symmetric_e5m2_graniteamx(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
+                                                     nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
+                                                     nk_size_t row_start, nk_size_t row_count);
+/** @copydoc nk_euclideans_packed_f16 */
+NK_PUBLIC void nk_euclideans_packed_e5m2_graniteamx(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
+                                                    nk_size_t rows, nk_size_t cols, nk_size_t depth,
+                                                    nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
+/** @copydoc nk_euclideans_symmetric_f16 */
+NK_PUBLIC void nk_euclideans_symmetric_e5m2_graniteamx(nk_e5m2_t const *vectors, nk_size_t vectors_count,
+                                                       nk_size_t depth, nk_size_t stride, nk_f32_t *result,
+                                                       nk_size_t result_stride, nk_size_t row_start,
+                                                       nk_size_t row_count);
 #endif // NK_TARGET_GRANITEAMX
 
 /*  ARM SME backends using Scalable Matrix Extension.