numkong 7.0.0

package/include/numkong/dot/rvvbb.h

@@ -0,0 +1,72 @@
+/**
+ *  @brief SIMD-accelerated Dot Products for RISC-V with Zvbb.
+ *  @file include/numkong/dot/rvvbb.h
+ *  @author Ash Vardanian
+ *  @date February 22, 2026
+ *
+ *  @sa include/numkong/dot.h
+ *
+ *  Zvbb (Vector Basic Bit-manipulation) provides native per-element popcount via `vcpop.v`,
+ *  replacing the 11-instruction SWAR approach with a single instruction for u1 dot products.
+ *
+ *  Only `nk_dot_u1` benefits from Zvbb (it needs byte-level popcount of AND results).
+ *  Requires: RVV 1.0 + Zvbb extension (GCC 14+ or Clang 18+)
+ */
+#ifndef NK_DOT_RVVBB_H
+#define NK_DOT_RVVBB_H
+
+#if NK_TARGET_RISCV_
+#if NK_TARGET_RVVBB
+
+#include "numkong/types.h"
+#include "numkong/set/rvvbb.h" // `nk_popcount_u8m4_rvvbb_`
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=+v,+zvbb"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=+v,+zvbb")
+#endif
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+NK_PUBLIC void nk_dot_u1_rvvbb(nk_u1x8_t const *a, nk_u1x8_t const *b, nk_size_t n_bits, nk_u32_t *result) {
+    nk_size_t count_bytes = nk_size_divide_round_up_(n_bits, NK_BITS_PER_BYTE);
+
+    vuint32m1_t sum_u32m1 = __riscv_vmv_v_x_u32m1(0, 1);
+
+    nk_size_t i = 0;
+    for (nk_size_t vector_length; i + 1 <= count_bytes; i += vector_length) {
+        vector_length = __riscv_vsetvl_e8m4(count_bytes - i);
+
+        // Load and AND to find shared bits (dot product of binary vectors)
+        vuint8m4_t a_u8m4 = __riscv_vle8_v_u8m4(a + i, vector_length);
+        vuint8m4_t b_u8m4 = __riscv_vle8_v_u8m4(b + i, vector_length);
+        vuint8m4_t and_u8m4 = __riscv_vand_vv_u8m4(a_u8m4, b_u8m4, vector_length);
+
+        // Native per-element popcount via Zvbb (1 instruction vs 11 SWAR)
+        vuint8m4_t popcount_u8m4 = nk_popcount_u8m4_rvvbb_(and_u8m4);
+
+        // Widen to u16 and accumulate via widening reduction sum
+        vuint16m8_t popcount_u16m8 = __riscv_vwaddu_vx_u16m8(popcount_u8m4, 0, vector_length);
+        sum_u32m1 = __riscv_vwredsumu_vs_u16m8_u32m1(popcount_u16m8, sum_u32m1, vector_length);
+    }
+
+    *result = __riscv_vmv_x_s_u32m1_u32(sum_u32m1);
+}
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#endif // NK_TARGET_RVVBB
+#endif // NK_TARGET_RISCV_
+#endif // NK_DOT_RVVBB_H

package/include/numkong/dot/rvvbf16.h

@@ -0,0 +1,123 @@
+/**
+ *  @brief SIMD-accelerated Dot Products for RISC-V BF16.
+ *  @file include/numkong/dot/rvvbf16.h
+ *  @author Ash Vardanian
+ *  @date January 5, 2026
+ *
+ *  @sa include/numkong/dot.h
+ *
+ *  Alibaba XuanTie C930 and similar chips implement RVV 1.0 with Zvfbfwma extension.
+ *  Zvfbfwma provides widening bf16 fused multiply-accumulate to f32:
+ *    vfwmaccbf16: f32 ← bf16 ⨯ bf16
+ *
+ *  All mini-float types use 256-entry VLUXEI16 LUT gathers from cast/rvv.h (3 instructions each).
+ *  All variants then use vfwmaccbf16_vv for fused bf16 ⨯ bf16 → f32 multiply-accumulate.
+ *
+ *  Requires: RVV 1.0 + Zvfbfwma extension (GCC 14+ or Clang 18+)
+ */
+#ifndef NK_DOT_RVVBF16_H
+#define NK_DOT_RVVBF16_H
+
+#if NK_TARGET_RISCV_
+#if NK_TARGET_RVVBF16
+
+#include "numkong/types.h"
+#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_bf16m2_rvv_`, `nk_e5m2m1_to_bf16m2_rvv_`, etc.
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=+v,+zvfbfwma"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=+v,+zvfbfwma")
+#endif
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+NK_PUBLIC void nk_dot_bf16_rvvbf16(nk_bf16_t const *a_scalars, nk_bf16_t const *b_scalars, nk_size_t count_scalars,
+                                   nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t sum_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e16m1(count_scalars);
+        vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)a_scalars, vector_length);
+        vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)b_scalars, vector_length);
+        vbfloat16m1_t a_bf16m1 = __riscv_vreinterpret_v_u16m1_bf16m1(a_u16m1);
+        vbfloat16m1_t b_bf16m1 = __riscv_vreinterpret_v_u16m1_bf16m1(b_u16m1);
+        // Widening bf16 FMA: f32 ← bf16 ⨯ bf16, per-lane accumulation
+        sum_f32m2 = __riscv_vfwmaccbf16_vv_f32m2_tu(sum_f32m2, a_bf16m1, b_bf16m1, vector_length);
+    }
+    // Single horizontal reduction at the end
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, vlmax);
+    *result = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(sum_f32m2, zero_f32m1, vlmax));
+}
+
+/** @brief Convert e2m3 to bf16 via 256-entry LUT in cast/rvv.h + reinterpret. */
+NK_INTERNAL vbfloat16m2_t nk_e2m3m1_to_bf16m2_rvvbf16_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
+    return __riscv_vreinterpret_v_u16m2_bf16m2(nk_e2m3m1_to_bf16m2_rvv_(raw_u8m1, vector_length));
+}
+
+/** @brief Convert e3m2 to bf16 via 256-entry LUT in cast/rvv.h + reinterpret. */
+NK_INTERNAL vbfloat16m2_t nk_e3m2m1_to_bf16m2_rvvbf16_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
+    return __riscv_vreinterpret_v_u16m2_bf16m2(nk_e3m2m1_to_bf16m2_rvv_(raw_u8m1, vector_length));
+}
+
+/** @brief Convert e4m3 to bf16 via 256-entry LUT in cast/rvv.h + reinterpret. */
+NK_INTERNAL vbfloat16m2_t nk_e4m3m1_to_bf16m2_rvvbf16_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
+    return __riscv_vreinterpret_v_u16m2_bf16m2(nk_e4m3m1_to_bf16m2_rvv_(raw_u8m1, vector_length));
+}
+
+/** @brief Convert e5m2 to bf16 via 256-entry LUT in cast/rvv.h + reinterpret. */
+NK_INTERNAL vbfloat16m2_t nk_e5m2m1_to_bf16m2_rvvbf16_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
+    return __riscv_vreinterpret_v_u16m2_bf16m2(nk_e5m2m1_to_bf16m2_rvv_(raw_u8m1, vector_length));
+}
+
+NK_PUBLIC void nk_dot_e4m3_rvvbf16(nk_e4m3_t const *a_scalars, nk_e4m3_t const *b_scalars, nk_size_t count_scalars,
+                                   nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m4();
+    vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, vlmax);
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e8m1(count_scalars);
+        vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a_scalars, vector_length);
+        vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b_scalars, vector_length);
+        vbfloat16m2_t a_bf16m2 = nk_e4m3m1_to_bf16m2_rvvbf16_(a_u8m1, vector_length);
+        vbfloat16m2_t b_bf16m2 = nk_e4m3m1_to_bf16m2_rvvbf16_(b_u8m1, vector_length);
+        sum_f32m4 = __riscv_vfwmaccbf16_vv_f32m4_tu(sum_f32m4, a_bf16m2, b_bf16m2, vector_length);
+    }
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, vlmax);
+    *result = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, vlmax));
+}
+
+NK_PUBLIC void nk_dot_e5m2_rvvbf16(nk_e5m2_t const *a_scalars, nk_e5m2_t const *b_scalars, nk_size_t count_scalars,
+                                   nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m4();
+    vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, vlmax);
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e8m1(count_scalars);
+        vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a_scalars, vector_length);
+        vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b_scalars, vector_length);
+        vbfloat16m2_t a_bf16m2 = nk_e5m2m1_to_bf16m2_rvvbf16_(a_u8m1, vector_length);
+        vbfloat16m2_t b_bf16m2 = nk_e5m2m1_to_bf16m2_rvvbf16_(b_u8m1, vector_length);
+        sum_f32m4 = __riscv_vfwmaccbf16_vv_f32m4_tu(sum_f32m4, a_bf16m2, b_bf16m2, vector_length);
+    }
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, vlmax);
+    *result = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, vlmax));
+}
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#endif // NK_TARGET_RVVBF16
+#endif // NK_TARGET_RISCV_
+#endif // NK_DOT_RVVBF16_H

package/include/numkong/dot/rvvhalf.h

@@ -0,0 +1,129 @@
+/**
+ *  @brief SIMD-accelerated Dot Products for RISC-V FP16.
+ *  @file include/numkong/dot/rvvhalf.h
+ *  @author Ash Vardanian
+ *  @date January 5, 2026
+ *
+ *  @sa include/numkong/dot.h
+ *
+ *  SiFive P670/X280 and similar chips implement RVV 1.0 with Zvfh extension.
+ *  Zvfh provides native half-precision (f16) vector operations.
+ *  Uses widening multiply (f16 ⨯ f16 → f32) for precision, then reduces to f32.
+ *
+ *  For e2m3, e3m2, e4m3: conversion uses 256-entry VLUXEI16 LUT gathers from cast/rvv.h (3 instructions each).
+ *  For e5m2: conversion uses pure shift (vzext + vsll) since e5m2 and f16 share the same exponent bias.
+ *  All variants then use vfwmacc_vv for widening fused f16 ⨯ f16 → f32 multiply-accumulate.
+ *
+ *  Requires: RVV 1.0 + Zvfh extension (GCC 14+ or Clang 18+)
+ */
+#ifndef NK_DOT_RVVHALF_H
+#define NK_DOT_RVVHALF_H
+
+#if NK_TARGET_RISCV_
+#if NK_TARGET_RVVHALF
+
+#include "numkong/types.h"
+#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_f16m2_rvv_`, `nk_e2m3m1_to_f16m2_rvv_`, etc.
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=+v,+zvfh"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=+v,+zvfh")
+#endif
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+NK_PUBLIC void nk_dot_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t const *b_scalars, nk_size_t count_scalars,
+                                  nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t sum_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e16m1(count_scalars);
+        vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)a_scalars, vector_length);
+        vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)b_scalars, vector_length);
+        vfloat16m1_t a_f16m1 = __riscv_vreinterpret_v_u16m1_f16m1(a_u16m1);
+        vfloat16m1_t b_f16m1 = __riscv_vreinterpret_v_u16m1_f16m1(b_u16m1);
+        // Widening FMA: f32 += f16 ⨯ f16, per-lane accumulation
+        sum_f32m2 = __riscv_vfwmacc_vv_f32m2_tu(sum_f32m2, a_f16m1, b_f16m1, vector_length);
+    }
+    // Single horizontal reduction at the end
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, vlmax);
+    *result = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(sum_f32m2, zero_f32m1, vlmax));
+}
+
+/** @brief Convert e2m3 to f16 via 256-entry LUT in cast/rvv.h + reinterpret. */
+NK_INTERNAL vfloat16m2_t nk_e2m3m1_to_f16m2_rvvhalf_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
+    return __riscv_vreinterpret_v_u16m2_f16m2(nk_e2m3m1_to_f16m2_rvv_(raw_u8m1, vector_length));
+}
+
+/** @brief Convert e3m2 to f16 via 256-entry LUT in cast/rvv.h + reinterpret. */
+NK_INTERNAL vfloat16m2_t nk_e3m2m1_to_f16m2_rvvhalf_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
+    return __riscv_vreinterpret_v_u16m2_f16m2(nk_e3m2m1_to_f16m2_rvv_(raw_u8m1, vector_length));
+}
+
+/** @brief Convert e4m3 to f16 via 256-entry LUT in cast/rvv.h + reinterpret. */
+NK_INTERNAL vfloat16m2_t nk_e4m3m1_to_f16m2_rvvhalf_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
+    return __riscv_vreinterpret_v_u16m2_f16m2(nk_e4m3m1_to_f16m2_rvv_(raw_u8m1, vector_length));
+}
+
+/**
+ *  @brief Convert e5m2 (1-5-2 sign-exp-mantissa, 8-bit) to f16 via pure shift (no LUT).
+ *  Same exponent bias (15) means f16 = (lower7 << 8) | (sign << 15). Handles all cases.
+ */
+NK_INTERNAL vfloat16m2_t nk_e5m2m1_to_f16m2_rvvhalf_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
+    vuint16m2_t wide_u16m2 = __riscv_vzext_vf2_u16m2(raw_u8m1, vector_length);
+    vuint16m2_t result_u16m2 = __riscv_vsll_vx_u16m2(wide_u16m2, 8, vector_length);
+    return __riscv_vreinterpret_v_u16m2_f16m2(result_u16m2);
+}
+
+NK_PUBLIC void nk_dot_e4m3_rvvhalf(nk_e4m3_t const *a_scalars, nk_e4m3_t const *b_scalars, nk_size_t count_scalars,
+                                   nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m4();
+    vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, vlmax);
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e8m1(count_scalars);
+        vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a_scalars, vector_length);
+        vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b_scalars, vector_length);
+        vfloat16m2_t a_f16m2 = nk_e4m3m1_to_f16m2_rvvhalf_(a_u8m1, vector_length);
+        vfloat16m2_t b_f16m2 = nk_e4m3m1_to_f16m2_rvvhalf_(b_u8m1, vector_length);
+        sum_f32m4 = __riscv_vfwmacc_vv_f32m4_tu(sum_f32m4, a_f16m2, b_f16m2, vector_length);
+    }
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, vlmax);
+    *result = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, vlmax));
+}
+
+NK_PUBLIC void nk_dot_e5m2_rvvhalf(nk_e5m2_t const *a_scalars, nk_e5m2_t const *b_scalars, nk_size_t count_scalars,
+                                   nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m4();
+    vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, vlmax);
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e8m1(count_scalars);
+        vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a_scalars, vector_length);
+        vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b_scalars, vector_length);
+        vfloat16m2_t a_f16m2 = nk_e5m2m1_to_f16m2_rvvhalf_(a_u8m1, vector_length);
+        vfloat16m2_t b_f16m2 = nk_e5m2m1_to_f16m2_rvvhalf_(b_u8m1, vector_length);
+        sum_f32m4 = __riscv_vfwmacc_vv_f32m4_tu(sum_f32m4, a_f16m2, b_f16m2, vector_length);
+    }
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, vlmax);
+    *result = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, vlmax));
+}
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#endif // NK_TARGET_RVVHALF
+#endif // NK_TARGET_RISCV_
+#endif // NK_DOT_RVVHALF_H

package/include/numkong/dot/sapphire.h

@@ -0,0 +1,141 @@
+/**
+ *  @brief SIMD-accelerated Dot Products for Sapphire Rapids.
+ *  @file include/numkong/dot/sapphire.h
+ *  @author Ash Vardanian
+ *  @date February 7, 2026
+ *
+ *  @sa include/numkong/dot.h
+ *
+ *  @section dot_sapphire_instructions Key AVX-512 FP16 Instructions
+ *
+ *      Intrinsic                   Instruction                     Latency     Throughput  Ports
+ *      _mm512_fmadd_ph             VFMADDPH (ZMM, ZMM, ZMM)        4cy         0.5/cy      p01
+ *      _mm512_fmadd_ps             VFMADD132PS (ZMM, ZMM, ZMM)     4cy         0.5/cy      p01
+ *      _mm512_cvtph_ps             VCVTPH2PS (ZMM, YMM)            7cy         1/cy        p01
+ *
+ *  Sapphire Rapids introduces native AVX-512 FP16 support, enabling 32 FP16 FMAs per instruction at the same
+ *  throughput as 16 FP32 FMAs — effectively 2x compute density. For FP6 types (E2M3 and E3M2) whose products
+ *  are small enough to accumulate safely in FP16, this provides near-2x speedup over the Genoa BF16 path.
+ *
+ *  @section dot_sapphire_accumulation Safe FP16 Accumulation
+ *
+ *  E2M3 max product: 7.5² = 56.25; flush every 4 iterations → max lane sum ~225, FP16 ULP ~0.125.
+ *  E3M2 max product: 28² = 784; flush every 4 iterations → max lane sum ~3136, FP16 ULP ~2.0.
+ *  After the flush window, we widen the FP16 accumulator to FP32 and reset.
+ *
+ *  @section dot_sapphire_stateful Stateful Streaming Logic
+ *
+ *  Typed wrappers control the flush cadence:
+ *  - nk_dot_e2m3x32_state_sapphire_t flushes every 4 iterations (128 elements)
+ *  - nk_dot_e3m2x32_state_sapphire_t flushes every 4 iterations (128 elements)
+ */
+#ifndef NK_DOT_SAPPHIRE_H
+#define NK_DOT_SAPPHIRE_H
+
+#if NK_TARGET_X86_
+#if NK_TARGET_SAPPHIRE
+
+#include "numkong/types.h"
+#include "numkong/reduce/skylake.h" // `nk_reduce_add_f32x16_skylake_`
+#include "numkong/dot/skylake.h"    // `nk_dot_through_f32_finalize_skylake_`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(                                                                        \
+    __attribute__((target("avx2,avx512f,avx512vl,avx512bw,avx512dq,avx512fp16,f16c,fma,bmi,bmi2"))), \
+    apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512fp16", "f16c", "fma", "bmi", "bmi2")
+#endif
+
+/** @brief Convert 32x e2m3 → 32x f16 via 64-entry signed LUT lookup (AVX-512BW).
+ *  E2M3 format: S EE MMM (bias=1, 6 bits total: sign at bit 5, magnitude bits 4-0).
+ *  F16: S EEEEE MMMMMMMMMM (bias=15).
+ *
+ *  Uses permutex2var with two 32-entry LUTs (positive and negative F16 values).
+ *  The E2M3 sign bit (bit 5) naturally becomes the source-select bit of the 6-bit index,
+ *  so no separate sign extraction, shift, or OR is needed. After cvtepu8_epi16, bits 15:6
+ *  are zero and permutex2var only reads bits 5:0, so no AND mask is required either. */
+NK_INTERNAL __m512h nk_e2m3x32_to_f16x32_sapphire_(__m256i e2m3x32) {
+    __m512i idx_i16x32 = _mm512_cvtepu8_epi16(e2m3x32);
+
+    // 32-entry LUT for positive E2M3 magnitudes → F16
+    __m512i const lut_pos_i16x32 = _mm512_set_epi16(                     //
+        0x4780, 0x4700, 0x4680, 0x4600, 0x4580, 0x4500, 0x4480, 0x4400,  // [31-24] exp=3: f16_exp=17
+        0x4380, 0x4300, 0x4280, 0x4200, 0x4180, 0x4100, 0x4080, 0x4000,  // [23-16] exp=2: f16_exp=16
+        0x3F80, 0x3F00, 0x3E80, 0x3E00, 0x3D80, 0x3D00, 0x3C80, 0x3C00,  // [15-8] exp=1: f16_exp=15
+        0x3B00, 0x3A00, 0x3900, 0x3800, 0x3600, 0x3400, 0x3000, 0x0000); // [7-0] exp=0: subnormals (0, 1/8..7/8)
+
+    // 32-entry LUT for negative E2M3 magnitudes → F16 (= positive | 0x8000)
+    __m512i const lut_neg_i16x32 = _mm512_set_epi16(                 //
+        (short)0xC780, (short)0xC700, (short)0xC680, (short)0xC600,  //
+        (short)0xC580, (short)0xC500, (short)0xC480, (short)0xC400,  // [31-24] exp=3
+        (short)0xC380, (short)0xC300, (short)0xC280, (short)0xC200,  //
+        (short)0xC180, (short)0xC100, (short)0xC080, (short)0xC000,  // [23-16] exp=2
+        (short)0xBF80, (short)0xBF00, (short)0xBE80, (short)0xBE00,  //
+        (short)0xBD80, (short)0xBD00, (short)0xBC80, (short)0xBC00,  // [15-8] exp=1
+        (short)0xBB00, (short)0xBA00, (short)0xB900, (short)0xB800,  //
+        (short)0xB600, (short)0xB400, (short)0xB000, (short)0x8000); // [7-0] exp=0
+
+    return nk_m512h_from_m512i_(_mm512_permutex2var_epi16(lut_pos_i16x32, idx_i16x32, lut_neg_i16x32));
+}
+
+/** @brief Convert 32x e3m2 → 32x f16 via 64-entry signed LUT lookup (AVX-512BW).
+ *  E3M2 format: S EEE MM (bias=3, 6 bits total: sign at bit 5, magnitude bits 4-0).
+ *  F16: S EEEEE MMMMMMMMMM (bias=15).
+ *
+ *  Same permutex2var technique as E2M3 — sign bit 5 selects the LUT source. */
+NK_INTERNAL __m512h nk_e3m2x32_to_f16x32_sapphire_(__m256i e3m2x32) {
+    __m512i idx_i16x32 = _mm512_cvtepu8_epi16(e3m2x32);
+
+    // 32-entry LUT for positive E3M2 magnitudes → F16
+    __m512i const lut_pos_i16x32 = _mm512_set_epi16( //
+        0x4F00, 0x4E00, 0x4D00, 0x4C00,              // [31-28] exp=7: f16_exp=19
+        0x4B00, 0x4A00, 0x4900, 0x4800,              // [27-24] exp=6: f16_exp=18
+        0x4700, 0x4600, 0x4500, 0x4400,              // [23-20] exp=5: f16_exp=17
+        0x4300, 0x4200, 0x4100, 0x4000,              // [19-16] exp=4: f16_exp=16
+        0x3F00, 0x3E00, 0x3D00, 0x3C00,              // [15-12] exp=3: f16_exp=15
+        0x3B00, 0x3A00, 0x3900, 0x3800,              // [11-8] exp=2: f16_exp=14
+        0x3700, 0x3600, 0x3500, 0x3400,              // [7-4] exp=1: f16_exp=13
+        0x3200, 0x3000, 0x2C00, 0x0000);             // [3-0] exp=0: subnormals
+
+    // 32-entry LUT for negative E3M2 magnitudes → F16 (= positive | 0x8000)
+    __m512i const lut_neg_i16x32 = _mm512_set_epi16(                 //
+        (short)0xCF00, (short)0xCE00, (short)0xCD00, (short)0xCC00,  // [31-28] exp=7
+        (short)0xCB00, (short)0xCA00, (short)0xC900, (short)0xC800,  // [27-24] exp=6
+        (short)0xC700, (short)0xC600, (short)0xC500, (short)0xC400,  // [23-20] exp=5
+        (short)0xC300, (short)0xC200, (short)0xC100, (short)0xC000,  // [19-16] exp=4
+        (short)0xBF00, (short)0xBE00, (short)0xBD00, (short)0xBC00,  // [15-12] exp=3
+        (short)0xBB00, (short)0xBA00, (short)0xB900, (short)0xB800,  // [11-8] exp=2
+        (short)0xB700, (short)0xB600, (short)0xB500, (short)0xB400,  // [7-4] exp=1
+        (short)0xB200, (short)0xB000, (short)0xAC00, (short)0x8000); // [3-0] exp=0
+
+    return nk_m512h_from_m512i_(_mm512_permutex2var_epi16(lut_pos_i16x32, idx_i16x32, lut_neg_i16x32));
+}
+
+/** @brief Flush 32 FP16 values to FP32 accumulator by splitting into 2x16 halves. */
+NK_INTERNAL __m512 nk_flush_f16_to_f32_sapphire_(__m512h acc_f16x32, __m512 sum_f32x16) {
+    __m256i low_f16x16 = _mm512_castsi512_si256(nk_m512i_from_m512h_(acc_f16x32));
+    __m256i high_f16x16 = _mm512_extracti64x4_epi64(nk_m512i_from_m512h_(acc_f16x32), 1);
+    sum_f32x16 = _mm512_add_ps(sum_f32x16, _mm512_cvtph_ps(low_f16x16));
+    sum_f32x16 = _mm512_add_ps(sum_f32x16, _mm512_cvtph_ps(high_f16x16));
+    return sum_f32x16;
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SAPPHIRE
+#endif // NK_TARGET_X86_
+#endif // NK_DOT_SAPPHIRE_H