numkong 7.0.0

package/include/numkong/types.h

@@ -0,0 +1,1384 @@
+/**
+ *  @brief Shared definitions for the NumKong library.
+ *  @file include/numkong/types.h
+ *  @author Ash Vardanian
+ *  @date October 2, 2023
+ *
+ *  Defines:
+ *
+ *  - Sized aliases for numeric types, like: `nk_i32_t` and `nk_f64_t`.
+ *  - Macros for internal compiler/hardware checks, like: `NK_TARGET_ARM_`.
+ *  - Macros for feature controls, like: `NK_TARGET_NEON`
+ *
+ *  @section fp8_types FP8 Numeric Types
+ *
+ *  There are several variants of 8-bit floating point types supported by different industry memebers
+ *  with different hardware support. None are part of the IEEE 754 standard, but some are part of the
+ *  Open Compute Project (OCP) 8-bit Floating Point Specification (OFP8):
+ *
+ *      Format    Bias  Sign  Exp  Mant  Range   Infinity            NaN               Standard
+ *      E4M3FN    7     1     4    3     ±448    ❌ No               Only 0x7F/0xFF    OCP, NVIDIA, ONNX
+ *      E5M2      15    1     5    2     ±57344  ✅ Yes (0x7C/0xFC)  0x7D-7F, 0xFD-FF  OCP, IEEE-like
+ *      E4M3FNUZ  8     1     4    3     ±240    ❌ No               0x80 only         GraphCore, ONNX
+ *      E5M2FNUZ  16    1     5    2     ±57344  ❌ No               0x80 only         GraphCore, ONNX
+ *
+ *  In currently available and soon incoming harware, only two series of models prioritze FNUZ over OCP:
+ *
+ *  - GraphCore IPUs were the original platform proposing FNUZ
+ *  - AMD MI300 series based on CDNA3 implements FNUZ, but not OCP
+ *  - AMD MI350+ series based on CDNA4 switch to OCP and remove FNUZ
+ *  - NVIDIA Hopper and Blackwell only support E4M3FN, E5M2
+ *  - Intel AVX10.2 defines HF8 (E4M3FN) and BF8 (E5M2) - OCP-aligned
+ *  - Arm implements E4M3 (meaning E4M3FN) and E5M2 with a shared `__mfp8` type and a `FPMR` format selector
+ *
+ *  For brevety, across NumKong, "E4M3" implies "E4M3FN".
+ *
+ *  @see https://www.opencompute.org/documents/ocp-8-bit-floating-point-specification-ofp8-revision-1-0-2023-12-01-pdf-1
+ *  @see FP8 Formats for Deep Learning: https://arxiv.org/pdf/2209.05433
+ *  @see ONNX Float8 Types: https://onnx.ai/onnx/technical/float8.html
+ */
+#ifndef NK_TYPES_H
+#define NK_TYPES_H
+
+// On Linux, `_GNU_SOURCE` must be defined before any system headers
+// to expose `syscall` and other GNU extensions when C extensions are disabled.
+#if defined(__linux__) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif
+
+// Inferring target OS: Windows, macOS, Linux, or FreeBSD
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
+#define NK_DEFINED_WINDOWS_ 1
+#elif defined(__APPLE__) && defined(__MACH__)
+#define NK_DEFINED_APPLE_ 1
+#elif defined(__linux__)
+#define NK_DEFINED_LINUX_ 1
+#elif defined(__FreeBSD__)
+#define NK_DEFINED_FREEBSD_ 1
+#endif
+
+// Annotation for the public API symbols:
+//
+// - `NK_PUBLIC` is used for functions that are part of the public API.
+// - `NK_INTERNAL` is used for internal helper functions with unstable APIs.
+// - `NK_DYNAMIC` is used for functions that are part of the public API, but are dispatched at runtime.
+//
+// On GCC we mark the functions as `nonnull` informing that none of the arguments can be `NULL`.
+// Marking with `pure` and `const` isn't possible as outputting to a pointer is a "side effect".
+#if defined(__GNUC__) || defined(__clang__)
+#define NK_PUBLIC   __attribute__((unused)) inline static
+#define NK_INTERNAL __attribute__((always_inline)) inline static
+#else
+#define NK_PUBLIC   inline static
+#define NK_INTERNAL inline static
+#endif // defined(__GNUC__) || defined(__clang__)
+
+#if NK_DYNAMIC_DISPATCH
+#if defined(_WIN32) || defined(__CYGWIN__)
+#define NK_DYNAMIC __declspec(dllexport)
+#elif defined(__GNUC__) || defined(__clang__)
+#define NK_DYNAMIC __attribute__((visibility("default")))
+#else
+#define NK_DYNAMIC NK_PUBLIC
+#endif
+#else
+#define NK_DYNAMIC NK_PUBLIC
+#endif // NK_DYNAMIC_DISPATCH
+
+// Allow SIMD kernels to redirect small inputs to serial implementations.
+// Enabled by default for production use. Tests and benchmarks may disable
+// this to isolate SIMD path behavior on small inputs.
+#if !defined(NK_ALLOW_ISA_REDIRECT)
+#define NK_ALLOW_ISA_REDIRECT 1
+#endif
+
+// Compiling for Arm: NK_TARGET_ARM_
+#if !defined(NK_TARGET_ARM_)
+#if defined(__aarch64__) || defined(_M_ARM64)
+#define NK_TARGET_ARM_ 1
+#else
+#define NK_TARGET_ARM_ 0
+#endif // defined(__aarch64__) || defined(_M_ARM64)
+#endif // !defined(NK_TARGET_ARM_)
+
+// Compiling for x86: NK_TARGET_X86_
+#if !defined(NK_TARGET_X86_)
+#if defined(__x86_64__) || defined(_M_X64)
+#define NK_TARGET_X86_ 1
+#else
+#define NK_TARGET_X86_ 0
+#endif // defined(__x86_64__) || defined(_M_X64)
+#endif // !defined(NK_TARGET_X86_)
+
+// Compiling for RISC-V: NK_TARGET_RISCV_
+#if !defined(NK_TARGET_RISCV_)
+#if defined(__riscv) && (__riscv_xlen == 64)
+#define NK_TARGET_RISCV_ 1
+#else
+#define NK_TARGET_RISCV_ 0
+#endif // defined(__riscv) && (__riscv_xlen == 64)
+#endif // !defined(NK_TARGET_RISCV_)
+
+// Compiling for WASM: NK_TARGET_WASM_
+#if !defined(NK_TARGET_WASM_)
+#if defined(__wasm__) || defined(__EMSCRIPTEN__)
+#define NK_TARGET_WASM_ 1
+#else
+#define NK_TARGET_WASM_ 0
+#endif
+#endif // !defined(NK_TARGET_WASM_)
+
+// WASI hosted mode: NK_DEFINED_WASI_
+// When NK_WASI_HOSTED=ON in CMake, this is predefined to 1 so the library
+// imports capability probes (nk_has_v128, nk_has_relaxed) from the host.
+// Standalone runtimes (Wasmer, Wasmtime CLI) cannot supply those imports,
+// so the default for plain __wasi__ builds is 0 (compile-time detection).
+#if !defined(NK_DEFINED_WASI_)
+#define NK_DEFINED_WASI_ 0
+#endif // !defined(NK_DEFINED_WASI_)
+
+// Compiling for WASM with Relaxed SIMD: NK_TARGET_V128RELAXED
+// Requires -mrelaxed-simd for FMA instructions (f32x4.relaxed_madd, f64x2.relaxed_madd)
+#if !defined(NK_TARGET_V128RELAXED) || (NK_TARGET_V128RELAXED && !NK_TARGET_WASM_)
+#if defined(__wasm_relaxed_simd__)
+#define NK_TARGET_V128RELAXED 1
+#else
+#undef NK_TARGET_V128RELAXED
+#define NK_TARGET_V128RELAXED 0
+#endif
+#endif // !defined(NK_TARGET_V128RELAXED) || ...
+
+// Compiling for RISC-V Vector: NK_TARGET_RVV
+#if !defined(NK_TARGET_RVV) || (NK_TARGET_RVV && !NK_TARGET_RISCV_)
+#if defined(__riscv_v) && (__riscv_v >= 1000000)
+#define NK_TARGET_RVV 1
+#else
+#undef NK_TARGET_RVV
+#define NK_TARGET_RVV 0
+#endif // defined(__riscv_v) && (__riscv_v >= 1000000)
+#endif // !defined(NK_TARGET_RVV) || ...
+
+// Compiling for RISC-V Vector with Zvfh (f16): NK_TARGET_RVVHALF
+// Requires GCC 14+ or Clang 18+ for full intrinsic support
+#if !defined(NK_TARGET_RVVHALF) || (NK_TARGET_RVVHALF && !NK_TARGET_RVV)
+#if defined(__riscv_zvfh) && (__riscv_zvfh > 0)
+#define NK_TARGET_RVVHALF 1
+#else
+#undef NK_TARGET_RVVHALF
+#define NK_TARGET_RVVHALF 0
+#endif // defined(__riscv_zvfh) && (__riscv_zvfh > 0)
+#endif // !defined(NK_TARGET_RVVHALF) || ...
+
+// Compiling for RISC-V Vector with Zvfbfwma (bf16 widening FMA): NK_TARGET_RVVBF16
+// Requires GCC 14+ or Clang 18+ for full intrinsic support
+#if !defined(NK_TARGET_RVVBF16) || (NK_TARGET_RVVBF16 && !NK_TARGET_RVV)
+#if defined(__riscv_zvfbfwma) && (__riscv_zvfbfwma > 0)
+#define NK_TARGET_RVVBF16 1
+#else
+#undef NK_TARGET_RVVBF16
+#define NK_TARGET_RVVBF16 0
+#endif // defined(__riscv_zvfbfwma) && (__riscv_zvfbfwma > 0)
+#endif // !defined(NK_TARGET_RVVBF16) || ...
+
+// Compiling for RISC-V Vector with Zvbb (basic bit-manipulation): NK_TARGET_RVVBB
+// Provides vcpop.v (per-element popcount), vclz.v, vctz.v, vbrev.v, vrol.v, vror.v
+#if !defined(NK_TARGET_RVVBB) || (NK_TARGET_RVVBB && !NK_TARGET_RVV)
+#if defined(__riscv_zvbb) && (__riscv_zvbb > 0)
+#define NK_TARGET_RVVBB 1
+#else
+#undef NK_TARGET_RVVBB
+#define NK_TARGET_RVVBB 0
+#endif // defined(__riscv_zvbb) && (__riscv_zvbb > 0)
+#endif // !defined(NK_TARGET_RVVBB) || ...
+
+// Compiling for Arm: NK_TARGET_NEON
+#if !defined(NK_TARGET_NEON) || (NK_TARGET_NEON && !NK_TARGET_ARM_)
+#if defined(__ARM_NEON)
+#define NK_TARGET_NEON 1
+#else
+#undef NK_TARGET_NEON
+#define NK_TARGET_NEON 0
+#endif // defined(__ARM_NEON)
+#endif // !defined(NK_TARGET_NEON) || ...
+
+// Compiling for Arm: NK_TARGET_NEONSDOT
+#if !defined(NK_TARGET_NEONSDOT) || (NK_TARGET_NEONSDOT && !NK_TARGET_ARM_)
+#if defined(__ARM_NEON)
+#define NK_TARGET_NEONSDOT 1
+#else
+#undef NK_TARGET_NEONSDOT
+#define NK_TARGET_NEONSDOT 0
+#endif // defined(__ARM_NEON)
+#endif // !defined(NK_TARGET_NEONSDOT) || ...
+
+// Compiling for Arm: NK_TARGET_NEONHALF
+#if !defined(NK_TARGET_NEONHALF) || (NK_TARGET_NEONHALF && !NK_TARGET_ARM_)
+#if defined(__ARM_NEON)
+#define NK_TARGET_NEONHALF 1
+#else
+#undef NK_TARGET_NEONHALF
+#define NK_TARGET_NEONHALF 0
+#endif // defined(__ARM_NEON)
+#endif // !defined(NK_TARGET_NEONHALF) || ...
+
+// Compiling for Arm: NK_TARGET_NEONFHM (FEAT_FHM - FMLAL/FMLSL widening ops)
+#if !defined(NK_TARGET_NEONFHM) || (NK_TARGET_NEONFHM && !NK_TARGET_ARM_)
+#if defined(__ARM_NEON)
+#define NK_TARGET_NEONFHM 1
+#else
+#undef NK_TARGET_NEONFHM
+#define NK_TARGET_NEONFHM 0
+#endif // defined(__ARM_NEON)
+#endif // !defined(NK_TARGET_NEONFHM) || ...
+
+// Compiling for Arm: NK_TARGET_NEONBFDOT
+#if !defined(NK_TARGET_NEONBFDOT) || (NK_TARGET_NEONBFDOT && !NK_TARGET_ARM_)
+#if defined(__ARM_NEON)
+#define NK_TARGET_NEONBFDOT 1
+#else
+#undef NK_TARGET_NEONBFDOT
+#define NK_TARGET_NEONBFDOT 0
+#endif // defined(__ARM_NEON)
+#endif // !defined(NK_TARGET_NEONBFDOT) || ...
+
+// Compiling for Arm: NK_TARGET_SVE
+#if !defined(NK_TARGET_SVE) || (NK_TARGET_SVE && !NK_TARGET_ARM_)
+#if defined(__ARM_FEATURE_SVE)
+#define NK_TARGET_SVE 1
+#else
+#undef NK_TARGET_SVE
+#define NK_TARGET_SVE 0
+#endif // defined(__ARM_FEATURE_SVE)
+#endif // !defined(NK_TARGET_SVE) || ...
+
+// Compiling for Arm: NK_TARGET_SVESDOT
+#if !defined(NK_TARGET_SVESDOT) || (NK_TARGET_SVESDOT && !NK_TARGET_ARM_)
+#if defined(__ARM_FEATURE_SVE)
+#define NK_TARGET_SVESDOT 1
+#else
+#undef NK_TARGET_SVESDOT
+#define NK_TARGET_SVESDOT 0
+#endif // defined(__ARM_FEATURE_SVE)
+#endif // !defined(NK_TARGET_SVESDOT) || ...
+
+// Compiling for Arm: NK_TARGET_SVEHALF
+#if !defined(NK_TARGET_SVEHALF) || (NK_TARGET_SVEHALF && !NK_TARGET_ARM_)
+#if defined(__ARM_FEATURE_SVE)
+#define NK_TARGET_SVEHALF 1
+#else
+#undef NK_TARGET_SVEHALF
+#define NK_TARGET_SVEHALF 0
+#endif // defined(__ARM_FEATURE_SVE)
+#endif // !defined(NK_TARGET_SVEHALF) || ...
+
+// Compiling for Arm: NK_TARGET_SVEBFDOT
+#if !defined(NK_TARGET_SVEBFDOT) || (NK_TARGET_SVEBFDOT && !NK_TARGET_ARM_)
+#if defined(__ARM_FEATURE_SVE)
+#define NK_TARGET_SVEBFDOT 1
+#else
+#undef NK_TARGET_SVEBFDOT
+#define NK_TARGET_SVEBFDOT 0
+#endif // defined(__ARM_FEATURE_SVE)
+#endif // !defined(NK_TARGET_SVEBFDOT) || ...
+
+// Compiling for Arm: NK_TARGET_SVE2
+#if !defined(NK_TARGET_SVE2) || (NK_TARGET_SVE2 && !NK_TARGET_ARM_)
+#if defined(__ARM_FEATURE_SVE2)
+#define NK_TARGET_SVE2 1
+#else
+#undef NK_TARGET_SVE2
+#define NK_TARGET_SVE2 0
+#endif // defined(__ARM_FEATURE_SVE2)
+#endif // !defined(NK_TARGET_SVE2) || ...
+
+// Compiling for Arm: NK_TARGET_SVE2P1
+#if !defined(NK_TARGET_SVE2P1) || (NK_TARGET_SVE2P1 && !NK_TARGET_ARM_)
+#undef NK_TARGET_SVE2P1
+#define NK_TARGET_SVE2P1 0
+#endif // !defined(NK_TARGET_SVE2P1) || ...
+
+// Compiling for Arm: NK_TARGET_SME (Scalable Matrix Extension)
+#if !defined(NK_TARGET_SME) || (NK_TARGET_SME && !NK_TARGET_ARM_)
+#if defined(__ARM_FEATURE_SME)
+#define NK_TARGET_SME 1
+#else
+#undef NK_TARGET_SME
+#define NK_TARGET_SME 0
+#endif // defined(__ARM_FEATURE_SME)
+#endif // !defined(NK_TARGET_SME) || ...
+
+#if !defined(NK_TARGET_SME2) || (NK_TARGET_SME2 && !NK_TARGET_ARM_)
+#if defined(__ARM_FEATURE_SME2)
+#define NK_TARGET_SME2 1
+#else
+#undef NK_TARGET_SME2
+#define NK_TARGET_SME2 0
+#endif // defined(__ARM_FEATURE_SME2)
+#endif // !defined(NK_TARGET_SME2) || ...
+
+#if !defined(NK_TARGET_SME2P1) || (NK_TARGET_SME2P1 && !NK_TARGET_ARM_)
+#undef NK_TARGET_SME2P1
+#define NK_TARGET_SME2P1 0
+#endif
+
+// AppleClang 17 exposes SME sub-features through `arm_sme.h` builtin aliases,
+// not dedicated `__ARM_FEATURE_*` predefines for every matrix subtype.
+#if !defined(NK_TARGET_SMEF64) || (NK_TARGET_SMEF64 && !NK_TARGET_ARM_)
+#if defined(__has_builtin) && __has_builtin(__builtin_sme_svmopa_za64_f64_m)
+#define NK_TARGET_SMEF64 1
+#else
+#undef NK_TARGET_SMEF64
+#define NK_TARGET_SMEF64 0
+#endif // defined(__has_builtin) && __has_builtin(__builtin_sme_svmopa_za64_f64_m)
+#endif // !defined(NK_TARGET_SMEF64) || ...
+
+#if !defined(NK_TARGET_SMEBI32) || (NK_TARGET_SMEBI32 && !NK_TARGET_ARM_)
+#if defined(__has_builtin) && __has_builtin(__builtin_sme_svbmopa_za32_u32_m)
+#define NK_TARGET_SMEBI32 1
+#else
+#undef NK_TARGET_SMEBI32
+#define NK_TARGET_SMEBI32 0
+#endif // defined(__has_builtin) && __has_builtin(__builtin_sme_svbmopa_za32_u32_m)
+#endif // !defined(NK_TARGET_SMEBI32) || ...
+
+#if !defined(NK_TARGET_SMEHALF) || (NK_TARGET_SMEHALF && !NK_TARGET_ARM_)
+#if defined(__has_builtin) && __has_builtin(__builtin_sme_svmopa_za32_f16_m)
+#define NK_TARGET_SMEHALF 1
+#else
+#undef NK_TARGET_SMEHALF
+#define NK_TARGET_SMEHALF 0
+#endif // defined(__has_builtin) && __has_builtin(__builtin_sme_svmopa_za32_f16_m)
+#endif // !defined(NK_TARGET_SMEHALF) || ...
+
+#if !defined(NK_TARGET_SMEBF16) || (NK_TARGET_SMEBF16 && !NK_TARGET_ARM_)
+#if defined(__has_builtin) && __has_builtin(__builtin_sme_svmopa_za32_bf16_m)
+#define NK_TARGET_SMEBF16 1
+#else
+#undef NK_TARGET_SMEBF16
+#define NK_TARGET_SMEBF16 0
+#endif // defined(__has_builtin) && __has_builtin(__builtin_sme_svmopa_za32_bf16_m)
+#endif // !defined(NK_TARGET_SMEBF16) || ...
+
+#if !defined(NK_TARGET_SMELUT2) || (NK_TARGET_SMELUT2 && !NK_TARGET_ARM_)
+#if defined(__has_builtin) && __has_builtin(__builtin_sme_svluti2_lane_zt_u8)
+#define NK_TARGET_SMELUT2 1
+#else
+#undef NK_TARGET_SMELUT2
+#define NK_TARGET_SMELUT2 0
+#endif // defined(__has_builtin) && __has_builtin(__builtin_sme_svluti2_lane_zt_u8)
+#endif // !defined(NK_TARGET_SMELUT2) || ...
+
+#if !defined(NK_TARGET_SMEFA64) || (NK_TARGET_SMEFA64 && !NK_TARGET_ARM_)
+#undef NK_TARGET_SMEFA64
+#define NK_TARGET_SMEFA64 0
+#endif
+
+// Compiling for x86: NK_TARGET_HASWELL
+//
+// Starting with Ivy Bridge, Intel supports the `F16C` extensions for fast half-precision
+// to single-precision floating-point conversions. On AMD those instructions
+// are supported on all CPUs starting with Jaguar 2009.
+// Starting with Sandy Bridge, Intel adds basic AVX support in their CPUs and in 2013
+// extends it with AVX2 in the Haswell generation. Moreover, Haswell adds FMA support.
+//
+// On MSVC, most GCC-style ISA macros are unavailable. MSVC defines __AVX__, __AVX2__,
+// __AVX512F/BW/CD/DQ/VL__, and __AVX10_VER__, but NOT __AVXVNNI__, __AVX512VNNI__,
+// __AVX512BF16__, __AVX512FP16__, __AMX_*__, etc.
+// Instead, MSVC makes all intrinsics available once the toolset version supports them,
+// without requiring `/arch:AVX512`. We gate on _MSC_VER to auto-enable targets:
+//   - _MSC_VER >= 1900 (VS 2015+): AVX2/FMA/F16C (Haswell)
+//   - _MSC_VER >= 1920 (VS 2019+): AVX-512 base (Skylake, Icelake), AVX-VNNI (Alder)
+//   - _MSC_VER >= 1944 (VS 2022 17.14+): BF16, FP16, VP2INTERSECT, VNNI-INT8 (Sierra), AMX
+#if !defined(NK_TARGET_HASWELL) || (NK_TARGET_HASWELL && !NK_TARGET_X86_)
+#if (defined(__AVX2__) && defined(__FMA__) && defined(__F16C__)) || (defined(_MSC_VER) && _MSC_VER >= 1900)
+#define NK_TARGET_HASWELL 1
+#else
+#undef NK_TARGET_HASWELL
+#define NK_TARGET_HASWELL 0
+#endif // defined(__AVX2__)
+#endif // !defined(NK_TARGET_HASWELL) || ...
+
+// Compiling for x86: NK_TARGET_SKYLAKE, NK_TARGET_ICELAKE, NK_TARGET_GENOA,
+// NK_TARGET_SAPPHIRE, NK_TARGET_TURIN, NK_TARGET_SIERRA
+//
+// To list all available macros for x86, take a recent compiler, like GCC 12 and run:
+//      gcc-12 -march=sapphirerapids -dM -E - < /dev/null | egrep "SSE|AVX" | sort
+// On Arm machines you may want to check for other flags:
+//      gcc-12 -march=native -dM -E - < /dev/null | egrep "NEON|SVE|FP16|FMA" | sort
+#if !defined(NK_TARGET_SKYLAKE) || (NK_TARGET_SKYLAKE && !NK_TARGET_X86_)
+#if (defined(__AVX512F__) && defined(__AVX512CD__) && defined(__AVX512VL__) && defined(__AVX512DQ__) && \
+     defined(__AVX512BW__)) ||                                                                          \
+    (defined(_MSC_VER) && _MSC_VER >= 1920)
+#define NK_TARGET_SKYLAKE 1
+#else
+#undef NK_TARGET_SKYLAKE
+#define NK_TARGET_SKYLAKE 0
+#endif
+#endif // !defined(NK_TARGET_SKYLAKE) || ...
+
+#if !defined(NK_TARGET_ICELAKE) || (NK_TARGET_ICELAKE && !NK_TARGET_X86_)
+#if (defined(__AVX512VNNI__) && defined(__AVX512IFMA__) && defined(__AVX512BITALG__) && defined(__AVX512VBMI__) && \
+     defined(__AVX512VBMI2__) && defined(__AVX512VPOPCNTDQ__)) ||                                                  \
+    (defined(_MSC_VER) && _MSC_VER >= 1920)
+#define NK_TARGET_ICELAKE 1
+#else
+#undef NK_TARGET_ICELAKE
+#define NK_TARGET_ICELAKE 0
+#endif
+#endif // !defined(NK_TARGET_ICELAKE) || ...
+
+#if !defined(NK_TARGET_GENOA) || (NK_TARGET_GENOA && !NK_TARGET_X86_)
+#if defined(__AVX512BF16__) || (defined(_MSC_VER) && _MSC_VER >= 1944)
+#define NK_TARGET_GENOA 1
+#else
+#undef NK_TARGET_GENOA
+#define NK_TARGET_GENOA 0
+#endif
+#endif // !defined(NK_TARGET_GENOA) || ...
+
+#if !defined(NK_TARGET_SAPPHIRE) || (NK_TARGET_SAPPHIRE && !NK_TARGET_X86_)
+#if defined(__AVX512FP16__) || (defined(_MSC_VER) && _MSC_VER >= 1944)
+#define NK_TARGET_SAPPHIRE 1
+#else
+#undef NK_TARGET_SAPPHIRE
+#define NK_TARGET_SAPPHIRE 0
+#endif
+#endif // !defined(NK_TARGET_SAPPHIRE) || ...
+
+#if !defined(NK_TARGET_SAPPHIREAMX) || (NK_TARGET_SAPPHIREAMX && !NK_TARGET_X86_)
+#if (defined(__AMX_TILE__) && defined(__AMX_BF16__) && defined(__AMX_INT8__)) || (defined(_MSC_VER) && _MSC_VER >= 1944)
+#define NK_TARGET_SAPPHIREAMX 1
+#else
+#undef NK_TARGET_SAPPHIREAMX
+#define NK_TARGET_SAPPHIREAMX 0
+#endif
+#endif // !defined(NK_TARGET_SAPPHIREAMX) || ...
+
+#if !defined(NK_TARGET_GRANITEAMX) || (NK_TARGET_GRANITEAMX && !NK_TARGET_X86_)
+#if (defined(__AMX_TILE__) && defined(__AMX_FP16__)) || (defined(_MSC_VER) && _MSC_VER >= 1944)
+#define NK_TARGET_GRANITEAMX 1
+#else
+#undef NK_TARGET_GRANITEAMX
+#define NK_TARGET_GRANITEAMX 0
+#endif
+#endif // !defined(NK_TARGET_GRANITEAMX) || ...
+
+#if !defined(NK_TARGET_TURIN) || (NK_TARGET_TURIN && !NK_TARGET_X86_)
+#if defined(__AVX512VP2INTERSECT__) || (defined(_MSC_VER) && _MSC_VER >= 1944)
+#define NK_TARGET_TURIN 1
+#else
+#undef NK_TARGET_TURIN
+#define NK_TARGET_TURIN 0
+#endif
+#endif // !defined(NK_TARGET_TURIN) || ...
+
+#if !defined(NK_TARGET_ALDER) || (NK_TARGET_ALDER && !NK_TARGET_X86_)
+#if defined(__AVXVNNI__) || (defined(_MSC_VER) && _MSC_VER >= 1920)
+#define NK_TARGET_ALDER 1
+#else
+#undef NK_TARGET_ALDER
+#define NK_TARGET_ALDER 0
+#endif
+#endif // !defined(NK_TARGET_ALDER) || ...
+
+#if !defined(NK_TARGET_SIERRA) || (NK_TARGET_SIERRA && !NK_TARGET_X86_)
+#if defined(__AVXVNNIINT8__) || (defined(_MSC_VER) && _MSC_VER >= 1944)
+#define NK_TARGET_SIERRA 1
+#else
+#undef NK_TARGET_SIERRA
+#define NK_TARGET_SIERRA 0
+#endif
+#endif // !defined(NK_TARGET_SIERRA) || ...
+
+// Include the relevant intrinsics file - different for different OSes and ISAs
+#if defined(_MSC_VER)
+#include <intrin.h>
+#elif NK_TARGET_ARM_
+#if NK_TARGET_NEON
+#include <arm_neon.h>
+#endif
+#if NK_TARGET_SVE || NK_TARGET_SVE2
+#include <arm_sve.h>
+#endif
+#if NK_TARGET_SME || NK_TARGET_SME2 || NK_TARGET_SMEBI32
+#include <arm_sme.h>
+#endif
+#elif NK_TARGET_HASWELL || NK_TARGET_SKYLAKE
+#include <immintrin.h>
+#elif NK_TARGET_RVV
+#include <riscv_vector.h>
+#elif NK_TARGET_V128RELAXED
+#include <wasm_simd128.h>
+#endif
+
+#if !defined(NK_F64_DIVISION_EPSILON)
+#define NK_F64_DIVISION_EPSILON (1e-15)
+#endif
+
+#if !defined(NK_F32_DIVISION_EPSILON)
+#define NK_F32_DIVISION_EPSILON (1e-7)
+#endif
+
+#if !defined(NK_F16_DIVISION_EPSILON)
+#define NK_F16_DIVISION_EPSILON (1e-3)
+#endif
+
+/**
+ *  @brief  The compile-time constant defining the capacity of `nk_tensor_position_t`.
+ *          Matches `PyBUF_MAX_NDIM` by default.
+ */
+#if !defined(NK_TENSOR_MAX_RANK)
+#define NK_TENSOR_MAX_RANK (64)
+#endif
+
+/**
+ *  @brief  Aligns a variable to a 64-byte boundary using compiler extensions for
+ *          compatibility with C 99, as `alignas(64)` is only available in C 11 or C++.
+ *          Used internally and recommended for external users.
+ */
+#if defined(_MSC_VER)
+#define NK_ALIGN64 __declspec(align(64))
+#elif defined(__GNUC__) || defined(__clang__)
+#define NK_ALIGN64 __attribute__((aligned(64)))
+#endif
+
+/**
+ *  ARM Streaming attributes (require SME-capable compiler: GCC 14+, Clang 16+).
+ *  NK_STREAMING_ marks functions that require streaming SVE mode (e.g. FCVTLT).
+ *  NK_STREAMING_COMPATIBLE_ marks helpers callable from both streaming and non-streaming mode.
+ */
+#if NK_TARGET_ARM_ && NK_TARGET_SME
+#define NK_STREAMING_            __arm_streaming
+#define NK_STREAMING_COMPATIBLE_ __arm_streaming_compatible
+#else
+#define NK_STREAMING_
+#define NK_STREAMING_COMPATIBLE_
+#endif
+
+/**
+ *  @brief  Portable casts between SIMD vector types.
+ *          MSVC typedefs `__m512bh`, `__m512h`, `__m256bh` as aliases for `__m512i`/`__m256i`,
+ *          but rejects C-style casts between them. GCC/Clang define them as distinct types.
+ */
+#if NK_TARGET_X86_
+#if defined(_MSC_VER)
+#define nk_m512bh_from_m512i_(x) (x)
+#define nk_m512h_from_m512i_(x)  (x)
+#define nk_m512i_from_m512h_(x)  (x)
+#define nk_m256bh_from_m256i_(x) (x)
+#define nk_m256i_from_m256bh_(x) (x)
+#else
+#define nk_m512bh_from_m512i_(x) ((__m512bh)(x))
+#define nk_m512h_from_m512i_(x)  ((__m512h)(x))
+#define nk_m512i_from_m512h_(x)  ((__m512i)(x))
+#define nk_m256bh_from_m256i_(x) ((__m256bh)(x))
+#define nk_m256i_from_m256bh_(x) ((__m256i)(x))
+#endif
+#endif
+
+/** Copy 16 bits (2 bytes) from source to destination */
+#if defined(__GNUC__) || defined(__clang__)
+#define nk_copy_bytes_(destination_ptr, source_ptr, count) __builtin_memcpy((destination_ptr), (source_ptr), count)
+#else
+#include <string.h> // `memcpy`
+#define nk_copy_bytes_(destination_ptr, source_ptr, count) memcpy((destination_ptr), (source_ptr), count)
+#endif
+
+/** Macro to mark unused parameters (cleaner than (void)variable) */
+#define nk_unused_(x) ((void)(x))
+
+/**
+ *  @brief C99 static array parameter annotation for minimum array size.
+ *
+ *  In C, expands to `static n` enabling compiler bounds checking.
+ *  In C++, expands to nothing as this syntax is not supported.
+ *  @see https://lwn.net/Articles/1046840/
+ *
+ *  Example usage:
+ *  @code{.c}
+ *      void hash_digest(uint8_t digest[nk_at_least_(32)]);
+ *      void lookup(uint8_t const lut[nk_at_least_(256)]);
+ *  @endcode
+ */
+#if defined(__cplusplus) || defined(_MSC_VER)
+#define nk_at_least_(n)
+#else
+#define nk_at_least_(n) static n
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** @brief Packed 8-bit bit-vector (8 booleans in one byte), LSB = dimension 0.
+ *  Used for Hamming distance and Jaccard similarity via popcount.
+ *  Dimension count must be a multiple of 8; unused bits in the final byte must be zeroed. */
+typedef unsigned char nk_u1x8_t;
+/** @brief Packed 4-bit signed integer pair (2 × i4 in one byte), [high nibble : low nibble].
+ *  Range per element: [−8, +7]. Elements sign-extended to i8 for arithmetic.
+ *  Dimension count must be a multiple of 2; unused nibbles in the final byte must be zeroed. */
+typedef unsigned char nk_i4x2_t;
+/** @brief Packed 4-bit unsigned integer pair (2 × u4 in one byte), [high nibble : low nibble].
+ *  Range per element: [0, 15]. Elements zero-extended to u8 for arithmetic.
+ *  Dimension count must be a multiple of 2; unused nibbles in the final byte must be zeroed. */
+typedef unsigned char nk_u4x2_t;
+
+/** @brief 8-bit E4M3 float (OCP FP8): sign(1) + exponent(4) + mantissa(3), bias=7.
+ *  Range: ±448, no infinities (all-ones exponent → NaN at 0x7F/0xFF).
+ *  114 of 254 finite values (44.9%) fall in [−1, +1]. */
+typedef unsigned char nk_e4m3_t;
+/** @brief 8-bit E5M2 float (OCP FP8): sign(1) + exponent(5) + mantissa(2), bias=15.
+ *  Range: ±57 344, supports infinities at 0x7C/0xFC.
+ *  122 of 248 finite values (49.2%) fall in [−1, +1]. */
+typedef unsigned char nk_e5m2_t;
+/** @brief 6-bit E2M3 micro-float (OCP MX v1.0): sign(1) + exponent(2) + mantissa(3), bias=1.
+ *  Range: ±7.5, no infinities or NaN. Only 64 total codes; 18 (28.1%) fall in [−1, +1]. */
+typedef unsigned char nk_e2m3_t;
+/** @brief 6-bit E3M2 micro-float (OCP MX v1.0): sign(1) + exponent(3) + mantissa(2), bias=3.
+ *  Range: ±28, supports infinities. Only 64 total codes; 26 (40.6%) fall in [−1, +1]. */
+typedef unsigned char nk_e3m2_t;
+
+/** @brief Signed 8-bit integer. Range: [−128, +127]. */
+typedef signed char nk_i8_t;
+/** @brief Unsigned 8-bit integer. Range: [0, 255]. */
+typedef unsigned char nk_u8_t;
+/** @brief Signed 16-bit integer. Range: [−32 768, +32 767]. */
+typedef signed short nk_i16_t;
+/** @brief Unsigned 16-bit integer. Range: [0, 65 535]. */
+typedef unsigned short nk_u16_t;
+/** @brief Signed 32-bit integer. Range: [−2³¹, +2³¹−1]. */
+typedef signed int nk_i32_t;
+/** @brief Unsigned 32-bit integer. Range: [0, 2³²−1]. */
+typedef unsigned int nk_u32_t;
+/*  On LP64 targets (Linux ARM64, RISC-V 64), `long` and `long long` are both 64-bit but distinct types.
+ *  NEON/RVV intrinsics on Linux expect `long*`, while Apple's NEON intrinsics expect `long long*`.
+ *  Windows uses LLP64 where `long` is 32-bit, so it must use `long long` for 64-bit types. */
+#if ((NK_TARGET_ARM_ && !defined(NK_DEFINED_APPLE_)) || NK_TARGET_RISCV_) && !defined(NK_DEFINED_WINDOWS_)
+/** @brief Signed 64-bit integer. Range: [−2⁶³, +2⁶³−1]. */
+typedef signed long nk_i64_t;
+/** @brief Unsigned 64-bit integer. Range: [0, 2⁶⁴−1]. */
+typedef unsigned long nk_u64_t;
+#else
+/** @brief Signed 64-bit integer. Range: [−2⁶³, +2⁶³−1]. */
+typedef signed long long nk_i64_t;
+/** @brief Unsigned 64-bit integer. Range: [0, 2⁶⁴−1]. */
+typedef unsigned long long nk_u64_t;
+#endif
+
+/** @brief Single-precision (32-bit) IEEE 754 float. sign(1) + exponent(8) + mantissa(23), bias=127. */
+typedef float nk_f32_t;
+/** @brief Double-precision (64-bit) IEEE 754 float. sign(1) + exponent(11) + mantissa(52), bias=1023. */
+typedef double nk_f64_t;
+
+#if NK_TARGET_X86_ || NK_TARGET_ARM_ || NK_TARGET_RISCV_
+#define NK_IS_64BIT_ 1
+#else
+#define NK_IS_64BIT_ 0
+#endif
+
+#if NK_IS_64BIT_
+typedef nk_u64_t nk_size_t;
+typedef nk_i64_t nk_ssize_t;
+#else
+typedef nk_u32_t nk_size_t;
+typedef nk_i32_t nk_ssize_t;
+#endif
+typedef nk_f64_t nk_fmax_t;
+
+#define NK_SIZE_MAX ((nk_size_t) - 1)
+
+#define NK_F64_MAX 1.7976931348623157e+308
+#define NK_F64_MIN (-1.7976931348623157e+308)
+#define NK_F32_MAX 3.402823466e+38f
+#define NK_F32_MIN (-3.402823466e+38f)
+
+#define NK_I64_MAX 9223372036854775807LL
+#define NK_I64_MIN (-9223372036854775807LL - 1LL)
+#define NK_U64_MAX 18446744073709551615ULL
+#define NK_U64_MIN 0x0ULL
+
+#define NK_I32_MAX 2147483647
+#define NK_I32_MIN (-2147483647 - 1)
+#define NK_U32_MAX 4294967295U
+#define NK_U32_MIN 0x0U
+
+#define NK_I16_MAX 32767
+#define NK_I16_MIN (-32767 - 1)
+#define NK_U16_MAX 65535U
+#define NK_U16_MIN 0x0U
+
+#define NK_I8_MAX 127
+#define NK_I8_MIN (-127 - 1)
+#define NK_U8_MAX 255U
+#define NK_U8_MIN 0x0U
+
+#define NK_F16_MAX 0x7BFF // IEEE 754 binary16: +65504.0
+#define NK_F16_MIN 0xFBFF // IEEE 754 binary16: -65504.0
+
+#define NK_BF16_MAX 0x7F7F // BFloat16: ~+3.39e38
+#define NK_BF16_MIN 0xFF7F // BFloat16: ~-3.39e38
+
+#define NK_E4M3_MAX 0x7E // FP8 E4M3: +448.0
+#define NK_E4M3_MIN 0xFE // FP8 E4M3: -448.0
+
+#define NK_E5M2_MAX 0x7B // FP8 E5M2: +57344.0
+#define NK_E5M2_MIN 0xFB // FP8 E5M2: -57344.0
+
+#define NK_E2M3_MAX 0x1F // FP6 E2M3: +7.5
+#define NK_E2M3_MIN 0x3F // FP6 E2M3: -7.5
+
+#define NK_E3M2_MAX 0x1F // FP6 E3M2: +28.0
+#define NK_E3M2_MIN 0x3F // FP6 E3M2: -28.0
+
+#define NK_BITS_PER_BYTE 8
+
+/**
+ *  @brief  Enumeration of supported scalar data types.
+ *
+ *  Includes complex type descriptors which in C code would use the real counterparts,
+ *  but the independent flags contain metadata to be passed between programming language
+ *  interfaces.
+ */
+typedef enum {
+    nk_dtype_unknown_k = 0, ///< Unknown data type
+    nk_u1_k = 1 << 1,       ///< Single-bit values packed into 8-bit words
+
+    nk_i8_k = 1 << 2,  ///< 8-bit signed integer
+    nk_i16_k = 1 << 3, ///< 16-bit signed integer
+    nk_i32_k = 1 << 4, ///< 32-bit signed integer
+    nk_i64_k = 1 << 5, ///< 64-bit signed integer
+
+    nk_u8_k = 1 << 6,  ///< 8-bit unsigned integer
+    nk_u16_k = 1 << 7, ///< 16-bit unsigned integer
+    nk_u32_k = 1 << 8, ///< 32-bit unsigned integer
+    nk_u64_k = 1 << 9, ///< 64-bit unsigned integer
+
+    nk_f64_k = 1 << 10,  ///< Double precision floating point
+    nk_f32_k = 1 << 11,  ///< Single precision floating point
+    nk_f16_k = 1 << 12,  ///< Half precision floating point
+    nk_bf16_k = 1 << 13, ///< Brain floating point
+
+    nk_e4m3_k = 1 << 14, ///< FP8 E4M3 floating point
+    nk_e5m2_k = 1 << 15, ///< FP8 E5M2 floating point
+    nk_i4_k = 1 << 16,   ///< 4-bit signed integers packed into 8-bit words
+    nk_u4_k = 1 << 17,   ///< 4-bit unsigned integers packed into 8-bit words
+    nk_e2m3_k = 1 << 18, ///< FP6 E2M3 floating point
+    nk_e3m2_k = 1 << 19, ///< FP6 E3M2 floating point
+
+    nk_f64c_k = 1 << 20,  ///< Complex double precision floating point
+    nk_f32c_k = 1 << 21,  ///< Complex single precision floating point
+    nk_f16c_k = 1 << 22,  ///< Complex half precision floating point
+    nk_bf16c_k = 1 << 23, ///< Complex brain floating point
+} nk_dtype_t;
+
+typedef enum {
+    nk_dtype_family_unknown_k = 0,
+    nk_dtype_family_float_k,
+    nk_dtype_family_complex_float_k,
+    nk_dtype_family_int_k,
+    nk_dtype_family_uint_k,
+} nk_dtype_family_t;
+
+/** @brief Classifies the family of the dtype. */
+NK_PUBLIC nk_dtype_family_t nk_dtype_family(nk_dtype_t dtype) {
+    switch (dtype) {
+    case nk_f64_k: return nk_dtype_family_float_k;
+    case nk_f32_k: return nk_dtype_family_float_k;
+    case nk_f16_k: return nk_dtype_family_float_k;
+    case nk_bf16_k: return nk_dtype_family_float_k;
+    case nk_e4m3_k: return nk_dtype_family_float_k;
+    case nk_e5m2_k: return nk_dtype_family_float_k;
+    case nk_e2m3_k: return nk_dtype_family_float_k;
+    case nk_e3m2_k: return nk_dtype_family_float_k;
+    case nk_f64c_k: return nk_dtype_family_complex_float_k;
+    case nk_f32c_k: return nk_dtype_family_complex_float_k;
+    case nk_f16c_k: return nk_dtype_family_complex_float_k;
+    case nk_bf16c_k: return nk_dtype_family_complex_float_k;
+    case nk_u1_k: return nk_dtype_family_uint_k;
+    case nk_u4_k: return nk_dtype_family_uint_k;
+    case nk_u8_k: return nk_dtype_family_uint_k;
+    case nk_u16_k: return nk_dtype_family_uint_k;
+    case nk_u32_k: return nk_dtype_family_uint_k;
+    case nk_u64_k: return nk_dtype_family_uint_k;
+    case nk_i4_k: return nk_dtype_family_int_k;
+    case nk_i8_k: return nk_dtype_family_int_k;
+    case nk_i16_k: return nk_dtype_family_int_k;
+    case nk_i32_k: return nk_dtype_family_int_k;
+    case nk_i64_k: return nk_dtype_family_int_k;
+    default: return nk_dtype_family_unknown_k;
+    }
+}
+
+/** @brief Returns the number of bits in a single scalar of a given type. */
+NK_PUBLIC nk_size_t nk_dtype_bits(nk_dtype_t dtype) {
+    switch (dtype) {
+    case nk_f64_k: return 64;
+    case nk_f32_k: return 32;
+    case nk_f16_k: return 16;
+    case nk_bf16_k: return 16;
+    case nk_e4m3_k: return 8;
+    case nk_e5m2_k: return 8;
+    case nk_e2m3_k: return 8;
+    case nk_e3m2_k: return 8;
+    case nk_f64c_k: return 128;
+    case nk_f32c_k: return 64;
+    case nk_f16c_k: return 32;
+    case nk_bf16c_k: return 32;
+    case nk_u1_k: return 1;
+    case nk_u4_k: return 4;
+    case nk_u8_k: return 8;
+    case nk_u16_k: return 16;
+    case nk_u32_k: return 32;
+    case nk_u64_k: return 64;
+    case nk_i4_k: return 4;
+    case nk_i8_k: return 8;
+    case nk_i16_k: return 16;
+    case nk_i32_k: return 32;
+    case nk_i64_k: return 64;
+    default: return 0;
+    }
+}
+
+/** @brief Returns how many logical dimensions are packed into one storage value.
+ *  For sub-byte types multiple dimensions share a single byte container.
+ *  For byte-or-larger types this is always 1. */
+NK_PUBLIC nk_size_t nk_dtype_dimensions_per_value(nk_dtype_t dtype) {
+    switch (dtype) {
+    case nk_u1_k: return 8;
+    case nk_i4_k: return 2;
+    case nk_u4_k: return 2;
+    default: return 1;
+    }
+}
+
+/** @brief Half-precision (16-bit) IEEE 754 float.
+ *
+ *  Layout: sign(1) + exponent(5) + mantissa(10), bias=15.
+ *  Range: ±65 504, epsilon at 1.0 ≈ 9.77×10⁻⁴. 30 722 of 63 488 finite values (48.4%) in [−1, +1].
+ *
+ *  - GCC or Clang on 64-bit Arm: `__fp16`, may require `-mfp16-format` option.
+ *  - GCC or Clang on 64-bit x86: `_Float16`.
+ *  - Default: `unsigned short`.
+ */
+#if !defined(NK_NATIVE_F16) || NK_NATIVE_F16
+#if (defined(__GNUC__) || defined(__clang__)) && (defined(__ARM_ARCH) || defined(__aarch64__)) && \
+    (defined(__ARM_FP16_FORMAT_IEEE))
+#undef NK_NATIVE_F16
+#define NK_NATIVE_F16 1
+typedef __fp16 nk_f16_t;
+#elif ((defined(__GNUC__) || defined(__clang__)) && (defined(__x86_64__) || defined(__i386__)) && \
+       (defined(__AVX512FP16__)))
+typedef _Float16 nk_f16_t;
+#undef NK_NATIVE_F16
+#define NK_NATIVE_F16 1
+#else // Unknown compiler or architecture
+#undef NK_NATIVE_F16
+#define NK_NATIVE_F16 0
+#endif // Unknown compiler or architecture
+#endif // !NK_NATIVE_F16
+
+#if !NK_NATIVE_F16
+typedef unsigned short nk_f16_t;
+#endif
+
+#if !defined(NK_NATIVE_BF16) || NK_NATIVE_BF16
+/** @brief BFloat16 (16-bit) float — truncated IEEE 754 single-precision.
+ *
+ *  Layout: sign(1) + exponent(8) + mantissa(7), bias=127.
+ *  Same dynamic range as f32, epsilon ≈ 7.81×10⁻³.
+ *  32 514 of 65 280 finite values (49.8%) in [−1, +1]. Wider range than f16 but lower precision.
+ *
+ *  - GCC or Clang: `__bf16`
+ *  - Default: `unsigned short`.
+ *
+ *  The compilers have added `__bf16` support in compliance with the x86-64 psABI spec.
+ *  The motivation for this new special type is summed up as:
+ *
+ *      Currently `__bfloat16` is a typedef of short, which creates a problem where the
+ *      compiler does not raise any alarms if it is used to add, subtract, multiply or
+ *      divide, but the result of the calculation is actually meaningless.
+ *      To solve this problem, a real scalar type `__Bfloat16` needs to be introduced.
+ *      It is mainly used for intrinsics, not available for C standard operators.
+ *      `__Bfloat16` will also be used for movement like passing parameter, load and store,
+ *      vector initialization, vector shuffle, and etc. It creates a need for a
+ *      corresponding psABI.
+ *
+ *  @warning Apple Clang has hard time with bf16.
+ *  https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms
+ *  https://forums.developer.apple.com/forums/thread/726201
+ *  https://www.phoronix.com/news/GCC-LLVM-bf16-BFloat16-Type
+ */
+#if (defined(__GNUC__) || defined(__clang__)) && ((defined(__ARM_BF16_FORMAT_ALTERNATIVE)) || (defined(__AVX512BF16__)))
+#undef NK_NATIVE_BF16
+#define NK_NATIVE_BF16 1
+typedef __bf16 nk_bf16_t;
+#else // Unknown compiler or architecture
+#undef NK_NATIVE_BF16
+#define NK_NATIVE_BF16 0
+#endif // Unknown compiler or architecture
+#endif // !NK_NATIVE_BF16
+
+#if !NK_NATIVE_BF16
+typedef unsigned short nk_bf16_t;
+#endif
+
+/**
+ *  @brief  Alias for the half-precision floating-point type on Arm.
+ *
+ *  Clang and GCC bring the `float16_t` symbol when you compile for Aarch64.
+ *  MSVC lacks it, and it's `vld1_f16`-like intrinsics are in reality macros,
+ *  that cast to 16-bit integers internally, instead of using floats.
+ *  Some of those are defined as aliases, so we use `#define` preprocessor
+ *  directives instead of `typedef` to avoid errors.
+ */
+#if NK_TARGET_ARM_
+#if defined(_MSC_VER)
+#define nk_f16_for_arm_simd_t  nk_f16_t
+#define nk_bf16_for_arm_simd_t nk_bf16_t
+#else
+#define nk_f16_for_arm_simd_t  float16_t
+#define nk_bf16_for_arm_simd_t bfloat16_t
+#endif
+#endif
+
+/**
+ *  RISC-V Vector (RVV) intrinsics use `_Float16` for half-precision floats.
+ *  This is the standard C23 type, also available in GCC/Clang with RVV extensions.
+ */
+#if NK_TARGET_RISCV_
+#define nk_f16_for_rvv_intrinsics_t _Float16
+#endif
+
+/*
+ *  Let's make sure the sizes of the types are as expected.
+ *  In C the `_Static_assert` is only available with C11 and later.
+ */
+#define NK_STATIC_ASSERT(cond, msg) typedef char static_assertion_##msg[(cond) ? 1 : -1]
+NK_STATIC_ASSERT(sizeof(nk_u1x8_t) == 1, nk_u1x8_t_must_be_1_byte);
+NK_STATIC_ASSERT(sizeof(nk_i4x2_t) == 1, nk_i4_t_must_be_1_byte);
+NK_STATIC_ASSERT(sizeof(nk_u4x2_t) == 1, nk_u4_t_must_be_1_byte);
+NK_STATIC_ASSERT(sizeof(nk_e4m3_t) == 1, nk_e4m3_t_must_be_1_byte);
+NK_STATIC_ASSERT(sizeof(nk_e5m2_t) == 1, nk_e5m2_t_must_be_1_byte);
+NK_STATIC_ASSERT(sizeof(nk_i8_t) == 1, nk_i8_t_must_be_1_byte);
+NK_STATIC_ASSERT(sizeof(nk_u8_t) == 1, nk_u8_t_must_be_1_byte);
+NK_STATIC_ASSERT(sizeof(nk_i16_t) == 2, nk_i16_t_must_be_2_bytes);
+NK_STATIC_ASSERT(sizeof(nk_u16_t) == 2, nk_u16_t_must_be_2_bytes);
+NK_STATIC_ASSERT(sizeof(nk_i32_t) == 4, nk_i32_t_must_be_4_bytes);
+NK_STATIC_ASSERT(sizeof(nk_u32_t) == 4, nk_u32_t_must_be_4_bytes);
+NK_STATIC_ASSERT(sizeof(nk_i64_t) == 8, nk_i64_t_must_be_8_bytes);
+NK_STATIC_ASSERT(sizeof(nk_u64_t) == 8, nk_u64_t_must_be_8_bytes);
+NK_STATIC_ASSERT(sizeof(nk_f32_t) == 4, nk_f32_t_must_be_4_bytes);
+NK_STATIC_ASSERT(sizeof(nk_f64_t) == 8, nk_f64_t_must_be_8_bytes);
+NK_STATIC_ASSERT(sizeof(nk_f16_t) == 2, nk_f16_t_must_be_2_bytes);
+NK_STATIC_ASSERT(sizeof(nk_bf16_t) == 2, nk_bf16_t_must_be_2_bytes);
+
+#define nk_assign_from_to_(src, dest) (*(dest) = *(src))
+
+/** @brief 16-bit union for f16/bf16/u16/i16 bit manipulation. */
+typedef union {
+    nk_u16_t u;
+    nk_i16_t i;
+    nk_f16_t f;
+    nk_bf16_t bf;
+} nk_fui16_t;
+
+/** @brief 32-bit union for f32/u32/i32 bit manipulation. */
+typedef union {
+    nk_u32_t u;
+    nk_i32_t i;
+    nk_f32_t f;
+} nk_fui32_t;
+
+/** @brief 64-bit union for f64/u64/i64 bit manipulation. */
+typedef union {
+    nk_u64_t u;
+    nk_i64_t i;
+    nk_f64_t f;
+} nk_fui64_t;
+
+/** @brief Half-precision (32-bit) complex number — {real: f16, imag: f16}. Kernel outputs widened to f32c. */
+typedef struct {
+    nk_f16_t real;
+    nk_f16_t imag;
+} nk_f16c_t;
+
+/** @brief BFloat16 (32-bit) complex number — {real: bf16, imag: bf16}. Kernel outputs widened to f32c. */
+typedef struct {
+    nk_bf16_t real;
+    nk_bf16_t imag;
+} nk_bf16c_t;
+
+/** @brief Single-precision (64-bit) complex number — {real: f32, imag: f32}. */
+typedef struct {
+    nk_f32_t real;
+    nk_f32_t imag;
+} nk_f32c_t;
+
+/** @brief Double-precision (128-bit) complex number — {real: f64, imag: f64}. */
+typedef struct {
+    nk_f64_t real;
+    nk_f64_t imag;
+} nk_f64c_t;
+
+/** @brief  Small 4-byte memory slice viewable as different types. */
+typedef union nk_b32_vec_t {
+    nk_u32_t u32;
+    nk_i32_t i32;
+    nk_f32_t f32;
+    nk_u8_t u8s[4];
+    nk_i8_t i8s[4];
+    nk_u16_t u16s[2];
+    nk_i16_t i16s[2];
+    nk_e4m3_t e4m3s[4];
+    nk_e5m2_t e5m2s[4];
+} nk_b32_vec_t;
+
+/** @brief  Small 8-byte memory slice viewable as different types. */
+typedef union nk_b64_vec_t {
+#if NK_TARGET_NEON
+    uint8x8_t u8x8;
+    uint16x4_t u16x4;
+    uint32x2_t u32x2;
+    int8x8_t i8x8;
+    int16x4_t i16x4;
+    int32x2_t i32x2;
+    float32x2_t f32x2;
+#endif
+#if NK_TARGET_NEONHALF
+    float16x4_t f16x4;
+#endif
+    nk_u8_t u8s[8];
+    nk_u16_t u16s[4];
+    nk_u32_t u32s[2];
+    nk_u64_t u64;
+    nk_i8_t i8s[8];
+    nk_i16_t i16s[4];
+    nk_i32_t i32s[2];
+    nk_i64_t i64;
+    nk_f16_t f16s[4];
+    nk_bf16_t bf16s[4];
+    nk_f32_t f32s[2];
+} nk_b64_vec_t;
+
+/** @brief  Small 16-byte memory slice viewable as different types. */
+typedef union nk_b128_vec_t {
+#if NK_TARGET_HASWELL
+    __m128i xmm;
+    __m128d xmm_pd;
+    __m128 xmm_ps;
+#endif
+#if NK_TARGET_V128RELAXED
+    v128_t v128;
+#endif
+#if NK_TARGET_NEON
+    uint8x16_t u8x16;
+    uint16x8_t u16x8;
+    uint32x4_t u32x4;
+    uint64x2_t u64x2;
+    int8x16_t i8x16;
+    int16x8_t i16x8;
+    int32x4_t i32x4;
+    int64x2_t i64x2;
+    float32x4_t f32x4;
+    float64x2_t f64x2;
+#endif
+    nk_u8_t u8s[16];
+    nk_u16_t u16s[8];
+    nk_u32_t u32s[4];
+    nk_u64_t u64s[2];
+    nk_i8_t i8s[16];
+    nk_i16_t i16s[8];
+    nk_i32_t i32s[4];
+    nk_i64_t i64s[2];
+    nk_f16_t f16s[8];
+    nk_bf16_t bf16s[8];
+    nk_e4m3_t e4m3s[16];
+    nk_e5m2_t e5m2s[16];
+    nk_e2m3_t e2m3s[16];
+    nk_e3m2_t e3m2s[16];
+    nk_f32_t f32s[4];
+    nk_f64_t f64s[2];
+} nk_b128_vec_t;
+
+/** @brief  Small 32-byte memory slice viewable as different types. */
+typedef union nk_b256_vec_t {
+#if NK_TARGET_HASWELL
+    __m256i ymm;
+    __m256d ymm_pd;
+    __m256 ymm_ps;
+    __m128i xmms[2];
+#endif
+#if NK_TARGET_V128RELAXED
+    v128_t v128s[2];
+#endif
+#if NK_TARGET_NEON
+    uint8x16_t u8x16s[2];
+    uint16x8_t u16x8s[2];
+    uint32x4_t u32x4s[2];
+    uint64x2_t u64x2s[2];
+    int8x16_t i8x16s[2];
+    int16x8_t i16x8s[2];
+    int32x4_t i32x4s[2];
+    int64x2_t i64x2s[2];
+    float32x4_t f32x4s[2];
+    float64x2_t f64x2s[2];
+#endif
+    nk_u8_t u8s[32];
+    nk_u16_t u16s[16];
+    nk_u32_t u32s[8];
+    nk_u64_t u64s[4];
+    nk_i8_t i8s[32];
+    nk_i16_t i16s[16];
+    nk_i32_t i32s[8];
+    nk_i64_t i64s[4];
+    nk_f16_t f16s[16];
+    nk_bf16_t bf16s[16];
+    nk_e4m3_t e4m3s[32];
+    nk_e5m2_t e5m2s[32];
+    nk_e2m3_t e2m3s[32];
+    nk_e3m2_t e3m2s[32];
+    nk_f32_t f32s[8];
+    nk_f64_t f64s[4];
+} nk_b256_vec_t;
+
+/** @brief  Small 64-byte memory slice viewable as different types.
+ *
+ *  TODO: On GCC and Clang we use `__transparent_union__` attribute to allow implicit conversions
+ *  between the different vector types when passing them as function arguments. The most important side-effect
+ *  of this is that the argument of such type is passed to functions using the calling convention of the first
+ *  member of the union, which in our case is a register-based calling convention for SIMD types.
+ */
+typedef union nk_b512_vec_t {
+#if NK_TARGET_SKYLAKE
+    __m512i zmm;
+    __m512d zmm_pd;
+    __m512 zmm_ps;
+#endif
+#if NK_TARGET_HASWELL
+    __m256i ymms[2];
+    __m256d ymms_pd[2];
+    __m256 ymms_ps[2];
+    __m128i xmms[4];
+    __m128d xmms_pd[4];
+    __m128 xmms_ps[4];
+#endif
+#if NK_TARGET_NEON
+    uint8x16_t u8x16s[4];
+    uint16x8_t u16x8s[4];
+    uint32x4_t u32x4s[4];
+    uint64x2_t u64x2s[4];
+#endif
+    nk_u8_t u8s[64];
+    nk_u16_t u16s[32];
+    nk_u32_t u32s[16];
+    nk_u64_t u64s[8];
+    nk_i8_t i8s[64];
+    nk_i16_t i16s[32];
+    nk_i32_t i32s[16];
+    nk_i64_t i64s[8];
+    nk_f16_t f16s[32];
+    nk_bf16_t bf16s[32];
+    nk_f32_t f32s[16];
+    nk_f64_t f64s[8];
+    nk_e4m3_t e4m3s[64];
+    nk_e5m2_t e5m2s[64];
+    nk_e2m3_t e2m3s[64];
+    nk_e3m2_t e3m2s[64];
+} nk_b512_vec_t;
+
+/**
+ *  @brief Advances the Multi-Dimensional iterator to the next set of indicies.
+ *  @param[in] extents The extents of the tensor, defined by an array of at least `rank` scalars.
+ *  @param[in] strides The @b signed strides of the tensor in bytes, defined by an array of at least `rank` scalars.
+ *  @param[in] rank The number of dimensions in the tensor (its rank).
+ *  @param[inout] coordinates The array of offsets along each of `rank` dimensions, which will be updated.
+ *  @param[inout] byte_offset The @b signed byte offset of the current element, which will be advanced.
+ *  @return 1 if the iterator was successfully advanced, 0 if the end of iteration was reached.
+ *
+ *  For flexibility, the API is decoupled from from the `nk_tensor_position_t` structure, and
+ *  can be used on any-rank tensors, independent of the `NK_TENSOR_MAX_RANK` constant.
+ */
+NK_PUBLIC int nk_tensor_position_next(                                   //
+    nk_size_t const *extents, nk_ssize_t const *strides, nk_size_t rank, //
+    nk_size_t *coordinates, nk_ssize_t *byte_offset) {
+    // Start from last dimension and move backward
+    for (nk_size_t i = rank; i-- > 0;) {
+        coordinates[i]++;
+        *byte_offset += strides[i];
+        if (coordinates[i] < extents[i]) return 1; // Successfully moved to the next index
+        coordinates[i] = 0;                        // Reset this dimension counter
+        *byte_offset -= strides[i] * extents[i];   // Discard the running progress along this dimension
+    }
+    // If we reach here, we've iterated over all elements
+    return 0; // End of iteration
+}
+
+/**
+ *  @brief Advances the Multi-Dimensional iterator to the provided coordinates, updating the byte offset.
+ *  @param[in] extents The extents of the tensor, defined by an array of at least `rank` scalars.
+ *  @param[in] strides The @b signed strides of the tensor in bytes, defined by an array of at least `rank` scalars.
+ *  @param[in] rank The number of dimensions in the tensor (its rank).
+ *  @param[in] coordinates The array of offsets along each of `rank` dimensions, which will be updated.
+ *  @param[out] byte_offset The byte offset of the current element, which will be advanced.
+ *  @return 1 if the offset was successfully advanced, 0 if the end of iteration was reached.
+ */
+NK_PUBLIC int nk_tensor_position_linearize(                              //
+    nk_size_t const *extents, nk_ssize_t const *strides, nk_size_t rank, //
+    nk_size_t const *coordinates, nk_ssize_t *byte_offset) {
+
+    nk_ssize_t result = 0;
+    for (nk_size_t i = 0; i < rank; i++) {
+        // Ensure the coordinates is within bounds for the given dimension
+        if (coordinates[i] >= extents[i]) return 0; // Invalid coordinates, out of bounds
+        // Update the byte offset by multiplying the coordinates by the stride
+        result += coordinates[i] * strides[i];
+    }
+    *byte_offset = result;
+    return 1; // Successfully calculated global and byte offsets
+}
+
+/**
+ *  @brief  A @b beefy structure to iterate through Multi-Dimensional arrays.
+ *          Occupies 512 + 8 = 520 bytes on a 64-bit machine, or @b 9 cache-lines, by default.
+ *
+ *  When advancing through a structure, its overall size and strides should be stored somewhere else.
+ *  The `byte_offset` starts at zero and grow monotonically during iteration, if the strides are positive.
+ */
+typedef struct nk_tensor_position_t {
+    nk_size_t coordinates[NK_TENSOR_MAX_RANK]; // Coordinate offsets along each dimension
+    nk_ssize_t byte_offset;                    // Byte offset of the current element
+} nk_tensor_position_t;
+
+NK_PUBLIC void nk_tensor_position_init(nk_tensor_position_t *tensor_position) {
+    for (nk_size_t i = 0; i < NK_TENSOR_MAX_RANK; i++) tensor_position->coordinates[i] = 0;
+    tensor_position->byte_offset = 0;
+}
+
+/**
+ *  @brief  A @b beefy structure describing the shape and memory layout of a Multi-Dimensional array.
+ *          Similar to `md::span` in C++20 and `numpy.ndarray` in Python, but with a focus on compatibility.
+ *          Occupies 512 + 512 + 8 = 2052 bytes on a 64-bit machine, or @b 17 cache-lines, by default.
+ *
+ *  Unlike NumPy and the CPython "Buffer Protocol", we don't use `suboffsets` for pointer indirection.
+ *  The logic is that such layouts aren't friendly to conventional SIMD operations and dense matrix algorithms.
+ *  If the tensor is sparse, consider using a different data structure or a different memory layout.
+ *
+ *  Most NumKong algorithms don't work with the entire structure, but expect the fields to be passed separately.
+ *  It would also require storing the @b start-pointer and the @b dtype/item-size separately, as it's not
+ *  stored inside the structure.
+ */
+typedef struct nk_tensor_shape_t {
+    nk_size_t extents[NK_TENSOR_MAX_RANK];  /// Number of elements along each dimension
+    nk_ssize_t strides[NK_TENSOR_MAX_RANK]; /// Strides of the tensor in bytes
+    nk_size_t rank;                         /// Number of dimensions in the tensor
+} nk_tensor_shape_t;
+
+NK_PUBLIC void nk_tensor_shape_init(nk_tensor_shape_t *tensor_shape) {
+    for (nk_size_t i = 0; i < NK_TENSOR_MAX_RANK; i++) tensor_shape->extents[i] = 0, tensor_shape->strides[i] = 0;
+    tensor_shape->rank = 0;
+}
+
+NK_INTERNAL nk_u32_t nk_u32_rol(nk_u32_t x, int n) { return (x << n) | (x >> (32 - n)); }
+NK_INTERNAL nk_u16_t nk_u16_rol(nk_u16_t x, int n) { return (x << n) | (x >> (16 - n)); }
+NK_INTERNAL nk_u8_t nk_u8_rol(nk_u8_t x, int n) { return (x << n) | (x >> (8 - n)); }
+NK_INTERNAL nk_u32_t nk_u32_ror(nk_u32_t x, int n) { return (x >> n) | (x << (32 - n)); }
+NK_INTERNAL nk_u16_t nk_u16_ror(nk_u16_t x, int n) { return (x >> n) | (x << (16 - n)); }
+NK_INTERNAL nk_u8_t nk_u8_ror(nk_u8_t x, int n) { return (x >> n) | (x << (8 - n)); }
+
+/**
+ *  @brief  SWAR population count for 64-bit integers.
+ *
+ *  Classic algorithm from Hacker's Delight using parallel bit summation:
+ *  - Step 1: Count bits in pairs (2-bit sums)
+ *  - Step 2: Count bits in nibbles (4-bit sums)
+ *  - Step 3: Count bits in bytes (8-bit sums)
+ *  - Step 4: Horizontal sum via multiply - each byte contributes to bits 56-63
+ *
+ *  Cost: ~12 ALU ops, zero memory access (vs 8 table lookups for byte-wise).
+ */
+NK_INTERNAL nk_u64_t nk_u64_popcount_(nk_u64_t x) {
+    x = x - ((x >> 1) & 0x5555555555555555ull);
+    x = (x & 0x3333333333333333ull) + ((x >> 2) & 0x3333333333333333ull);
+    x = (x + (x >> 4)) & 0x0F0F0F0F0F0F0F0Full;
+    return (x * 0x0101010101010101ull) >> 56;
+}
+
+NK_INTERNAL unsigned char nk_u1x8_popcount_(nk_u1x8_t x) {
+    static unsigned char lookup_table[256] = {
+        0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, //
+        1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+        1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+        2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+        1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+        2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+        2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+        3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8};
+    return lookup_table[x];
+}
+
+/** @brief Divides the number rounding up to the next multiple of the given divisor. */
+NK_PUBLIC nk_size_t nk_size_divide_round_up_(nk_size_t number, nk_size_t divisor) NK_STREAMING_COMPATIBLE_ {
+    return (number + divisor - 1) / divisor;
+}
+
+/** @brief Rounds up the number to the next multiple of the given divisor. */
+NK_PUBLIC nk_size_t nk_size_round_up_to_multiple_(nk_size_t number, nk_size_t divisor) NK_STREAMING_COMPATIBLE_ {
+    return nk_size_divide_round_up_(number, divisor) * divisor;
+}
+
+NK_INTERNAL nk_f32_t nk_f32_abs_(nk_f32_t x) { return x < 0 ? -x : x; }
+NK_INTERNAL nk_f64_t nk_f64_abs_(nk_f64_t x) { return x < 0 ? -x : x; }
+NK_INTERNAL nk_i64_t nk_i64_abs_(nk_i64_t x) { return x < 0 ? -x : x; }
+NK_INTERNAL nk_u64_t nk_u64_abs_(nk_u64_t x) { return x; }
+NK_INTERNAL nk_i64_t nk_i32_abs_(nk_i32_t x) { return x < 0 ? -x : x; }
+NK_INTERNAL nk_u32_t nk_u32_abs_(nk_u32_t x) { return x; }
+
+/** @brief Extract low (bits 0-3) unsigned nibble from packed u4x2 byte. */
+NK_INTERNAL nk_u8_t nk_u4x2_low_(nk_u4x2_t byte_val) { return byte_val & 0x0F; }
+/** @brief Extract high (bits 4-7) unsigned nibble from packed u4x2 byte. */
+NK_INTERNAL nk_u8_t nk_u4x2_high_(nk_u4x2_t byte_val) { return (byte_val >> 4) & 0x0F; }
+
+/** @brief Extract low (bits 0-3) signed nibble from packed i4x2 byte as i8. */
+NK_INTERNAL nk_i8_t nk_i4x2_low_(nk_i4x2_t byte_val) { return (nk_i8_t)(((byte_val & 0x0F) ^ 8) - 8); }
+/** @brief Extract high (bits 4-7) signed nibble from packed i4x2 byte as i8. */
+NK_INTERNAL nk_i8_t nk_i4x2_high_(nk_i4x2_t byte_val) { return (nk_i8_t)((((byte_val >> 4) & 0x0F) ^ 8) - 8); }
+
+/** @brief Extract n-th nibble (n=0: low, n=1: high) — branchless. */
+NK_INTERNAL nk_u8_t nk_u4x2_get_(nk_u4x2_t byte_val, int n) { return (byte_val >> ((n & 1) * 4)) & 0x0F; }
+NK_INTERNAL nk_i8_t nk_i4x2_get_(nk_i4x2_t byte_val, int n) {
+    nk_u8_t nibble = (byte_val >> ((n & 1) * 4)) & 0x0F;
+    return (nk_i8_t)((nibble ^ 8) - 8);
+}
+
+/** @brief Extract bit at position n (0-7) from packed u1x8 byte. */
+NK_INTERNAL nk_u8_t nk_u1x8_get_(nk_u1x8_t byte_val, int n) { return (byte_val >> (n & 7)) & 1; }
+
+NK_INTERNAL nk_f16_t nk_f16_from_u16_(nk_u16_t bits) {
+    nk_fui16_t c;
+    c.u = bits;
+    return c.f;
+}
+NK_INTERNAL nk_bf16_t nk_bf16_from_u16_(nk_u16_t bits) {
+    nk_fui16_t c;
+    c.u = bits;
+    return c.bf;
+}
+
+/** @brief E4M3: NaN when (raw & 0x7F) == 0x7F  (two NaN values: 0x7F, 0xFF). */
+NK_INTERNAL int nk_e4m3_is_nan_(nk_e4m3_t x) { return (x & 0x7F) == 0x7F; }
+
+/** @brief E5M2: NaN when exponent=31 and mantissa!=0, i.e. (raw & 0x7F) > 0x7C.
+ *  Values: 0x7D-0x7F (positive), 0xFD-0xFF (negative). Infinity = 0x7C/0xFC is NOT NaN. */
+NK_INTERNAL int nk_e5m2_is_nan_(nk_e5m2_t x) { return (x & 0x7F) > 0x7C; }
+
+/** @brief F16: NaN when (raw & 0x7FFF) > 0x7C00. */
+NK_INTERNAL int nk_f16_is_nan_(nk_u16_t x) { return (x & 0x7FFF) > 0x7C00; }
+
+/** @brief BF16: NaN when (raw & 0x7FFF) > 0x7F80. */
+NK_INTERNAL int nk_bf16_is_nan_(nk_u16_t x) { return (x & 0x7FFF) > 0x7F80; }
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // NK_TYPES_H