numkong 7.0.0 → 7.4.1

package/include/numkong/curved.h

@@ -40,7 +40,7 @@
  *
  *  @section references References
  *
- *  - x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
+ *  - x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html
  *  - Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
  *  - Neumaier, A. (1974). "Rundungsfehleranalyse einiger Verfahren zur Summation endlicher Summen"
  *  - Ogita, T., Rump, S.M., Oishi, S. (2005). "Accurate Sum and Dot Product"
@@ -173,19 +173,16 @@ NK_PUBLIC void nk_bilinear_f32c_neon(nk_f32c_t const *a, nk_f32c_t const *b, nk_
 /** @copydoc nk_mahalanobis_f32 */
 NK_PUBLIC void nk_mahalanobis_f32_neon(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t n,
                                        nk_f64_t *result);
-#endif // NK_TARGET_NEON
-
-#if NK_TARGET_NEONHALF
 /** @copydoc nk_bilinear_f16 */
-NK_PUBLIC void nk_bilinear_f16_neonhalf(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n,
-                                        nk_f32_t *result);
+NK_PUBLIC void nk_bilinear_f16_neon(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n,
+                                    nk_f32_t *result);
 /** @copydoc nk_bilinear_f16c */
-NK_PUBLIC void nk_bilinear_f16c_neonhalf(nk_f16c_t const *a, nk_f16c_t const *b, nk_f16c_t const *c, nk_size_t n,
-                                         nk_f32c_t *results);
+NK_PUBLIC void nk_bilinear_f16c_neon(nk_f16c_t const *a, nk_f16c_t const *b, nk_f16c_t const *c, nk_size_t n,
+                                     nk_f32c_t *results);
 /** @copydoc nk_mahalanobis_f16 */
-NK_PUBLIC void nk_mahalanobis_f16_neonhalf(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n,
-                                           nk_f32_t *result);
-#endif // NK_TARGET_NEONHALF
+NK_PUBLIC void nk_mahalanobis_f16_neon(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n,
+                                       nk_f32_t *result);
+#endif // NK_TARGET_NEON
 
 #if NK_TARGET_NEONBFDOT
 /** @copydoc nk_bilinear_bf16 */
@@ -337,7 +334,6 @@ NK_INTERNAL nk_dtype_t nk_mahalanobis_output_dtype(nk_dtype_t dtype) {
 
 #include "numkong/curved/serial.h"
 #include "numkong/curved/neon.h"
-#include "numkong/curved/neonhalf.h"
 #include "numkong/curved/neonbfdot.h"
 #include "numkong/curved/smef64.h"
 #include "numkong/curved/haswell.h"
@@ -382,8 +378,8 @@ NK_PUBLIC void nk_bilinear_f32(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t co
 NK_PUBLIC void nk_bilinear_f16(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n, nk_f32_t *result) {
 #if NK_TARGET_HASWELL
     nk_bilinear_f16_haswell(a, b, c, n, result);
-#elif NK_TARGET_NEONHALF
-    nk_bilinear_f16_neonhalf(a, b, c, n, result);
+#elif NK_TARGET_NEON
+    nk_bilinear_f16_neon(a, b, c, n, result);
 #elif NK_TARGET_RVV
     nk_bilinear_f16_rvv(a, b, c, n, result);
 #else
@@ -432,8 +428,8 @@ NK_PUBLIC void nk_bilinear_f32c(nk_f32c_t const *a, nk_f32c_t const *b, nk_f32c_
 
 NK_PUBLIC void nk_bilinear_f16c(nk_f16c_t const *a, nk_f16c_t const *b, nk_f16c_t const *c, nk_size_t n,
                                 nk_f32c_t *results) {
-#if NK_TARGET_NEONHALF
-    nk_bilinear_f16c_neonhalf(a, b, c, n, results);
+#if NK_TARGET_NEON
+    nk_bilinear_f16c_neon(a, b, c, n, results);
 #else
     nk_bilinear_f16c_serial(a, b, c, n, results);
 #endif
@@ -484,8 +480,8 @@ NK_PUBLIC void nk_mahalanobis_f16(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t
                                   nk_f32_t *result) {
 #if NK_TARGET_HASWELL
     nk_mahalanobis_f16_haswell(a, b, c, n, result);
-#elif NK_TARGET_NEONHALF
-    nk_mahalanobis_f16_neonhalf(a, b, c, n, result);
+#elif NK_TARGET_NEON
+    nk_mahalanobis_f16_neon(a, b, c, n, result);
 #elif NK_TARGET_RVV
     nk_mahalanobis_f16_rvv(a, b, c, n, result);
 #else

package/include/numkong/dot/README.md

@@ -1,24 +1,24 @@
 # Vector-Vector Dot Products in NumKong
 
-NumKong implements dot products for every numeric type supported by the library, as the most important building block of higher-level functionality for vectors and higher rank tensors.
+NumKong implements dot products for every numeric type supported by the library, as a core building block of higher-level functionality for vectors and higher rank tensors.
 
 Dot product for real numbers and integers is defined as:
 
-```math
+$$
 \text{dot}(a, b) = \sum_{i=0}^{n-1} a_i \cdot b_i
-```
+$$
 
 For complex numbers, the dot product expands via the distributive property of complex multiplication:
 
-```math
+$$
 \text{dot}(a, b) = \sum_{i=0}^{n-1} (a_{i,re} \cdot b_{i,re} - a_{i,im} \cdot b_{i,im}) + j \sum_{i=0}^{n-1} (a_{i,re} \cdot b_{i,im} + a_{i,im} \cdot b_{i,re})
-```
+$$
 
 The conjugate dot product negates the imaginary part of $b$:
 
-```math
+$$
 \text{vdot}(a, b) = \sum_{i=0}^{n-1} a_i \cdot \bar{b_i} = \sum_{i=0}^{n-1} (a_{i,re} \cdot b_{i,re} + a_{i,im} \cdot b_{i,im}) + j \sum_{i=0}^{n-1} (a_{i,im} \cdot b_{i,re} - a_{i,re} \cdot b_{i,im})
-```
+$$
 
 Where $\bar{b_i}$ is the complex conjugate of $b_i$.
 Reformulating as Python pseudocode for interleaved real/imaginary scalar arrays:
@@ -97,9 +97,19 @@ For Int8 × Int8, one operand is XORed with `0x80` to shift to unsigned, and the
 For Int8 × Int8, the sign bit of $b$ is cleared to produce a 7-bit value, and a windowed correction $-128 \cdot \sum_{b_i < 0} a_i$ is accumulated in Int16 and flushed every 127 iterations to prevent overflow.
 For UInt8 × UInt8, $b$ is XORed with `0x80` to shift into signed range, same as Ice Lake, with the correction $128 \cdot \sum a_i$ computed via pairwise widening adds.
 
+### Octave Decomposition for E4M3 via VNNI
+
+`nk_dot_e4m3_icelake` splits the 4-bit E4M3 exponent into 2 "octave" bits (top) and 2 "remainder" bits (bottom).
+The bottom 5 bits (2 remainder + 3 mantissa) map via `VPERMB` to u8 integers in [0, 120] — identical structure to the E2M3 $\times 16$ LUT.
+A subnormal fixup replaces LUT entries for magnitude < 8 with $2 \times \text{mantissa}$ via a second masked `VPERMB`, avoiding `VPADDB` on the VPDPBUSD execution ports.
+Sign is computed via `VPTERNLOGD` with immediate 0x14, fusing `(a \oplus b) \wedge \lnot \text{0x7F}` in one instruction.
+The 4 octave bins per operand produce $4 \times 4 = 16$ `VPDPBUSD` cross-products accumulated into 7 registers grouped by octave sum $k = o_a + o_b \in [0, 6]$.
+Each accumulator is scaled by $2^{4k-20}$ — an exact power of two, introducing no rounding.
+This processes 64 E4M3 bytes per iteration in u8, doubling the element density of the BF16 upcast path.
+
 ### Widening Fusion Through BFloat16 on x86
 
-`nk_dot_e4m3_genoa`, `nk_dot_e5m2_genoa` convert FP8 values to BF16, then accumulate via `VDPBF16PS` — repurposing Genoa's BF16 dot-product hardware for types it was never designed for.
+`nk_dot_e5m2_genoa` converts FP8 values to BF16, then accumulates via `VDPBF16PS`, reusing Genoa's BF16 dot-product instruction for FP8 types.
 Each `VDPBF16PS` fuses two BF16 multiply-adds per 32-bit lane at 6-cycle throughput.
 `nk_dot_bf16c_genoa` uses the same instruction for complex BF16, preparing operands with `VPSHUFB` for lane swapping and `VPXORD` with `0x80000000` for sign flips before feeding into `VDPBF16PS`.
 
@@ -170,66 +180,68 @@ Workloads that significantly degrade CPU frequencies (Intel AMX, Apple SME) run
 | `nk_vdot_f16c_haswell`  |      24.0 gb/s, 11.1 ulp |      20.0 gb/s, 17.4 ulp |      17.1 gb/s, 29.2 ulp |
 | __f64__                 | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `dot_f64_with_blas` 🧩   |       27.8 gb/s, 6.9 ulp |       30.1 gb/s, 9.3 ulp |        15.7 gb/s, 20 ulp |
-| `nk_dot_f64_serial`     |       5.41 gb/s, 2.2 ulp |       6.31 gb/s, 2.0 ulp |       6.77 gb/s, 3.3 ulp |
-| `nk_dot_f64_haswell`    |         21.9 gb/s, 0 ulp |         26.1 gb/s, 0 ulp |         12.4 gb/s, 0 ulp |
-| `nk_dot_f64_skylake`    |         23.9 gb/s, 0 ulp |         27.0 gb/s, 0 ulp |         16.4 gb/s, 0 ulp |
+| `nk_dot_f64_serial`     |       4.28 gb/s, 2.2 ulp |       4.39 gb/s, 2.0 ulp |       4.42 gb/s, 3.3 ulp |
+| `nk_dot_f64_haswell`    |         24.2 gb/s, 0 ulp |         25.7 gb/s, 0 ulp |         18.3 gb/s, 0 ulp |
+| `nk_dot_f64_skylake`    |         29.0 gb/s, 0 ulp |         28.6 gb/s, 0 ulp |         24.9 gb/s, 0 ulp |
 | __f32__                 | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `dot_f32_with_blas` 🧩   |        47.8 gb/s, 14 ulp |        30.7 gb/s, 14 ulp |        29.7 gb/s, 15 ulp |
-| `nk_dot_f32_serial`     |         11.3 gb/s, 0 ulp |         11.7 gb/s, 0 ulp |         10.7 gb/s, 0 ulp |
-| `nk_dot_f32_haswell`    |         28.0 gb/s, 0 ulp |         23.6 gb/s, 0 ulp |         21.3 gb/s, 0 ulp |
-| `nk_dot_f32_skylake`    |         36.3 gb/s, 0 ulp |         29.2 gb/s, 0 ulp |         23.7 gb/s, 0 ulp |
+| `nk_dot_f32_serial`     |         11.0 gb/s, 0 ulp |         11.2 gb/s, 0 ulp |         11.5 gb/s, 0 ulp |
+| `nk_dot_f32_haswell`    |         30.5 gb/s, 0 ulp |         23.9 gb/s, 0 ulp |         24.4 gb/s, 0 ulp |
+| `nk_dot_f32_skylake`    |         44.2 gb/s, 0 ulp |         29.8 gb/s, 0 ulp |         30.0 gb/s, 0 ulp |
 | __bf16__                | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_bf16_serial`    |        0.655 gb/s, 0 ulp |      0.644 gb/s, 0.6 ulp |      0.651 gb/s, 5.2 ulp |
-| `nk_dot_bf16_haswell`   |         30.1 gb/s, 0 ulp |       20.3 gb/s, 0.2 ulp |      19.4 gb/s, 41.3 ulp |
-| `nk_dot_bf16_skylake`   |         53.6 gb/s, 0 ulp |       30.4 gb/s, 0.2 ulp |      29.6 gb/s, 21.8 ulp |
-| `nk_dot_bf16_genoa`     |         88.1 gb/s, 0 ulp |       31.6 gb/s, 0.2 ulp |       31.1 gb/s, 2.2 ulp |
+| `nk_dot_bf16_serial`    |        0.633 gb/s, 0 ulp |      0.630 gb/s, 0.5 ulp |      0.638 gb/s, 5.4 ulp |
+| `nk_dot_bf16_haswell`   |         39.3 gb/s, 0 ulp |       25.5 gb/s, 0.2 ulp |      20.2 gb/s, 25.3 ulp |
+| `nk_dot_bf16_skylake`   |         62.7 gb/s, 0 ulp |       30.2 gb/s, 0.2 ulp |       29.5 gb/s, 2.3 ulp |
+| `nk_dot_bf16_genoa`     |         88.8 gb/s, 0 ulp |       29.7 gb/s, 0.2 ulp |       31.2 gb/s, 2.2 ulp |
 | __f16__                 | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f16_serial`     |      1.38 gb/s, 11.5 ulp |      1.37 gb/s, 33.7 ulp |      1.32 gb/s, 59.7 ulp |
-| `nk_dot_f16_haswell`    |       30.6 gb/s, 7.0 ulp |      23.3 gb/s, 14.0 ulp |      20.0 gb/s, 29.8 ulp |
-| `nk_dot_f16_skylake`    |       54.4 gb/s, 6.2 ulp |       31.4 gb/s, 8.6 ulp |      30.0 gb/s, 22.8 ulp |
+| `nk_dot_f16_serial`     |      1.31 gb/s, 11.5 ulp |      1.32 gb/s, 33.7 ulp |      1.30 gb/s, 59.7 ulp |
+| `nk_dot_f16_haswell`    |       31.3 gb/s, 7.0 ulp |      22.8 gb/s, 14.0 ulp |      19.8 gb/s, 29.8 ulp |
+| `nk_dot_f16_skylake`    |       54.9 gb/s, 6.2 ulp |       31.7 gb/s, 8.6 ulp |      30.9 gb/s, 22.8 ulp |
 | __e5m2__                | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e5m2_serial`    |         1.99 gb/s, 0 ulp |         2.11 gb/s, 0 ulp |         2.13 gb/s, 0 ulp |
-| `nk_dot_e5m2_haswell`   |         4.90 gb/s, 0 ulp |         4.87 gb/s, 0 ulp |         4.56 gb/s, 0 ulp |
-| `nk_dot_e5m2_skylake`   |         6.34 gb/s, 0 ulp |         6.45 gb/s, 0 ulp |         6.17 gb/s, 0 ulp |
-| `nk_dot_e5m2_genoa`     |         12.6 gb/s, 0 ulp |         12.7 gb/s, 0 ulp |         12.8 gb/s, 0 ulp |
+| `nk_dot_e5m2_serial`    |         1.90 gb/s, 0 ulp |         1.07 gb/s, 0 ulp |         1.08 gb/s, 0 ulp |
+| `nk_dot_e5m2_haswell`   |         4.92 gb/s, 0 ulp |         4.95 gb/s, 0 ulp |         4.80 gb/s, 0 ulp |
+| `nk_dot_e5m2_skylake`   |         6.20 gb/s, 0 ulp |         6.36 gb/s, 0 ulp |         6.25 gb/s, 0 ulp |
+| `nk_dot_e5m2_genoa`     |         12.1 gb/s, 0 ulp |         12.6 gb/s, 0 ulp |         12.6 gb/s, 0 ulp |
 | __e4m3__                | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e4m3_serial`    |        0.797 gb/s, 0 ulp |        0.801 gb/s, 0 ulp |        0.816 gb/s, 0 ulp |
-| `nk_dot_e4m3_haswell`   |         3.25 gb/s, 0 ulp |         3.25 gb/s, 0 ulp |         2.53 gb/s, 0 ulp |
-| `nk_dot_e4m3_skylake`   |         4.99 gb/s, 0 ulp |         5.03 gb/s, 0 ulp |         4.94 gb/s, 0 ulp |
-| `nk_dot_e4m3_genoa`     |         12.4 gb/s, 0 ulp |         13.0 gb/s, 0 ulp |         12.7 gb/s, 0 ulp |
+| `nk_dot_e4m3_serial`    |        0.762 gb/s, 0 ulp |        0.424 gb/s, 0 ulp |        0.420 gb/s, 0 ulp |
+| `nk_dot_e4m3_haswell`   |         3.78 gb/s, 0 ulp |         3.77 gb/s, 0 ulp |         3.75 gb/s, 0 ulp |
+| `nk_dot_e4m3_skylake`   |         5.10 gb/s, 0 ulp |         5.16 gb/s, 0 ulp |         5.21 gb/s, 0 ulp |
+| `nk_dot_e4m3_icelake`   |         13.2 gb/s, 0 ulp |         14.9 gb/s, 0 ulp |         14.7 gb/s, 0 ulp |
 | __e3m2__                | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e3m2_serial`    |         2.02 gb/s, 0 ulp |         2.08 gb/s, 0 ulp |         2.14 gb/s, 0 ulp |
-| `nk_dot_e3m2_haswell`   |         11.9 gb/s, 0 ulp |         12.0 gb/s, 0 ulp |         11.5 gb/s, 0 ulp |
-| `nk_dot_e3m2_icelake`   |         22.6 gb/s, 0 ulp |         24.1 gb/s, 0 ulp |         22.5 gb/s, 0 ulp |
+| `nk_dot_e3m2_serial`    |         1.47 gb/s, 0 ulp |         1.05 gb/s, 0 ulp |         1.04 gb/s, 0 ulp |
+| `nk_dot_e3m2_haswell`   |         12.0 gb/s, 0 ulp |         12.2 gb/s, 0 ulp |         12.2 gb/s, 0 ulp |
+| `nk_dot_e3m2_skylake`   |         21.6 gb/s, 0 ulp |         23.1 gb/s, 0 ulp |         23.2 gb/s, 0 ulp |
+| `nk_dot_e3m2_icelake`   |         23.1 gb/s, 0 ulp |         24.3 gb/s, 0 ulp |         23.9 gb/s, 0 ulp |
 | __e2m3__                | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e2m3_serial`    |         2.07 gb/s, 0 ulp |         2.05 gb/s, 0 ulp |         2.14 gb/s, 0 ulp |
-| `nk_dot_e2m3_haswell`   |         20.0 gb/s, 0 ulp |         19.5 gb/s, 0 ulp |         18.7 gb/s, 0 ulp |
-| `nk_dot_e2m3_icelake`   |         56.9 gb/s, 0 ulp |         43.1 gb/s, 0 ulp |         30.1 gb/s, 0 ulp |
-| `nk_dot_e2m3_alder`     |         29.8 gb/s, 0 ulp |         30.2 gb/s, 0 ulp |         25.6 gb/s, 0 ulp |
+| `nk_dot_e2m3_serial`    |         1.87 gb/s, 0 ulp |         1.25 gb/s, 0 ulp |         1.96 gb/s, 0 ulp |
+| `nk_dot_e2m3_haswell`   |         20.5 gb/s, 0 ulp |         20.4 gb/s, 0 ulp |         19.3 gb/s, 0 ulp |
+| `nk_dot_e2m3_skylake`   |         35.7 gb/s, 0 ulp |         33.2 gb/s, 0 ulp |         30.7 gb/s, 0 ulp |
+| `nk_dot_e2m3_icelake`   |         58.0 gb/s, 0 ulp |         46.0 gb/s, 0 ulp |         31.5 gb/s, 0 ulp |
+| `nk_dot_e2m3_alder`     |         29.9 gb/s, 0 ulp |         30.8 gb/s, 0 ulp |         29.1 gb/s, 0 ulp |
 | __i8__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_i8_serial`      |                17.4 gb/s |                17.2 gb/s |                16.0 gb/s |
-| `nk_dot_i8_haswell`     |                33.4 gb/s |                23.5 gb/s |                24.9 gb/s |
-| `nk_dot_i8_skylake`     |                53.6 gb/s |                39.9 gb/s |                29.7 gb/s |
-| `nk_dot_i8_icelake`     |                63.3 gb/s |                49.5 gb/s |                30.4 gb/s |
-| `nk_dot_i8_alder`       |                43.8 gb/s |                43.0 gb/s |                30.4 gb/s |
+| `nk_dot_i8_serial`      |                16.9 gb/s |                16.8 gb/s |                15.6 gb/s |
+| `nk_dot_i8_haswell`     |                43.2 gb/s |                35.8 gb/s |                29.1 gb/s |
+| `nk_dot_i8_skylake`     |                52.9 gb/s |                36.5 gb/s |                28.5 gb/s |
+| `nk_dot_i8_icelake`     |                64.0 gb/s |                46.2 gb/s |                26.8 gb/s |
+| `nk_dot_i8_alder`       |                42.8 gb/s |                40.4 gb/s |                31.1 gb/s |
 | __u8__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_u8_serial`      |                17.4 gb/s |                17.1 gb/s |                16.3 gb/s |
-| `nk_dot_u8_haswell`     |                32.2 gb/s |                37.5 gb/s |                28.3 gb/s |
-| `nk_dot_u8_skylake`     |                54.6 gb/s |                41.0 gb/s |                28.4 gb/s |
-| `nk_dot_u8_icelake`     |                74.4 gb/s |                48.4 gb/s |                30.3 gb/s |
-| `nk_dot_u8_alder`       |                54.2 gb/s |                43.9 gb/s |                32.3 gb/s |
+| `nk_dot_u8_serial`      |                16.9 gb/s |                16.5 gb/s |                15.8 gb/s |
+| `nk_dot_u8_haswell`     |                47.7 gb/s |                37.7 gb/s |                29.1 gb/s |
+| `nk_dot_u8_skylake`     |                48.7 gb/s |                32.6 gb/s |                27.5 gb/s |
+| `nk_dot_u8_icelake`     |                68.4 gb/s |                46.9 gb/s |                30.2 gb/s |
+| `nk_dot_u8_alder`       |                42.1 gb/s |                41.8 gb/s |                31.6 gb/s |
 | __i4__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_dot_i4_serial`      |                9.37 gb/s |                11.8 gb/s |                11.8 gb/s |
-| `nk_dot_i4_haswell`     |                8.39 gb/s |                8.47 gb/s |                8.30 gb/s |
-| `nk_dot_i4_icelake`     |                24.9 gb/s |                35.9 gb/s |                25.8 gb/s |
+| `nk_dot_i4_haswell`     |                8.22 gb/s |                8.53 gb/s |                8.23 gb/s |
+| `nk_dot_i4_icelake`     |                24.3 gb/s |                36.3 gb/s |                25.5 gb/s |
 | __u4__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_dot_u4_serial`      |                10.6 gb/s |                12.0 gb/s |                11.9 gb/s |
-| `nk_dot_u4_haswell`     |                15.2 gb/s |                16.0 gb/s |                14.4 gb/s |
-| `nk_dot_u4_icelake`     |                49.6 gb/s |                58.3 gb/s |                29.6 gb/s |
+| `nk_dot_u4_haswell`     |                15.0 gb/s |                16.4 gb/s |                14.3 gb/s |
+| `nk_dot_u4_icelake`     |                48.1 gb/s |                64.4 gb/s |                30.9 gb/s |
 | __u1__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_dot_u1_serial`      |                3.92 gb/s |                5.04 gb/s |                4.97 gb/s |
-| `nk_dot_u1_haswell`     |                14.2 gb/s |                46.7 gb/s |                70.9 gb/s |
-| `nk_dot_u1_icelake`     |                21.1 gb/s |                70.9 gb/s |                 109 gb/s |
+| `nk_dot_u1_haswell`     |                14.7 gb/s |                43.2 gb/s |                69.4 gb/s |
+| `nk_dot_u1_icelake`     |                17.9 gb/s |                68.8 gb/s |                 110 gb/s |
 
 #### WASM
 
@@ -293,133 +305,138 @@ Measured with Wasmtime v42 (Cranelift backend).
 | `nk_dot_u1_serial`         |                1.95 gb/s |                1.53 gb/s |                0.09 gb/s |
 | `nk_dot_u1_v128relaxed`    |               0.548 gb/s |                1.88 gb/s |                0.13 gb/s |
 
-### Apple M4
+### Apple M5
 
 #### Native
 
 | Kernel                    |                      256 |                     1024 |                     4096 |
 | :------------------------ | -----------------------: | -----------------------: | -----------------------: |
 | __f64c__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f64c_serial`      |         12.5 gb/s, 5 ulp |         12.4 gb/s, 3 ulp |       11.9 gb/s, 9.7 ulp |
-| `nk_vdot_f64c_serial`     |       12.4 gb/s, 4.2 ulp |       12.3 gb/s, 3.3 ulp |       11.7 gb/s, 3.3 ulp |
-| `nk_dot_f64c_neon`        |         8.06 gb/s, 0 ulp |         8.05 gb/s, 0 ulp |         7.85 gb/s, 0 ulp |
-| `nk_vdot_f64c_neon`       |         7.79 gb/s, 0 ulp |         8.05 gb/s, 0 ulp |         7.88 gb/s, 0 ulp |
+| `nk_dot_f64c_serial`      |         8.02 gb/s, 5 ulp |         7.30 gb/s, 3 ulp |       7.25 gb/s, 9.7 ulp |
+| `nk_vdot_f64c_serial`     |       8.29 gb/s, 4.2 ulp |       7.53 gb/s, 3.3 ulp |       7.38 gb/s, 3.3 ulp |
+| `nk_dot_f64c_neon`        |         23.7 gb/s, 0 ulp |         21.6 gb/s, 0 ulp |         21.3 gb/s, 0 ulp |
+| `nk_vdot_f64c_neon`       |         23.6 gb/s, 0 ulp |         21.8 gb/s, 0 ulp |         20.9 gb/s, 0 ulp |
 | __f32c__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f32c_serial`      |         12.9 gb/s, 0 ulp |         12.4 gb/s, 0 ulp |         11.9 gb/s, 0 ulp |
-| `nk_vdot_f32c_serial`     |         12.8 gb/s, 0 ulp |         12.4 gb/s, 0 ulp |         12.1 gb/s, 0 ulp |
-| `nk_dot_f32c_neon`        |         7.97 gb/s, 0 ulp |         7.26 gb/s, 0 ulp |         7.00 gb/s, 0 ulp |
-| `nk_vdot_f32c_neon`       |         8.29 gb/s, 0 ulp |         7.58 gb/s, 0 ulp |         7.36 gb/s, 0 ulp |
+| `nk_dot_f32c_serial`      |         27.8 gb/s, 0 ulp |         24.6 gb/s, 0 ulp |         23.2 gb/s, 0 ulp |
+| `nk_vdot_f32c_serial`     |         27.2 gb/s, 0 ulp |         24.0 gb/s, 0 ulp |         22.6 gb/s, 0 ulp |
+| `nk_dot_f32c_neon`        |         22.8 gb/s, 0 ulp |         18.2 gb/s, 0 ulp |         16.9 gb/s, 0 ulp |
+| `nk_vdot_f32c_neon`       |         22.7 gb/s, 0 ulp |         17.5 gb/s, 0 ulp |         16.7 gb/s, 0 ulp |
 | __bf16c__                 | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_bf16c_serial`     |       7.47 gb/s, 0.2 ulp |       6.33 gb/s, 2.8 ulp |      6.08 gb/s, 15.8 ulp |
-| `nk_vdot_bf16c_serial`    |       7.47 gb/s, 0.2 ulp |       6.43 gb/s, 2.6 ulp |      6.08 gb/s, 11.4 ulp |
-| `nk_dot_bf16c_neonbfdot`  |       12.9 gb/s, 0.1 ulp |         9.31 gb/s, 2 ulp |       8.56 gb/s, 8.8 ulp |
-| `nk_vdot_bf16c_neonbfdot` |       12.9 gb/s, 0.1 ulp |       9.29 gb/s, 1.8 ulp |       8.56 gb/s, 8.8 ulp |
+| `nk_dot_bf16c_serial`     |       15.6 gb/s, 0.2 ulp |       12.5 gb/s, 2.8 ulp |      12.5 gb/s, 15.8 ulp |
+| `nk_vdot_bf16c_serial`    |       15.9 gb/s, 0.2 ulp |       12.9 gb/s, 2.6 ulp |      11.7 gb/s, 11.4 ulp |
+| `nk_dot_bf16c_neonbfdot`  |       26.3 gb/s, 0.1 ulp |         18.5 gb/s, 2 ulp |       17.6 gb/s, 8.8 ulp |
+| `nk_vdot_bf16c_neonbfdot` |       26.5 gb/s, 0.1 ulp |       18.2 gb/s, 1.8 ulp |       17.3 gb/s, 8.8 ulp |
 | __f16c__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f16c_serial`      |      7.53 gb/s, 20.8 ulp |      6.34 gb/s, 64.1 ulp |      6.07 gb/s, 73.1 ulp |
-| `nk_vdot_f16c_serial`     |      7.53 gb/s, 24.8 ulp |      6.34 gb/s, 31.9 ulp |       6.07 gb/s, 137 ulp |
-| `nk_dot_f16c_neonhalf`    |       9.94 gb/s, 3.0 ulp |       7.94 gb/s, 6.5 ulp |      7.60 gb/s, 20.5 ulp |
-| `nk_vdot_f16c_neonhalf`   |      9.85 gb/s, 34.9 ulp |      7.79 gb/s, 40.7 ulp |      7.57 gb/s, 73.1 ulp |
-| `nk_dot_f16c_neonfhm`     |       9.39 gb/s, 3.0 ulp |       7.46 gb/s, 6.5 ulp |      7.19 gb/s, 20.5 ulp |
-| `nk_vdot_f16c_neonfhm`    |      9.75 gb/s, 31.4 ulp |      7.50 gb/s, 38.6 ulp |      7.29 gb/s, 67.6 ulp |
+| `nk_dot_f16c_serial`      |      15.8 gb/s, 20.8 ulp |      13.0 gb/s, 64.1 ulp |      12.3 gb/s, 73.1 ulp |
+| `nk_vdot_f16c_serial`     |      15.8 gb/s, 24.8 ulp |      13.0 gb/s, 31.9 ulp |       12.3 gb/s, 137 ulp |
+| `nk_dot_f16c_neonhalf`    |       26.1 gb/s, 3.0 ulp |       18.4 gb/s, 6.5 ulp |      16.8 gb/s, 20.5 ulp |
+| `nk_vdot_f16c_neonhalf`   |      26.1 gb/s, 34.9 ulp |      18.5 gb/s, 40.7 ulp |      17.0 gb/s, 73.1 ulp |
+| `nk_dot_f16c_neonfhm`     |       25.3 gb/s, 3.0 ulp |       17.1 gb/s, 6.5 ulp |      15.9 gb/s, 20.5 ulp |
+| `nk_vdot_f16c_neonfhm`    |      25.0 gb/s, 31.4 ulp |      17.0 gb/s, 38.6 ulp |      15.8 gb/s, 67.6 ulp |
 | __f64__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f64_serial`       |       7.79 gb/s, 2.4 ulp |       7.79 gb/s, 175 ulp |       7.74 gb/s, 2.7 ulp |
-| `nk_dot_f64_neon`         |         14.8 gb/s, 0 ulp |         15.1 gb/s, 0 ulp |         14.7 gb/s, 0 ulp |
+| `nk_dot_f64_serial`       |       8.11 gb/s, 2.4 ulp |       8.13 gb/s, 175 ulp |       8.09 gb/s, 2.7 ulp |
+| `nk_dot_f64_neon`         |         44.2 gb/s, 0 ulp |         42.3 gb/s, 0 ulp |         38.4 gb/s, 0 ulp |
 | __f32__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f32_serial`       |         11.0 gb/s, 0 ulp |         7.77 gb/s, 0 ulp |         7.18 gb/s, 0 ulp |
-| `nk_dot_f32_neon`         |         9.23 gb/s, 0 ulp |         7.30 gb/s, 0 ulp |         6.96 gb/s, 0 ulp |
+| `nk_dot_f32_serial`       |         23.3 gb/s, 0 ulp |         15.8 gb/s, 0 ulp |         14.6 gb/s, 0 ulp |
+| `nk_dot_f32_neon`         |         46.4 gb/s, 0 ulp |         38.0 gb/s, 0 ulp |         34.8 gb/s, 0 ulp |
 | __bf16__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_bf16_serial`      |         5.76 gb/s, 0 ulp |       4.10 gb/s, 0.9 ulp |         3.62 gb/s, 6 ulp |
-| `nk_dot_bf16_neonbfdot`   |         35.1 gb/s, 0 ulp |       28.6 gb/s, 0.6 ulp |       22.9 gb/s, 4.5 ulp |
+| `nk_dot_bf16_serial`      |         12.4 gb/s, 0 ulp |       8.59 gb/s, 0.9 ulp |         7.36 gb/s, 6 ulp |
+| `nk_dot_bf16_neon`        |       39.0 gb/s, 3.7 ulp |       27.2 gb/s, 3.7 ulp |       19.9 gb/s, 3.7 ulp |
+| `nk_dot_bf16_neonbfdot`   |         70.8 gb/s, 0 ulp |       60.8 gb/s, 0.6 ulp |       47.8 gb/s, 4.5 ulp |
 | __f16__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f16_serial`       |        5.66 gb/s, 19 ulp |      3.99 gb/s, 31.1 ulp |      3.51 gb/s, 57.8 ulp |
-| `nk_dot_f16_neonhalf`     |      12.1 gb/s, 19.4 ulp |      9.18 gb/s, 21.5 ulp |      7.72 gb/s, 36.3 ulp |
-| `nk_dot_f16_neonfhm`      |      16.3 gb/s, 14.9 ulp |      10.7 gb/s, 26.7 ulp |      7.95 gb/s, 39.9 ulp |
+| `nk_dot_f16_serial`       |        12.0 gb/s, 19 ulp |      8.33 gb/s, 31.1 ulp |      7.11 gb/s, 57.8 ulp |
+| `nk_dot_f16_neon`         |      35.7 gb/s, 33.4 ulp |      25.8 gb/s, 37.4 ulp |      21.3 gb/s, 23.1 ulp |
+| `nk_dot_f16_neonfhm`      |      48.7 gb/s, 14.9 ulp |      27.5 gb/s, 26.7 ulp |      18.8 gb/s, 39.9 ulp |
 | __e5m2__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e5m2_serial`      |         1.87 gb/s, 0 ulp |         1.76 gb/s, 0 ulp |         1.74 gb/s, 0 ulp |
-| `nk_dot_e5m2_neon`        |         7.15 gb/s, 0 ulp |         4.95 gb/s, 0 ulp |         4.23 gb/s, 0 ulp |
+| `nk_dot_e5m2_serial`      |         3.80 gb/s, 0 ulp |         3.41 gb/s, 0 ulp |         3.41 gb/s, 0 ulp |
+| `nk_dot_e5m2_neon`        |         19.0 gb/s, 0 ulp |         13.2 gb/s, 0 ulp |         10.5 gb/s, 0 ulp |
+| `nk_dot_e5m2_neonfhm`     |         25.6 gb/s, 0 ulp |         15.3 gb/s, 0 ulp |         9.55 gb/s, 0 ulp |
+| `nk_dot_e5m2_neonbfdot`   |         3.65 gb/s, 0 ulp |         3.82 gb/s, 0 ulp |         3.68 gb/s, 0 ulp |
 | __e4m3__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e4m3_serial`      |        0.874 gb/s, 0 ulp |        0.873 gb/s, 0 ulp |        0.871 gb/s, 0 ulp |
-| `nk_dot_e4m3_neon`        |         1.62 gb/s, 0 ulp |         1.62 gb/s, 0 ulp |         1.60 gb/s, 0 ulp |
+| `nk_dot_e4m3_serial`      |         1.74 gb/s, 0 ulp |         1.72 gb/s, 0 ulp |         1.71 gb/s, 0 ulp |
+| `nk_dot_e4m3_neon`        |         4.44 gb/s, 0 ulp |         4.51 gb/s, 0 ulp |         4.57 gb/s, 0 ulp |
+| `nk_dot_e4m3_neonfhm`     |         10.1 gb/s, 0 ulp |         8.51 gb/s, 0 ulp |         7.96 gb/s, 0 ulp |
+| `nk_dot_e4m3_neonbfdot`   |         3.59 gb/s, 0 ulp |         3.68 gb/s, 0 ulp |         3.64 gb/s, 0 ulp |
 | __e3m2__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e3m2_serial`      |         1.24 gb/s, 0 ulp |         1.15 gb/s, 0 ulp |         1.13 gb/s, 0 ulp |
-| `nk_dot_e3m2_neonsdot`    |         7.69 gb/s, 0 ulp |         7.64 gb/s, 0 ulp |         7.70 gb/s, 0 ulp |
+| `nk_dot_e3m2_serial`      |         2.51 gb/s, 0 ulp |         2.33 gb/s, 0 ulp |         2.24 gb/s, 0 ulp |
+| `nk_dot_e3m2_neonsdot`    |         20.5 gb/s, 0 ulp |         20.7 gb/s, 0 ulp |         20.1 gb/s, 0 ulp |
 | __e2m3__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e2m3_serial`      |         1.23 gb/s, 0 ulp |         1.15 gb/s, 0 ulp |         1.13 gb/s, 0 ulp |
-| `nk_dot_e2m3_neonsdot`    |         16.6 gb/s, 0 ulp |         16.8 gb/s, 0 ulp |         16.4 gb/s, 0 ulp |
+| `nk_dot_e2m3_serial`      |         2.54 gb/s, 0 ulp |         2.27 gb/s, 0 ulp |         2.29 gb/s, 0 ulp |
+| `nk_dot_e2m3_neonsdot`    |         47.3 gb/s, 0 ulp |         47.5 gb/s, 0 ulp |         43.4 gb/s, 0 ulp |
 | __i8__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_i8_serial`        |                43.2 gb/s |                48.5 gb/s |                48.0 gb/s |
-| `nk_dot_i8_neonsdot`      |                29.8 gb/s |                29.4 gb/s |                22.9 gb/s |
+| `nk_dot_i8_serial`        |                 115 gb/s |                 102 gb/s |                92.3 gb/s |
+| `nk_dot_i8_neonsdot`      |                92.8 gb/s |                87.4 gb/s |                59.9 gb/s |
 | __u8__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_u8_serial`        |                44.1 gb/s |                45.7 gb/s |                50.0 gb/s |
-| `nk_dot_u8_neonsdot`      |                30.6 gb/s |                28.4 gb/s |                22.3 gb/s |
+| `nk_dot_u8_serial`        |                 110 gb/s |                99.2 gb/s |                94.9 gb/s |
+| `nk_dot_u8_neonsdot`      |                92.5 gb/s |                86.6 gb/s |                59.5 gb/s |
 | __i4__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_i4_serial`        |                11.9 gb/s |                12.4 gb/s |                12.6 gb/s |
-| `nk_dot_i4_neonsdot`      |                19.5 gb/s |                15.5 gb/s |                11.3 gb/s |
+| `nk_dot_i4_serial`        |                23.0 gb/s |                24.4 gb/s |                24.2 gb/s |
+| `nk_dot_i4_neonsdot`      |                58.2 gb/s |                44.7 gb/s |                30.4 gb/s |
 | __u4__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_u4_serial`        |                12.9 gb/s |                13.7 gb/s |                13.9 gb/s |
-| `nk_dot_u4_neonsdot`      |                21.7 gb/s |                16.1 gb/s |                11.4 gb/s |
+| `nk_dot_u4_serial`        |                25.3 gb/s |                27.2 gb/s |                26.9 gb/s |
+| `nk_dot_u4_neonsdot`      |                67.3 gb/s |                47.4 gb/s |                29.4 gb/s |
 | __u1__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_u1_serial`        |                3.32 gb/s |                3.56 gb/s |                3.58 gb/s |
-| `nk_dot_u1_neon`          |                11.2 gb/s |                21.8 gb/s |                27.4 gb/s |
+| `nk_dot_u1_serial`        |                7.01 gb/s |                7.63 gb/s |                7.19 gb/s |
+| `nk_dot_u1_neon`          |                33.3 gb/s |                64.6 gb/s |                88.0 gb/s |
 
 #### WASM
 
-Measured with Wasmtime v42 (Cranelift backend).
+Measured with Wasmtime v43 (Cranelift backend).
 
 | Kernel                     |                      256 |                     1024 |                     4096 |
 | :------------------------- | -----------------------: | -----------------------: | -----------------------: |
 | __f64c__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f64c_serial`       |       27.7 gb/s, 3.8 ulp |       24.1 gb/s, 3.9 ulp |       26.9 gb/s, 3.2 ulp |
-| `nk_vdot_f64c_serial`      |       5.87 gb/s, 3.8 ulp |       5.47 gb/s, 3.4 ulp |      5.66 gb/s, 15.1 ulp |
-| `nk_dot_f64c_v128relaxed`  |        45.0 gb/s, 26 ulp |        34.0 gb/s, 42 ulp |        35.9 gb/s, 88 ulp |
-| `nk_vdot_f64c_v128relaxed` |      22.0 gb/s, 22.8 ulp |      19.0 gb/s, 37.3 ulp |      17.6 gb/s, 43.6 ulp |
+| `nk_dot_f64c_serial`       |       4.17 gb/s, 3.8 ulp |       6.03 gb/s, 3.9 ulp |       6.37 gb/s, 3.2 ulp |
+| `nk_vdot_f64c_serial`      |       6.00 gb/s, 3.8 ulp |       6.55 gb/s, 3.4 ulp |      6.83 gb/s, 15.1 ulp |
+| `nk_dot_f64c_v128relaxed`  |        46.7 gb/s, 26 ulp |        38.2 gb/s, 42 ulp |        40.5 gb/s, 88 ulp |
+| `nk_vdot_f64c_v128relaxed` |      46.1 gb/s, 22.8 ulp |      39.8 gb/s, 37.3 ulp |      39.9 gb/s, 43.6 ulp |
 | __f32c__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f32c_serial`       |         21.3 gb/s, 0 ulp |         19.8 gb/s, 0 ulp |         20.4 gb/s, 0 ulp |
-| `nk_vdot_f32c_serial`      |         11.1 gb/s, 0 ulp |         10.7 gb/s, 0 ulp |         10.7 gb/s, 0 ulp |
-| `nk_dot_f32c_v128relaxed`  |         21.4 gb/s, 0 ulp |         17.2 gb/s, 0 ulp |         18.0 gb/s, 0 ulp |
-| `nk_vdot_f32c_v128relaxed` |         10.4 gb/s, 0 ulp |         9.47 gb/s, 0 ulp |         8.71 gb/s, 0 ulp |
+| `nk_dot_f32c_serial`       |         20.9 gb/s, 0 ulp |         21.3 gb/s, 0 ulp |         22.5 gb/s, 0 ulp |
+| `nk_vdot_f32c_serial`      |         19.9 gb/s, 0 ulp |         21.5 gb/s, 0 ulp |         22.4 gb/s, 0 ulp |
+| `nk_dot_f32c_v128relaxed`  |         22.4 gb/s, 0 ulp |         20.3 gb/s, 0 ulp |         19.8 gb/s, 0 ulp |
+| `nk_vdot_f32c_v128relaxed` |         21.9 gb/s, 0 ulp |         20.3 gb/s, 0 ulp |         19.9 gb/s, 0 ulp |
 | __bf16c__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_bf16c_serial`      |       11.1 gb/s, 0.1 ulp |       11.2 gb/s, 2.5 ulp |        11.2 gb/s, 10 ulp |
-| `nk_vdot_bf16c_serial`     |       5.79 gb/s, 0.2 ulp |       5.68 gb/s, 2.1 ulp |      5.80 gb/s, 11.4 ulp |
+| `nk_dot_bf16c_serial`      |       10.3 gb/s, 0.1 ulp |       11.6 gb/s, 2.5 ulp |        11.6 gb/s, 10 ulp |
+| `nk_vdot_bf16c_serial`     |       10.7 gb/s, 0.2 ulp |       11.7 gb/s, 2.1 ulp |      11.6 gb/s, 11.4 ulp |
 | __f16c__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f16c_serial`       |        3.52 gb/s, 13 ulp |        3.48 gb/s, 20 ulp |        3.49 gb/s, 90 ulp |
-| `nk_vdot_f16c_serial`      |      1.84 gb/s, 13.9 ulp |      1.79 gb/s, 35.5 ulp |      1.85 gb/s, 42.4 ulp |
+| `nk_dot_f16c_serial`       |        3.76 gb/s, 13 ulp |        3.83 gb/s, 20 ulp |        3.86 gb/s, 90 ulp |
+| `nk_vdot_f16c_serial`      |      3.81 gb/s, 13.9 ulp |      3.89 gb/s, 35.5 ulp |      3.85 gb/s, 42.4 ulp |
 | __f64__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f64_serial`        |       22.2 gb/s, 2.4 ulp |       19.3 gb/s, 2.6 ulp |       21.1 gb/s, 2.2 ulp |
-| `nk_dot_f64_v128relaxed`   |       39.6 gb/s, 2.6 ulp |       41.0 gb/s, 3.2 ulp |       32.9 gb/s, 2.6 ulp |
+| `nk_dot_f64_serial`        |       7.26 gb/s, 2.4 ulp |       7.45 gb/s, 2.6 ulp |       7.96 gb/s, 2.2 ulp |
+| `nk_dot_f64_v128relaxed`   |       38.7 gb/s, 2.6 ulp |       42.0 gb/s, 3.2 ulp |       43.9 gb/s, 2.6 ulp |
 | __f32__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f32_serial`        |        17.9 gb/s, 16 ulp |        12.6 gb/s, 69 ulp |       12.7 gb/s, 104 ulp |
-| `nk_dot_f32_v128relaxed`   |         19.5 gb/s, 0 ulp |         17.5 gb/s, 0 ulp |         17.2 gb/s, 0 ulp |
+| `nk_dot_f32_serial`        |        19.0 gb/s, 16 ulp |        14.6 gb/s, 69 ulp |       14.0 gb/s, 104 ulp |
+| `nk_dot_f32_v128relaxed`   |         20.4 gb/s, 0 ulp |         18.9 gb/s, 0 ulp |         18.7 gb/s, 0 ulp |
 | __bf16__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_bf16_serial`       |         8.70 gb/s, 0 ulp |       6.36 gb/s, 0.6 ulp |       6.57 gb/s, 5.9 ulp |
-| `nk_dot_bf16_v128relaxed`  |         9.07 gb/s, 0 ulp |       7.98 gb/s, 0.4 ulp |       8.24 gb/s, 3.7 ulp |
+| `nk_dot_bf16_serial`       |         9.53 gb/s, 0 ulp |       7.42 gb/s, 0.6 ulp |       7.20 gb/s, 5.9 ulp |
+| `nk_dot_bf16_v128relaxed`  |         41.9 gb/s, 0 ulp |       28.3 gb/s, 0.4 ulp |       21.5 gb/s, 3.7 ulp |
 | __f16__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_f16_serial`        |        3.15 gb/s, 16 ulp |        2.74 gb/s, 26 ulp |        3.14 gb/s, 53 ulp |
-| `nk_dot_f16_v128relaxed`   |       4.78 gb/s, 9.0 ulp |        4.70 gb/s, 23 ulp |        4.92 gb/s, 39 ulp |
+| `nk_dot_f16_serial`        |        3.31 gb/s, 16 ulp |        3.63 gb/s, 26 ulp |        3.66 gb/s, 53 ulp |
+| `nk_dot_f16_v128relaxed`   |       11.4 gb/s, 9.0 ulp |        11.2 gb/s, 23 ulp |        12.0 gb/s, 39 ulp |
 | __e5m2__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e5m2_serial`       |         2.90 gb/s, 0 ulp |         2.51 gb/s, 0 ulp |         2.88 gb/s, 0 ulp |
-| `nk_dot_e5m2_v128relaxed`  |         3.05 gb/s, 0 ulp |         2.65 gb/s, 0 ulp |         2.99 gb/s, 0 ulp |
+| `nk_dot_e5m2_serial`       |         3.02 gb/s, 0 ulp |         2.95 gb/s, 0 ulp |         3.16 gb/s, 0 ulp |
+| `nk_dot_e5m2_v128relaxed`  |         3.47 gb/s, 0 ulp |         3.45 gb/s, 0 ulp |         3.48 gb/s, 0 ulp |
 | __e4m3__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e4m3_serial`       |        0.903 gb/s, 0 ulp |        0.776 gb/s, 0 ulp |        0.874 gb/s, 0 ulp |
-| `nk_dot_e4m3_v128relaxed`  |         2.42 gb/s, 0 ulp |         2.12 gb/s, 0 ulp |         2.36 gb/s, 0 ulp |
+| `nk_dot_e4m3_serial`       |        0.978 gb/s, 0 ulp |        0.893 gb/s, 0 ulp |        0.936 gb/s, 0 ulp |
+| `nk_dot_e4m3_v128relaxed`  |         2.78 gb/s, 0 ulp |         2.75 gb/s, 0 ulp |         2.78 gb/s, 0 ulp |
 | __e3m2__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e3m2_serial`       |         2.90 gb/s, 0 ulp |         2.53 gb/s, 0 ulp |         2.88 gb/s, 0 ulp |
-| `nk_dot_e3m2_v128relaxed`  |         11.8 gb/s, 0 ulp |         10.5 gb/s, 0 ulp |         11.7 gb/s, 0 ulp |
+| `nk_dot_e3m2_serial`       |         3.13 gb/s, 0 ulp |         2.95 gb/s, 0 ulp |         3.16 gb/s, 0 ulp |
+| `nk_dot_e3m2_v128relaxed`  |         12.1 gb/s, 0 ulp |         11.9 gb/s, 0 ulp |         12.6 gb/s, 0 ulp |
 | __e2m3__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_e2m3_serial`       |         2.90 gb/s, 0 ulp |         2.53 gb/s, 0 ulp |         2.88 gb/s, 0 ulp |
-| `nk_dot_e2m3_v128relaxed`  |         20.0 gb/s, 0 ulp |         20.0 gb/s, 0 ulp |         20.0 gb/s, 0 ulp |
+| `nk_dot_e2m3_serial`       |         2.99 gb/s, 0 ulp |         3.00 gb/s, 0 ulp |         3.17 gb/s, 0 ulp |
+| `nk_dot_e2m3_v128relaxed`  |         20.4 gb/s, 0 ulp |         20.6 gb/s, 0 ulp |         21.7 gb/s, 0 ulp |
 | __i8__                     | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_i8_serial`         |                21.7 gb/s |                16.8 gb/s |                16.3 gb/s |
-| `nk_dot_i8_v128relaxed`    |                42.0 gb/s |                47.7 gb/s |                46.3 gb/s |
+| `nk_dot_i8_serial`         |                22.2 gb/s |                19.6 gb/s |                17.8 gb/s |
+| `nk_dot_i8_v128relaxed`    |                42.0 gb/s |                49.0 gb/s |                49.7 gb/s |
 | __u8__                     | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_u8_serial`         |                21.4 gb/s |                16.8 gb/s |                16.3 gb/s |
-| `nk_dot_u8_v128relaxed`    |                43.8 gb/s |                51.2 gb/s |                49.2 gb/s |
+| `nk_dot_u8_serial`         |                23.0 gb/s |                19.9 gb/s |                17.8 gb/s |
+| `nk_dot_u8_v128relaxed`    |                29.3 gb/s |                33.0 gb/s |                35.1 gb/s |
 | __i4__                     | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_i4_serial`         |               0.984 gb/s |               0.824 gb/s |               0.957 gb/s |
-| `nk_dot_i4_v128relaxed`    |                14.8 gb/s |                15.0 gb/s |                17.9 gb/s |
+| `nk_dot_i4_serial`         |               0.990 gb/s |               0.923 gb/s |               0.985 gb/s |
+| `nk_dot_i4_v128relaxed`    |                15.2 gb/s |                17.9 gb/s |                19.2 gb/s |
 | __u4__                     | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_u4_serial`         |               0.988 gb/s |               0.957 gb/s |               0.959 gb/s |
-| `nk_dot_u4_v128relaxed`    |                30.5 gb/s |                27.5 gb/s |                31.4 gb/s |
+| `nk_dot_u4_serial`         |               0.992 gb/s |               0.933 gb/s |               0.988 gb/s |
+| `nk_dot_u4_v128relaxed`    |                30.2 gb/s |                32.1 gb/s |                33.8 gb/s |
 | __u1__                     | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_dot_u1_serial`         |                4.85 gb/s |                5.38 gb/s |                5.67 gb/s |
-| `nk_dot_u1_v128relaxed`    |                21.8 gb/s |                29.8 gb/s |                52.0 gb/s |
+| `nk_dot_u1_serial`         |                5.26 gb/s |                5.80 gb/s |                6.48 gb/s |
+| `nk_dot_u1_v128relaxed`    |                21.2 gb/s |                47.4 gb/s |                67.3 gb/s |