pq_crypto 0.4.2 → 0.5.1

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/x86_64/src/poly_use_hint_32_avx2.c

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/* References
+ * ==========
+ *
+ * - [REF_AVX2]
+ *   CRYSTALS-Dilithium optimized AVX2 implementation
+ *   Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé
+ *   https://github.com/pq-crystals/dilithium/tree/master/avx2
+ */
+
+/*
+ * This file is derived from the public domain
+ * AVX2 Dilithium implementation @[REF_AVX2].
+ */
+
+#include "../../../common.h"
+
+#if defined(MLD_ARITH_BACKEND_X86_64_DEFAULT) &&   \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) &&   \
+    (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \
+     (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87))
+
+#include <immintrin.h>
+#include "arith_native_x86_64.h"
+#include "consts.h"
+
+#define MLD_MM256_BLENDV_EPI32(a, b, mask)                     \
+  _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(a), \
+                                       _mm256_castsi256_ps(b), \
+                                       _mm256_castsi256_ps(mask)))
+
+void mld_poly_use_hint_32_avx2(int32_t *b, const int32_t *a,
+                               const int32_t *hint)
+{
+  unsigned int i;
+  __m256i f, f0, f1, h, t;
+  const __m256i q_bound = _mm256_set1_epi32(87 * ((MLDSA_Q - 1) / 32));
+  /* check-magic: 1025 == floor(2**22 / 4092) */
+  const __m256i v = _mm256_set1_epi32(1025);
+  const __m256i alpha = _mm256_set1_epi32(2 * ((MLDSA_Q - 1) / 32));
+  const __m256i off = _mm256_set1_epi32(127);
+  const __m256i shift = _mm256_set1_epi32(512);
+  const __m256i mask = _mm256_set1_epi32(15);
+  const __m256i zero = _mm256_setzero_si256();
+
+  for (i = 0; i < MLDSA_N / 8; i++)
+  {
+    f = _mm256_load_si256((const __m256i *)&a[8 * i]);
+    h = _mm256_load_si256((const __m256i *)&hint[8 * i]);
+
+    /* Reference:
+     * - @[REF_AVX2] calls poly_decompose to compute all a1, a0 before the loop.
+     * - Our implementation of decompose() is slightly different from that in
+     *   @[REF_AVX2]. See poly_decompose_32_avx2.c for more information.
+     */
+    /* f1, f2 = decompose(f) */
+    f1 = _mm256_add_epi32(f, off);
+    f1 = _mm256_srli_epi32(f1, 7);
+    f1 = _mm256_mulhi_epu16(f1, v);
+    f1 = _mm256_mulhrs_epi16(f1, shift);
+    t = _mm256_cmpgt_epi32(f, q_bound);
+    f0 = _mm256_mullo_epi32(f1, alpha);
+    f0 = _mm256_sub_epi32(f, f0);
+    f1 = _mm256_andnot_si256(t, f1);
+    f0 = _mm256_add_epi32(f0, t);
+
+    /* Reference: The reference avx2 implementation checks a0 >= 0, which is
+     * different from the specification and the reference C implementation. We
+     * follow the specification and check a0 > 0.
+     */
+    /* t = (f0 > 0) ? h : -h */
+    f0 = _mm256_cmpgt_epi32(f0, zero);
+    t = MLD_MM256_BLENDV_EPI32(h, zero, f0);
+    t = _mm256_slli_epi32(t, 1);
+    h = _mm256_sub_epi32(h, t);
+
+    /* f1 = (f1 + t) % 16 */
+    f1 = _mm256_add_epi32(f1, h);
+    f1 = _mm256_and_si256(f1, mask);
+
+    _mm256_store_si256((__m256i *)&b[8 * i], f1);
+  }
+}
+
+#else /* MLD_ARITH_BACKEND_X86_64_DEFAULT && !MLD_CONFIG_MULTILEVEL_NO_SHARED \
+         && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == \
+         65 || MLD_CONFIG_PARAMETER_SET == 87) */
+
+MLD_EMPTY_CU(avx2_poly_use_hint_32)
+
+#endif /* !(MLD_ARITH_BACKEND_X86_64_DEFAULT &&                                \
+          !MLD_CONFIG_MULTILEVEL_NO_SHARED &&                                  \
+          (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \
+          || MLD_CONFIG_PARAMETER_SET == 87)) */
+
+/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
+ * Don't modify by hand -- this is auto-generated by scripts/autogen. */
+#undef MLD_MM256_BLENDV_EPI32

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/x86_64/src/poly_use_hint_88_avx2.c

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/* References
+ * ==========
+ *
+ * - [REF_AVX2]
+ *   CRYSTALS-Dilithium optimized AVX2 implementation
+ *   Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé
+ *   https://github.com/pq-crystals/dilithium/tree/master/avx2
+ */
+
+/*
+ * This file is derived from the public domain
+ * AVX2 Dilithium implementation @[REF_AVX2].
+ */
+
+#include "../../../common.h"
+
+#if defined(MLD_ARITH_BACKEND_X86_64_DEFAULT) &&   \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) &&   \
+    (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \
+     MLD_CONFIG_PARAMETER_SET == 44)
+
+#include <immintrin.h>
+#include "arith_native_x86_64.h"
+#include "consts.h"
+
+#define MLD_MM256_BLENDV_EPI32(a, b, mask)                     \
+  _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(a), \
+                                       _mm256_castsi256_ps(b), \
+                                       _mm256_castsi256_ps(mask)))
+
+void mld_poly_use_hint_88_avx2(int32_t *b, const int32_t *a,
+                               const int32_t *hint)
+{
+  unsigned int i;
+  __m256i f, f0, f1, h, t;
+  const __m256i q_bound = _mm256_set1_epi32(87 * ((MLDSA_Q - 1) / 88));
+  /* check-magic: 11275 == floor(2**24 / 1488) */
+  const __m256i v = _mm256_set1_epi32(11275);
+  const __m256i alpha = _mm256_set1_epi32(2 * ((MLDSA_Q - 1) / 88));
+  const __m256i off = _mm256_set1_epi32(127);
+  const __m256i shift = _mm256_set1_epi32(128);
+  const __m256i max = _mm256_set1_epi32(43);
+  const __m256i zero = _mm256_setzero_si256();
+
+  for (i = 0; i < MLDSA_N / 8; i++)
+  {
+    f = _mm256_load_si256((const __m256i *)&a[8 * i]);
+    h = _mm256_load_si256((const __m256i *)&hint[8 * i]);
+
+    /* Reference:
+     * - @[REF_AVX2] calls poly_decompose to compute all a1, a0 before the loop.
+     * - Our implementation of decompose() is slightly different from that in
+     *   @[REF_AVX2]. See poly_decompose_88_avx2.c for more information.
+     */
+    /* f1, f2 = decompose(f) */
+    f1 = _mm256_add_epi32(f, off);
+    f1 = _mm256_srli_epi32(f1, 7);
+    f1 = _mm256_mulhi_epu16(f1, v);
+    f1 = _mm256_mulhrs_epi16(f1, shift);
+    t = _mm256_cmpgt_epi32(f, q_bound);
+    f0 = _mm256_mullo_epi32(f1, alpha);
+    f0 = _mm256_sub_epi32(f, f0);
+    f1 = _mm256_andnot_si256(t, f1);
+    f0 = _mm256_add_epi32(f0, t);
+
+    /* Reference: The reference avx2 implementation checks a0 >= 0, which is
+     * different from the specification and the reference C implementation. We
+     * follow the specification and check a0 > 0.
+     */
+    /* t = (f0 > 0) ? h : -h */
+    f0 = _mm256_cmpgt_epi32(f0, zero);
+    t = MLD_MM256_BLENDV_EPI32(h, zero, f0);
+    t = _mm256_slli_epi32(t, 1);
+    h = _mm256_sub_epi32(h, t);
+
+    /* f1 = (f1 + t) % 44 */
+    f1 = _mm256_add_epi32(f1, h);
+    f1 = MLD_MM256_BLENDV_EPI32(f1, max, f1);
+    f = _mm256_cmpgt_epi32(f1, max);
+    f1 = MLD_MM256_BLENDV_EPI32(f1, zero, f);
+
+    _mm256_store_si256((__m256i *)&b[8 * i], f1);
+  }
+}
+
+#else /* MLD_ARITH_BACKEND_X86_64_DEFAULT && !MLD_CONFIG_MULTILEVEL_NO_SHARED \
+         && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == \
+         44) */
+
+MLD_EMPTY_CU(avx2_poly_use_hint_88)
+
+#endif /* !(MLD_ARITH_BACKEND_X86_64_DEFAULT &&                             \
+          !MLD_CONFIG_MULTILEVEL_NO_SHARED &&                               \
+          (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == \
+          44)) */
+
+/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
+ * Don't modify by hand -- this is auto-generated by scripts/autogen. */
+#undef MLD_MM256_BLENDV_EPI32

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/x86_64/src/polyz_unpack_17_avx2.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/* References
+ * ==========
+ *
+ * - [REF_AVX2]
+ *   CRYSTALS-Dilithium optimized AVX2 implementation
+ *   Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé
+ *   https://github.com/pq-crystals/dilithium/tree/master/avx2
+ */
+
+/*
+ * This file is derived from the public domain
+ * AVX2 Dilithium implementation @[REF_AVX2].
+ */
+
+#include "../../../common.h"
+
+#if defined(MLD_ARITH_BACKEND_X86_64_DEFAULT) &&   \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) &&   \
+    (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \
+     MLD_CONFIG_PARAMETER_SET == 44)
+
+#include <immintrin.h>
+#include "arith_native_x86_64.h"
+
+void mld_polyz_unpack_17_avx2(int32_t *r, const uint8_t *a)
+{
+  unsigned int i;
+  __m256i f;
+  __m128i low, high;
+
+  const __m256i shufbidx = _mm256_set_epi8(
+      -1, 31, 30, 29, -1, 29, 28, 27, -1, 27, 26, 25, -1, 25, 24, 23, -1, 8, 7,
+      6, -1, 6, 5, 4, -1, 4, 3, 2, -1, 2, 1, 0);
+  const __m256i srlvdidx = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);
+  const __m256i mask = _mm256_set1_epi32(0x3FFFF);
+  const __m256i gamma1 = _mm256_set1_epi32((1 << 17));
+
+  for (i = 0; i < MLDSA_N / 8; i++)
+  {
+    /* Load bytes 0..15 into low 128-bit vector */
+    low = _mm_loadu_si128((__m128i *)&a[18 * i]);
+    /* Load bytes 2..17 into high 128-bit vector */
+    high = _mm_loadu_si128((__m128i *)&a[18 * i + 2]);
+    /* Combine into 256-bit vector */
+    f = _mm256_inserti128_si256(_mm256_castsi128_si256(low), high, 1);
+
+    /* Shuffling 8-bit lanes
+     *
+     * ┌─ Indices 0-8 into low 128-bit half ───────────────────────────────────┐
+     * │ Shuffle: [-1, 8, 7, 6, -1, 6, 5, 4, -1, 4, 3, 2, -1, 2, 1, 0]         │
+     * │ Result:  [0, byte8, byte7, byte6, ..., 0, byte2, byte1, byte0]        │
+     * └───────────────────────────────────────────────────────────────────────┘
+     *
+     * ┌─ Indices 16-31 into high 128-bit half ────────────────────────────────┐
+     * │ Shuffle: [-1,31, 30, 29, -1,29, 28, 27, -1,27, 26, 25, -1,25, 24, 23] │
+     * │ Result:  [0, byte17, byte16, byte15, ..., 0, byte11, byte10, byte9]   │
+     * └───────────────────────────────────────────────────────────────────────┘
+     */
+    f = _mm256_shuffle_epi8(f, shufbidx);
+
+    /* Keep only 18 out of 24 bits in each 32-bit lane */
+    /* Bits   0..23     16..39    32..55    48..71
+     *        72..95    88..111   104..127  120..143 */
+    f = _mm256_srlv_epi32(f, srlvdidx);
+    /* Bits   0..23     18..39    36..55    54..71
+     *        72..95    90..111   108..127  126..143 */
+    f = _mm256_and_si256(f, mask);
+    /* Bits   0..17     18..35    36..53    54..71
+     *        72..89    90..107   108..125  126..143 */
+
+    /* Map [0, 1, ..., 2^18-1] to [2^17, 2^17-1, ..., -2^17+1] */
+    f = _mm256_sub_epi32(gamma1, f);
+
+    _mm256_store_si256((__m256i *)&r[8 * i], f);
+  }
+}
+#else /* MLD_ARITH_BACKEND_X86_64_DEFAULT && !MLD_CONFIG_MULTILEVEL_NO_SHARED \
+         && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == \
+         44) */
+
+MLD_EMPTY_CU(avx2_polyz_unpack_17)
+
+#endif /* !(MLD_ARITH_BACKEND_X86_64_DEFAULT &&                             \
+          !MLD_CONFIG_MULTILEVEL_NO_SHARED &&                               \
+          (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == \
+          44)) */

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/x86_64/src/polyz_unpack_19_avx2.c

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/* References
+ * ==========
+ *
+ * - [REF_AVX2]
+ *   CRYSTALS-Dilithium optimized AVX2 implementation
+ *   Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé
+ *   https://github.com/pq-crystals/dilithium/tree/master/avx2
+ */
+
+/*
+ * This file is derived from the public domain
+ * AVX2 Dilithium implementation @[REF_AVX2].
+ */
+
+#include "../../../common.h"
+
+#if defined(MLD_ARITH_BACKEND_X86_64_DEFAULT) &&   \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) &&   \
+    (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \
+     (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87))
+
+#include <immintrin.h>
+#include "arith_native_x86_64.h"
+
+void mld_polyz_unpack_19_avx2(int32_t *r, const uint8_t *a)
+{
+  unsigned int i;
+  __m256i f;
+  __m128i low, high;
+
+  const __m256i shufbidx = _mm256_set_epi8(
+      -1, 31, 30, 29, -1, 29, 28, 27, -1, 26, 25, 24, -1, 24, 23, 22, -1, 9, 8,
+      7, -1, 7, 6, 5, -1, 4, 3, 2, -1, 2, 1, 0);
+  /* Equivalent to _mm256_set_epi32(4, 0, 4, 0, 4, 0, 4, 0) */
+  const __m256i srlvdidx = _mm256_set1_epi64x((uint64_t)4 << 32);
+  const __m256i mask = _mm256_set1_epi32(0xFFFFF);
+  const __m256i gamma1 = _mm256_set1_epi32((1 << 19));
+
+  for (i = 0; i < MLDSA_N / 8; i++)
+  {
+    /* Load bytes 0..15 into low 128-bit vector */
+    low = _mm_loadu_si128((__m128i *)&a[20 * i]);
+    /* Load bytes 4..19 into high 128-bit vector */
+    high = _mm_loadu_si128((__m128i *)&a[20 * i + 4]);
+    /* Combine into 256-bit vector */
+    f = _mm256_inserti128_si256(_mm256_castsi128_si256(low), high, 1);
+
+    /* Shuffling 8-bit lanes
+     *
+     * ┌─ Indices 0-9 into low 128-bit half ───────────────────────────────────┐
+     * │ Shuffle: [-1, 9, 8, 7, -1, 7, 6, 5, -1, 4, 3, 2, -1, 2, 1, 0]         │
+     * │ Result:  [0, byte9, byte8, byte7, ..., 0, byte2, byte1, byte0]        │
+     * └───────────────────────────────────────────────────────────────────────┘
+     *
+     * ┌─ Indices 16-31 into high 128-bit half ────────────────────────────────┐
+     * │ Shuffle: [-1,31, 30, 29, -1,29, 28, 27, -1,26, 25, 24, -1,24, 23, 22] │
+     * │ Result:  [0, byte19, byte18, byte17, ..., 0, byte12, byte11, byte10]  │
+     * └───────────────────────────────────────────────────────────────────────┘
+     */
+    f = _mm256_shuffle_epi8(f, shufbidx);
+
+    /* Keep only 20 out of 24 bits in each 32-bit lane */
+    /* Bits   0..23     16..39    40..63    56..79
+     *        80..103   96..119   120..143  136..159 */
+    f = _mm256_srlv_epi32(f, srlvdidx);
+    /* Bits   0..23     20..39    40..63    60..79
+     *        80..103   100..119  120..143  140..159 */
+    f = _mm256_and_si256(f, mask);
+    /* Bits   0..19     20..39    40..59    60..79
+     *        80..99    100..119  120..139  140..159 */
+
+    /* Map [0, 1, ..., 2^20-1] to [2^19, 2^19-1, ..., -2^19+1] */
+    f = _mm256_sub_epi32(gamma1, f);
+
+    _mm256_store_si256((__m256i *)&r[8 * i], f);
+  }
+}
+
+#else /* MLD_ARITH_BACKEND_X86_64_DEFAULT && !MLD_CONFIG_MULTILEVEL_NO_SHARED \
+         && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == \
+         65 || MLD_CONFIG_PARAMETER_SET == 87) */
+
+MLD_EMPTY_CU(avx2_polyz_unpack_19)
+
+#endif /* !(MLD_ARITH_BACKEND_X86_64_DEFAULT &&                                \
+          !MLD_CONFIG_MULTILEVEL_NO_SHARED &&                                  \
+          (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \
+          || MLD_CONFIG_PARAMETER_SET == 87)) */

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/x86_64/src/rej_uniform_avx2.c

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/* References
+ * ==========
+ *
+ * - [REF_AVX2]
+ *   CRYSTALS-Dilithium optimized AVX2 implementation
+ *   Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé
+ *   https://github.com/pq-crystals/dilithium/tree/master/avx2
+ */
+
+/*
+ * This file is derived from the public domain
+ * AVX2 Dilithium implementation @[REF_AVX2].
+ */
+
+#include "../../../common.h"
+
+#if defined(MLD_ARITH_BACKEND_X86_64_DEFAULT) && \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)
+
+#include <immintrin.h>
+#include "arith_native_x86_64.h"
+#include "consts.h"
+
+/*
+ * Reference: The pqcrystals implementation assumes a buffer that is 8 bytes
+ *.           larger as the first loop overreads by 8 bytes that are then
+ *            discarded. We instead do not pad the buffer and do not overread.
+ *            The performance impact is negligible and it does not force the
+ *            frontend to perform the unintuitive padding.
+ */
+
+unsigned int mld_rej_uniform_avx2(
+    int32_t *MLD_RESTRICT r, const uint8_t buf[MLD_AVX2_REJ_UNIFORM_BUFLEN])
+{
+  unsigned int ctr, pos;
+  uint32_t good;
+  __m256i d, tmp;
+  const __m256i bound = _mm256_set1_epi32(MLDSA_Q);
+  const __m256i mask = _mm256_set1_epi32(0x7FFFFF);
+  const __m256i idx8 =
+      _mm256_set_epi8(-1, 15, 14, 13, -1, 12, 11, 10, -1, 9, 8, 7, -1, 6, 5, 4,
+                      -1, 11, 10, 9, -1, 8, 7, 6, -1, 5, 4, 3, -1, 2, 1, 0);
+
+  ctr = pos = 0;
+  while (ctr <= MLDSA_N - 8 && pos <= MLD_AVX2_REJ_UNIFORM_BUFLEN - 32)
+  {
+    d = _mm256_loadu_si256((__m256i *)&buf[pos]);
+
+    /* Permute 64-bit lanes
+     * 0x94 = 10010100b rearranges 64-bit lanes as: [3,2,1,0] -> [2,1,1,0]
+     *
+     * ╔═══════════════════════════════════════════════════════════════════════╗
+     * ║                         Original Layout                               ║
+     * ╚═══════════════════════════════════════════════════════════════════════╝
+     * ┌─────────────────┬─────────────────┬─────────────────┬─────────────────┐
+     * │     Lane 0      │     Lane 1      │     Lane 2      │     Lane 3      │
+     * │   bytes 0..7    │   bytes 8..15   │   bytes 16..23  │   bytes 24..31  │
+     * └─────────────────┴─────────────────┴─────────────────┴─────────────────┘
+     *
+     * ╔═══════════════════════════════════════════════════════════════════════╗
+     * ║                        Layout after permute                           ║
+     * ║        Byte indices in high half shifted down by 8 positions          ║
+     * ╚═══════════════════════════════════════════════════════════════════════╝
+     * ┌───────────────┬─────────────────┐ ┌─────────────────┬─────────────────┐
+     * │   Lane 0      │     Lane 1      │ │     Lane 2      │     Lane 3      │
+     * │ bytes 0..7    │   bytes 8..15   │ │   bytes 8..15   │   bytes 16..23  │
+     * └───────────────┴─────────────────┘ └─────────────────┴─────────────────┘
+     *   Lower 128-bit lane (bytes 0-15)      Upper 128-bit lane (bytes 16-31)
+     */
+    d = _mm256_permute4x64_epi64(d, 0x94);
+
+    /* Shuffling 8-bit lanes
+     *
+     * ┌─ Indices 0-11 into low 128-bit half of permuted vector────────────────┐
+     * │ Shuffle: [-1, 11, 10, 9, -1, 8, 7, 6, -1, 5, 4, 3, -1, 2, 1, 0]       │
+     * │ Result:  [0, byte11, byte10, byte9, ..., 0, byte2, byte1, byte0]      │
+     * └───────────────────────────────────────────────────────────────────────┘
+     *
+     * ┌─ Indices 4-15 into high 128-bit half of permuted vector ──────────────┐
+     * │ Shuffle: [-1, 15, 14, 13, -1, 12, 11, 10, -1, 9, 8, 7, -1, 6, 5, 4]   │
+     * │ Result:  [0, byte23, byte22, byte21, ..., 0, byte14, byte13, byte12   │
+     * └───────────────────────────────────────────────────────────────────────┘
+     */
+    d = _mm256_shuffle_epi8(d, idx8);
+    d = _mm256_and_si256(d, mask);
+    pos += 24;
+
+    tmp = _mm256_sub_epi32(d, bound);
+    good = (uint32_t)_mm256_movemask_ps((__m256)tmp);
+    tmp = _mm256_cvtepu8_epi32(
+        _mm_loadl_epi64((__m128i *)&mld_rej_uniform_table[good]));
+    d = _mm256_permutevar8x32_epi32(d, tmp);
+
+    _mm256_storeu_si256((__m256i *)&r[ctr], d);
+    ctr += (unsigned)_mm_popcnt_u32(good);
+  }
+
+  while (ctr < MLDSA_N && pos <= MLD_AVX2_REJ_UNIFORM_BUFLEN - 3)
+  {
+    uint32_t t = buf[pos++];
+    t |= (uint32_t)buf[pos++] << 8;
+    t |= (uint32_t)buf[pos++] << 16;
+    t &= 0x7FFFFF;
+
+    if (t < MLDSA_Q)
+    {
+      /* Safe because t < MLDSA_Q. */
+      r[ctr++] = (int32_t)t;
+    }
+  }
+
+  return ctr;
+}
+
+#else /* MLD_ARITH_BACKEND_X86_64_DEFAULT && !MLD_CONFIG_MULTILEVEL_NO_SHARED \
+       */
+
+MLD_EMPTY_CU(avx2_rej_uniform)
+
+#endif /* !(MLD_ARITH_BACKEND_X86_64_DEFAULT && \
+          !MLD_CONFIG_MULTILEVEL_NO_SHARED) */

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/x86_64/src/rej_uniform_eta2_avx2.c

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/* References
+ * ==========
+ *
+ * - [REF_AVX2]
+ *   CRYSTALS-Dilithium optimized AVX2 implementation
+ *   Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé
+ *   https://github.com/pq-crystals/dilithium/tree/master/avx2
+ */
+
+/*
+ * This file is derived from the public domain
+ * AVX2 Dilithium implementation @[REF_AVX2].
+ */
+
+#include "../../../common.h"
+
+#if defined(MLD_ARITH_BACKEND_X86_64_DEFAULT) && \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \
+    (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2)
+
+#include <immintrin.h>
+#include "arith_native_x86_64.h"
+#include "consts.h"
+
+#define MLD_AVX2_ETA2 2
+
+/*
+ * Reference: In the pqcrystals implementation this function is called
+ *            rej_eta_avx and supports multiple values for ETA via preprocessor
+ *            conditionals. We move the conditionals to the frontend.
+ */
+unsigned int mld_rej_uniform_eta2_avx2(
+    int32_t *MLD_RESTRICT r,
+    const uint8_t buf[MLD_AVX2_REJ_UNIFORM_ETA2_BUFLEN])
+{
+  unsigned int ctr, pos;
+  uint32_t good;
+  __m256i f0, f1, f2;
+  __m128i g0, g1;
+  const __m256i mask = _mm256_set1_epi8(15);
+  const __m256i eta = _mm256_set1_epi8(MLD_AVX2_ETA2);
+  const __m256i bound = mask;
+  /* check-magic: -6560 ==  32*round(-2**10 / 5) */
+  const __m256i v = _mm256_set1_epi32(-6560);
+  const __m256i p = _mm256_set1_epi32(5);
+
+  ctr = pos = 0;
+  while (ctr <= MLDSA_N - 8 && pos <= MLD_AVX2_REJ_UNIFORM_ETA2_BUFLEN - 16)
+  {
+    f0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((__m128i *)&buf[pos]));
+    f1 = _mm256_slli_epi16(f0, 4);
+    f0 = _mm256_or_si256(f0, f1);
+    f0 = _mm256_and_si256(f0, mask);
+
+    f1 = _mm256_sub_epi8(f0, bound);
+    f0 = _mm256_sub_epi8(eta, f0);
+    good = (uint32_t)_mm256_movemask_epi8(f1);
+
+    g0 = _mm256_castsi256_si128(f0);
+    g1 = _mm_loadl_epi64((__m128i *)&mld_rej_uniform_table[good & 0xFF]);
+    g1 = _mm_shuffle_epi8(g0, g1);
+    f1 = _mm256_cvtepi8_epi32(g1);
+    f2 = _mm256_mulhrs_epi16(f1, v);
+    f2 = _mm256_mullo_epi16(f2, p);
+    f1 = _mm256_add_epi32(f1, f2);
+    _mm256_storeu_si256((__m256i *)&r[ctr], f1);
+    ctr += (unsigned)_mm_popcnt_u32(good & 0xFF);
+    good >>= 8;
+    pos += 4;
+
+    if (ctr > MLDSA_N - 8)
+    {
+      break;
+    }
+    g0 = _mm_bsrli_si128(g0, 8);
+    g1 = _mm_loadl_epi64((__m128i *)&mld_rej_uniform_table[good & 0xFF]);
+    g1 = _mm_shuffle_epi8(g0, g1);
+    f1 = _mm256_cvtepi8_epi32(g1);
+    f2 = _mm256_mulhrs_epi16(f1, v);
+    f2 = _mm256_mullo_epi16(f2, p);
+    f1 = _mm256_add_epi32(f1, f2);
+    _mm256_storeu_si256((__m256i *)&r[ctr], f1);
+    ctr += (unsigned)_mm_popcnt_u32(good & 0xFF);
+    good >>= 8;
+    pos += 4;
+
+    if (ctr > MLDSA_N - 8)
+    {
+      break;
+    }
+    g0 = _mm256_extracti128_si256(f0, 1);
+    g1 = _mm_loadl_epi64((__m128i *)&mld_rej_uniform_table[good & 0xFF]);
+    g1 = _mm_shuffle_epi8(g0, g1);
+    f1 = _mm256_cvtepi8_epi32(g1);
+    f2 = _mm256_mulhrs_epi16(f1, v);
+    f2 = _mm256_mullo_epi16(f2, p);
+    f1 = _mm256_add_epi32(f1, f2);
+    _mm256_storeu_si256((__m256i *)&r[ctr], f1);
+    ctr += (unsigned)_mm_popcnt_u32(good & 0xFF);
+    good >>= 8;
+    pos += 4;
+
+    if (ctr > MLDSA_N - 8)
+    {
+      break;
+    }
+    g0 = _mm_bsrli_si128(g0, 8);
+    g1 = _mm_loadl_epi64((__m128i *)&mld_rej_uniform_table[good]);
+    g1 = _mm_shuffle_epi8(g0, g1);
+    f1 = _mm256_cvtepi8_epi32(g1);
+    f2 = _mm256_mulhrs_epi16(f1, v);
+    f2 = _mm256_mullo_epi16(f2, p);
+    f1 = _mm256_add_epi32(f1, f2);
+    _mm256_storeu_si256((__m256i *)&r[ctr], f1);
+    ctr += (unsigned)_mm_popcnt_u32(good);
+    pos += 4;
+  }
+
+  while (ctr < MLDSA_N && pos < MLD_AVX2_REJ_UNIFORM_ETA2_BUFLEN)
+  {
+    uint32_t t0 = buf[pos] & 0x0F;
+    uint32_t t1 = buf[pos++] >> 4;
+
+    if (t0 < 15)
+    {
+      t0 = t0 - (205 * t0 >> 10) * 5;
+      r[ctr++] = (int32_t)(2 - t0);
+    }
+    if (t1 < 15 && ctr < MLDSA_N)
+    {
+      t1 = t1 - (205 * t1 >> 10) * 5;
+      r[ctr++] = (int32_t)(2 - t1);
+    }
+  }
+
+  return ctr;
+}
+
+#else /* MLD_ARITH_BACKEND_X86_64_DEFAULT && !MLD_CONFIG_MULTILEVEL_NO_SHARED \
+         && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 2) */
+
+MLD_EMPTY_CU(avx2_rej_uniform_eta2)
+
+#endif /* !(MLD_ARITH_BACKEND_X86_64_DEFAULT && \
+          !MLD_CONFIG_MULTILEVEL_NO_SHARED &&   \
+          (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 2)) */
+
+/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
+ * Don't modify by hand -- this is auto-generated by scripts/autogen. */
+#undef MLD_AVX2_ETA2