pq_crypto 0.4.2 → 0.5.0

data/ext/pqcrypto/vendor/mlkem-native/mlkem/src/native/x86_64/src/reduce.S

@@ -0,0 +1,218 @@
+/*
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/* References
+ * ==========
+ *
+ * - [REF_AVX2]
+ *   CRYSTALS-Kyber optimized AVX2 implementation
+ *   Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
+ *   https://github.com/pq-crystals/kyber/tree/main/avx2
+ */
+
+/*
+ * This file is derived from the public domain
+ * AVX2 Kyber implementation @[REF_AVX2].
+ *
+ * Changes:
+ * - Add call to csub in reduce128_avx to produce outputs
+ *   in [0,1,...,q-1] rather than [0,1,...,q], matching the
+ *   semantics of mlk_poly_reduce(),
+ * - Use a macro instead of a local function call.
+ */
+
+#include "../../../common.h"
+
+#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
+    !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
+
+/*
+ * WARNING: This file is auto-derived from the mlkem-native source file
+ *   dev/x86_64/src/reduce.S using scripts/simpasm. Do not modify it directly.
+ */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",@progbits
+#endif
+
+.text
+.balign 4
+.global MLK_ASM_NAMESPACE(reduce_avx2)
+MLK_ASM_FN_SYMBOL(reduce_avx2)
+
+        .cfi_startproc
+        movl $0xd010d01, %eax        # imm = 0xD010D01
+        vmovd %eax, %xmm0
+        vpbroadcastd %xmm0, %ymm0
+        movl $0x4ebf4ebf, %eax       # imm = 0x4EBF4EBF
+        vmovd %eax, %xmm1
+        vpbroadcastd %xmm1, %ymm1
+        vmovdqa (%rdi), %ymm2
+        vmovdqa 0x20(%rdi), %ymm3
+        vmovdqa 0x40(%rdi), %ymm4
+        vmovdqa 0x60(%rdi), %ymm5
+        vmovdqa 0x80(%rdi), %ymm6
+        vmovdqa 0xa0(%rdi), %ymm7
+        vmovdqa 0xc0(%rdi), %ymm8
+        vmovdqa 0xe0(%rdi), %ymm9
+        vpmulhw %ymm1, %ymm2, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm2, %ymm2
+        vpmulhw %ymm1, %ymm3, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm3, %ymm3
+        vpmulhw %ymm1, %ymm4, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm4, %ymm4
+        vpmulhw %ymm1, %ymm5, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm5, %ymm5
+        vpmulhw %ymm1, %ymm6, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm6, %ymm6
+        vpmulhw %ymm1, %ymm7, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm7, %ymm7
+        vpmulhw %ymm1, %ymm8, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm8, %ymm8
+        vpmulhw %ymm1, %ymm9, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm9, %ymm9
+        vpsubw %ymm0, %ymm2, %ymm2
+        vpsraw $0xf, %ymm2, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm2, %ymm2
+        vpsubw %ymm0, %ymm3, %ymm3
+        vpsraw $0xf, %ymm3, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm3, %ymm3
+        vpsubw %ymm0, %ymm4, %ymm4
+        vpsraw $0xf, %ymm4, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm4, %ymm4
+        vpsubw %ymm0, %ymm5, %ymm5
+        vpsraw $0xf, %ymm5, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm5, %ymm5
+        vpsubw %ymm0, %ymm6, %ymm6
+        vpsraw $0xf, %ymm6, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm6, %ymm6
+        vpsubw %ymm0, %ymm7, %ymm7
+        vpsraw $0xf, %ymm7, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm7, %ymm7
+        vpsubw %ymm0, %ymm8, %ymm8
+        vpsraw $0xf, %ymm8, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm8, %ymm8
+        vpsubw %ymm0, %ymm9, %ymm9
+        vpsraw $0xf, %ymm9, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm9, %ymm9
+        vmovdqa %ymm2, (%rdi)
+        vmovdqa %ymm3, 0x20(%rdi)
+        vmovdqa %ymm4, 0x40(%rdi)
+        vmovdqa %ymm5, 0x60(%rdi)
+        vmovdqa %ymm6, 0x80(%rdi)
+        vmovdqa %ymm7, 0xa0(%rdi)
+        vmovdqa %ymm8, 0xc0(%rdi)
+        vmovdqa %ymm9, 0xe0(%rdi)
+        vmovdqa 0x100(%rdi), %ymm2
+        vmovdqa 0x120(%rdi), %ymm3
+        vmovdqa 0x140(%rdi), %ymm4
+        vmovdqa 0x160(%rdi), %ymm5
+        vmovdqa 0x180(%rdi), %ymm6
+        vmovdqa 0x1a0(%rdi), %ymm7
+        vmovdqa 0x1c0(%rdi), %ymm8
+        vmovdqa 0x1e0(%rdi), %ymm9
+        vpmulhw %ymm1, %ymm2, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm2, %ymm2
+        vpmulhw %ymm1, %ymm3, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm3, %ymm3
+        vpmulhw %ymm1, %ymm4, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm4, %ymm4
+        vpmulhw %ymm1, %ymm5, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm5, %ymm5
+        vpmulhw %ymm1, %ymm6, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm6, %ymm6
+        vpmulhw %ymm1, %ymm7, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm7, %ymm7
+        vpmulhw %ymm1, %ymm8, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm8, %ymm8
+        vpmulhw %ymm1, %ymm9, %ymm12
+        vpsraw $0xa, %ymm12, %ymm12
+        vpmullw %ymm0, %ymm12, %ymm12
+        vpsubw %ymm12, %ymm9, %ymm9
+        vpsubw %ymm0, %ymm2, %ymm2
+        vpsraw $0xf, %ymm2, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm2, %ymm2
+        vpsubw %ymm0, %ymm3, %ymm3
+        vpsraw $0xf, %ymm3, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm3, %ymm3
+        vpsubw %ymm0, %ymm4, %ymm4
+        vpsraw $0xf, %ymm4, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm4, %ymm4
+        vpsubw %ymm0, %ymm5, %ymm5
+        vpsraw $0xf, %ymm5, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm5, %ymm5
+        vpsubw %ymm0, %ymm6, %ymm6
+        vpsraw $0xf, %ymm6, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm6, %ymm6
+        vpsubw %ymm0, %ymm7, %ymm7
+        vpsraw $0xf, %ymm7, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm7, %ymm7
+        vpsubw %ymm0, %ymm8, %ymm8
+        vpsraw $0xf, %ymm8, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm8, %ymm8
+        vpsubw %ymm0, %ymm9, %ymm9
+        vpsraw $0xf, %ymm9, %ymm12
+        vpand %ymm0, %ymm12, %ymm12
+        vpaddw %ymm12, %ymm9, %ymm9
+        vmovdqa %ymm2, 0x100(%rdi)
+        vmovdqa %ymm3, 0x120(%rdi)
+        vmovdqa %ymm4, 0x140(%rdi)
+        vmovdqa %ymm5, 0x160(%rdi)
+        vmovdqa %ymm6, 0x180(%rdi)
+        vmovdqa %ymm7, 0x1a0(%rdi)
+        vmovdqa %ymm8, 0x1c0(%rdi)
+        vmovdqa %ymm9, 0x1e0(%rdi)
+        retq
+        .cfi_endproc
+
+MLK_ASM_FN_SIZE(reduce_avx2)
+
+#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
+        */

data/ext/pqcrypto/vendor/mlkem-native/mlkem/src/native/x86_64/src/rej_uniform_asm.S

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/*************************************************
+ * Name:        mlk_rej_uniform_asm
+ *
+ * Description: Run rejection sampling on uniform random bytes to generate
+ *              uniform random integers mod q
+ *
+ * Arguments:   - int16_t *r:          pointer to output buffer of MLKEM_N
+ *                                     16-bit coefficients.
+ *              - const uint8_t *buf:  pointer to input buffer
+ *                                     (assumed to be uniform random bytes)
+ *              - unsigned buflen:     length of input buffer in bytes.
+ *                                     Must be a multiple of 12.
+ *
+ * Returns number of sampled 16-bit integers (at most MLKEM_N).
+ **************************************************/
+#include "../../../common.h"
+
+#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
+    !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
+
+/*
+ * WARNING: This file is auto-derived from the mlkem-native source file
+ *   dev/x86_64/src/rej_uniform_asm.S using scripts/simpasm. Do not modify it directly.
+ */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",@progbits
+#endif
+
+.text
+.balign 4
+.global MLK_ASM_NAMESPACE(rej_uniform_asm)
+MLK_ASM_FN_SYMBOL(rej_uniform_asm)
+
+        .cfi_startproc
+        subq $0x210, %rsp            # imm = 0x210
+        .cfi_adjust_cfa_offset 0x210
+        xorl %eax, %eax
+        testq %rdx, %rdx
+        je Lrej_uniform_asm_end
+        movabsq $0xd010d010d010d01, %rax # imm = 0xD010D010D010D01
+        movq %rax, %xmm0
+        pinsrq $0x1, %rax, %xmm0
+        movabsq $0xfff0fff0fff0fff, %rax # imm = 0xFFF0FFF0FFF0FFF
+        movq %rax, %xmm5
+        pinsrq $0x1, %rax, %xmm5
+        movabsq $0x504040302010100, %rax # imm = 0x504040302010100
+        movq %rax, %xmm4
+        movabsq $0xb0a0a0908070706, %rax # imm = 0xB0A0A0908070706
+        pinsrq $0x1, %rax, %xmm4
+        movq $0x0, %rax
+        movq $0x0, %r8
+        movq $0x5555, %r9            # imm = 0x5555
+
+Lrej_uniform_asm_loop_start:
+        movq (%rsi,%r8), %xmm2
+        pinsrd $0x2, 0x8(%rsi,%r8), %xmm2
+        pshufb %xmm4, %xmm2
+        movdqa %xmm2, %xmm3
+        psrlw $0x4, %xmm3
+        pblendw $0xaa, %xmm3, %xmm2     # xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+        pand %xmm5, %xmm2
+        movdqa %xmm0, %xmm1
+        pcmpgtw %xmm2, %xmm1
+        pmovmskb %xmm1, %r11d
+        pextq %r9, %r11, %r11
+        movq %r11, %r10
+        shlq $0x4, %r10
+        movdqu (%rcx,%r10), %xmm3
+        pshufb %xmm3, %xmm2
+        movdqu %xmm2, (%rsp,%rax,2)
+        popcntq %r11, %r11
+        addq %r11, %rax
+        cmpq $0x100, %rax            # imm = 0x100
+        jae Lrej_uniform_asm_final_copy
+        addq $0xc, %r8
+        cmpq %r8, %rdx
+        ja Lrej_uniform_asm_loop_start
+
+Lrej_uniform_asm_final_copy:
+        movq $0x100, %rcx            # imm = 0x100
+        cmpq $0x100, %rax            # imm = 0x100
+        cmovaq %rcx, %rax
+        movq %rsp, %rsi
+        movq %rax, %rcx
+        shlq %rcx
+        rep  movsb (%rsi), %es:(%rdi)
+
+Lrej_uniform_asm_end:
+        addq $0x210, %rsp            # imm = 0x210
+        .cfi_adjust_cfa_offset -0x210
+        retq
+        .cfi_endproc
+
+MLK_ASM_FN_SIZE(rej_uniform_asm)
+
+#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
+        */