mldsa_gh 0.1.0

data/ext/mldsa_gh_native/vendor/mldsa-native/mldsa/src/native/aarch64/src/mldsa_rej_uniform_eta4_aarch64_asm.S

@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/*yaml
+  Name: rej_uniform_eta4_aarch64_asm
+  Description: AArch64 rejection sampling of eta=4 secret coefficients
+  Signature: uint64_t mld_rej_uniform_eta4_aarch64_asm(int32_t r[256], const uint8_t *buf, unsigned buflen, const uint8_t table[4096])
+  ABI:
+    Architecture: aarch64
+    CallingConvention: AAPCS64
+    Features: [NEON]
+    x0:
+      type: buffer
+      size_bytes: 1024
+      permissions: write-only
+      c_parameter: int32_t r[256]
+      description: Output buffer (256 x int32_t)
+    x1:
+      type: buffer
+      size_bytes: x2
+      permissions: read-only
+      c_parameter: const uint8_t *buf
+      description: Input buffer
+    x2:
+      type: scalar
+      c_parameter: unsigned buflen
+      description: Length of input buffer (must be a multiple of 8)
+      test_with: 272  # MLD_AARCH64_REJ_UNIFORM_ETA4_BUFLEN
+    x3:
+      type: buffer
+      size_bytes: 4096
+      permissions: read-only
+      c_parameter: const uint8_t table[4096]
+      description: Lookup table (4096 x uint8_t)
+*/
+
+#include "../../../common.h"
+#if defined(MLD_ARITH_BACKEND_AARCH64) && \
+    !defined(MLD_CONFIG_NO_KEYPAIR_API) && \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \
+    (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4)
+
+/*
+ * WARNING: This file is auto-derived from the mldsa-native source file
+ *   dev/aarch64_opt/src/mldsa_rej_uniform_eta4_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
+ */
+
+.text
+.balign 4
+.global MLD_ASM_NAMESPACE(rej_uniform_eta4_aarch64_asm)
+MLD_ASM_FN_SYMBOL(rej_uniform_eta4_aarch64_asm)
+
+        .cfi_startproc
+        sub sp, sp, #0x240
+        .cfi_adjust_cfa_offset 0x240
+        mov x7, #0x1                // =1
+        movk x7, #0x2, lsl #16
+        movk x7, #0x4, lsl #32
+        movk x7, #0x8, lsl #48
+        mov v31.d[0], x7
+        mov x7, #0x10               // =16
+        movk x7, #0x20, lsl #16
+        movk x7, #0x40, lsl #32
+        movk x7, #0x80, lsl #48
+        mov v31.d[1], x7
+        movi v30.8h, #0x9
+        movi v7.8h, #0x4
+        mov x8, sp
+        mov x7, x8
+        mov x11, #0x0               // =0
+        eor v16.16b, v16.16b, v16.16b
+
+Lmld_rej_uniform_eta4_initial_zero:
+        str q16, [x7], #0x40
+        stur q16, [x7, #-0x30]
+        stur q16, [x7, #-0x20]
+        stur q16, [x7, #-0x10]
+        add x11, x11, #0x20
+        cmp x11, #0x100
+        b.lt Lmld_rej_uniform_eta4_initial_zero
+        mov x7, x8
+        mov x9, #0x0                // =0
+        mov x4, #0x100              // =256
+
+Lmld_rej_uniform_eta4_loop8:
+        cmp x9, x4
+        b.hs Lmld_rej_uniform_eta4_memory_copy
+        sub x2, x2, #0x8
+        ld1 { v0.8b }, [x1], #8
+        movi v26.8b, #0xf
+        and v27.8b, v0.8b, v26.8b
+        ushr v28.8b, v0.8b, #0x4
+        zip1 v26.8b, v27.8b, v28.8b
+        zip2 v29.8b, v27.8b, v28.8b
+        ushll v16.8h, v26.8b, #0x0
+        ushll v17.8h, v29.8b, #0x0
+        cmhi v4.8h, v30.8h, v16.8h
+        cmhi v5.8h, v30.8h, v17.8h
+        and v4.16b, v4.16b, v31.16b
+        and v5.16b, v5.16b, v31.16b
+        uaddlv s20, v4.8h
+        uaddlv s21, v5.8h
+        fmov w12, s20
+        fmov w13, s21
+        ldr q24, [x3, x12, lsl #4]
+        ldr q25, [x3, x13, lsl #4]
+        cnt v4.16b, v4.16b
+        cnt v5.16b, v5.16b
+        uaddlv s20, v4.8h
+        uaddlv s21, v5.8h
+        fmov w12, s20
+        fmov w13, s21
+        tbl v16.16b, { v16.16b }, v24.16b
+        tbl v17.16b, { v17.16b }, v25.16b
+        st1 { v16.8h }, [x7]
+        add x7, x7, x12, lsl #1
+        st1 { v17.8h }, [x7]
+        add x7, x7, x13, lsl #1
+        add x12, x12, x13
+        add x9, x9, x12
+        cmp x2, #0x8
+        b.hs Lmld_rej_uniform_eta4_loop8
+
+Lmld_rej_uniform_eta4_memory_copy:
+        cmp x9, x4
+        csel x9, x9, x4, lo
+        mov x11, #0x0               // =0
+        mov x7, x8
+
+Lmld_rej_uniform_eta4_final_copy:
+        ldr q16, [x7], #0x20
+        ldur q18, [x7, #-0x10]
+        sub v16.8h, v7.8h, v16.8h
+        sub v18.8h, v7.8h, v18.8h
+        sshll2 v17.4s, v16.8h, #0x0
+        sshll v16.4s, v16.4h, #0x0
+        sshll2 v19.4s, v18.8h, #0x0
+        sshll v18.4s, v18.4h, #0x0
+        str q16, [x0], #0x40
+        stur q17, [x0, #-0x30]
+        stur q18, [x0, #-0x20]
+        stur q19, [x0, #-0x10]
+        add x11, x11, #0x10
+        cmp x11, #0x100
+        b.lt Lmld_rej_uniform_eta4_final_copy
+        mov x0, x9
+        add sp, sp, #0x240
+        .cfi_adjust_cfa_offset -0x240
+        ret
+        .cfi_endproc
+
+MLD_ASM_FN_SIZE(rej_uniform_eta4_aarch64_asm)
+
+#endif /* MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_NO_KEYPAIR_API && \
+          !MLD_CONFIG_MULTILEVEL_NO_SHARED && \
+          (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 4) */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif

data/ext/mldsa_gh_native/vendor/mldsa-native/mldsa/src/native/aarch64/src/polyz_unpack_table.c

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/*
+ * WARNING: This file is auto-generated from scripts/autogen
+ *          in the mldsa-native repository.
+ *          Do not modify it directly.
+ */
+
+#include "../../../common.h"
+
+#if defined(MLD_ARITH_BACKEND_AARCH64) && \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)
+
+#include "arith_native_aarch64.h"
+
+#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API)
+/* Table of indices used for tbl instructions in polyz_unpack_{17,19}.
+ * See autogen for details. */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44
+MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint8_t
+    mld_polyz_unpack_17_indices[64] = {
+        0,  1,  2,  255, 2,  3,  4,  255, 4,  5,  6,  255, 6,  7,  8,  255,
+        9,  10, 11, 255, 11, 12, 13, 255, 13, 14, 15, 255, 15, 16, 17, 255,
+        2,  3,  4,  255, 4,  5,  6,  255, 6,  7,  8,  255, 8,  9,  10, 255,
+        11, 12, 13, 255, 13, 14, 15, 255, 15, 28, 29, 255, 29, 30, 31, 255,
+};
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \
+        */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \
+    (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)
+MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint8_t
+    mld_polyz_unpack_19_indices[64] = {
+        0,  1,  2,  255, 2,  3,  4,  255, 5,  6,  7,  255, 7,  8,  9,  255,
+        10, 11, 12, 255, 12, 13, 14, 255, 15, 16, 17, 255, 17, 18, 19, 255,
+        4,  5,  6,  255, 6,  7,  8,  255, 9,  10, 11, 255, 11, 12, 13, 255,
+        14, 15, 24, 255, 24, 25, 26, 255, 27, 28, 29, 255, 29, 30, 31, 255,
+};
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \
+          || MLD_CONFIG_PARAMETER_SET == 87 */
+
+#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */
+
+#else /* MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED */
+
+MLD_EMPTY_CU(aarch64_polyz_unpack_table)
+
+#endif /* !(MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED) */