pq_crypto 0.6.1 → 0.6.2

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/aarch64/src/polyz_unpack_19_aarch64_asm.S

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+ #include "../../../common.h"
+#if defined(MLD_ARITH_BACKEND_AARCH64) && \
+           (!defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API)) && \
+           !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \
+           (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87))
+
+/*
+ * WARNING: This file is auto-derived from the mldsa-native source file
+ *   dev/aarch64_opt/src/polyz_unpack_19_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
+ */
+
+.text
+.balign 4
+.global MLD_ASM_NAMESPACE(polyz_unpack_19_aarch64_asm)
+MLD_ASM_FN_SYMBOL(polyz_unpack_19_aarch64_asm)
+
+        .cfi_startproc
+        ldr q24, [x2]
+        ldr q25, [x2, #0x10]
+        ldr q26, [x2, #0x20]
+        ldr q27, [x2, #0x30]
+        mov x3, #0xfc00000000       // =1082331758592
+        dup v28.2d, x3
+        movi v29.4s, #0xf, msl #16
+        movi v30.4s, #0x8, lsl #16
+        mov x9, #0x10               // =16
+
+Lpolyz_unpack_19_loop:
+        ld1 { v0.16b, v1.16b }, [x1]
+        add x1, x1, #0x18
+        ld1 { v2.16b }, [x1], #16
+        tbl v4.16b, { v0.16b }, v24.16b
+        tbl v5.16b, { v0.16b, v1.16b }, v25.16b
+        tbl v6.16b, { v1.16b }, v26.16b
+        tbl v7.16b, { v1.16b, v2.16b }, v27.16b
+        ushl v4.4s, v4.4s, v28.4s
+        and v4.16b, v4.16b, v29.16b
+        sub v4.4s, v30.4s, v4.4s
+        ushl v5.4s, v5.4s, v28.4s
+        and v5.16b, v5.16b, v29.16b
+        sub v5.4s, v30.4s, v5.4s
+        ushl v6.4s, v6.4s, v28.4s
+        and v6.16b, v6.16b, v29.16b
+        sub v6.4s, v30.4s, v6.4s
+        ushl v7.4s, v7.4s, v28.4s
+        and v7.16b, v7.16b, v29.16b
+        sub v7.4s, v30.4s, v7.4s
+        str q5, [x0, #0x10]
+        str q6, [x0, #0x20]
+        str q7, [x0, #0x30]
+        str q4, [x0], #0x40
+        subs x9, x9, #0x1
+        b.ne Lpolyz_unpack_19_loop
+        ret
+        .cfi_endproc
+
+MLD_ASM_FN_SIZE(polyz_unpack_19_aarch64_asm)
+
+#endif /* MLD_ARITH_BACKEND_AARCH64 && (!MLD_CONFIG_NO_SIGN_API || \
+          !MLD_CONFIG_NO_VERIFY_API) && !MLD_CONFIG_MULTILEVEL_NO_SHARED && \
+          (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \
+          || MLD_CONFIG_PARAMETER_SET == 87) */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/aarch64/src/polyz_unpack_table.c

@@ -16,22 +16,34 @@
 
 #include "arith_native_aarch64.h"
 
+#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API)
 /* Table of indices used for tbl instructions in polyz_unpack_{17,19}.
  * See autogen for details. */
 
-MLD_ALIGN const uint8_t mld_polyz_unpack_17_indices[] = {
-    0,  1,  2,  255, 2,  3,  4,  255, 4,  5,  6,  255, 6,  7,  8,  255,
-    9,  10, 11, 255, 11, 12, 13, 255, 13, 14, 15, 255, 15, 16, 17, 255,
-    2,  3,  4,  255, 4,  5,  6,  255, 6,  7,  8,  255, 8,  9,  10, 255,
-    11, 12, 13, 255, 13, 14, 15, 255, 15, 28, 29, 255, 29, 30, 31, 255,
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44
+MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint8_t
+    mld_polyz_unpack_17_indices[64] = {
+        0,  1,  2,  255, 2,  3,  4,  255, 4,  5,  6,  255, 6,  7,  8,  255,
+        9,  10, 11, 255, 11, 12, 13, 255, 13, 14, 15, 255, 15, 16, 17, 255,
+        2,  3,  4,  255, 4,  5,  6,  255, 6,  7,  8,  255, 8,  9,  10, 255,
+        11, 12, 13, 255, 13, 14, 15, 255, 15, 28, 29, 255, 29, 30, 31, 255,
 };
-
-MLD_ALIGN const uint8_t mld_polyz_unpack_19_indices[] = {
-    0,  1,  2,  255, 2,  3,  4,  255, 5,  6,  7,  255, 7,  8,  9,  255,
-    10, 11, 12, 255, 12, 13, 14, 255, 15, 16, 17, 255, 17, 18, 19, 255,
-    4,  5,  6,  255, 6,  7,  8,  255, 9,  10, 11, 255, 11, 12, 13, 255,
-    14, 15, 24, 255, 24, 25, 26, 255, 27, 28, 29, 255, 29, 30, 31, 255,
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \
+        */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \
+    (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)
+MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint8_t
+    mld_polyz_unpack_19_indices[64] = {
+        0,  1,  2,  255, 2,  3,  4,  255, 5,  6,  7,  255, 7,  8,  9,  255,
+        10, 11, 12, 255, 12, 13, 14, 255, 15, 16, 17, 255, 17, 18, 19, 255,
+        4,  5,  6,  255, 6,  7,  8,  255, 9,  10, 11, 255, 11, 12, 13, 255,
+        14, 15, 24, 255, 24, 25, 26, 255, 27, 28, 29, 255, 29, 30, 31, 255,
 };
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \
+          || MLD_CONFIG_PARAMETER_SET == 87 */
+
+#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */
 
 #else /* MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED */
 

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/aarch64/src/rej_uniform_aarch64_asm.S

@@ -0,0 +1,189 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+ #include "../../../common.h"
+#if defined(MLD_ARITH_BACKEND_AARCH64) && \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)
+
+/*
+ * WARNING: This file is auto-derived from the mldsa-native source file
+ *   dev/aarch64_opt/src/rej_uniform_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
+ */
+
+.text
+.balign 4
+.global MLD_ASM_NAMESPACE(rej_uniform_aarch64_asm)
+MLD_ASM_FN_SYMBOL(rej_uniform_aarch64_asm)
+
+        .cfi_startproc
+        sub sp, sp, #0x440
+        .cfi_adjust_cfa_offset 0x440
+        mov x7, #0x1                // =1
+        movk x7, #0x2, lsl #32
+        mov v31.d[0], x7
+        mov x7, #0x4                // =4
+        movk x7, #0x8, lsl #32
+        mov v31.d[1], x7
+        mov w7, #0xe001             // =57345
+        movk w7, #0x7f, lsl #16
+        dup v30.4s, w7
+        mov x8, sp
+        mov x7, x8
+        mov x11, #0x0               // =0
+        eor v16.16b, v16.16b, v16.16b
+
+Lrej_uniform_initial_zero:
+        str q16, [x7], #0x40
+        stur q16, [x7, #-0x30]
+        stur q16, [x7, #-0x20]
+        stur q16, [x7, #-0x10]
+        add x11, x11, #0x10
+        cmp x11, #0x100
+        b.lt Lrej_uniform_initial_zero
+        mov x7, x8
+        mov x9, #0x0                // =0
+        mov x4, #0x100              // =256
+        cmp x2, #0x30
+        b.lo Lrej_uniform_loop48_end
+
+Lrej_uniform_loop48:
+        cmp x9, x4
+        b.hs Lrej_uniform_memory_copy
+        sub x2, x2, #0x30
+        ld3 { v0.16b, v1.16b, v2.16b }, [x1], #48
+        movi v4.16b, #0x80
+        bic v2.16b, v2.16b, v4.16b
+        zip1 v4.16b, v0.16b, v1.16b
+        zip2 v5.16b, v0.16b, v1.16b
+        ushll v6.8h, v2.8b, #0x0
+        ushll2 v7.8h, v2.16b, #0x0
+        zip1 v16.8h, v4.8h, v6.8h
+        zip2 v17.8h, v4.8h, v6.8h
+        zip1 v18.8h, v5.8h, v7.8h
+        zip2 v19.8h, v5.8h, v7.8h
+        cmhi v4.4s, v30.4s, v16.4s
+        cmhi v5.4s, v30.4s, v17.4s
+        cmhi v6.4s, v30.4s, v18.4s
+        cmhi v7.4s, v30.4s, v19.4s
+        and v4.16b, v4.16b, v31.16b
+        and v5.16b, v5.16b, v31.16b
+        and v6.16b, v6.16b, v31.16b
+        and v7.16b, v7.16b, v31.16b
+        uaddlv d20, v4.4s
+        uaddlv d21, v5.4s
+        uaddlv d22, v6.4s
+        uaddlv d23, v7.4s
+        fmov x12, d20
+        fmov x13, d21
+        fmov x14, d22
+        fmov x15, d23
+        ldr q24, [x3, x12, lsl #4]
+        ldr q25, [x3, x13, lsl #4]
+        ldr q26, [x3, x14, lsl #4]
+        ldr q27, [x3, x15, lsl #4]
+        cnt v4.16b, v4.16b
+        cnt v5.16b, v5.16b
+        cnt v6.16b, v6.16b
+        cnt v7.16b, v7.16b
+        uaddlv d20, v4.4s
+        uaddlv d21, v5.4s
+        uaddlv d22, v6.4s
+        uaddlv d23, v7.4s
+        fmov x12, d20
+        fmov x13, d21
+        fmov x14, d22
+        fmov x15, d23
+        tbl v16.16b, { v16.16b }, v24.16b
+        tbl v17.16b, { v17.16b }, v25.16b
+        tbl v18.16b, { v18.16b }, v26.16b
+        tbl v19.16b, { v19.16b }, v27.16b
+        st1 { v16.4s }, [x7]
+        add x7, x7, x12, lsl #2
+        st1 { v17.4s }, [x7]
+        add x7, x7, x13, lsl #2
+        st1 { v18.4s }, [x7]
+        add x7, x7, x14, lsl #2
+        st1 { v19.4s }, [x7]
+        add x7, x7, x15, lsl #2
+        add x12, x12, x13
+        add x14, x14, x15
+        add x9, x9, x12
+        add x9, x9, x14
+        cmp x2, #0x30
+        b.hs Lrej_uniform_loop48
+
+Lrej_uniform_loop48_end:
+        cmp x9, x4
+        b.hs Lrej_uniform_memory_copy
+        cmp x2, #0x18
+        b.lo Lrej_uniform_memory_copy
+        sub x2, x2, #0x18
+        ld3 { v0.8b, v1.8b, v2.8b }, [x1], #24
+        movi v4.16b, #0x80
+        bic v2.16b, v2.16b, v4.16b
+        zip1 v4.16b, v0.16b, v1.16b
+        ushll v6.8h, v2.8b, #0x0
+        zip1 v16.8h, v4.8h, v6.8h
+        zip2 v17.8h, v4.8h, v6.8h
+        cmhi v4.4s, v30.4s, v16.4s
+        cmhi v5.4s, v30.4s, v17.4s
+        and v4.16b, v4.16b, v31.16b
+        and v5.16b, v5.16b, v31.16b
+        uaddlv d20, v4.4s
+        uaddlv d21, v5.4s
+        fmov x12, d20
+        fmov x13, d21
+        ldr q24, [x3, x12, lsl #4]
+        ldr q25, [x3, x13, lsl #4]
+        cnt v4.16b, v4.16b
+        cnt v5.16b, v5.16b
+        uaddlv d20, v4.4s
+        uaddlv d21, v5.4s
+        fmov x12, d20
+        fmov x13, d21
+        tbl v16.16b, { v16.16b }, v24.16b
+        tbl v17.16b, { v17.16b }, v25.16b
+        st1 { v16.4s }, [x7]
+        add x7, x7, x12, lsl #2
+        st1 { v17.4s }, [x7]
+        add x7, x7, x13, lsl #2
+        add x9, x9, x12
+        add x9, x9, x13
+
+Lrej_uniform_memory_copy:
+        cmp x9, x4
+        csel x9, x9, x4, lo
+        mov x11, #0x0               // =0
+        mov x7, x8
+
+Lrej_uniform_final_copy:
+        ldr q16, [x7], #0x40
+        ldur q17, [x7, #-0x30]
+        ldur q18, [x7, #-0x20]
+        ldur q19, [x7, #-0x10]
+        str q16, [x0], #0x40
+        stur q17, [x0, #-0x30]
+        stur q18, [x0, #-0x20]
+        stur q19, [x0, #-0x10]
+        add x11, x11, #0x10
+        cmp x11, #0x100
+        b.lt Lrej_uniform_final_copy
+        mov x0, x9
+        b Lrej_uniform_return
+
+Lrej_uniform_return:
+        add sp, sp, #0x440
+        .cfi_adjust_cfa_offset -0x440
+        ret
+        .cfi_endproc
+
+MLD_ASM_FN_SIZE(rej_uniform_aarch64_asm)
+
+#endif /* MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/aarch64/src/rej_uniform_eta2_aarch64_asm.S

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+#include "../../../common.h"
+#if defined(MLD_ARITH_BACKEND_AARCH64) && \
+    !defined(MLD_CONFIG_NO_KEYPAIR_API) && \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \
+    (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2)
+
+/*
+ * WARNING: This file is auto-derived from the mldsa-native source file
+ *   dev/aarch64_opt/src/rej_uniform_eta2_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
+ */
+
+.text
+.balign 4
+.global MLD_ASM_NAMESPACE(rej_uniform_eta2_aarch64_asm)
+MLD_ASM_FN_SYMBOL(rej_uniform_eta2_aarch64_asm)
+
+        .cfi_startproc
+        sub sp, sp, #0x240
+        .cfi_adjust_cfa_offset 0x240
+        mov x7, #0x1                // =1
+        movk x7, #0x2, lsl #16
+        movk x7, #0x4, lsl #32
+        movk x7, #0x8, lsl #48
+        mov v31.d[0], x7
+        mov x7, #0x10               // =16
+        movk x7, #0x20, lsl #16
+        movk x7, #0x40, lsl #32
+        movk x7, #0x80, lsl #48
+        mov v31.d[1], x7
+        movi v30.8h, #0xf
+        mov x8, sp
+        mov x7, x8
+        mov x11, #0x0               // =0
+        eor v16.16b, v16.16b, v16.16b
+
+Lrej_uniform_eta2_initial_zero:
+        str q16, [x7], #0x40
+        stur q16, [x7, #-0x30]
+        stur q16, [x7, #-0x20]
+        stur q16, [x7, #-0x10]
+        add x11, x11, #0x20
+        cmp x11, #0x100
+        b.lt Lrej_uniform_eta2_initial_zero
+        mov x7, x8
+        mov x9, #0x0                // =0
+        mov x4, #0x100              // =256
+
+Lrej_uniform_eta2_loop8:
+        cmp x9, x4
+        b.hs Lrej_uniform_eta2_memory_copy
+        sub x2, x2, #0x8
+        ld1 { v0.8b }, [x1], #8
+        movi v26.8b, #0xf
+        and v27.8b, v0.8b, v26.8b
+        ushr v28.8b, v0.8b, #0x4
+        zip1 v26.8b, v27.8b, v28.8b
+        zip2 v29.8b, v27.8b, v28.8b
+        ushll v16.8h, v26.8b, #0x0
+        ushll v17.8h, v29.8b, #0x0
+        cmhi v4.8h, v30.8h, v16.8h
+        cmhi v5.8h, v30.8h, v17.8h
+        and v4.16b, v4.16b, v31.16b
+        and v5.16b, v5.16b, v31.16b
+        uaddlv s20, v4.8h
+        uaddlv s21, v5.8h
+        fmov w12, s20
+        fmov w13, s21
+        ldr q24, [x3, x12, lsl #4]
+        ldr q25, [x3, x13, lsl #4]
+        cnt v4.16b, v4.16b
+        cnt v5.16b, v5.16b
+        uaddlv s20, v4.8h
+        uaddlv s21, v5.8h
+        fmov w12, s20
+        fmov w13, s21
+        tbl v16.16b, { v16.16b }, v24.16b
+        tbl v17.16b, { v17.16b }, v25.16b
+        st1 { v16.8h }, [x7]
+        add x7, x7, x12, lsl #1
+        st1 { v17.8h }, [x7]
+        add x7, x7, x13, lsl #1
+        add x12, x12, x13
+        add x9, x9, x12
+        cmp x2, #0x8
+        b.hs Lrej_uniform_eta2_loop8
+
+Lrej_uniform_eta2_memory_copy:
+        cmp x9, x4
+        csel x9, x9, x4, lo
+        mov w7, #0x199a             // =6554
+        dup v26.8h, w7
+        movi v27.8h, #0x5
+        movi v7.8h, #0x2
+        mov x11, #0x0               // =0
+        mov x7, x8
+
+Lrej_uniform_eta2_final_copy:
+        ldr q16, [x7], #0x20
+        ldur q18, [x7, #-0x10]
+        sqdmulh v28.8h, v16.8h, v26.8h
+        mls v16.8h, v28.8h, v27.8h
+        sqdmulh v28.8h, v18.8h, v26.8h
+        mls v18.8h, v28.8h, v27.8h
+        sub v16.8h, v7.8h, v16.8h
+        sub v18.8h, v7.8h, v18.8h
+        sshll2 v17.4s, v16.8h, #0x0
+        sshll v16.4s, v16.4h, #0x0
+        sshll2 v19.4s, v18.8h, #0x0
+        sshll v18.4s, v18.4h, #0x0
+        str q16, [x0], #0x40
+        stur q17, [x0, #-0x30]
+        stur q18, [x0, #-0x20]
+        stur q19, [x0, #-0x10]
+        add x11, x11, #0x10
+        cmp x11, #0x100
+        b.lt Lrej_uniform_eta2_final_copy
+        mov x0, x9
+        add sp, sp, #0x240
+        .cfi_adjust_cfa_offset -0x240
+        ret
+        .cfi_endproc
+
+MLD_ASM_FN_SIZE(rej_uniform_eta2_aarch64_asm)
+
+#endif /* MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_NO_KEYPAIR_API && \
+          !MLD_CONFIG_MULTILEVEL_NO_SHARED && \
+          (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 2) */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/native/aarch64/src/rej_uniform_eta4_aarch64_asm.S

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+#include "../../../common.h"
+#if defined(MLD_ARITH_BACKEND_AARCH64) && \
+    !defined(MLD_CONFIG_NO_KEYPAIR_API) && \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \
+    (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4)
+
+/*
+ * WARNING: This file is auto-derived from the mldsa-native source file
+ *   dev/aarch64_opt/src/rej_uniform_eta4_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
+ */
+
+.text
+.balign 4
+.global MLD_ASM_NAMESPACE(rej_uniform_eta4_aarch64_asm)
+MLD_ASM_FN_SYMBOL(rej_uniform_eta4_aarch64_asm)
+
+        .cfi_startproc
+        sub sp, sp, #0x240
+        .cfi_adjust_cfa_offset 0x240
+        mov x7, #0x1                // =1
+        movk x7, #0x2, lsl #16
+        movk x7, #0x4, lsl #32
+        movk x7, #0x8, lsl #48
+        mov v31.d[0], x7
+        mov x7, #0x10               // =16
+        movk x7, #0x20, lsl #16
+        movk x7, #0x40, lsl #32
+        movk x7, #0x80, lsl #48
+        mov v31.d[1], x7
+        movi v30.8h, #0x9
+        movi v7.8h, #0x4
+        mov x8, sp
+        mov x7, x8
+        mov x11, #0x0               // =0
+        eor v16.16b, v16.16b, v16.16b
+
+Lrej_uniform_eta4_initial_zero:
+        str q16, [x7], #0x40
+        stur q16, [x7, #-0x30]
+        stur q16, [x7, #-0x20]
+        stur q16, [x7, #-0x10]
+        add x11, x11, #0x20
+        cmp x11, #0x100
+        b.lt Lrej_uniform_eta4_initial_zero
+        mov x7, x8
+        mov x9, #0x0                // =0
+        mov x4, #0x100              // =256
+
+Lrej_uniform_eta4_loop8:
+        cmp x9, x4
+        b.hs Lrej_uniform_eta4_memory_copy
+        sub x2, x2, #0x8
+        ld1 { v0.8b }, [x1], #8
+        movi v26.8b, #0xf
+        and v27.8b, v0.8b, v26.8b
+        ushr v28.8b, v0.8b, #0x4
+        zip1 v26.8b, v27.8b, v28.8b
+        zip2 v29.8b, v27.8b, v28.8b
+        ushll v16.8h, v26.8b, #0x0
+        ushll v17.8h, v29.8b, #0x0
+        cmhi v4.8h, v30.8h, v16.8h
+        cmhi v5.8h, v30.8h, v17.8h
+        and v4.16b, v4.16b, v31.16b
+        and v5.16b, v5.16b, v31.16b
+        uaddlv s20, v4.8h
+        uaddlv s21, v5.8h
+        fmov w12, s20
+        fmov w13, s21
+        ldr q24, [x3, x12, lsl #4]
+        ldr q25, [x3, x13, lsl #4]
+        cnt v4.16b, v4.16b
+        cnt v5.16b, v5.16b
+        uaddlv s20, v4.8h
+        uaddlv s21, v5.8h
+        fmov w12, s20
+        fmov w13, s21
+        tbl v16.16b, { v16.16b }, v24.16b
+        tbl v17.16b, { v17.16b }, v25.16b
+        st1 { v16.8h }, [x7]
+        add x7, x7, x12, lsl #1
+        st1 { v17.8h }, [x7]
+        add x7, x7, x13, lsl #1
+        add x12, x12, x13
+        add x9, x9, x12
+        cmp x2, #0x8
+        b.hs Lrej_uniform_eta4_loop8
+
+Lrej_uniform_eta4_memory_copy:
+        cmp x9, x4
+        csel x9, x9, x4, lo
+        mov x11, #0x0               // =0
+        mov x7, x8
+
+Lrej_uniform_eta4_final_copy:
+        ldr q16, [x7], #0x20
+        ldur q18, [x7, #-0x10]
+        sub v16.8h, v7.8h, v16.8h
+        sub v18.8h, v7.8h, v18.8h
+        sshll2 v17.4s, v16.8h, #0x0
+        sshll v16.4s, v16.4h, #0x0
+        sshll2 v19.4s, v18.8h, #0x0
+        sshll v18.4s, v18.4h, #0x0
+        str q16, [x0], #0x40
+        stur q17, [x0, #-0x30]
+        stur q18, [x0, #-0x20]
+        stur q19, [x0, #-0x10]
+        add x11, x11, #0x10
+        cmp x11, #0x100
+        b.lt Lrej_uniform_eta4_final_copy
+        mov x0, x9
+        add sp, sp, #0x240
+        .cfi_adjust_cfa_offset -0x240
+        ret
+        .cfi_endproc
+
+MLD_ASM_FN_SIZE(rej_uniform_eta4_aarch64_asm)
+
+#endif /* MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_NO_KEYPAIR_API && \
+          !MLD_CONFIG_MULTILEVEL_NO_SHARED && \
+          (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 4) */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif