pq_crypto 0.4.2 → 0.5.0

data/ext/pqcrypto/vendor/mlkem-native/mlkem/src/native/aarch64/src/poly_reduce_asm.S

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/*yaml
+  Name: poly_reduce_asm
+  Description: Barrett reduction of polynomial coefficients
+  Signature: void mlk_poly_reduce_asm(int16_t p[256])
+  ABI:
+    x0:
+      type: buffer
+      size_bytes: 512
+      permissions: read/write
+      c_parameter: int16_t p[256]
+      description: Input/output polynomial
+  Stack:
+    bytes: 0
+*/
+
+#include "../../../common.h"
+#if defined(MLK_ARITH_BACKEND_AARCH64) &&  !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
+
+/*
+ * WARNING: This file is auto-derived from the mlkem-native source file
+ *   dev/aarch64_opt/src/poly_reduce_asm.S using scripts/simpasm. Do not modify it directly.
+ */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",@progbits
+#endif
+
+.text
+.balign 4
+.global MLK_ASM_NAMESPACE(poly_reduce_asm)
+MLK_ASM_FN_SYMBOL(poly_reduce_asm)
+
+        .cfi_startproc
+        mov w2, #0xd01              // =3329
+        dup v3.8h, w2
+        mov w2, #0x4ebf             // =20159
+        dup v4.8h, w2
+        mov x1, #0x8                // =8
+        ldr q21, [x0], #0x40
+        ldur q18, [x0, #-0x20]
+        ldur q0, [x0, #-0x30]
+        ldur q5, [x0, #-0x10]
+        ldr q26, [x0], #0x40
+        sqdmulh v17.8h, v21.8h, v4.h[0]
+        sqdmulh v27.8h, v18.8h, v4.h[0]
+        sqdmulh v22.8h, v0.8h, v4.h[0]
+        srshr v17.8h, v17.8h, #0xb
+        sqdmulh v23.8h, v5.8h, v4.h[0]
+        srshr v29.8h, v27.8h, #0xb
+        mls v21.8h, v17.8h, v3.h[0]
+        srshr v17.8h, v22.8h, #0xb
+        mls v18.8h, v29.8h, v3.h[0]
+        srshr v22.8h, v23.8h, #0xb
+        mls v0.8h, v17.8h, v3.h[0]
+        sshr v2.8h, v21.8h, #0xf
+        mls v5.8h, v22.8h, v3.h[0]
+        sshr v29.8h, v18.8h, #0xf
+        and v19.16b, v3.16b, v2.16b
+        sqdmulh v2.8h, v26.8h, v4.h[0]
+        sshr v31.8h, v0.8h, #0xf
+        add v17.8h, v21.8h, v19.8h
+        and v21.16b, v3.16b, v29.16b
+        and v31.16b, v3.16b, v31.16b
+        sub x1, x1, #0x2
+
+Lpoly_reduce_loop_start:
+        add v21.8h, v18.8h, v21.8h
+        ldur q18, [x0, #-0x20]
+        add v25.8h, v0.8h, v31.8h
+        ldur q0, [x0, #-0x30]
+        stur q21, [x0, #-0x60]
+        sshr v28.8h, v5.8h, #0xf
+        stur q17, [x0, #-0x80]
+        srshr v23.8h, v2.8h, #0xb
+        sqdmulh v30.8h, v18.8h, v4.h[0]
+        stur q25, [x0, #-0x70]
+        and v22.16b, v3.16b, v28.16b
+        sqdmulh v7.8h, v0.8h, v4.h[0]
+        add v16.8h, v5.8h, v22.8h
+        ldur q5, [x0, #-0x10]
+        mls v26.8h, v23.8h, v3.h[0]
+        stur q16, [x0, #-0x50]
+        srshr v6.8h, v30.8h, #0xb
+        srshr v1.8h, v7.8h, #0xb
+        sqdmulh v19.8h, v5.8h, v4.h[0]
+        mls v18.8h, v6.8h, v3.h[0]
+        sshr v24.8h, v26.8h, #0xf
+        mls v0.8h, v1.8h, v3.h[0]
+        and v27.16b, v3.16b, v24.16b
+        srshr v29.8h, v19.8h, #0xb
+        add v17.8h, v26.8h, v27.8h
+        ldr q26, [x0], #0x40
+        sshr v1.8h, v18.8h, #0xf
+        mls v5.8h, v29.8h, v3.h[0]
+        sshr v20.8h, v0.8h, #0xf
+        and v21.16b, v3.16b, v1.16b
+        and v31.16b, v3.16b, v20.16b
+        sqdmulh v2.8h, v26.8h, v4.h[0]
+        subs x1, x1, #0x1
+        cbnz x1, Lpoly_reduce_loop_start
+        add v28.8h, v0.8h, v31.8h
+        ldur q29, [x0, #-0x10]
+        add v21.8h, v18.8h, v21.8h
+        srshr v18.8h, v2.8h, #0xb
+        sshr v2.8h, v5.8h, #0xf
+        ldur q16, [x0, #-0x20]
+        stur q17, [x0, #-0x80]
+        ldur q0, [x0, #-0x30]
+        and v2.16b, v3.16b, v2.16b
+        sqdmulh v24.8h, v29.8h, v4.h[0]
+        stur q28, [x0, #-0x70]
+        stur q21, [x0, #-0x60]
+        add v31.8h, v5.8h, v2.8h
+        sqdmulh v6.8h, v16.8h, v4.h[0]
+        stur q31, [x0, #-0x50]
+        sqdmulh v17.8h, v0.8h, v4.h[0]
+        srshr v22.8h, v24.8h, #0xb
+        mls v26.8h, v18.8h, v3.h[0]
+        srshr v31.8h, v6.8h, #0xb
+        mls v29.8h, v22.8h, v3.h[0]
+        srshr v19.8h, v17.8h, #0xb
+        mls v16.8h, v31.8h, v3.h[0]
+        sshr v7.8h, v26.8h, #0xf
+        mls v0.8h, v19.8h, v3.h[0]
+        and v5.16b, v3.16b, v7.16b
+        sshr v22.8h, v29.8h, #0xf
+        add v27.8h, v26.8h, v5.8h
+        and v26.16b, v3.16b, v22.16b
+        sshr v20.8h, v16.8h, #0xf
+        stur q27, [x0, #-0x40]
+        and v2.16b, v3.16b, v20.16b
+        sshr v23.8h, v0.8h, #0xf
+        add v18.8h, v29.8h, v26.8h
+        add v31.8h, v16.8h, v2.8h
+        and v29.16b, v3.16b, v23.16b
+        stur q18, [x0, #-0x10]
+        add v25.8h, v0.8h, v29.8h
+        stur q31, [x0, #-0x20]
+        stur q25, [x0, #-0x30]
+        ret
+        .cfi_endproc
+
+MLK_ASM_FN_SIZE(poly_reduce_asm)
+
+#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */

data/ext/pqcrypto/vendor/mlkem-native/mlkem/src/native/aarch64/src/poly_tobytes_asm.S

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/*yaml
+  Name: poly_tobytes_asm
+  Description: Convert polynomial to byte representation
+  Signature: void mlk_poly_tobytes_asm(uint8_t r[384], const int16_t a[256])
+  ABI:
+    x0:
+      type: buffer
+      size_bytes: 384
+      permissions: write-only
+      c_parameter: uint8_t r[384]
+      description: Output byte array
+    x1:
+      type: buffer
+      size_bytes: 512
+      permissions: read-only
+      c_parameter: const int16_t a[256]
+      description: Input polynomial
+  Stack:
+    bytes: 0
+*/
+
+#include "../../../common.h"
+#if defined(MLK_ARITH_BACKEND_AARCH64) &&  !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
+
+/*
+ * WARNING: This file is auto-derived from the mlkem-native source file
+ *   dev/aarch64_opt/src/poly_tobytes_asm.S using scripts/simpasm. Do not modify it directly.
+ */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",@progbits
+#endif
+
+.text
+.balign 4
+.global MLK_ASM_NAMESPACE(poly_tobytes_asm)
+MLK_ASM_FN_SYMBOL(poly_tobytes_asm)
+
+        .cfi_startproc
+        mov x2, #0x10               // =16
+        ldr q5, [x1, #0x10]
+        ldr q3, [x1], #0x20
+        ldr q29, [x1], #0x20
+        ldur q2, [x1, #-0x10]
+        ldr q27, [x1, #0x10]
+        ldr q23, [x1, #0x30]
+        ldr q17, [x1], #0x20
+        ldr q16, [x1], #0x20
+        uzp2 v26.8h, v3.8h, v5.8h
+        uzp1 v19.8h, v3.8h, v5.8h
+        uzp2 v0.8h, v29.8h, v2.8h
+        uzp1 v1.8h, v29.8h, v2.8h
+        xtn v5.8b, v26.8h
+        shrn v3.8b, v19.8h, #0x8
+        shrn v4.8b, v26.8h, #0x4
+        xtn v18.8b, v0.8h
+        shrn v30.8b, v0.8h, #0x4
+        xtn v28.8b, v1.8h
+        shrn v29.8b, v1.8h, #0x8
+        sli v3.8b, v5.8b, #0x4
+        xtn v2.8b, v19.8h
+        sli v29.8b, v18.8b, #0x4
+        lsr x2, x2, #1
+        sub x2, x2, #0x2
+
+Lpoly_tobytes_loop_start:
+        uzp1 v25.8h, v17.8h, v27.8h
+        uzp2 v31.8h, v17.8h, v27.8h
+        uzp1 v24.8h, v16.8h, v23.8h
+        uzp2 v6.8h, v16.8h, v23.8h
+        st3 { v2.8b, v3.8b, v4.8b }, [x0], #24
+        shrn v3.8b, v25.8h, #0x8
+        ldr q17, [x1], #0x20
+        shrn v4.8b, v31.8h, #0x4
+        xtn v21.8b, v6.8h
+        ldr q23, [x1, #0x10]
+        st3 { v28.8b, v29.8b, v30.8b }, [x0], #24
+        shrn v29.8b, v24.8h, #0x8
+        ldur q27, [x1, #-0x10]
+        xtn v20.8b, v31.8h
+        ldr q16, [x1], #0x20
+        sli v29.8b, v21.8b, #0x4
+        xtn v2.8b, v25.8h
+        sli v3.8b, v20.8b, #0x4
+        xtn v28.8b, v24.8h
+        shrn v30.8b, v6.8h, #0x4
+        subs x2, x2, #0x1
+        cbnz x2, Lpoly_tobytes_loop_start
+        uzp2 v7.8h, v17.8h, v27.8h
+        uzp1 v25.8h, v17.8h, v27.8h
+        uzp2 v0.8h, v16.8h, v23.8h
+        st3 { v2.8b, v3.8b, v4.8b }, [x0], #24
+        st3 { v28.8b, v29.8b, v30.8b }, [x0], #24
+        shrn v21.8b, v25.8h, #0x8
+        uzp1 v2.8h, v16.8h, v23.8h
+        shrn v22.8b, v7.8h, #0x4
+        shrn v4.8b, v0.8h, #0x4
+        xtn v28.8b, v7.8h
+        xtn v27.8b, v0.8h
+        shrn v3.8b, v2.8h, #0x8
+        sli v21.8b, v28.8b, #0x4
+        xtn v2.8b, v2.8h
+        sli v3.8b, v27.8b, #0x4
+        xtn v20.8b, v25.8h
+        st3 { v20.8b, v21.8b, v22.8b }, [x0], #24
+        st3 { v2.8b, v3.8b, v4.8b }, [x0], #24
+        ret
+        .cfi_endproc
+
+MLK_ASM_FN_SIZE(poly_tobytes_asm)
+
+#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */

data/ext/pqcrypto/vendor/mlkem-native/mlkem/src/native/aarch64/src/poly_tomont_asm.S

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/*yaml
+  Name: poly_tomont_asm
+  Description: Convert polynomial to Montgomery domain
+  Signature: void mlk_poly_tomont_asm(int16_t p[256])
+  ABI:
+    x0:
+      type: buffer
+      size_bytes: 512
+      permissions: read/write
+      c_parameter: int16_t p[256]
+      description: Input/output polynomial
+  Stack:
+    bytes: 0
+*/
+
+#include "../../../common.h"
+#if defined(MLK_ARITH_BACKEND_AARCH64) &&  !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
+
+/*
+ * WARNING: This file is auto-derived from the mlkem-native source file
+ *   dev/aarch64_opt/src/poly_tomont_asm.S using scripts/simpasm. Do not modify it directly.
+ */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",@progbits
+#endif
+
+.text
+.balign 4
+.global MLK_ASM_NAMESPACE(poly_tomont_asm)
+MLK_ASM_FN_SYMBOL(poly_tomont_asm)
+
+        .cfi_startproc
+        mov w2, #0xd01              // =3329
+        dup v4.8h, w2
+        mov w2, #0x4ebf             // =20159
+        dup v5.8h, w2
+        mov w2, #-0x414             // =-1044
+        dup v2.8h, w2
+        mov w2, #-0x2824            // =-10276
+        dup v3.8h, w2
+        mov x1, #0x8                // =8
+        ldr q18, [x0, #0x20]
+        ldr q0, [x0, #0x10]
+        ldr q16, [x0], #0x40
+        sqrdmulh v23.8h, v0.8h, v3.8h
+        mul v26.8h, v0.8h, v2.8h
+        sqrdmulh v19.8h, v16.8h, v3.8h
+        mls v26.8h, v23.8h, v4.h[0]
+        mul v29.8h, v16.8h, v2.8h
+        ldur q16, [x0, #-0x10]
+        mls v29.8h, v19.8h, v4.h[0]
+        stur q26, [x0, #-0x30]
+        sqrdmulh v26.8h, v18.8h, v3.8h
+        mul v18.8h, v18.8h, v2.8h
+        stur q29, [x0, #-0x40]
+        sqrdmulh v29.8h, v16.8h, v3.8h
+        mls v18.8h, v26.8h, v4.h[0]
+        sub x1, x1, #0x1
+
+Lpoly_tomont_loop:
+        ldr q19, [x0, #0x10]
+        mul v26.8h, v16.8h, v2.8h
+        ldr q23, [x0, #0x20]
+        ldr q17, [x0], #0x40
+        mls v26.8h, v29.8h, v4.h[0]
+        ldur q16, [x0, #-0x10]
+        sqrdmulh v28.8h, v19.8h, v3.8h
+        stur q18, [x0, #-0x60]
+        mul v0.8h, v19.8h, v2.8h
+        stur q26, [x0, #-0x50]
+        sqrdmulh v24.8h, v23.8h, v3.8h
+        mul v18.8h, v23.8h, v2.8h
+        sqrdmulh v22.8h, v17.8h, v3.8h
+        mul v26.8h, v17.8h, v2.8h
+        mls v0.8h, v28.8h, v4.h[0]
+        mls v26.8h, v22.8h, v4.h[0]
+        sqrdmulh v29.8h, v16.8h, v3.8h
+        stur q0, [x0, #-0x30]
+        mls v18.8h, v24.8h, v4.h[0]
+        stur q26, [x0, #-0x40]
+        sub x1, x1, #0x1
+        cbnz x1, Lpoly_tomont_loop
+        mul v16.8h, v16.8h, v2.8h
+        stur q18, [x0, #-0x20]
+        mls v16.8h, v29.8h, v4.h[0]
+        stur q16, [x0, #-0x10]
+        ret
+        .cfi_endproc
+
+MLK_ASM_FN_SIZE(poly_tomont_asm)
+
+#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */

data/ext/pqcrypto/vendor/mlkem-native/mlkem/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_asm_k2.S

@@ -0,0 +1,261 @@
+/*
+ * Copyright (c) The mlkem-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/* References
+ * ==========
+ *
+ * - [NeonNTT]
+ *   Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1
+ *   Becker, Hwang, Kannwischer, Yang, Yang
+ *   https://eprint.iacr.org/2021/986
+ */
+
+/*yaml
+  Name: polyvec_basemul_acc_montgomery_cached_asm_k2
+  Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=2
+  Signature: void mlk_polyvec_basemul_acc_montgomery_cached_asm_k2(int16_t r[256], const int16_t a[512], const int16_t b[512], const int16_t b_cache[256])
+  ABI:
+    x0:
+      type: buffer
+      size_bytes: 512
+      permissions: write-only
+      c_parameter: int16_t r[256]
+      description: Output polynomial
+    x1:
+      type: buffer
+      size_bytes: 1024
+      permissions: read-only
+      c_parameter: const int16_t a[512]
+      description: Input polynomial vector a
+    x2:
+      type: buffer
+      size_bytes: 1024
+      permissions: read-only
+      c_parameter: const int16_t b[512]
+      description: Input polynomial vector b
+    x3:
+      type: buffer
+      size_bytes: 512
+      permissions: read-only
+      c_parameter: const int16_t b_cache[256]
+      description: Cached values for b
+  Stack:
+    bytes: 64
+    description: saving callee-saved Neon registers
+*/
+
+/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */
+
+#include "../../../common.h"
+#if defined(MLK_ARITH_BACKEND_AARCH64) &&  !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) &&  (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2)
+
+/*
+ * WARNING: This file is auto-derived from the mlkem-native source file
+ *   dev/aarch64_opt/src/polyvec_basemul_acc_montgomery_cached_asm_k2.S using scripts/simpasm. Do not modify it directly.
+ */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",@progbits
+#endif
+
+.text
+.balign 4
+.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_asm_k2)
+MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_asm_k2)
+
+        .cfi_startproc
+        sub sp, sp, #0x40
+        .cfi_adjust_cfa_offset 0x40
+        stp d8, d9, [sp]
+        .cfi_rel_offset d8, 0x0
+        .cfi_rel_offset d9, 0x8
+        stp d10, d11, [sp, #0x10]
+        .cfi_rel_offset d10, 0x10
+        .cfi_rel_offset d11, 0x18
+        stp d12, d13, [sp, #0x20]
+        .cfi_rel_offset d12, 0x20
+        .cfi_rel_offset d13, 0x28
+        stp d14, d15, [sp, #0x30]
+        .cfi_rel_offset d14, 0x30
+        .cfi_rel_offset d15, 0x38
+        mov w14, #0xd01             // =3329
+        dup v0.8h, w14
+        mov w14, #0xcff             // =3327
+        dup v2.8h, w14
+        add x4, x1, #0x200
+        add x5, x2, #0x200
+        add x6, x3, #0x100
+        mov x13, #0x10              // =16
+        ldr q12, [x1], #0x20
+        ldur q9, [x1, #-0x10]
+        ldr q22, [x2], #0x20
+        ldur q30, [x2, #-0x10]
+        ldr q6, [x5], #0x20
+        ldr q7, [x4, #0x10]
+        ldr q8, [x4], #0x20
+        ldur q23, [x5, #-0x10]
+        uzp1 v16.8h, v12.8h, v9.8h
+        uzp2 v14.8h, v12.8h, v9.8h
+        uzp2 v13.8h, v22.8h, v30.8h
+        uzp1 v18.8h, v22.8h, v30.8h
+        ld1 { v27.8h }, [x3], #16
+        ld1 { v17.8h }, [x6], #16
+        smull2 v4.4s, v16.8h, v18.8h
+        ldr q31, [x1, #0x10]
+        smull v19.4s, v16.4h, v13.4h
+        ldr q24, [x1], #0x20
+        smlal v19.4s, v14.4h, v18.4h
+        ldr q22, [x2], #0x20
+        smlal2 v4.4s, v14.8h, v27.8h
+        uzp2 v5.8h, v6.8h, v23.8h
+        smull2 v29.4s, v16.8h, v13.8h
+        uzp2 v26.8h, v8.8h, v7.8h
+        smlal2 v29.4s, v14.8h, v18.8h
+        uzp1 v30.8h, v24.8h, v31.8h
+        uzp1 v8.8h, v8.8h, v7.8h
+        smull v11.4s, v16.4h, v18.4h
+        smlal v11.4s, v14.4h, v27.4h
+        ldur q1, [x2, #-0x10]
+        uzp1 v28.8h, v6.8h, v23.8h
+        smlal2 v29.4s, v8.8h, v5.8h
+        ldr q25, [x5], #0x20
+        smlal v19.4s, v8.4h, v5.4h
+        ldr q3, [x4, #0x10]
+        smlal2 v29.4s, v26.8h, v28.8h
+        uzp1 v27.8h, v22.8h, v1.8h
+        smlal v19.4s, v26.4h, v28.4h
+        ldr q12, [x4], #0x20
+        smlal2 v4.4s, v8.8h, v28.8h
+        ldur q21, [x5, #-0x10]
+        smlal2 v4.4s, v26.8h, v17.8h
+        smlal v11.4s, v8.4h, v28.4h
+        ld1 { v15.8h }, [x6], #16
+        smlal v11.4s, v26.4h, v17.4h
+        ld1 { v20.8h }, [x3], #16
+        uzp1 v28.8h, v19.8h, v29.8h
+        smull2 v23.4s, v30.8h, v27.8h
+        smull v26.4s, v30.4h, v27.4h
+        uzp2 v16.8h, v22.8h, v1.8h
+        mul v28.8h, v28.8h, v2.8h
+        uzp1 v10.8h, v11.8h, v4.8h
+        smull2 v8.4s, v30.8h, v16.8h
+        mul v13.8h, v10.8h, v2.8h
+        smlal v19.4s, v28.4h, v0.4h
+        smlal2 v29.4s, v28.8h, v0.8h
+        smull v18.4s, v30.4h, v16.4h
+        uzp1 v30.8h, v25.8h, v21.8h
+        smlal v11.4s, v13.4h, v0.4h
+        uzp2 v6.8h, v24.8h, v31.8h
+        uzp1 v16.8h, v12.8h, v3.8h
+        smlal2 v4.4s, v13.8h, v0.8h
+        uzp2 v17.8h, v25.8h, v21.8h
+        smlal2 v8.4s, v6.8h, v27.8h
+        uzp2 v12.8h, v12.8h, v3.8h
+        smlal v18.4s, v6.4h, v27.4h
+        uzp2 v9.8h, v19.8h, v29.8h
+        smlal2 v8.4s, v16.8h, v17.8h
+        smlal2 v8.4s, v12.8h, v30.8h
+        uzp2 v19.8h, v11.8h, v4.8h
+        sub x13, x13, #0x2
+
+Lpolyvec_basemul_acc_montgomery_cached_k2_loop_start:
+        smlal v18.4s, v16.4h, v17.4h
+        ldr q7, [x4], #0x20
+        ldr q10, [x2, #0x10]
+        smlal v18.4s, v12.4h, v30.4h
+        smlal2 v23.4s, v6.8h, v20.8h
+        ldr q14, [x2], #0x20
+        smlal2 v23.4s, v16.8h, v30.8h
+        zip1 v25.8h, v19.8h, v9.8h
+        zip2 v3.8h, v19.8h, v9.8h
+        smlal2 v23.4s, v12.8h, v15.8h
+        smlal v26.4s, v6.4h, v20.4h
+        uzp1 v5.8h, v18.8h, v8.8h
+        uzp2 v21.8h, v14.8h, v10.8h
+        smlal v26.4s, v16.4h, v30.4h
+        str q25, [x0], #0x20
+        mul v29.8h, v5.8h, v2.8h
+        uzp1 v24.8h, v14.8h, v10.8h
+        stur q3, [x0, #-0x10]
+        smlal v26.4s, v12.4h, v15.4h
+        ld1 { v15.8h }, [x6], #16
+        ldr q28, [x1, #0x10]
+        ldr q11, [x1], #0x20
+        ldr q13, [x5], #0x20
+        ldur q27, [x4, #-0x10]
+        smlal2 v8.4s, v29.8h, v0.8h
+        ldur q22, [x5, #-0x10]
+        smlal v18.4s, v29.4h, v0.4h
+        uzp1 v4.8h, v26.8h, v23.8h
+        uzp1 v1.8h, v11.8h, v28.8h
+        uzp2 v6.8h, v11.8h, v28.8h
+        uzp1 v16.8h, v7.8h, v27.8h
+        mul v31.8h, v4.8h, v2.8h
+        uzp2 v17.8h, v13.8h, v22.8h
+        ld1 { v20.8h }, [x3], #16
+        uzp2 v9.8h, v18.8h, v8.8h
+        smull2 v8.4s, v1.8h, v21.8h
+        uzp1 v30.8h, v13.8h, v22.8h
+        smlal2 v8.4s, v6.8h, v24.8h
+        smlal2 v8.4s, v16.8h, v17.8h
+        uzp2 v12.8h, v7.8h, v27.8h
+        smlal v26.4s, v31.4h, v0.4h
+        smlal2 v23.4s, v31.8h, v0.8h
+        smull v18.4s, v1.4h, v21.4h
+        smlal v18.4s, v6.4h, v24.4h
+        smlal2 v8.4s, v12.8h, v30.8h
+        uzp2 v19.8h, v26.8h, v23.8h
+        smull2 v23.4s, v1.8h, v24.8h
+        smull v26.4s, v1.4h, v24.4h
+        subs x13, x13, #0x1
+        cbnz x13, Lpolyvec_basemul_acc_montgomery_cached_k2_loop_start
+        smlal v26.4s, v6.4h, v20.4h
+        smlal2 v23.4s, v6.8h, v20.8h
+        smlal v26.4s, v16.4h, v30.4h
+        smlal2 v23.4s, v16.8h, v30.8h
+        smlal v26.4s, v12.4h, v15.4h
+        smlal2 v23.4s, v12.8h, v15.8h
+        smlal v18.4s, v16.4h, v17.4h
+        smlal v18.4s, v12.4h, v30.4h
+        zip1 v12.8h, v19.8h, v9.8h
+        str q12, [x0], #0x20
+        uzp1 v12.8h, v26.8h, v23.8h
+        mul v6.8h, v12.8h, v2.8h
+        uzp1 v12.8h, v18.8h, v8.8h
+        mul v12.8h, v12.8h, v2.8h
+        smlal v26.4s, v6.4h, v0.4h
+        smlal2 v23.4s, v6.8h, v0.8h
+        smlal2 v8.4s, v12.8h, v0.8h
+        smlal v18.4s, v12.4h, v0.4h
+        zip2 v12.8h, v19.8h, v9.8h
+        uzp2 v6.8h, v26.8h, v23.8h
+        stur q12, [x0, #-0x10]
+        uzp2 v12.8h, v18.8h, v8.8h
+        zip2 v1.8h, v6.8h, v12.8h
+        zip1 v12.8h, v6.8h, v12.8h
+        str q1, [x0, #0x10]
+        str q12, [x0], #0x20
+        ldp d8, d9, [sp]
+        .cfi_restore d8
+        .cfi_restore d9
+        ldp d10, d11, [sp, #0x10]
+        .cfi_restore d10
+        .cfi_restore d11
+        ldp d12, d13, [sp, #0x20]
+        .cfi_restore d12
+        .cfi_restore d13
+        ldp d14, d15, [sp, #0x30]
+        .cfi_restore d14
+        .cfi_restore d15
+        add sp, sp, #0x40
+        .cfi_adjust_cfa_offset -0x40
+        ret
+        .cfi_endproc
+
+MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_asm_k2)
+
+
+#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \
+          (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2) */