aws-crt 0.1.9 → 0.2.0

data/aws-crt-ffi/crt/aws-lc/third_party/s2n-bignum/arm/curve25519/bignum_madd_n25519.S

@@ -0,0 +1,284 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR ISC
+
+// ----------------------------------------------------------------------------
+// Multiply-add modulo the order of the curve25519/edwards25519 basepoint
+// Inputs x[4], y[4], c[4]; output z[4]
+//
+//    extern void bignum_madd_n25519
+//     (uint64_t z[static 4], uint64_t x[static 4],
+//      uint64_t y[static 4], uint64_t c[static 4]);
+//
+// Performs z := (x * y + c) mod n_25519, where the modulus is
+// n_25519 = 2^252 + 27742317777372353535851937790883648493, the
+// order of the curve25519/edwards25519 basepoint. The result z
+// and the inputs x, y and c are all 4 digits (256 bits).
+//
+// Standard ARM ABI: X0 = z, X1 = x, X2 = y, X3 = c
+// ----------------------------------------------------------------------------
+#include "_internal_s2n_bignum.h"
+
+        S2N_BN_SYM_VISIBILITY_DIRECTIVE(bignum_madd_n25519)
+        S2N_BN_SYM_PRIVACY_DIRECTIVE(bignum_madd_n25519)
+        .text
+        .balign 4
+
+// Backup of the input pointer so we can modify x0
+
+#define z x19
+
+// Temporaries for reduction phase
+
+#define q   x2
+#define n0  x3
+#define n1  x4
+#define t0  x5
+#define t1  x6
+#define t2  x7
+
+// Loading large constants
+
+#define movbig(nn,n3,n2,n1,n0)                                      \
+        movz    nn, n0;                                             \
+        movk    nn, n1, lsl #16;                                    \
+        movk    nn, n2, lsl #32;                                    \
+        movk    nn, n3, lsl #48
+
+// Single round of modular reduction mod_n25519, mapping
+// [m4;m3;m2;m1;m0] = m to [m3;m2;m1;m0] = m mod n_25519,
+// *assuming* the input m < 2^64 * n_25519. This is very
+// close to the loop body of the bignum_mod_n25519 function.
+
+#define reduce(m4,m3,m2,m1,m0)                          \
+        extr    q, m4, m3, #60;                         \
+        and     m3, m3, #0x0FFFFFFFFFFFFFFF;            \
+        sub     q, q, m4, lsr #60;                      \
+        and     t0, m4, #0xF000000000000000;            \
+        add     m3, m3, t0;                             \
+        mul     t0, n0, q;                              \
+        mul     t1, n1, q;                              \
+        umulh   t2, n0, q;                              \
+        adds    t1, t1, t2;                             \
+        umulh   t2, n1, q;                              \
+        adc     t2, t2, xzr;                            \
+        subs    m0, m0, t0;                             \
+        sbcs    m1, m1, t1;                             \
+        sbcs    m2, m2, t2;                             \
+        sbcs    m3, m3, xzr;                            \
+        csel    t0, n0, xzr, cc;                        \
+        csel    t1, n1, xzr, cc;                        \
+        adds    m0, m0, t0;                             \
+        and     t2, t0, #0x1000000000000000;            \
+        adcs    m1, m1, t1;                             \
+        adcs    m2, m2, xzr;                            \
+        adc     m3, m3, t2
+
+// Special case of "reduce" with m4 = 0. As well as not using m4,
+// the quotient selection is slightly simpler, just floor(m/2^252)
+// versus min (floor(m/2^252)) (2^63-1).
+
+#define reduce0(m3,m2,m1,m0)                            \
+        lsr     q, m3, #60;                             \
+        and     m3, m3, #0x0FFFFFFFFFFFFFFF;            \
+        mul     t0, n0, q;                              \
+        mul     t1, n1, q;                              \
+        umulh   t2, n0, q;                              \
+        adds    t1, t1, t2;                             \
+        umulh   t2, n1, q;                              \
+        adc     t2, t2, xzr;                            \
+        subs    m0, m0, t0;                             \
+        sbcs    m1, m1, t1;                             \
+        sbcs    m2, m2, t2;                             \
+        sbcs    m3, m3, xzr;                            \
+        csel    t0, n0, xzr, cc;                        \
+        csel    t1, n1, xzr, cc;                        \
+        adds    m0, m0, t0;                             \
+        and     t2, t0, #0x1000000000000000;            \
+        adcs    m1, m1, t1;                             \
+        adcs    m2, m2, xzr;                            \
+        adc     m3, m3, t2
+
+S2N_BN_SYMBOL(bignum_madd_n25519):
+
+        stp     x19, x20, [sp, -16]!
+
+// Back up the result pointer so we can overwrite x0 in intermediate steps
+
+        mov     z, x0
+
+// First compute [x15;x14;x13;x12;x11;x10;x9;x8] = x * y. This is
+// a basic 2-level Karatsuba multiplier, similar to the start of
+// bignum_mul_p25519, but with changes to the register allocation,
+// which in particular preserve x3/w3 for the next step.
+
+        ldp     x0, x4, [x1]
+        ldp     x5, x6, [x2]
+        umull   x8, w0, w5
+        lsr     x17, x0, #32
+        umull   x7, w17, w5
+        lsr     x16, x5, #32
+        umull   x9, w16, w17
+        umull   x16, w0, w16
+        adds    x8, x8, x7, lsl #32
+        lsr     x7, x7, #32
+        adc     x9, x9, x7
+        adds    x8, x8, x16, lsl #32
+        lsr     x16, x16, #32
+        adc     x9, x9, x16
+        mul     x10, x4, x6
+        umulh   x11, x4, x6
+        subs    x4, x4, x0
+        cneg    x4, x4, cc
+        csetm   x16, cc
+        adds    x10, x10, x9
+        adc     x11, x11, xzr
+        subs    x0, x5, x6
+        cneg    x0, x0, cc
+        cinv    x16, x16, cc
+        mul     x7, x4, x0
+        umulh   x0, x4, x0
+        adds    x9, x8, x10
+        adcs    x10, x10, x11
+        adc     x11, x11, xzr
+        cmn     x16, #0x1
+        eor     x7, x7, x16
+        adcs    x9, x7, x9
+        eor     x0, x0, x16
+        adcs    x10, x0, x10
+        adc     x11, x11, x16
+        ldp     x0, x4, [x1, #16]
+        ldp     x5, x6, [x2, #16]
+        umull   x12, w0, w5
+        lsr     x17, x0, #32
+        umull   x7, w17, w5
+        lsr     x16, x5, #32
+        umull   x13, w16, w17
+        umull   x16, w0, w16
+        adds    x12, x12, x7, lsl #32
+        lsr     x7, x7, #32
+        adc     x13, x13, x7
+        adds    x12, x12, x16, lsl #32
+        lsr     x16, x16, #32
+        adc     x13, x13, x16
+        mul     x14, x4, x6
+        umulh   x15, x4, x6
+        subs    x4, x4, x0
+        cneg    x4, x4, cc
+        csetm   x16, cc
+        adds    x14, x14, x13
+        adc     x15, x15, xzr
+        subs    x0, x5, x6
+        cneg    x0, x0, cc
+        cinv    x16, x16, cc
+        mul     x7, x4, x0
+        umulh   x0, x4, x0
+        adds    x13, x12, x14
+        adcs    x14, x14, x15
+        adc     x15, x15, xzr
+        cmn     x16, #0x1
+        eor     x7, x7, x16
+        adcs    x13, x7, x13
+        eor     x0, x0, x16
+        adcs    x14, x0, x14
+        adc     x15, x15, x16
+        ldp     x0, x4, [x1, #16]
+        ldp     x7, x16, [x1]
+        subs    x0, x0, x7
+        sbcs    x4, x4, x16
+        csetm   x16, cc
+        ldp     x7, x17, [x2]
+        subs    x5, x7, x5
+        sbcs    x6, x17, x6
+        csetm   x17, cc
+        eor     x0, x0, x16
+        subs    x0, x0, x16
+        eor     x4, x4, x16
+        sbc     x4, x4, x16
+        eor     x5, x5, x17
+        subs    x5, x5, x17
+        eor     x6, x6, x17
+        sbc     x6, x6, x17
+        eor     x16, x17, x16
+        adds    x12, x12, x10
+        adcs    x13, x13, x11
+        adcs    x14, x14, xzr
+        adc     x15, x15, xzr
+        mul     x2, x0, x5
+        umulh   x17, x0, x5
+        mul     x7, x4, x6
+        umulh   x1, x4, x6
+        subs    x4, x4, x0
+        cneg    x4, x4, cc
+        csetm   x10, cc
+        adds    x7, x7, x17
+        adc     x1, x1, xzr
+        subs    x6, x5, x6
+        cneg    x6, x6, cc
+        cinv    x10, x10, cc
+        mul     x5, x4, x6
+        umulh   x6, x4, x6
+        adds    x17, x2, x7
+        adcs    x7, x7, x1
+        adc     x1, x1, xzr
+        cmn     x10, #0x1
+        eor     x5, x5, x10
+        adcs    x17, x5, x17
+        eor     x6, x6, x10
+        adcs    x7, x6, x7
+        adc     x1, x1, x10
+        adds    x10, x12, x8
+        adcs    x11, x13, x9
+        adcs    x12, x14, x12
+        adcs    x13, x15, x13
+        adcs    x14, x14, xzr
+        adc     x15, x15, xzr
+        cmn     x16, #0x1
+        eor     x2, x2, x16
+        adcs    x10, x2, x10
+        eor     x17, x17, x16
+        adcs    x11, x17, x11
+        eor     x7, x7, x16
+        adcs    x12, x7, x12
+        eor     x1, x1, x16
+        adcs    x13, x1, x13
+        adcs    x14, x14, x16
+        adc     x15, x15, x16
+
+// Add the constant term, so [x15;x14;x13;x12;x11;x10;x9;x8] = x * y + c
+// It's easier to just do this now versus incorporating it into the
+// Karatsuba steps above or deferring it until partway through the
+// reduction, though it does result in a long carry propagation here.
+
+        ldp     x0, x1, [x3]
+        adds    x8, x8, x0
+        adcs    x9, x9, x1
+        ldp     x0, x1, [x3, #16]
+        adcs    x10, x10, x0
+        adcs    x11, x11, x1
+        adcs    x12, x12, xzr
+        adcs    x13, x13, xzr
+        adcs    x14, x14, xzr
+        adc     x15, x15, xzr
+
+// Now do the modular reduction and write back
+
+        movbig( n0, #0x5812, #0x631a, #0x5cf5, #0xd3ed)
+        movbig( n1, #0x14de, #0xf9de, #0xa2f7, #0x9cd6)
+
+        reduce0(x15,x14,x13,x12)
+        reduce(x15,x14,x13,x12,x11)
+        reduce(x14,x13,x12,x11,x10)
+        reduce(x13,x12,x11,x10,x9)
+        reduce(x12,x11,x10,x9,x8)
+
+        stp     x8, x9, [z]
+        stp     x10, x11, [z, #16]
+
+// Restore registers and return
+
+        ldp     x19, x20, [sp], 16
+        ret
+
+#if defined(__linux__) && defined(__ELF__)
+.section .note.GNU-stacz,"",%progbits
+#endif

data/aws-crt-ffi/crt/aws-lc/third_party/s2n-bignum/arm/curve25519/bignum_madd_n25519_alt.S

@@ -0,0 +1,210 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR ISC
+
+// ----------------------------------------------------------------------------
+// Multiply-add modulo the order of the curve25519/edwards25519 basepoint
+// Inputs x[4], y[4], c[4]; output z[4]
+//
+//    extern void bignum_madd_n25519_alt
+//     (uint64_t z[static 4], uint64_t x[static 4],
+//      uint64_t y[static 4], uint64_t c[static 4]);
+//
+// Performs z := (x * y + c) mod n_25519, where the modulus is
+// n_25519 = 2^252 + 27742317777372353535851937790883648493, the
+// order of the curve25519/edwards25519 basepoint. The result z
+// and the inputs x, y and c are all 4 digits (256 bits).
+//
+// Standard ARM ABI: X0 = z, X1 = x, X2 = y, X3 = c
+// ----------------------------------------------------------------------------
+#include "_internal_s2n_bignum.h"
+
+        S2N_BN_SYM_VISIBILITY_DIRECTIVE(bignum_madd_n25519_alt)
+        S2N_BN_SYM_PRIVACY_DIRECTIVE(bignum_madd_n25519_alt)
+        .text
+        .balign 4
+
+// Backup of the input pointer so we can modify x0
+
+#define z x19
+
+// Temporaries for reduction phase
+
+#define q   x2
+#define n0  x3
+#define n1  x4
+#define t0  x5
+#define t1  x6
+#define t2  x7
+
+// Loading large constants
+
+#define movbig(nn,n3,n2,n1,n0)                                      \
+        movz    nn, n0;                                             \
+        movk    nn, n1, lsl #16;                                    \
+        movk    nn, n2, lsl #32;                                    \
+        movk    nn, n3, lsl #48
+
+// Single round of modular reduction mod_n25519, mapping
+// [m4;m3;m2;m1;m0] = m to [m3;m2;m1;m0] = m mod n_25519,
+// *assuming* the input m < 2^64 * n_25519. This is very
+// close to the loop body of the bignum_mod_n25519 function.
+
+#define reduce(m4,m3,m2,m1,m0)                          \
+        extr    q, m4, m3, #60;                         \
+        and     m3, m3, #0x0FFFFFFFFFFFFFFF;            \
+        sub     q, q, m4, lsr #60;                      \
+        and     t0, m4, #0xF000000000000000;            \
+        add     m3, m3, t0;                             \
+        mul     t0, n0, q;                              \
+        mul     t1, n1, q;                              \
+        umulh   t2, n0, q;                              \
+        adds    t1, t1, t2;                             \
+        umulh   t2, n1, q;                              \
+        adc     t2, t2, xzr;                            \
+        subs    m0, m0, t0;                             \
+        sbcs    m1, m1, t1;                             \
+        sbcs    m2, m2, t2;                             \
+        sbcs    m3, m3, xzr;                            \
+        csel    t0, n0, xzr, cc;                        \
+        csel    t1, n1, xzr, cc;                        \
+        adds    m0, m0, t0;                             \
+        and     t2, t0, #0x1000000000000000;            \
+        adcs    m1, m1, t1;                             \
+        adcs    m2, m2, xzr;                            \
+        adc     m3, m3, t2
+
+// Special case of "reduce" with m4 = 0. As well as not using m4,
+// the quotient selection is slightly simpler, just floor(m/2^252)
+// versus min (floor(m/2^252)) (2^63-1).
+
+#define reduce0(m3,m2,m1,m0)                            \
+        lsr     q, m3, #60;                             \
+        and     m3, m3, #0x0FFFFFFFFFFFFFFF;            \
+        mul     t0, n0, q;                              \
+        mul     t1, n1, q;                              \
+        umulh   t2, n0, q;                              \
+        adds    t1, t1, t2;                             \
+        umulh   t2, n1, q;                              \
+        adc     t2, t2, xzr;                            \
+        subs    m0, m0, t0;                             \
+        sbcs    m1, m1, t1;                             \
+        sbcs    m2, m2, t2;                             \
+        sbcs    m3, m3, xzr;                            \
+        csel    t0, n0, xzr, cc;                        \
+        csel    t1, n1, xzr, cc;                        \
+        adds    m0, m0, t0;                             \
+        and     t2, t0, #0x1000000000000000;            \
+        adcs    m1, m1, t1;                             \
+        adcs    m2, m2, xzr;                            \
+        adc     m3, m3, t2
+
+S2N_BN_SYMBOL(bignum_madd_n25519_alt):
+
+        stp     x19, x20, [sp, -16]!
+
+// Back up the result pointer so we can overwrite x0 in intermediate steps
+
+        mov     z, x0
+
+// First compute [x15;x14;x13;x12;x11;x10;x9;x8] = x * y + c. This
+// is a basic schoolbook multiplier similar to the start of
+// bignum_mul_p25519_alt except for different registers, but it
+// also adds in the c term after the first row accumulation.
+
+        ldp     x13, x14, [x1]
+        ldp     x7, x0, [x2]
+        mul     x8, x13, x7
+        umulh   x9, x13, x7
+        mul     x16, x13, x0
+        umulh   x10, x13, x0
+        adds    x9, x9, x16
+        ldp     x4, x5, [x2, #16]
+        mul     x16, x13, x4
+        umulh   x11, x13, x4
+        adcs    x10, x10, x16
+        mul     x16, x13, x5
+        umulh   x12, x13, x5
+        adcs    x11, x11, x16
+        adc     x12, x12, xzr
+        ldp     x15, x6, [x3]
+        adds    x8, x8, x15
+        adcs    x9, x9, x6
+        ldp     x15, x6, [x3, #16]
+        adcs    x10, x10, x15
+        adcs    x11, x11, x6
+        adc     x12, x12, xzr
+        ldp     x15, x6, [x1, #16]
+        mul     x16, x14, x7
+        adds    x9, x9, x16
+        mul     x16, x14, x0
+        adcs    x10, x10, x16
+        mul     x16, x14, x4
+        adcs    x11, x11, x16
+        mul     x16, x14, x5
+        adcs    x12, x12, x16
+        umulh   x13, x14, x5
+        adc     x13, x13, xzr
+        umulh   x16, x14, x7
+        adds    x10, x10, x16
+        umulh   x16, x14, x0
+        adcs    x11, x11, x16
+        umulh   x16, x14, x4
+        adcs    x12, x12, x16
+        adc     x13, x13, xzr
+        mul     x16, x15, x7
+        adds    x10, x10, x16
+        mul     x16, x15, x0
+        adcs    x11, x11, x16
+        mul     x16, x15, x4
+        adcs    x12, x12, x16
+        mul     x16, x15, x5
+        adcs    x13, x13, x16
+        umulh   x14, x15, x5
+        adc     x14, x14, xzr
+        umulh   x16, x15, x7
+        adds    x11, x11, x16
+        umulh   x16, x15, x0
+        adcs    x12, x12, x16
+        umulh   x16, x15, x4
+        adcs    x13, x13, x16
+        adc     x14, x14, xzr
+        mul     x16, x6, x7
+        adds    x11, x11, x16
+        mul     x16, x6, x0
+        adcs    x12, x12, x16
+        mul     x16, x6, x4
+        adcs    x13, x13, x16
+        mul     x16, x6, x5
+        adcs    x14, x14, x16
+        umulh   x15, x6, x5
+        adc     x15, x15, xzr
+        umulh   x16, x6, x7
+        adds    x12, x12, x16
+        umulh   x16, x6, x0
+        adcs    x13, x13, x16
+        umulh   x16, x6, x4
+        adcs    x14, x14, x16
+        adc     x15, x15, xzr
+
+// Now do the modular reduction and write back
+
+        movbig( n0, #0x5812, #0x631a, #0x5cf5, #0xd3ed)
+        movbig( n1, #0x14de, #0xf9de, #0xa2f7, #0x9cd6)
+
+        reduce0(x15,x14,x13,x12)
+        reduce(x15,x14,x13,x12,x11)
+        reduce(x14,x13,x12,x11,x10)
+        reduce(x13,x12,x11,x10,x9)
+        reduce(x12,x11,x10,x9,x8)
+
+        stp     x8, x9, [z]
+        stp     x10, x11, [z, #16]
+
+// Restore registers and return
+
+        ldp     x19, x20, [sp], 16
+        ret
+
+#if defined(__linux__) && defined(__ELF__)
+.section .note.GNU-stacz,"",%progbits
+#endif

data/aws-crt-ffi/crt/aws-lc/third_party/s2n-bignum/arm/curve25519/bignum_mod_n25519.S

@@ -0,0 +1,186 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR ISC
+
+// ----------------------------------------------------------------------------
+// Reduce modulo basepoint order, z := x mod n_25519
+// Input x[k]; output z[4]
+//
+//    extern void bignum_mod_n25519
+//     (uint64_t z[static 4], uint64_t k, uint64_t *x);
+//
+// Reduction is modulo the order of the curve25519/edwards25519 basepoint,
+// which is n_25519 = 2^252 + 27742317777372353535851937790883648493
+//
+// Standard ARM ABI: X0 = z, X1 = k, X2 = x
+// ----------------------------------------------------------------------------
+#include "_internal_s2n_bignum.h"
+
+        S2N_BN_SYM_VISIBILITY_DIRECTIVE(bignum_mod_n25519)
+        S2N_BN_SYM_PRIVACY_DIRECTIVE(bignum_mod_n25519)
+        .text
+        .balign 4
+
+#define z x0
+#define k x1
+#define x x2
+
+#define m0 x3
+#define m1 x4
+#define m2 x5
+#define m3 x6
+
+#define t0 x7
+#define t1 x8
+#define t2 x9
+#define t3 x10
+
+#define n0 x11
+#define n1 x12
+
+// These two are aliased: we only load d when finished with q
+
+#define q x13
+#define d x13
+
+// Loading large constants
+
+#define movbig(nn,n3,n2,n1,n0)                                      \
+        movz    nn, n0;                                             \
+        movk    nn, n1, lsl #16;                                    \
+        movk    nn, n2, lsl #32;                                    \
+        movk    nn, n3, lsl #48
+
+S2N_BN_SYMBOL(bignum_mod_n25519):
+
+// If the input is already <= 3 words long, go to a trivial "copy" path
+
+        cmp     k, #4
+        bcc     short
+
+// Otherwise load the top 4 digits (top-down) and reduce k by 4
+// This [m3;m2;m1;m0] is the initial x where we begin reduction.
+
+        sub     k, k, #4
+        lsl     t0, k, #3
+        add     t0, t0, x
+        ldp     m2, m3, [t0, #16]
+        ldp     m0, m1, [t0]
+
+// Load the complicated two words of n_25519 = 2^252 + [n1; n0]
+
+        movbig( n0, #0x5812, #0x631a, #0x5cf5, #0xd3ed)
+        movbig( n1, #0x14de, #0xf9de, #0xa2f7, #0x9cd6)
+
+// Get the quotient estimate q = floor(x/2^252).
+// Also delete it from m3, in effect doing x' = x - q * 2^252
+
+        lsr     q, m3, #60
+        and     m3, m3, #0x0FFFFFFFFFFFFFFF
+
+// Multiply [t2;t1;t0] = q * [n1;n0]
+
+        mul     t0, n0, q
+        mul     t1, n1, q
+        umulh   t2, n0, q
+        adds    t1, t1, t2
+        umulh   t2, n1, q
+        adc     t2, t2, xzr
+
+// Subtract [m3;m2;m1;m0] = x' - q * [n1;n0] = x - q * n_25519
+
+        subs    m0, m0, t0
+        sbcs    m1, m1, t1
+        sbcs    m2, m2, t2
+        sbcs    m3, m3, xzr
+
+// If this borrows (CF = 0 because of inversion), add back n_25519.
+// The masked n3 digit exploits the fact that bit 60 of n0 is set.
+
+        csel    t0, n0, xzr, cc
+        csel    t1, n1, xzr, cc
+        adds    m0, m0, t0
+        adcs    m1, m1, t1
+        and     t0, t0, #0x1000000000000000
+        adcs    m2, m2, xzr
+        adc     m3, m3, t0
+
+// Now do (k-4) iterations of 5->4 word modular reduction. Each one
+// is similar to the sequence above except for the more refined quotient
+// estimation process.
+
+        cbz     k, writeback
+
+loop:
+
+// Assume that the new 5-digit x is 2^64 * previous_x + next_digit.
+// Get the quotient estimate q = max (floor(x/2^252)) (2^64 - 1)
+// and first compute x' = x - 2^252 * q.
+
+        extr    q, m3, m2, #60
+        and     m2, m2, #0x0FFFFFFFFFFFFFFF
+        sub     q, q, m3, lsr #60
+        and     m3, m3, #0xF000000000000000
+        add     m2, m2, m3
+
+// Multiply [t2;t1;t0] = q * [n1;n0]
+
+        mul     t0, n0, q
+        mul     t1, n1, q
+        umulh   t2, n0, q
+        adds    t1, t1, t2
+        umulh   t2, n1, q
+        adc     t2, t2, xzr
+
+// Decrement k and load the next digit (note that d aliases to q)
+
+        sub     k, k, #1
+        ldr     d, [x, k, lsl #3]
+
+// Subtract [t3;t2;t1;t0] = x' - q * [n1;n0] = x - q * n_25519
+
+        subs    t0, d, t0
+        sbcs    t1, m0, t1
+        sbcs    t2, m1, t2
+        sbcs    t3, m2, xzr
+
+// If this borrows (CF = 0 because of inversion), add back n_25519.
+// The masked n3 digit exploits the fact that bit 60 of n1 is set.
+
+        csel    m0, n0, xzr, cc
+        csel    m1, n1, xzr, cc
+        adds    m0, t0, m0
+        and     m3, m1, #0x1000000000000000
+        adcs    m1, t1, m1
+        adcs    m2, t2, xzr
+        adc     m3, t3, m3
+
+        cbnz    k, loop
+
+// Finally write back [m3;m2;m1;m0] and return
+
+writeback:
+        stp     m0, m1, [z]
+        stp     m2, m3, [z, #16]
+        ret
+
+// Short case: just copy the input with zero-padding
+
+short:
+        mov     m0, xzr
+        mov     m1, xzr
+        mov     m2, xzr
+        mov     m3, xzr
+
+        cbz     k, writeback
+        ldr     m0, [x]
+        subs    k, k, #1
+        beq     writeback
+        ldr     m1, [x, #8]
+        subs    k, k, #1
+        beq     writeback
+        ldr     m2, [x, #16]
+        b       writeback
+
+#if defined(__linux__) && defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif