aws-crt 0.1.9 → 0.2.0

data/aws-crt-ffi/crt/aws-lc/third_party/s2n-bignum/arm/curve25519/curve25519_x25519.S

@@ -13,7 +13,8 @@
 // this returns the X coordinate of n * P = (X, Y), or 0 when n * P is the
 // point at infinity. Both n and X inputs are first slightly modified/mangled
 // as specified in the relevant RFC (https://www.rfc-editor.org/rfc/rfc7748);
-// in particular the lower three bits of n are set to zero.
+// in particular the lower three bits of n are set to zero. Does not implement
+// the zero-check specified in Section 6.1.
 //
 // Standard ARM ABI: X0 = res, X1 = scalar, X2 = point
 // ----------------------------------------------------------------------------
@@ -849,356 +850,1046 @@ curve25519_x25519_scalarloop:
         mul_p25519(zn,p,e)
 
 // The projective result of the scalar multiplication is now (xn,zn).
-// First set up the constant sn = 2^255 - 19 for the modular inverse.
-
-        mov     x0, #-19
-        mov     x1, #-1
-        mov     x2, #0x7fffffffffffffff
-        stp     x0, x1, [sn]
-        stp     x1, x2, [sn+16]
-
-// Prepare to call the modular inverse function to get zm = 1/zn
-
-        mov     x0, #4
-        add     x1, zm
-        add     x2, zn
-        add     x3, sn
-        add     x4, p
-
-// Inline copy of bignum_modinv, identical except for stripping out the
-// prologue and epilogue saving and restoring registers and the initial
-// test for k = 0 (which is trivially false here since k = 4). For more
-// details and explanations see "arm/generic/bignum_modinv.S".
-
-        lsl     x10, x0, #3
-        add     x21, x4, x10
-        add     x22, x21, x10
-        mov     x10, xzr
-curve25519_x25519_copyloop:
-        ldr     x11, [x2, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        str     x11, [x21, x10, lsl #3]
-        str     x12, [x22, x10, lsl #3]
-        str     x12, [x4, x10, lsl #3]
-        str     xzr, [x1, x10, lsl #3]
-        add     x10, x10, #0x1
-        cmp     x10, x0
-        b.cc    curve25519_x25519_copyloop
-        ldr     x11, [x4]
-        sub     x12, x11, #0x1
-        str     x12, [x4]
-        lsl     x20, x11, #2
-        sub     x20, x11, x20
-        eor     x20, x20, #0x2
-        mov     x12, #0x1
-        madd    x12, x11, x20, x12
-        mul     x11, x12, x12
-        madd    x20, x12, x20, x20
-        mul     x12, x11, x11
-        madd    x20, x11, x20, x20
-        mul     x11, x12, x12
-        madd    x20, x12, x20, x20
-        madd    x20, x11, x20, x20
-        lsl     x2, x0, #7
-curve25519_x25519_outerloop:
-        add     x10, x2, #0x3f
-        lsr     x5, x10, #6
-        cmp     x5, x0
-        csel    x5, x0, x5, cs
-        mov     x13, xzr
-        mov     x15, xzr
-        mov     x14, xzr
-        mov     x16, xzr
-        mov     x19, xzr
-        mov     x10, xzr
-curve25519_x25519_toploop:
-        ldr     x11, [x21, x10, lsl #3]
-        ldr     x12, [x22, x10, lsl #3]
-        orr     x17, x11, x12
-        cmp     x17, xzr
-        and     x17, x19, x13
-        csel    x15, x17, x15, ne
-        and     x17, x19, x14
-        csel    x16, x17, x16, ne
-        csel    x13, x11, x13, ne
-        csel    x14, x12, x14, ne
-        csetm   x19, ne
-        add     x10, x10, #0x1
-        cmp     x10, x5
-        b.cc    curve25519_x25519_toploop
-        orr     x11, x13, x14
-        clz     x12, x11
-        negs    x17, x12
-        lsl     x13, x13, x12
-        csel    x15, x15, xzr, ne
-        lsl     x14, x14, x12
-        csel    x16, x16, xzr, ne
-        lsr     x15, x15, x17
-        lsr     x16, x16, x17
-        orr     x13, x13, x15
-        orr     x14, x14, x16
-        ldr     x15, [x21]
-        ldr     x16, [x22]
-        mov     x6, #0x1
-        mov     x7, xzr
-        mov     x8, xzr
-        mov     x9, #0x1
-        mov     x10, #0x3a
-        tst     x15, #0x1
-curve25519_x25519_innerloop:
-        csel    x11, x14, xzr, ne
-        csel    x12, x16, xzr, ne
-        csel    x17, x8, xzr, ne
-        csel    x19, x9, xzr, ne
-        ccmp    x13, x14, #0x2, ne
-        sub     x11, x13, x11
-        sub     x12, x15, x12
-        csel    x14, x14, x13, cs
-        cneg    x11, x11, cc
-        csel    x16, x16, x15, cs
-        cneg    x15, x12, cc
-        csel    x8, x8, x6, cs
-        csel    x9, x9, x7, cs
-        tst     x12, #0x2
-        add     x6, x6, x17
-        add     x7, x7, x19
-        lsr     x13, x11, #1
-        lsr     x15, x15, #1
-        add     x8, x8, x8
-        add     x9, x9, x9
-        sub     x10, x10, #0x1
-        cbnz    x10, curve25519_x25519_innerloop
-        mov     x13, xzr
-        mov     x14, xzr
-        mov     x17, xzr
-        mov     x19, xzr
-        mov     x10, xzr
-curve25519_x25519_congloop:
-        ldr     x11, [x4, x10, lsl #3]
-        ldr     x12, [x1, x10, lsl #3]
-        mul     x15, x6, x11
-        mul     x16, x7, x12
-        adds    x15, x15, x13
-        umulh   x13, x6, x11
-        adc     x13, x13, xzr
-        adds    x15, x15, x16
-        extr    x17, x15, x17, #58
-        str     x17, [x4, x10, lsl #3]
-        mov     x17, x15
-        umulh   x15, x7, x12
-        adc     x13, x13, x15
-        mul     x15, x8, x11
-        mul     x16, x9, x12
-        adds    x15, x15, x14
-        umulh   x14, x8, x11
-        adc     x14, x14, xzr
-        adds    x15, x15, x16
-        extr    x19, x15, x19, #58
-        str     x19, [x1, x10, lsl #3]
-        mov     x19, x15
-        umulh   x15, x9, x12
-        adc     x14, x14, x15
-        add     x10, x10, #0x1
-        cmp     x10, x0
-        b.cc    curve25519_x25519_congloop
-        extr    x13, x13, x17, #58
-        extr    x14, x14, x19, #58
-        ldr     x11, [x4]
-        mul     x17, x11, x20
-        ldr     x12, [x3]
-        mul     x15, x17, x12
-        umulh   x16, x17, x12
-        adds    x11, x11, x15
-        mov     x10, #0x1
-        sub     x11, x0, #0x1
-        cbz     x11, curve25519_x25519_wmontend
-curve25519_x25519_wmontloop:
-        ldr     x11, [x3, x10, lsl #3]
-        ldr     x12, [x4, x10, lsl #3]
-        mul     x15, x17, x11
-        adcs    x12, x12, x16
-        umulh   x16, x17, x11
-        adc     x16, x16, xzr
-        adds    x12, x12, x15
-        sub     x15, x10, #0x1
-        str     x12, [x4, x15, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519_wmontloop
-curve25519_x25519_wmontend:
-        adcs    x16, x16, x13
-        adc     x13, xzr, xzr
-        sub     x15, x10, #0x1
-        str     x16, [x4, x15, lsl #3]
-        negs    x10, xzr
-curve25519_x25519_wcmploop:
-        ldr     x11, [x4, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        sbcs    xzr, x11, x12
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519_wcmploop
-        sbcs    xzr, x13, xzr
-        csetm   x13, cs
-        negs    x10, xzr
-curve25519_x25519_wcorrloop:
-        ldr     x11, [x4, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        and     x12, x12, x13
-        sbcs    x11, x11, x12
-        str     x11, [x4, x10, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519_wcorrloop
-        ldr     x11, [x1]
-        mul     x17, x11, x20
-        ldr     x12, [x3]
-        mul     x15, x17, x12
-        umulh   x16, x17, x12
-        adds    x11, x11, x15
-        mov     x10, #0x1
-        sub     x11, x0, #0x1
-        cbz     x11, curve25519_x25519_zmontend
-curve25519_x25519_zmontloop:
-        ldr     x11, [x3, x10, lsl #3]
-        ldr     x12, [x1, x10, lsl #3]
-        mul     x15, x17, x11
-        adcs    x12, x12, x16
-        umulh   x16, x17, x11
-        adc     x16, x16, xzr
-        adds    x12, x12, x15
-        sub     x15, x10, #0x1
-        str     x12, [x1, x15, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519_zmontloop
-curve25519_x25519_zmontend:
-        adcs    x16, x16, x14
-        adc     x14, xzr, xzr
-        sub     x15, x10, #0x1
-        str     x16, [x1, x15, lsl #3]
-        negs    x10, xzr
-curve25519_x25519_zcmploop:
-        ldr     x11, [x1, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        sbcs    xzr, x11, x12
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519_zcmploop
-        sbcs    xzr, x14, xzr
-        csetm   x14, cs
-        negs    x10, xzr
-curve25519_x25519_zcorrloop:
-        ldr     x11, [x1, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        and     x12, x12, x14
-        sbcs    x11, x11, x12
-        str     x11, [x1, x10, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519_zcorrloop
-        mov     x13, xzr
-        mov     x14, xzr
-        mov     x17, xzr
-        mov     x19, xzr
-        mov     x10, xzr
-curve25519_x25519_crossloop:
-        ldr     x11, [x21, x10, lsl #3]
-        ldr     x12, [x22, x10, lsl #3]
-        mul     x15, x6, x11
-        mul     x16, x7, x12
-        adds    x15, x15, x13
-        umulh   x13, x6, x11
-        adc     x13, x13, xzr
-        subs    x15, x15, x16
-        str     x15, [x21, x10, lsl #3]
-        umulh   x15, x7, x12
-        sub     x17, x15, x17
-        sbcs    x13, x13, x17
-        csetm   x17, cc
-        mul     x15, x8, x11
-        mul     x16, x9, x12
-        adds    x15, x15, x14
-        umulh   x14, x8, x11
-        adc     x14, x14, xzr
-        subs    x15, x15, x16
-        str     x15, [x22, x10, lsl #3]
-        umulh   x15, x9, x12
-        sub     x19, x15, x19
-        sbcs    x14, x14, x19
-        csetm   x19, cc
-        add     x10, x10, #0x1
-        cmp     x10, x5
-        b.cc    curve25519_x25519_crossloop
-        cmn     x17, x17
-        ldr     x15, [x21]
-        mov     x10, xzr
-        sub     x6, x5, #0x1
-        cbz     x6, curve25519_x25519_negskip1
-curve25519_x25519_negloop1:
-        add     x11, x10, #0x8
-        ldr     x12, [x21, x11]
-        extr    x15, x12, x15, #58
-        eor     x15, x15, x17
-        adcs    x15, x15, xzr
-        str     x15, [x21, x10]
-        mov     x15, x12
-        add     x10, x10, #0x8
-        sub     x6, x6, #0x1
-        cbnz    x6, curve25519_x25519_negloop1
-curve25519_x25519_negskip1:
-        extr    x15, x13, x15, #58
-        eor     x15, x15, x17
-        adcs    x15, x15, xzr
-        str     x15, [x21, x10]
-        cmn     x19, x19
-        ldr     x15, [x22]
-        mov     x10, xzr
-        sub     x6, x5, #0x1
-        cbz     x6, curve25519_x25519_negskip2
-curve25519_x25519_negloop2:
-        add     x11, x10, #0x8
-        ldr     x12, [x22, x11]
-        extr    x15, x12, x15, #58
-        eor     x15, x15, x19
-        adcs    x15, x15, xzr
-        str     x15, [x22, x10]
-        mov     x15, x12
-        add     x10, x10, #0x8
-        sub     x6, x6, #0x1
-        cbnz    x6, curve25519_x25519_negloop2
-curve25519_x25519_negskip2:
-        extr    x15, x14, x15, #58
-        eor     x15, x15, x19
-        adcs    x15, x15, xzr
-        str     x15, [x22, x10]
-        mov     x10, xzr
-        cmn     x17, x17
-curve25519_x25519_wfliploop:
-        ldr     x11, [x3, x10, lsl #3]
-        ldr     x12, [x4, x10, lsl #3]
-        and     x11, x11, x17
-        eor     x12, x12, x17
-        adcs    x11, x11, x12
-        str     x11, [x4, x10, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519_wfliploop
-        mvn     x19, x19
-        mov     x10, xzr
-        cmn     x19, x19
-curve25519_x25519_zfliploop:
-        ldr     x11, [x3, x10, lsl #3]
-        ldr     x12, [x1, x10, lsl #3]
-        and     x11, x11, x19
-        eor     x12, x12, x19
-        adcs    x11, x11, x12
-        str     x11, [x1, x10, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519_zfliploop
-        subs    x2, x2, #0x3a
-        b.hi    curve25519_x25519_outerloop
+// Prepare to call the modular inverse function to get xm = 1/zn
+
+        add     x0, xm
+        add     x1, zn
+
+// Inline copy of bignum_inv_p25519, identical except for stripping out
+// the prologue and epilogue saving and restoring registers and making
+// and reclaiming room on the stack. For more details and explanations see
+// "arm/curve25519/bignum_inv_p25519.S". Note that the stack it uses for
+// its own temporaries is 128 bytes, so it has no effect on variables
+// that are needed in the rest of our computation here: res, xm and zn.
+
+        mov     x20, x0
+        mov     x10, #0xffffffffffffffed
+        mov     x11, #0xffffffffffffffff
+        stp     x10, x11, [sp]
+        mov     x12, #0x7fffffffffffffff
+        stp     x11, x12, [sp, #16]
+        ldp     x2, x3, [x1]
+        ldp     x4, x5, [x1, #16]
+        mov     x7, #0x13
+        lsr     x6, x5, #63
+        madd    x6, x7, x6, x7
+        adds    x2, x2, x6
+        adcs    x3, x3, xzr
+        adcs    x4, x4, xzr
+        orr     x5, x5, #0x8000000000000000
+        adcs    x5, x5, xzr
+        csel    x6, x7, xzr, cc
+        subs    x2, x2, x6
+        sbcs    x3, x3, xzr
+        sbcs    x4, x4, xzr
+        sbc     x5, x5, xzr
+        and     x5, x5, #0x7fffffffffffffff
+        stp     x2, x3, [sp, #32]
+        stp     x4, x5, [sp, #48]
+        stp     xzr, xzr, [sp, #64]
+        stp     xzr, xzr, [sp, #80]
+        mov     x10, #0x2099
+        movk    x10, #0x7502, lsl #16
+        movk    x10, #0x9e23, lsl #32
+        movk    x10, #0xa0f9, lsl #48
+        mov     x11, #0x2595
+        movk    x11, #0x1d13, lsl #16
+        movk    x11, #0x8f3f, lsl #32
+        movk    x11, #0xa8c6, lsl #48
+        mov     x12, #0x5242
+        movk    x12, #0x5ac, lsl #16
+        movk    x12, #0x8938, lsl #32
+        movk    x12, #0x6c6c, lsl #48
+        mov     x13, #0x615
+        movk    x13, #0x4177, lsl #16
+        movk    x13, #0x8b2, lsl #32
+        movk    x13, #0x2765, lsl #48
+        stp     x10, x11, [sp, #96]
+        stp     x12, x13, [sp, #112]
+        mov     x21, #0xa
+        mov     x22, #0x1
+        b       curve25519_x25519_invmidloop
+curve25519_x25519_invloop:
+        cmp     x10, xzr
+        csetm   x14, mi
+        cneg    x10, x10, mi
+        cmp     x11, xzr
+        csetm   x15, mi
+        cneg    x11, x11, mi
+        cmp     x12, xzr
+        csetm   x16, mi
+        cneg    x12, x12, mi
+        cmp     x13, xzr
+        csetm   x17, mi
+        cneg    x13, x13, mi
+        and     x0, x10, x14
+        and     x1, x11, x15
+        add     x9, x0, x1
+        and     x0, x12, x16
+        and     x1, x13, x17
+        add     x19, x0, x1
+        ldr     x7, [sp]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x4, x9, x0
+        adc     x2, xzr, x1
+        ldr     x8, [sp, #32]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x4, x4, x0
+        adc     x2, x2, x1
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x5, x19, x0
+        adc     x3, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        ldr     x7, [sp, #8]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x2, x2, x0
+        adc     x6, xzr, x1
+        ldr     x8, [sp, #40]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x2, x2, x0
+        adc     x6, x6, x1
+        extr    x4, x2, x4, #59
+        str     x4, [sp]
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x3, x3, x0
+        adc     x4, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x3, x3, x0
+        adc     x4, x4, x1
+        extr    x5, x3, x5, #59
+        str     x5, [sp, #32]
+        ldr     x7, [sp, #16]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x6, x6, x0
+        adc     x5, xzr, x1
+        ldr     x8, [sp, #48]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x6, x6, x0
+        adc     x5, x5, x1
+        extr    x2, x6, x2, #59
+        str     x2, [sp, #8]
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x4, x4, x0
+        adc     x2, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x4, x4, x0
+        adc     x2, x2, x1
+        extr    x3, x4, x3, #59
+        str     x3, [sp, #40]
+        ldr     x7, [sp, #24]
+        eor     x1, x7, x14
+        asr     x3, x1, #63
+        and     x3, x3, x10
+        neg     x3, x3
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        ldr     x8, [sp, #56]
+        eor     x1, x8, x15
+        asr     x0, x1, #63
+        and     x0, x0, x11
+        sub     x3, x3, x0
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        extr    x6, x5, x6, #59
+        str     x6, [sp, #16]
+        extr    x5, x3, x5, #59
+        str     x5, [sp, #24]
+        eor     x1, x7, x16
+        asr     x5, x1, #63
+        and     x5, x5, x12
+        neg     x5, x5
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x2, x2, x0
+        adc     x5, x5, x1
+        eor     x1, x8, x17
+        asr     x0, x1, #63
+        and     x0, x0, x13
+        sub     x5, x5, x0
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x2, x2, x0
+        adc     x5, x5, x1
+        extr    x4, x2, x4, #59
+        str     x4, [sp, #48]
+        extr    x2, x5, x2, #59
+        str     x2, [sp, #56]
+        ldr     x7, [sp, #64]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x4, x9, x0
+        adc     x2, xzr, x1
+        ldr     x8, [sp, #96]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x4, x4, x0
+        str     x4, [sp, #64]
+        adc     x2, x2, x1
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x5, x19, x0
+        adc     x3, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x5, x5, x0
+        str     x5, [sp, #96]
+        adc     x3, x3, x1
+        ldr     x7, [sp, #72]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x2, x2, x0
+        adc     x6, xzr, x1
+        ldr     x8, [sp, #104]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x2, x2, x0
+        str     x2, [sp, #72]
+        adc     x6, x6, x1
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x3, x3, x0
+        adc     x4, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x3, x3, x0
+        str     x3, [sp, #104]
+        adc     x4, x4, x1
+        ldr     x7, [sp, #80]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x6, x6, x0
+        adc     x5, xzr, x1
+        ldr     x8, [sp, #112]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x6, x6, x0
+        str     x6, [sp, #80]
+        adc     x5, x5, x1
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x4, x4, x0
+        adc     x2, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x4, x4, x0
+        str     x4, [sp, #112]
+        adc     x2, x2, x1
+        ldr     x7, [sp, #88]
+        eor     x1, x7, x14
+        and     x3, x14, x10
+        neg     x3, x3
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        ldr     x8, [sp, #120]
+        eor     x1, x8, x15
+        and     x0, x15, x11
+        sub     x3, x3, x0
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        extr    x6, x3, x5, #63
+        ldp     x0, x1, [sp, #64]
+        add     x6, x6, x3, asr #63
+        mov     x3, #0x13
+        mul     x4, x6, x3
+        add     x5, x5, x6, lsl #63
+        smulh   x3, x6, x3
+        ldr     x6, [sp, #80]
+        adds    x0, x0, x4
+        adcs    x1, x1, x3
+        asr     x3, x3, #63
+        adcs    x6, x6, x3
+        adc     x5, x5, x3
+        stp     x0, x1, [sp, #64]
+        stp     x6, x5, [sp, #80]
+        eor     x1, x7, x16
+        and     x5, x16, x12
+        neg     x5, x5
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x2, x2, x0
+        adc     x5, x5, x1
+        eor     x1, x8, x17
+        and     x0, x17, x13
+        sub     x5, x5, x0
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x2, x2, x0
+        adc     x5, x5, x1
+        extr    x6, x5, x2, #63
+        ldp     x0, x1, [sp, #96]
+        add     x6, x6, x5, asr #63
+        mov     x5, #0x13
+        mul     x4, x6, x5
+        add     x2, x2, x6, lsl #63
+        smulh   x5, x6, x5
+        ldr     x3, [sp, #112]
+        adds    x0, x0, x4
+        adcs    x1, x1, x5
+        asr     x5, x5, #63
+        adcs    x3, x3, x5
+        adc     x2, x2, x5
+        stp     x0, x1, [sp, #96]
+        stp     x3, x2, [sp, #112]
+curve25519_x25519_invmidloop:
+        mov     x1, x22
+        ldr     x2, [sp]
+        ldr     x3, [sp, #32]
+        and     x4, x2, #0xfffff
+        orr     x4, x4, #0xfffffe0000000000
+        and     x5, x3, #0xfffff
+        orr     x5, x5, #0xc000000000000000
+        tst     x5, #0x1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        asr     x5, x5, #1
+        add     x8, x4, #0x100, lsl #12
+        sbfx    x8, x8, #21, #21
+        mov     x11, #0x100000
+        add     x11, x11, x11, lsl #21
+        add     x9, x4, x11
+        asr     x9, x9, #42
+        add     x10, x5, #0x100, lsl #12
+        sbfx    x10, x10, #21, #21
+        add     x11, x5, x11
+        asr     x11, x11, #42
+        mul     x6, x8, x2
+        mul     x7, x9, x3
+        mul     x2, x10, x2
+        mul     x3, x11, x3
+        add     x4, x6, x7
+        add     x5, x2, x3
+        asr     x2, x4, #20
+        asr     x3, x5, #20
+        and     x4, x2, #0xfffff
+        orr     x4, x4, #0xfffffe0000000000
+        and     x5, x3, #0xfffff
+        orr     x5, x5, #0xc000000000000000
+        tst     x5, #0x1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        asr     x5, x5, #1
+        add     x12, x4, #0x100, lsl #12
+        sbfx    x12, x12, #21, #21
+        mov     x15, #0x100000
+        add     x15, x15, x15, lsl #21
+        add     x13, x4, x15
+        asr     x13, x13, #42
+        add     x14, x5, #0x100, lsl #12
+        sbfx    x14, x14, #21, #21
+        add     x15, x5, x15
+        asr     x15, x15, #42
+        mul     x6, x12, x2
+        mul     x7, x13, x3
+        mul     x2, x14, x2
+        mul     x3, x15, x3
+        add     x4, x6, x7
+        add     x5, x2, x3
+        asr     x2, x4, #20
+        asr     x3, x5, #20
+        and     x4, x2, #0xfffff
+        orr     x4, x4, #0xfffffe0000000000
+        and     x5, x3, #0xfffff
+        orr     x5, x5, #0xc000000000000000
+        tst     x5, #0x1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        mul     x2, x12, x8
+        mul     x3, x12, x9
+        mul     x6, x14, x8
+        mul     x7, x14, x9
+        madd    x8, x13, x10, x2
+        madd    x9, x13, x11, x3
+        madd    x16, x15, x10, x6
+        madd    x17, x15, x11, x7
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        asr     x5, x5, #1
+        add     x12, x4, #0x100, lsl #12
+        sbfx    x12, x12, #22, #21
+        mov     x15, #0x100000
+        add     x15, x15, x15, lsl #21
+        add     x13, x4, x15
+        asr     x13, x13, #43
+        add     x14, x5, #0x100, lsl #12
+        sbfx    x14, x14, #22, #21
+        add     x15, x5, x15
+        asr     x15, x15, #43
+        mneg    x2, x12, x8
+        mneg    x3, x12, x9
+        mneg    x4, x14, x8
+        mneg    x5, x14, x9
+        msub    x10, x13, x16, x2
+        msub    x11, x13, x17, x3
+        msub    x12, x15, x16, x4
+        msub    x13, x15, x17, x5
+        mov     x22, x1
+        subs    x21, x21, #0x1
+        b.ne    curve25519_x25519_invloop
+        ldr     x0, [sp]
+        ldr     x1, [sp, #32]
+        mul     x0, x0, x10
+        madd    x1, x1, x11, x0
+        asr     x0, x1, #63
+        cmp     x10, xzr
+        csetm   x14, mi
+        cneg    x10, x10, mi
+        eor     x14, x14, x0
+        cmp     x11, xzr
+        csetm   x15, mi
+        cneg    x11, x11, mi
+        eor     x15, x15, x0
+        cmp     x12, xzr
+        csetm   x16, mi
+        cneg    x12, x12, mi
+        eor     x16, x16, x0
+        cmp     x13, xzr
+        csetm   x17, mi
+        cneg    x13, x13, mi
+        eor     x17, x17, x0
+        and     x0, x10, x14
+        and     x1, x11, x15
+        add     x9, x0, x1
+        ldr     x7, [sp, #64]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x4, x9, x0
+        adc     x2, xzr, x1
+        ldr     x8, [sp, #96]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x4, x4, x0
+        str     x4, [sp, #64]
+        adc     x2, x2, x1
+        ldr     x7, [sp, #72]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x2, x2, x0
+        adc     x6, xzr, x1
+        ldr     x8, [sp, #104]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x2, x2, x0
+        str     x2, [sp, #72]
+        adc     x6, x6, x1
+        ldr     x7, [sp, #80]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x6, x6, x0
+        adc     x5, xzr, x1
+        ldr     x8, [sp, #112]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x6, x6, x0
+        str     x6, [sp, #80]
+        adc     x5, x5, x1
+        ldr     x7, [sp, #88]
+        eor     x1, x7, x14
+        and     x3, x14, x10
+        neg     x3, x3
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        ldr     x8, [sp, #120]
+        eor     x1, x8, x15
+        and     x0, x15, x11
+        sub     x3, x3, x0
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        extr    x6, x3, x5, #63
+        ldp     x0, x1, [sp, #64]
+        tst     x3, x3
+        cinc    x6, x6, pl
+        mov     x3, #0x13
+        mul     x4, x6, x3
+        add     x5, x5, x6, lsl #63
+        smulh   x6, x6, x3
+        ldr     x2, [sp, #80]
+        adds    x0, x0, x4
+        adcs    x1, x1, x6
+        asr     x6, x6, #63
+        adcs    x2, x2, x6
+        adcs    x5, x5, x6
+        csel    x3, x3, xzr, mi
+        subs    x0, x0, x3
+        sbcs    x1, x1, xzr
+        sbcs    x2, x2, xzr
+        sbc     x5, x5, xzr
+        and     x5, x5, #0x7fffffffffffffff
+        mov     x4, x20
+        stp     x0, x1, [x4]
+        stp     x2, x5, [x4, #16]
 
 // Since we eventually want to return 0 when the result is the point at
 // infinity, we force xn = 0 whenever zn = 0. This avoids building in a
@@ -1221,7 +1912,7 @@ curve25519_x25519_zfliploop:
 
 // Now the result is xn * (1/zn), fully reduced modulo p.
 
-        mul_p25519(resx,xn,zm)
+        mul_p25519(resx,xn,xm)
 
 // Restore stack and registers