aws-crt 0.1.9 → 0.2.0

data/aws-crt-ffi/crt/aws-lc/third_party/s2n-bignum/arm/curve25519/curve25519_x25519base_byte.S

@@ -966,360 +966,1058 @@ curve25519_x25519base_byte_scalarloop:
 //
 // First the addition and subtraction:
 
-        add_twice4(y_3,x_3,w_3)
-        sub_twice4(z_3,x_3,w_3)
+        add_twice4(t1,x_3,w_3)
+        sub_twice4(t2,x_3,w_3)
 
-// Prepare to call the modular inverse function to get x_3 = 1/z_3
+// Prepare to call the modular inverse function to get t0 = 1/t2
 // Note that this works for the weakly normalized z_3 equally well.
 // The non-coprime case z_3 == 0 (mod p_25519) cannot arise anyway.
 
-        mov     x0, 4
-        add     x1, x_3
-        add     x2, z_3
-        adr     x3, curve25519_x25519base_byte_p_25519
-        add     x4, tmpspace
-
-// Inline copy of bignum_modinv, identical except for stripping out the
-// prologue and epilogue saving and restoring registers and the initial
-// test for k = 0 (which is trivially false here since k = 4). For more
-// details and explanations see "arm/generic/bignum_modinv.S".
-
-        lsl     x10, x0, #3
-        add     x21, x4, x10
-        add     x22, x21, x10
-        mov     x10, xzr
-curve25519_x25519base_byte_copyloop:
-        ldr     x11, [x2, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        str     x11, [x21, x10, lsl #3]
-        str     x12, [x22, x10, lsl #3]
-        str     x12, [x4, x10, lsl #3]
-        str     xzr, [x1, x10, lsl #3]
-        add     x10, x10, #0x1
-        cmp     x10, x0
-        b.cc    curve25519_x25519base_byte_copyloop
-        ldr     x11, [x4]
-        sub     x12, x11, #0x1
-        str     x12, [x4]
-        lsl     x20, x11, #2
-        sub     x20, x11, x20
-        eor     x20, x20, #0x2
-        mov     x12, #0x1
-        madd    x12, x11, x20, x12
-        mul     x11, x12, x12
-        madd    x20, x12, x20, x20
-        mul     x12, x11, x11
-        madd    x20, x11, x20, x20
-        mul     x11, x12, x12
-        madd    x20, x12, x20, x20
-        madd    x20, x11, x20, x20
-        lsl     x2, x0, #7
-curve25519_x25519base_byte_outerloop:
-        add     x10, x2, #0x3f
-        lsr     x5, x10, #6
-        cmp     x5, x0
-        csel    x5, x0, x5, cs
-        mov     x13, xzr
-        mov     x15, xzr
-        mov     x14, xzr
-        mov     x16, xzr
-        mov     x19, xzr
-        mov     x10, xzr
-curve25519_x25519base_byte_toploop:
-        ldr     x11, [x21, x10, lsl #3]
-        ldr     x12, [x22, x10, lsl #3]
-        orr     x17, x11, x12
-        cmp     x17, xzr
-        and     x17, x19, x13
-        csel    x15, x17, x15, ne
-        and     x17, x19, x14
-        csel    x16, x17, x16, ne
-        csel    x13, x11, x13, ne
-        csel    x14, x12, x14, ne
-        csetm   x19, ne
-        add     x10, x10, #0x1
-        cmp     x10, x5
-        b.cc    curve25519_x25519base_byte_toploop
-        orr     x11, x13, x14
-        clz     x12, x11
-        negs    x17, x12
-        lsl     x13, x13, x12
-        csel    x15, x15, xzr, ne
-        lsl     x14, x14, x12
-        csel    x16, x16, xzr, ne
-        lsr     x15, x15, x17
-        lsr     x16, x16, x17
-        orr     x13, x13, x15
-        orr     x14, x14, x16
-        ldr     x15, [x21]
-        ldr     x16, [x22]
-        mov     x6, #0x1
-        mov     x7, xzr
-        mov     x8, xzr
-        mov     x9, #0x1
-        mov     x10, #0x3a
-        tst     x15, #0x1
-curve25519_x25519base_byte_innerloop:
-        csel    x11, x14, xzr, ne
-        csel    x12, x16, xzr, ne
-        csel    x17, x8, xzr, ne
-        csel    x19, x9, xzr, ne
-        ccmp    x13, x14, #0x2, ne
-        sub     x11, x13, x11
-        sub     x12, x15, x12
-        csel    x14, x14, x13, cs
-        cneg    x11, x11, cc
-        csel    x16, x16, x15, cs
-        cneg    x15, x12, cc
-        csel    x8, x8, x6, cs
-        csel    x9, x9, x7, cs
-        tst     x12, #0x2
-        add     x6, x6, x17
-        add     x7, x7, x19
-        lsr     x13, x11, #1
-        lsr     x15, x15, #1
-        add     x8, x8, x8
-        add     x9, x9, x9
-        sub     x10, x10, #0x1
-        cbnz    x10, curve25519_x25519base_byte_innerloop
-        mov     x13, xzr
-        mov     x14, xzr
-        mov     x17, xzr
-        mov     x19, xzr
-        mov     x10, xzr
-curve25519_x25519base_byte_congloop:
-        ldr     x11, [x4, x10, lsl #3]
-        ldr     x12, [x1, x10, lsl #3]
-        mul     x15, x6, x11
-        mul     x16, x7, x12
-        adds    x15, x15, x13
-        umulh   x13, x6, x11
-        adc     x13, x13, xzr
-        adds    x15, x15, x16
-        extr    x17, x15, x17, #58
-        str     x17, [x4, x10, lsl #3]
-        mov     x17, x15
-        umulh   x15, x7, x12
-        adc     x13, x13, x15
-        mul     x15, x8, x11
-        mul     x16, x9, x12
-        adds    x15, x15, x14
-        umulh   x14, x8, x11
-        adc     x14, x14, xzr
-        adds    x15, x15, x16
-        extr    x19, x15, x19, #58
-        str     x19, [x1, x10, lsl #3]
-        mov     x19, x15
-        umulh   x15, x9, x12
-        adc     x14, x14, x15
-        add     x10, x10, #0x1
-        cmp     x10, x0
-        b.cc    curve25519_x25519base_byte_congloop
-        extr    x13, x13, x17, #58
-        extr    x14, x14, x19, #58
-        ldr     x11, [x4]
-        mul     x17, x11, x20
-        ldr     x12, [x3]
-        mul     x15, x17, x12
-        umulh   x16, x17, x12
-        adds    x11, x11, x15
-        mov     x10, #0x1
-        sub     x11, x0, #0x1
-        cbz     x11, curve25519_x25519base_byte_wmontend
-curve25519_x25519base_byte_wmontloop:
-        ldr     x11, [x3, x10, lsl #3]
-        ldr     x12, [x4, x10, lsl #3]
-        mul     x15, x17, x11
-        adcs    x12, x12, x16
-        umulh   x16, x17, x11
-        adc     x16, x16, xzr
-        adds    x12, x12, x15
-        sub     x15, x10, #0x1
-        str     x12, [x4, x15, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519base_byte_wmontloop
-curve25519_x25519base_byte_wmontend:
-        adcs    x16, x16, x13
-        adc     x13, xzr, xzr
-        sub     x15, x10, #0x1
-        str     x16, [x4, x15, lsl #3]
-        negs    x10, xzr
-curve25519_x25519base_byte_wcmploop:
-        ldr     x11, [x4, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        sbcs    xzr, x11, x12
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519base_byte_wcmploop
-        sbcs    xzr, x13, xzr
-        csetm   x13, cs
-        negs    x10, xzr
-curve25519_x25519base_byte_wcorrloop:
-        ldr     x11, [x4, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        and     x12, x12, x13
-        sbcs    x11, x11, x12
-        str     x11, [x4, x10, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519base_byte_wcorrloop
-        ldr     x11, [x1]
-        mul     x17, x11, x20
-        ldr     x12, [x3]
-        mul     x15, x17, x12
-        umulh   x16, x17, x12
-        adds    x11, x11, x15
-        mov     x10, #0x1
-        sub     x11, x0, #0x1
-        cbz     x11, curve25519_x25519base_byte_zmontend
-curve25519_x25519base_byte_zmontloop:
-        ldr     x11, [x3, x10, lsl #3]
-        ldr     x12, [x1, x10, lsl #3]
-        mul     x15, x17, x11
-        adcs    x12, x12, x16
-        umulh   x16, x17, x11
-        adc     x16, x16, xzr
-        adds    x12, x12, x15
-        sub     x15, x10, #0x1
-        str     x12, [x1, x15, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519base_byte_zmontloop
-curve25519_x25519base_byte_zmontend:
-        adcs    x16, x16, x14
-        adc     x14, xzr, xzr
-        sub     x15, x10, #0x1
-        str     x16, [x1, x15, lsl #3]
-        negs    x10, xzr
-curve25519_x25519base_byte_zcmploop:
-        ldr     x11, [x1, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        sbcs    xzr, x11, x12
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519base_byte_zcmploop
-        sbcs    xzr, x14, xzr
-        csetm   x14, cs
-        negs    x10, xzr
-curve25519_x25519base_byte_zcorrloop:
-        ldr     x11, [x1, x10, lsl #3]
-        ldr     x12, [x3, x10, lsl #3]
-        and     x12, x12, x14
-        sbcs    x11, x11, x12
-        str     x11, [x1, x10, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519base_byte_zcorrloop
-        mov     x13, xzr
-        mov     x14, xzr
-        mov     x17, xzr
-        mov     x19, xzr
-        mov     x10, xzr
-curve25519_x25519base_byte_crossloop:
-        ldr     x11, [x21, x10, lsl #3]
-        ldr     x12, [x22, x10, lsl #3]
-        mul     x15, x6, x11
-        mul     x16, x7, x12
-        adds    x15, x15, x13
-        umulh   x13, x6, x11
-        adc     x13, x13, xzr
-        subs    x15, x15, x16
-        str     x15, [x21, x10, lsl #3]
-        umulh   x15, x7, x12
-        sub     x17, x15, x17
-        sbcs    x13, x13, x17
-        csetm   x17, cc
-        mul     x15, x8, x11
-        mul     x16, x9, x12
-        adds    x15, x15, x14
-        umulh   x14, x8, x11
-        adc     x14, x14, xzr
-        subs    x15, x15, x16
-        str     x15, [x22, x10, lsl #3]
-        umulh   x15, x9, x12
-        sub     x19, x15, x19
-        sbcs    x14, x14, x19
-        csetm   x19, cc
-        add     x10, x10, #0x1
-        cmp     x10, x5
-        b.cc    curve25519_x25519base_byte_crossloop
-        cmn     x17, x17
-        ldr     x15, [x21]
-        mov     x10, xzr
-        sub     x6, x5, #0x1
-        cbz     x6, curve25519_x25519base_byte_negskip1
-curve25519_x25519base_byte_negloop1:
-        add     x11, x10, #0x8
-        ldr     x12, [x21, x11]
-        extr    x15, x12, x15, #58
-        eor     x15, x15, x17
-        adcs    x15, x15, xzr
-        str     x15, [x21, x10]
-        mov     x15, x12
-        add     x10, x10, #0x8
-        sub     x6, x6, #0x1
-        cbnz    x6, curve25519_x25519base_byte_negloop1
-curve25519_x25519base_byte_negskip1:
-        extr    x15, x13, x15, #58
-        eor     x15, x15, x17
-        adcs    x15, x15, xzr
-        str     x15, [x21, x10]
-        cmn     x19, x19
-        ldr     x15, [x22]
-        mov     x10, xzr
-        sub     x6, x5, #0x1
-        cbz     x6, curve25519_x25519base_byte_negskip2
-curve25519_x25519base_byte_negloop2:
-        add     x11, x10, #0x8
-        ldr     x12, [x22, x11]
-        extr    x15, x12, x15, #58
-        eor     x15, x15, x19
-        adcs    x15, x15, xzr
-        str     x15, [x22, x10]
-        mov     x15, x12
-        add     x10, x10, #0x8
-        sub     x6, x6, #0x1
-        cbnz    x6, curve25519_x25519base_byte_negloop2
-curve25519_x25519base_byte_negskip2:
-        extr    x15, x14, x15, #58
-        eor     x15, x15, x19
-        adcs    x15, x15, xzr
-        str     x15, [x22, x10]
-        mov     x10, xzr
-        cmn     x17, x17
-curve25519_x25519base_byte_wfliploop:
-        ldr     x11, [x3, x10, lsl #3]
-        ldr     x12, [x4, x10, lsl #3]
-        and     x11, x11, x17
-        eor     x12, x12, x17
-        adcs    x11, x11, x12
-        str     x11, [x4, x10, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519base_byte_wfliploop
-        mvn     x19, x19
-        mov     x10, xzr
-        cmn     x19, x19
-curve25519_x25519base_byte_zfliploop:
-        ldr     x11, [x3, x10, lsl #3]
-        ldr     x12, [x1, x10, lsl #3]
-        and     x11, x11, x19
-        eor     x12, x12, x19
-        adcs    x11, x11, x12
-        str     x11, [x1, x10, lsl #3]
-        add     x10, x10, #0x1
-        sub     x11, x10, x0
-        cbnz    x11, curve25519_x25519base_byte_zfliploop
-        subs    x2, x2, #0x3a
-        b.hi    curve25519_x25519base_byte_outerloop
+        add     x0, t0
+        add     x1, t2
+
+// Inline copy of bignum_inv_p25519, identical except for stripping out
+// the prologue and epilogue saving and restoring registers and making
+// and reclaiming room on the stack. For more details and explanations see
+// "arm/curve25519/bignum_inv_p25519.S". Note that the stack it uses for
+// its own temporaries is 128 bytes, so it has no effect on variables
+// that are needed in the rest of our computation here: res, t0, t1, t2.
+
+        mov     x20, x0
+        mov     x10, #0xffffffffffffffed
+        mov     x11, #0xffffffffffffffff
+        stp     x10, x11, [sp]
+        mov     x12, #0x7fffffffffffffff
+        stp     x11, x12, [sp, #16]
+        ldp     x2, x3, [x1]
+        ldp     x4, x5, [x1, #16]
+        mov     x7, #0x13
+        lsr     x6, x5, #63
+        madd    x6, x7, x6, x7
+        adds    x2, x2, x6
+        adcs    x3, x3, xzr
+        adcs    x4, x4, xzr
+        orr     x5, x5, #0x8000000000000000
+        adcs    x5, x5, xzr
+        csel    x6, x7, xzr, cc
+        subs    x2, x2, x6
+        sbcs    x3, x3, xzr
+        sbcs    x4, x4, xzr
+        sbc     x5, x5, xzr
+        and     x5, x5, #0x7fffffffffffffff
+        stp     x2, x3, [sp, #32]
+        stp     x4, x5, [sp, #48]
+        stp     xzr, xzr, [sp, #64]
+        stp     xzr, xzr, [sp, #80]
+        mov     x10, #0x2099
+        movk    x10, #0x7502, lsl #16
+        movk    x10, #0x9e23, lsl #32
+        movk    x10, #0xa0f9, lsl #48
+        mov     x11, #0x2595
+        movk    x11, #0x1d13, lsl #16
+        movk    x11, #0x8f3f, lsl #32
+        movk    x11, #0xa8c6, lsl #48
+        mov     x12, #0x5242
+        movk    x12, #0x5ac, lsl #16
+        movk    x12, #0x8938, lsl #32
+        movk    x12, #0x6c6c, lsl #48
+        mov     x13, #0x615
+        movk    x13, #0x4177, lsl #16
+        movk    x13, #0x8b2, lsl #32
+        movk    x13, #0x2765, lsl #48
+        stp     x10, x11, [sp, #96]
+        stp     x12, x13, [sp, #112]
+        mov     x21, #0xa
+        mov     x22, #0x1
+        b       curve25519_x25519base_byte_invmidloop
+curve25519_x25519base_byte_invloop:
+        cmp     x10, xzr
+        csetm   x14, mi
+        cneg    x10, x10, mi
+        cmp     x11, xzr
+        csetm   x15, mi
+        cneg    x11, x11, mi
+        cmp     x12, xzr
+        csetm   x16, mi
+        cneg    x12, x12, mi
+        cmp     x13, xzr
+        csetm   x17, mi
+        cneg    x13, x13, mi
+        and     x0, x10, x14
+        and     x1, x11, x15
+        add     x9, x0, x1
+        and     x0, x12, x16
+        and     x1, x13, x17
+        add     x19, x0, x1
+        ldr     x7, [sp]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x4, x9, x0
+        adc     x2, xzr, x1
+        ldr     x8, [sp, #32]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x4, x4, x0
+        adc     x2, x2, x1
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x5, x19, x0
+        adc     x3, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        ldr     x7, [sp, #8]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x2, x2, x0
+        adc     x6, xzr, x1
+        ldr     x8, [sp, #40]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x2, x2, x0
+        adc     x6, x6, x1
+        extr    x4, x2, x4, #59
+        str     x4, [sp]
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x3, x3, x0
+        adc     x4, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x3, x3, x0
+        adc     x4, x4, x1
+        extr    x5, x3, x5, #59
+        str     x5, [sp, #32]
+        ldr     x7, [sp, #16]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x6, x6, x0
+        adc     x5, xzr, x1
+        ldr     x8, [sp, #48]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x6, x6, x0
+        adc     x5, x5, x1
+        extr    x2, x6, x2, #59
+        str     x2, [sp, #8]
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x4, x4, x0
+        adc     x2, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x4, x4, x0
+        adc     x2, x2, x1
+        extr    x3, x4, x3, #59
+        str     x3, [sp, #40]
+        ldr     x7, [sp, #24]
+        eor     x1, x7, x14
+        asr     x3, x1, #63
+        and     x3, x3, x10
+        neg     x3, x3
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        ldr     x8, [sp, #56]
+        eor     x1, x8, x15
+        asr     x0, x1, #63
+        and     x0, x0, x11
+        sub     x3, x3, x0
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        extr    x6, x5, x6, #59
+        str     x6, [sp, #16]
+        extr    x5, x3, x5, #59
+        str     x5, [sp, #24]
+        eor     x1, x7, x16
+        asr     x5, x1, #63
+        and     x5, x5, x12
+        neg     x5, x5
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x2, x2, x0
+        adc     x5, x5, x1
+        eor     x1, x8, x17
+        asr     x0, x1, #63
+        and     x0, x0, x13
+        sub     x5, x5, x0
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x2, x2, x0
+        adc     x5, x5, x1
+        extr    x4, x2, x4, #59
+        str     x4, [sp, #48]
+        extr    x2, x5, x2, #59
+        str     x2, [sp, #56]
+        ldr     x7, [sp, #64]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x4, x9, x0
+        adc     x2, xzr, x1
+        ldr     x8, [sp, #96]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x4, x4, x0
+        str     x4, [sp, #64]
+        adc     x2, x2, x1
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x5, x19, x0
+        adc     x3, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x5, x5, x0
+        str     x5, [sp, #96]
+        adc     x3, x3, x1
+        ldr     x7, [sp, #72]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x2, x2, x0
+        adc     x6, xzr, x1
+        ldr     x8, [sp, #104]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x2, x2, x0
+        str     x2, [sp, #72]
+        adc     x6, x6, x1
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x3, x3, x0
+        adc     x4, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x3, x3, x0
+        str     x3, [sp, #104]
+        adc     x4, x4, x1
+        ldr     x7, [sp, #80]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x6, x6, x0
+        adc     x5, xzr, x1
+        ldr     x8, [sp, #112]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x6, x6, x0
+        str     x6, [sp, #80]
+        adc     x5, x5, x1
+        eor     x1, x7, x16
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x4, x4, x0
+        adc     x2, xzr, x1
+        eor     x1, x8, x17
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x4, x4, x0
+        str     x4, [sp, #112]
+        adc     x2, x2, x1
+        ldr     x7, [sp, #88]
+        eor     x1, x7, x14
+        and     x3, x14, x10
+        neg     x3, x3
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        ldr     x8, [sp, #120]
+        eor     x1, x8, x15
+        and     x0, x15, x11
+        sub     x3, x3, x0
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        extr    x6, x3, x5, #63
+        ldp     x0, x1, [sp, #64]
+        add     x6, x6, x3, asr #63
+        mov     x3, #0x13
+        mul     x4, x6, x3
+        add     x5, x5, x6, lsl #63
+        smulh   x3, x6, x3
+        ldr     x6, [sp, #80]
+        adds    x0, x0, x4
+        adcs    x1, x1, x3
+        asr     x3, x3, #63
+        adcs    x6, x6, x3
+        adc     x5, x5, x3
+        stp     x0, x1, [sp, #64]
+        stp     x6, x5, [sp, #80]
+        eor     x1, x7, x16
+        and     x5, x16, x12
+        neg     x5, x5
+        mul     x0, x1, x12
+        umulh   x1, x1, x12
+        adds    x2, x2, x0
+        adc     x5, x5, x1
+        eor     x1, x8, x17
+        and     x0, x17, x13
+        sub     x5, x5, x0
+        mul     x0, x1, x13
+        umulh   x1, x1, x13
+        adds    x2, x2, x0
+        adc     x5, x5, x1
+        extr    x6, x5, x2, #63
+        ldp     x0, x1, [sp, #96]
+        add     x6, x6, x5, asr #63
+        mov     x5, #0x13
+        mul     x4, x6, x5
+        add     x2, x2, x6, lsl #63
+        smulh   x5, x6, x5
+        ldr     x3, [sp, #112]
+        adds    x0, x0, x4
+        adcs    x1, x1, x5
+        asr     x5, x5, #63
+        adcs    x3, x3, x5
+        adc     x2, x2, x5
+        stp     x0, x1, [sp, #96]
+        stp     x3, x2, [sp, #112]
+curve25519_x25519base_byte_invmidloop:
+        mov     x1, x22
+        ldr     x2, [sp]
+        ldr     x3, [sp, #32]
+        and     x4, x2, #0xfffff
+        orr     x4, x4, #0xfffffe0000000000
+        and     x5, x3, #0xfffff
+        orr     x5, x5, #0xc000000000000000
+        tst     x5, #0x1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        asr     x5, x5, #1
+        add     x8, x4, #0x100, lsl #12
+        sbfx    x8, x8, #21, #21
+        mov     x11, #0x100000
+        add     x11, x11, x11, lsl #21
+        add     x9, x4, x11
+        asr     x9, x9, #42
+        add     x10, x5, #0x100, lsl #12
+        sbfx    x10, x10, #21, #21
+        add     x11, x5, x11
+        asr     x11, x11, #42
+        mul     x6, x8, x2
+        mul     x7, x9, x3
+        mul     x2, x10, x2
+        mul     x3, x11, x3
+        add     x4, x6, x7
+        add     x5, x2, x3
+        asr     x2, x4, #20
+        asr     x3, x5, #20
+        and     x4, x2, #0xfffff
+        orr     x4, x4, #0xfffffe0000000000
+        and     x5, x3, #0xfffff
+        orr     x5, x5, #0xc000000000000000
+        tst     x5, #0x1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        asr     x5, x5, #1
+        add     x12, x4, #0x100, lsl #12
+        sbfx    x12, x12, #21, #21
+        mov     x15, #0x100000
+        add     x15, x15, x15, lsl #21
+        add     x13, x4, x15
+        asr     x13, x13, #42
+        add     x14, x5, #0x100, lsl #12
+        sbfx    x14, x14, #21, #21
+        add     x15, x5, x15
+        asr     x15, x15, #42
+        mul     x6, x12, x2
+        mul     x7, x13, x3
+        mul     x2, x14, x2
+        mul     x3, x15, x3
+        add     x4, x6, x7
+        add     x5, x2, x3
+        asr     x2, x4, #20
+        asr     x3, x5, #20
+        and     x4, x2, #0xfffff
+        orr     x4, x4, #0xfffffe0000000000
+        and     x5, x3, #0xfffff
+        orr     x5, x5, #0xc000000000000000
+        tst     x5, #0x1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        mul     x2, x12, x8
+        mul     x3, x12, x9
+        mul     x6, x14, x8
+        mul     x7, x14, x9
+        madd    x8, x13, x10, x2
+        madd    x9, x13, x11, x3
+        madd    x16, x15, x10, x6
+        madd    x17, x15, x11, x7
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        tst     x5, #0x2
+        asr     x5, x5, #1
+        csel    x6, x4, xzr, ne
+        ccmp    x1, xzr, #0x8, ne
+        cneg    x1, x1, ge
+        cneg    x6, x6, ge
+        csel    x4, x5, x4, ge
+        add     x5, x5, x6
+        add     x1, x1, #0x2
+        asr     x5, x5, #1
+        add     x12, x4, #0x100, lsl #12
+        sbfx    x12, x12, #22, #21
+        mov     x15, #0x100000
+        add     x15, x15, x15, lsl #21
+        add     x13, x4, x15
+        asr     x13, x13, #43
+        add     x14, x5, #0x100, lsl #12
+        sbfx    x14, x14, #22, #21
+        add     x15, x5, x15
+        asr     x15, x15, #43
+        mneg    x2, x12, x8
+        mneg    x3, x12, x9
+        mneg    x4, x14, x8
+        mneg    x5, x14, x9
+        msub    x10, x13, x16, x2
+        msub    x11, x13, x17, x3
+        msub    x12, x15, x16, x4
+        msub    x13, x15, x17, x5
+        mov     x22, x1
+        subs    x21, x21, #0x1
+        b.ne    curve25519_x25519base_byte_invloop
+        ldr     x0, [sp]
+        ldr     x1, [sp, #32]
+        mul     x0, x0, x10
+        madd    x1, x1, x11, x0
+        asr     x0, x1, #63
+        cmp     x10, xzr
+        csetm   x14, mi
+        cneg    x10, x10, mi
+        eor     x14, x14, x0
+        cmp     x11, xzr
+        csetm   x15, mi
+        cneg    x11, x11, mi
+        eor     x15, x15, x0
+        cmp     x12, xzr
+        csetm   x16, mi
+        cneg    x12, x12, mi
+        eor     x16, x16, x0
+        cmp     x13, xzr
+        csetm   x17, mi
+        cneg    x13, x13, mi
+        eor     x17, x17, x0
+        and     x0, x10, x14
+        and     x1, x11, x15
+        add     x9, x0, x1
+        ldr     x7, [sp, #64]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x4, x9, x0
+        adc     x2, xzr, x1
+        ldr     x8, [sp, #96]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x4, x4, x0
+        str     x4, [sp, #64]
+        adc     x2, x2, x1
+        ldr     x7, [sp, #72]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x2, x2, x0
+        adc     x6, xzr, x1
+        ldr     x8, [sp, #104]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x2, x2, x0
+        str     x2, [sp, #72]
+        adc     x6, x6, x1
+        ldr     x7, [sp, #80]
+        eor     x1, x7, x14
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x6, x6, x0
+        adc     x5, xzr, x1
+        ldr     x8, [sp, #112]
+        eor     x1, x8, x15
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x6, x6, x0
+        str     x6, [sp, #80]
+        adc     x5, x5, x1
+        ldr     x7, [sp, #88]
+        eor     x1, x7, x14
+        and     x3, x14, x10
+        neg     x3, x3
+        mul     x0, x1, x10
+        umulh   x1, x1, x10
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        ldr     x8, [sp, #120]
+        eor     x1, x8, x15
+        and     x0, x15, x11
+        sub     x3, x3, x0
+        mul     x0, x1, x11
+        umulh   x1, x1, x11
+        adds    x5, x5, x0
+        adc     x3, x3, x1
+        extr    x6, x3, x5, #63
+        ldp     x0, x1, [sp, #64]
+        tst     x3, x3
+        cinc    x6, x6, pl
+        mov     x3, #0x13
+        mul     x4, x6, x3
+        add     x5, x5, x6, lsl #63
+        smulh   x6, x6, x3
+        ldr     x2, [sp, #80]
+        adds    x0, x0, x4
+        adcs    x1, x1, x6
+        asr     x6, x6, #63
+        adcs    x2, x2, x6
+        adcs    x5, x5, x6
+        csel    x3, x3, xzr, mi
+        subs    x0, x0, x3
+        sbcs    x1, x1, xzr
+        sbcs    x2, x2, xzr
+        sbc     x5, x5, xzr
+        and     x5, x5, #0x7fffffffffffffff
+        mov     x4, x20
+        stp     x0, x1, [x4]
+        stp     x2, x5, [x4, #16]
 
 // The final result is (X + T) / (X - T)
 // This is the only operation in the whole computation that
 // fully reduces modulo p_25519 since now we want the canonical
 // answer as output.
 
-        mul_p25519(x_1,y_3,x_3)
+        mul_p25519(x_1,t1,t0)
 
         ldp     x10, x11, [x_1]
         strb    w10, [resx]
@@ -1405,14 +2103,6 @@ curve25519_x25519base_byte_zfliploop:
 // .section .rodata
 // ****************************************************************************
 
-// The modulus p_25519 = 2^255 - 19, for the modular inverse
-
-curve25519_x25519base_byte_p_25519:
-        .quad   0xffffffffffffffed
-        .quad   0xffffffffffffffff
-        .quad   0xffffffffffffffff
-        .quad   0x7fffffffffffffff
-
 // 2^254 * G and (2^254 + 8) * G in extended-projective coordinates
 // but with Z = 1 assumed and hence left out, so they are (X,Y,T) only.