@pinkparrot/qsafe-mayo-wasm 0.0.3

package/mayo-c/src/mayo_1/api.c

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#include <api.h>
+#include <mayo.h>
+
+#ifdef ENABLE_PARAMS_DYNAMIC
+#define MAYO_PARAMS &MAYO_1
+#else
+#define MAYO_PARAMS 0
+#endif
+
+int
+crypto_sign_keypair(unsigned char *pk, unsigned char *sk) {
+    return mayo_keypair(MAYO_PARAMS, pk, sk);
+}
+
+int
+crypto_sign(unsigned char *sm, size_t *smlen,
+            const unsigned char *m, size_t mlen,
+            const unsigned char *sk) {
+    return mayo_sign(MAYO_PARAMS, sm, smlen, m, mlen, sk);
+}
+
+int
+crypto_sign_signature(unsigned char *sig,
+              size_t *siglen, const unsigned char *m,
+              size_t mlen, const unsigned char *sk) {
+    return mayo_sign_signature(MAYO_PARAMS, sig, siglen, m, mlen, sk);
+}
+
+int
+crypto_sign_open(unsigned char *m, size_t *mlen,
+                 const unsigned char *sm, size_t smlen,
+                 const unsigned char *pk) {
+    return mayo_open(MAYO_PARAMS, m, mlen, sm, smlen, pk);
+}
+
+int
+crypto_sign_verify(const unsigned char *sig, size_t siglen,
+                   const unsigned char *m, size_t mlen,
+                   const unsigned char *pk) {
+    if (siglen != CRYPTO_BYTES)
+        return -1;
+    return mayo_verify(MAYO_PARAMS, m, mlen, sig, pk);
+}
+

package/mayo-c/src/mayo_1/api.h

@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef api_h
+#define api_h
+
+#include <mayo.h>
+
+#define CRYPTO_SECRETKEYBYTES 24
+#define CRYPTO_PUBLICKEYBYTES 1420
+#define CRYPTO_BYTES 454
+
+#define CRYPTO_ALGNAME "MAYO-1"
+
+#define crypto_sign_keypair MAYO_NAMESPACE(crypto_sign_keypair)
+int
+crypto_sign_keypair(unsigned char *pk, unsigned char *sk);
+
+#define crypto_sign MAYO_NAMESPACE(crypto_sign)
+int
+crypto_sign(unsigned char *sm, size_t *smlen,
+            const unsigned char *m, size_t mlen,
+            const unsigned char *sk);
+
+#define crypto_sign_signature MAYO_NAMESPACE(crypto_sign_signature)
+int
+crypto_sign_signature(unsigned char *sig,
+              size_t *siglen, const unsigned char *m,
+              size_t mlen, const unsigned char *sk);
+
+#define crypto_sign_open MAYO_NAMESPACE(crypto_sign_open)
+int
+crypto_sign_open(unsigned char *m, size_t *mlen,
+                 const unsigned char *sm, size_t smlen,
+                 const unsigned char *pk);
+
+#define crypto_sign_verify MAYO_NAMESPACE(crypto_sign_verify)
+int
+crypto_sign_verify(const unsigned char *sig, size_t siglen,
+                   const unsigned char *m, size_t mlen,
+                   const unsigned char *pk);
+
+#endif /* api_h */
+

package/mayo-c/src/mayo_2/api.c

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#include <api.h>
+#include <mayo.h>
+
+#ifdef ENABLE_PARAMS_DYNAMIC
+#define MAYO_PARAMS &MAYO_2
+#else
+#define MAYO_PARAMS 0
+#endif
+
+int
+crypto_sign_keypair(unsigned char *pk, unsigned char *sk) {
+    return mayo_keypair(MAYO_PARAMS, pk, sk);
+}
+
+int
+crypto_sign(unsigned char *sm, size_t *smlen,
+            const unsigned char *m, size_t mlen,
+            const unsigned char *sk) {
+    return mayo_sign(MAYO_PARAMS, sm, smlen, m, mlen, sk);
+}
+
+int
+crypto_sign_signature(unsigned char *sig,
+              size_t *siglen, const unsigned char *m,
+              size_t mlen, const unsigned char *sk) {
+    return mayo_sign_signature(MAYO_PARAMS, sig, siglen, m, mlen, sk);
+}
+
+int
+crypto_sign_open(unsigned char *m, size_t *mlen,
+                 const unsigned char *sm, size_t smlen,
+                 const unsigned char *pk) {
+    return mayo_open(MAYO_PARAMS, m, mlen, sm, smlen, pk);
+}
+
+int
+crypto_sign_verify(const unsigned char *sig, size_t siglen,
+                   const unsigned char *m, size_t mlen,
+                   const unsigned char *pk) {
+    if (siglen != CRYPTO_BYTES)
+        return -1;
+    return mayo_verify(MAYO_PARAMS, m, mlen, sig, pk);
+}
+

package/mayo-c/src/mayo_2/api.h

@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef api_h
+#define api_h
+
+#include <mayo.h>
+
+#define CRYPTO_SECRETKEYBYTES 24
+#define CRYPTO_PUBLICKEYBYTES 4912
+#define CRYPTO_BYTES 186
+
+#define CRYPTO_ALGNAME "MAYO-2"
+
+#define crypto_sign_keypair MAYO_NAMESPACE(crypto_sign_keypair)
+int
+crypto_sign_keypair(unsigned char *pk, unsigned char *sk);
+
+#define crypto_sign MAYO_NAMESPACE(crypto_sign)
+int
+crypto_sign(unsigned char *sm, size_t *smlen,
+            const unsigned char *m, size_t mlen,
+            const unsigned char *sk);
+
+#define crypto_sign_signature MAYO_NAMESPACE(crypto_sign_signature)
+int
+crypto_sign_signature(unsigned char *sig,
+              size_t *siglen, const unsigned char *m,
+              size_t mlen, const unsigned char *sk);
+
+#define crypto_sign_open MAYO_NAMESPACE(crypto_sign_open)
+int
+crypto_sign_open(unsigned char *m, size_t *mlen,
+                 const unsigned char *sm, size_t smlen,
+                 const unsigned char *pk);
+
+#define crypto_sign_verify MAYO_NAMESPACE(crypto_sign_verify)
+int
+crypto_sign_verify(const unsigned char *sig, size_t siglen,
+                   const unsigned char *m, size_t mlen,
+                   const unsigned char *pk);
+
+#endif /* api_h */
+

package/mayo-c/src/mayo_3/api.c

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#include <api.h>
+#include <mayo.h>
+
+#ifdef ENABLE_PARAMS_DYNAMIC
+#define MAYO_PARAMS &MAYO_3
+#else
+#define MAYO_PARAMS 0
+#endif
+
+int
+crypto_sign_keypair(unsigned char *pk, unsigned char *sk) {
+    return mayo_keypair(MAYO_PARAMS, pk, sk);
+}
+
+int
+crypto_sign(unsigned char *sm, size_t *smlen,
+            const unsigned char *m, size_t mlen,
+            const unsigned char *sk) {
+    return mayo_sign(MAYO_PARAMS, sm, smlen, m, mlen, sk);
+}
+
+int
+crypto_sign_signature(unsigned char *sig,
+              size_t *siglen, const unsigned char *m,
+              size_t mlen, const unsigned char *sk) {
+    return mayo_sign_signature(MAYO_PARAMS, sig, siglen, m, mlen, sk);
+}
+
+int
+crypto_sign_open(unsigned char *m, size_t *mlen,
+                 const unsigned char *sm, size_t smlen,
+                 const unsigned char *pk) {
+    return mayo_open(MAYO_PARAMS, m, mlen, sm, smlen, pk);
+}
+
+int
+crypto_sign_verify(const unsigned char *sig, size_t siglen,
+                   const unsigned char *m, size_t mlen,
+                   const unsigned char *pk) {
+    if (siglen != CRYPTO_BYTES)
+        return -1;
+    return mayo_verify(MAYO_PARAMS, m, mlen, sig, pk);
+}
+

package/mayo-c/src/mayo_3/api.h

@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef api_h
+#define api_h
+
+#include <mayo.h>
+
+#define CRYPTO_SECRETKEYBYTES 32
+#define CRYPTO_PUBLICKEYBYTES 2986
+#define CRYPTO_BYTES 681
+
+#define CRYPTO_ALGNAME "MAYO-3"
+
+#define crypto_sign_keypair MAYO_NAMESPACE(crypto_sign_keypair)
+int
+crypto_sign_keypair(unsigned char *pk, unsigned char *sk);
+
+#define crypto_sign MAYO_NAMESPACE(crypto_sign)
+int
+crypto_sign(unsigned char *sm, size_t *smlen,
+            const unsigned char *m, size_t mlen,
+            const unsigned char *sk);
+
+#define crypto_sign_signature MAYO_NAMESPACE(crypto_sign_signature)
+int
+crypto_sign_signature(unsigned char *sig,
+              size_t *siglen, const unsigned char *m,
+              size_t mlen, const unsigned char *sk);
+
+#define crypto_sign_open MAYO_NAMESPACE(crypto_sign_open)
+int
+crypto_sign_open(unsigned char *m, size_t *mlen,
+                 const unsigned char *sm, size_t smlen,
+                 const unsigned char *pk);
+
+#define crypto_sign_verify MAYO_NAMESPACE(crypto_sign_verify)
+int
+crypto_sign_verify(const unsigned char *sig, size_t siglen,
+                   const unsigned char *m, size_t mlen,
+                   const unsigned char *pk);
+
+#endif /* api_h */
+

package/mayo-c/src/mayo_5/api.c

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#include <api.h>
+#include <mayo.h>
+
+#ifdef ENABLE_PARAMS_DYNAMIC
+#define MAYO_PARAMS &MAYO_5
+#else
+#define MAYO_PARAMS 0
+#endif
+
+int
+crypto_sign_keypair(unsigned char *pk, unsigned char *sk) {
+    return mayo_keypair(MAYO_PARAMS, pk, sk);
+}
+
+int
+crypto_sign(unsigned char *sm, size_t *smlen,
+            const unsigned char *m, size_t mlen,
+            const unsigned char *sk) {
+    return mayo_sign(MAYO_PARAMS, sm, smlen, m, mlen, sk);
+}
+
+int
+crypto_sign_signature(unsigned char *sig,
+              size_t *siglen, const unsigned char *m,
+              size_t mlen, const unsigned char *sk) {
+    return mayo_sign_signature(MAYO_PARAMS, sig, siglen, m, mlen, sk);
+}
+
+int
+crypto_sign_open(unsigned char *m, size_t *mlen,
+                 const unsigned char *sm, size_t smlen,
+                 const unsigned char *pk) {
+    return mayo_open(MAYO_PARAMS, m, mlen, sm, smlen, pk);
+}
+
+int
+crypto_sign_verify(const unsigned char *sig, size_t siglen,
+                   const unsigned char *m, size_t mlen,
+                   const unsigned char *pk) {
+    if (siglen != CRYPTO_BYTES)
+        return -1;
+    return mayo_verify(MAYO_PARAMS, m, mlen, sig, pk);
+}
+

package/mayo-c/src/mayo_5/api.h

@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef api_h
+#define api_h
+
+#include <mayo.h>
+
+#define CRYPTO_SECRETKEYBYTES 40
+#define CRYPTO_PUBLICKEYBYTES 5554
+#define CRYPTO_BYTES 964
+
+#define CRYPTO_ALGNAME "MAYO-5"
+
+#define crypto_sign_keypair MAYO_NAMESPACE(crypto_sign_keypair)
+int
+crypto_sign_keypair(unsigned char *pk, unsigned char *sk);
+
+#define crypto_sign MAYO_NAMESPACE(crypto_sign)
+int
+crypto_sign(unsigned char *sm, size_t *smlen,
+            const unsigned char *m, size_t mlen,
+            const unsigned char *sk);
+
+#define crypto_sign_signature MAYO_NAMESPACE(crypto_sign_signature)
+int
+crypto_sign_signature(unsigned char *sig,
+              size_t *siglen, const unsigned char *m,
+              size_t mlen, const unsigned char *sk);
+
+#define crypto_sign_open MAYO_NAMESPACE(crypto_sign_open)
+int
+crypto_sign_open(unsigned char *m, size_t *mlen,
+                 const unsigned char *sm, size_t smlen,
+                 const unsigned char *pk);
+
+#define crypto_sign_verify MAYO_NAMESPACE(crypto_sign_verify)
+int
+crypto_sign_verify(const unsigned char *sig, size_t siglen,
+                   const unsigned char *m, size_t mlen,
+                   const unsigned char *pk);
+
+#endif /* api_h */
+

package/mayo-c/src/neon/arithmetic_common.h

@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef ARITHMETIC_COMMON_H
+#define ARITHMETIC_COMMON_H
+
+#include <mayo.h>
+#include <stdalign.h>
+#include <stdint.h>
+
+#include <arm_neon.h>
+
+#define K_OVER_2 ((K_MAX+1)/2)
+
+static const unsigned char __0_f[16] __attribute__((aligned(16))) = {
+0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07, 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f
+};
+
+static const unsigned char __gf16_reduce[16] __attribute__((aligned(16))) = {
+0x00,0x13,0x26,0x35,0x4c,0x5f,0x6a,0x79, 0x8b,0x98,0xad,0xbe,0xc7,0xd4,0xe1,0xf2
+};
+
+static inline
+uint8x16_t _gf16v_mul_unpack( uint8x16_t a0 , uint8x16_t b0 , uint8x16_t tab_reduce )
+{
+    uint8x16_t ab = vreinterpretq_u8_p8(vmulq_p8(vreinterpretq_p8_u8(a0) , vreinterpretq_p8_u8(b0)));
+    return ab^vqtbl1q_u8( tab_reduce , vshrq_n_u8(ab,4) );
+}
+
+static inline
+uint8x16_t _gf16v_get_multab( uint8x16_t b , uint8x16_t tab_reduce , uint8x16_t tab_0_f ) { return _gf16v_mul_unpack(b,tab_0_f,tab_reduce); }
+
+static inline
+uint8x16_t gf16v_get_multab( uint8_t b )
+{
+    uint8x16_t tab_reduce = vld1q_u8(__gf16_reduce);
+    uint8x16_t tab_0_f = vld1q_u8(__0_f);
+
+    uint8x16_t bb = vdupq_n_u8(b);
+    return _gf16v_get_multab(bb,tab_reduce,tab_0_f);
+}
+
+#define O_NEON_ROUND_UP_ ((O_MAX + 1)/2*2)
+
+static
+inline void mayo_O_multabs(const unsigned char *O, uint8x16_t *O_multabs){
+    // build multiplication tables
+    for (size_t r = 0; r < V_MAX; r++)
+    {
+        size_t c = 0;
+        for (; c + 1 < O_MAX; c+=2)
+        {
+            O_multabs[O_NEON_ROUND_UP_/2*r + c/2] = gf16v_get_multab(O[O_MAX*r + c]) ^ (gf16v_get_multab(O[O_MAX*r + c + 1]) << 4);
+        }
+#if O_MAX % 2 == 1
+        {
+            O_multabs[O_NEON_ROUND_UP_/2*r + c/2] = gf16v_get_multab(O[O_MAX*r + c]);
+        }
+#endif
+    }
+}
+
+static
+inline void mayo_V_multabs(const unsigned char *V, uint8x16_t *V_multabs){
+    // build multiplication tables
+    size_t r;
+    for (size_t c = 0; c < V_MAX; c++)
+    {
+        for (r = 0; r+1 < K_MAX; r+= 2)
+        {
+            V_multabs[K_OVER_2*c +  r/2] = gf16v_get_multab(V[V_MAX*r + c]) ^ (gf16v_get_multab(V[V_MAX*(r+1) + c]) << 4);
+        }
+#if K_MAX % 2 == 1
+        V_multabs[K_OVER_2*c + r/2] = gf16v_get_multab(V[V_MAX*r + c]);
+#endif
+    }
+}
+
+
+static const unsigned char mayo_gf16_mul[256] __attribute__((aligned(32))) = {
+    0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+    0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07, 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,
+    0x00,0x02,0x04,0x06,0x08,0x0a,0x0c,0x0e, 0x03,0x01,0x07,0x05,0x0b,0x09,0x0f,0x0d,
+    0x00,0x03,0x06,0x05,0x0c,0x0f,0x0a,0x09, 0x0b,0x08,0x0d,0x0e,0x07,0x04,0x01,0x02,
+    0x00,0x04,0x08,0x0c,0x03,0x07,0x0b,0x0f, 0x06,0x02,0x0e,0x0a,0x05,0x01,0x0d,0x09,
+    0x00,0x05,0x0a,0x0f,0x07,0x02,0x0d,0x08, 0x0e,0x0b,0x04,0x01,0x09,0x0c,0x03,0x06,
+    0x00,0x06,0x0c,0x0a,0x0b,0x0d,0x07,0x01, 0x05,0x03,0x09,0x0f,0x0e,0x08,0x02,0x04,
+    0x00,0x07,0x0e,0x09,0x0f,0x08,0x01,0x06, 0x0d,0x0a,0x03,0x04,0x02,0x05,0x0c,0x0b,
+    0x00,0x08,0x03,0x0b,0x06,0x0e,0x05,0x0d, 0x0c,0x04,0x0f,0x07,0x0a,0x02,0x09,0x01,
+    0x00,0x09,0x01,0x08,0x02,0x0b,0x03,0x0a, 0x04,0x0d,0x05,0x0c,0x06,0x0f,0x07,0x0e,
+    0x00,0x0a,0x07,0x0d,0x0e,0x04,0x09,0x03, 0x0f,0x05,0x08,0x02,0x01,0x0b,0x06,0x0c,
+    0x00,0x0b,0x05,0x0e,0x0a,0x01,0x0f,0x04, 0x07,0x0c,0x02,0x09,0x0d,0x06,0x08,0x03,
+    0x00,0x0c,0x0b,0x07,0x05,0x09,0x0e,0x02, 0x0a,0x06,0x01,0x0d,0x0f,0x03,0x04,0x08,
+    0x00,0x0d,0x09,0x04,0x01,0x0c,0x08,0x05, 0x02,0x0f,0x0b,0x06,0x03,0x0e,0x0a,0x07,
+    0x00,0x0e,0x0f,0x01,0x0d,0x03,0x02,0x0c, 0x09,0x07,0x06,0x08,0x04,0x0a,0x0b,0x05,
+    0x00,0x0f,0x0d,0x02,0x09,0x06,0x04,0x0b, 0x01,0x0e,0x0c,0x03,0x08,0x07,0x05,0x0a
+};
+
+static
+inline void mayo_S1_multabs(const unsigned char *S1, uint8x16_t *S1_multabs) {
+    size_t r;
+    for (size_t c = 0; c < V_MAX; c++)
+    {
+        for (r = 0; r+1 < K_MAX; r+= 2)
+        {
+            S1_multabs[K_OVER_2*c +  r/2] = *((uint8x16_t *)(mayo_gf16_mul + 16*S1[V_MAX*r + c]))
+                                          ^ (*((uint8x16_t *)(mayo_gf16_mul + 16*S1[V_MAX*(r+1) + c])) << 4);
+        }
+#if K_MAX % 2 == 1
+        S1_multabs[K_OVER_2*c +  r/2] = *((uint8x16_t *)(mayo_gf16_mul + 16*S1[V_MAX*r + c]));
+#endif
+    }
+}
+
+static
+inline void mayo_S2_multabs(const unsigned char *S2, uint8x16_t *S2_multabs) {
+    // build multiplication tables
+    size_t r;
+    for (size_t c = 0; c < O_MAX; c++)
+    {
+        for (r = 0; r+1 < K_MAX; r+= 2)
+        {
+            S2_multabs[K_OVER_2*c +  r/2] = *((uint8x16_t *)(mayo_gf16_mul + 16*S2[O_MAX*r + c]))
+                                          ^ (*((uint8x16_t *)(mayo_gf16_mul + 16*S2[O_MAX*(r+1) + c])) << 4);
+        }
+#if K_MAX % 2 == 1
+        S2_multabs[K_OVER_2*c +  r/2] = *((uint8x16_t *)(mayo_gf16_mul + 16*S2[O_MAX*r + c])) ;
+#endif
+    }
+}
+
+#endif
+

package/mayo-c/src/neon/echelon_form.h

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#include <arithmetic_common.h>
+#include <mem.h>
+#include <arm_neon.h>
+#include <stdint.h>
+
+
+#define MAYO_MAX(x, y) (((x) > (y)) ? (x) : (y))
+#define MAYO_MIN(x, y) (((x) < (y)) ? (x) : (y))
+
+/* put matrix in row echelon form with ones on first nonzero entries in constant time*/
+static inline void EF(unsigned char *A, int _nrows, int _ncols) {
+
+    (void) _nrows;
+    (void) _ncols;
+
+    #define nrows M_MAX
+    #define ncols (K_MAX * O_MAX + 1)
+
+    #define NEON_REGS_PER_ROW ((K_MAX * O_MAX + 1 + 15) / 16)
+    #define MAX_COLS (NEON_REGS_PER_ROW * 16)
+
+    uint8x16_t _pivot_row[NEON_REGS_PER_ROW];
+    uint8x16_t A_neon[NEON_REGS_PER_ROW* M_MAX];
+
+    unsigned char* pivot_row_bytes = (unsigned char*) _pivot_row;
+    unsigned char* A_bytes = (unsigned char*) A_neon;
+
+    // load A in the tail of NEON registers
+    for (int i = 0; i < nrows; i++) {
+        for (int j = 0; j < ncols; j++)
+        {
+            A_bytes[i*MAX_COLS + (MAX_COLS - ncols) + j] = A[ i*ncols + j ];
+        }
+    }
+
+    // pivot row is secret, pivot col is not
+    unsigned char inverse;
+    int pivot_row = 0;
+    int pivot_col = MAYO_MAX(MAX_COLS - ncols,0);
+    for (; pivot_col < MAX_COLS; pivot_col++) {
+        #include "echelon_form_loop.h"
+    }
+
+    // write the matrix A back
+    for (int i = 0; i < nrows; i++) {
+        for (int j = 0; j < ncols; j++) {
+            A[i * ncols + j] = A_bytes[i*NEON_REGS_PER_ROW*16 + (MAX_COLS - ncols) + j];
+        }
+    }
+    mayo_secure_clear(_pivot_row, NEON_REGS_PER_ROW * 16);
+    mayo_secure_clear(A_neon, NEON_REGS_PER_ROW * 16 * nrows);
+}
+

package/mayo-c/src/neon/echelon_form_loop.h

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: Apache-2.0
+
+int pivot_col_rounded = pivot_col/16;
+
+int pivot_row_lower_bound = MAYO_MAX(0, pivot_col + nrows - MAX_COLS);
+int pivot_row_upper_bound = MAYO_MIN(nrows - 1, pivot_col - MAX_COLS + ncols);
+/* the pivot row is guaranteed to be between these lower and upper bounds if A has full rank*/
+
+/* zero out pivot row */
+for (int i = pivot_col_rounded; i < NEON_REGS_PER_ROW; i++) {
+    _pivot_row[i] = vmovq_n_u8(0);
+}
+
+/* try to get a pivot row in constant time */
+unsigned char pivot = 0;
+uint32_t pivot_is_zero = -1;
+for (int row = pivot_row_lower_bound;
+        row <= MAYO_MIN(nrows - 1, pivot_row_upper_bound + 32); row++) {
+    uint32_t is_pivot_row = ~ct_compare_32(row, pivot_row);
+    uint32_t below_pivot_row = ct_is_greater_than(row, pivot_row);
+    uint8x16_t mask = vmovq_n_u8( is_pivot_row | (below_pivot_row & pivot_is_zero) );
+    for (int j = pivot_col_rounded; j < NEON_REGS_PER_ROW; j++) {
+        _pivot_row[j] ^= mask & A_neon[row * NEON_REGS_PER_ROW + j];
+    }
+    pivot = pivot_row_bytes[pivot_col];
+    pivot_is_zero = ~ct_compare_32((int) pivot, 0);
+}
+
+/* multiply pivot row by inverse of pivot */
+inverse = inverse_f(pivot);
+uint8x16_t inverse_multab = gf16v_get_multab(inverse);
+
+for (int j = pivot_col_rounded; j < NEON_REGS_PER_ROW; j++) {
+    _pivot_row[j] = vqtbl1q_u8(inverse_multab, _pivot_row[j]);
+}
+
+/* conditionally write pivot row to the correct row, if there is a nonzero pivot */
+/* eliminate entries below pivot */
+for (int row = pivot_row_lower_bound; row < nrows; row++) {
+    unsigned char below_pivot =  (unsigned char) (ct_is_greater_than(row, pivot_row));
+    unsigned char elt_to_elim = A_bytes[row*NEON_REGS_PER_ROW*16 + pivot_col];
+
+    uint8x16_t multab = gf16v_get_multab(below_pivot & elt_to_elim);
+    if (row <= pivot_row_upper_bound) {
+        uint8x16_t mask = vmovq_n_u8(~ct_compare_32(row, pivot_row) & ~pivot_is_zero);
+        for (int col = pivot_col_rounded; col < NEON_REGS_PER_ROW; col++) {
+            A_neon[row*NEON_REGS_PER_ROW + col] = 	vbslq_u8(mask, _pivot_row[col], A_neon[row*NEON_REGS_PER_ROW + col]) ^
+                                                    vqtbl1q_u8(multab, _pivot_row[col]);
+        }
+    } else {
+        for (int j = pivot_col_rounded; j < NEON_REGS_PER_ROW; j++) {
+            A_neon[row*NEON_REGS_PER_ROW + j] ^= vqtbl1q_u8(multab, _pivot_row[j]);
+        }
+    }
+}
+
+pivot_row += (-(int32_t)(~pivot_is_zero));
+