grpc-flamingo 1.11.0 → 1.15.0

data/third_party/boringssl/crypto/fipsmodule/bn/exponentiation.c

@@ -622,8 +622,8 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
 
   // Allocate a montgomery context if it was not supplied by the caller.
   if (mont == NULL) {
-    new_mont = BN_MONT_CTX_new();
-    if (new_mont == NULL || !BN_MONT_CTX_set(new_mont, m, ctx)) {
+    new_mont = BN_MONT_CTX_new_for_modulus(m, ctx);
+    if (new_mont == NULL) {
       goto err;
     }
     mont = new_mont;
@@ -666,22 +666,7 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
     }
   }
 
-  // Set |r| to one in Montgomery form. If the high bit of |m| is set, |m| is
-  // close to R and we subtract rather than perform Montgomery reduction.
-  if (m->d[m->top - 1] & (((BN_ULONG)1) << (BN_BITS2 - 1))) {
-    if (!bn_wexpand(r, m->top)) {
-      goto err;
-    }
-    // r = 2^(top*BN_BITS2) - m
-    r->d[0] = 0 - m->d[0];
-    for (int i = 1; i < m->top; i++) {
-      r->d[i] = ~m->d[i];
-    }
-    r->top = m->top;
-    // The upper words will be zero if the corresponding words of |m| were
-    // 0xfff[...], so call |bn_correct_top|.
-    bn_correct_top(r);
-  } else if (!BN_to_montgomery(r, BN_value_one(), mont, ctx)) {
+  if (!bn_one_to_montgomery(r, mont, ctx)) {
     goto err;
   }
 
@@ -746,8 +731,7 @@ err:
 int bn_mod_exp_mont_small(BN_ULONG *r, size_t num_r, const BN_ULONG *a,
                           size_t num_a, const BN_ULONG *p, size_t num_p,
                           const BN_MONT_CTX *mont) {
-  const BN_ULONG *n = mont->N.d;
-  size_t num_n = mont->N.top;
+  size_t num_n = mont->N.width;
   if (num_n != num_a || num_n != num_r || num_n > BN_SMALL_MAX_WORDS) {
     OPENSSL_PUT_ERROR(BN, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED);
     return 0;
@@ -793,16 +777,7 @@ int bn_mod_exp_mont_small(BN_ULONG *r, size_t num_r, const BN_ULONG *a,
     }
   }
 
-  // Set |r| to one in Montgomery form. If the high bit of |m| is set, |m| is
-  // close to R and we subtract rather than perform Montgomery reduction.
-  if (n[num_n - 1] & (((BN_ULONG)1) << (BN_BITS2 - 1))) {
-    // r = 2^(top*BN_BITS2) - m
-    r[0] = 0 - n[0];
-    for (size_t i = 1; i < num_n; i++) {
-      r[i] = ~n[i];
-    }
-  } else if (!bn_from_montgomery_small(r, num_r, mont->RR.d, mont->RR.top,
-                                       mont)) {
+  if (!bn_one_to_montgomery_small(r, num_r, mont)) {
     goto err;
   }
 
@@ -866,7 +841,7 @@ int bn_mod_inverse_prime_mont_small(BN_ULONG *r, size_t num_r,
                                     const BN_ULONG *a, size_t num_a,
                                     const BN_MONT_CTX *mont) {
   const BN_ULONG *p = mont->N.d;
-  size_t num_p = mont->N.top;
+  size_t num_p = mont->N.width;
   if (num_p > BN_SMALL_MAX_WORDS || num_p == 0) {
     OPENSSL_PUT_ERROR(BN, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED);
     return 0;
@@ -901,8 +876,8 @@ static void copy_to_prebuf(const BIGNUM *b, int top, unsigned char *buf,
   const int width = 1 << window;
   BN_ULONG *table = (BN_ULONG *) buf;
 
-  if (top > b->top) {
-    top = b->top;  // this works because 'buf' is explicitly zeroed
+  if (top > b->width) {
+    top = b->width;  // this works because 'buf' is explicitly zeroed
   }
 
   for (i = 0, j = idx; i < top; i++, j += width)  {
@@ -955,8 +930,7 @@ static int copy_from_prebuf(BIGNUM *b, int top, unsigned char *buf, int idx,
     }
   }
 
-  b->top = top;
-  bn_correct_top(b);
+  b->width = top;
   return 1;
 }
 
@@ -1006,8 +980,7 @@ static int copy_from_prebuf(BIGNUM *b, int top, unsigned char *buf, int idx,
 int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
                               const BIGNUM *m, BN_CTX *ctx,
                               const BN_MONT_CTX *mont) {
-  int i, bits, ret = 0, window, wvalue;
-  int top;
+  int i, ret = 0, window, wvalue;
   BN_MONT_CTX *new_mont = NULL;
 
   int numPowers;
@@ -1022,9 +995,10 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
     return 0;
   }
 
-  top = m->top;
-
-  bits = BN_num_bits(p);
+  // Use all bits stored in |p|, rather than |BN_num_bits|, so we do not leak
+  // whether the top bits are zero.
+  int max_bits = p->width * BN_BITS2;
+  int bits = max_bits;
   if (bits == 0) {
     // x**0 mod 1 is still zero.
     if (BN_is_one(m)) {
@@ -1036,13 +1010,17 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
 
   // Allocate a montgomery context if it was not supplied by the caller.
   if (mont == NULL) {
-    new_mont = BN_MONT_CTX_new();
-    if (new_mont == NULL || !BN_MONT_CTX_set(new_mont, m, ctx)) {
+    new_mont = BN_MONT_CTX_new_for_modulus(m, ctx);
+    if (new_mont == NULL) {
       goto err;
     }
     mont = new_mont;
   }
 
+  // Use the width in |mont->N|, rather than the copy in |m|. The assembly
+  // implementation assumes it can use |top| to size R.
+  int top = mont->N.width;
+
   if (a->neg || BN_ucmp(a, m) >= 0) {
     new_a = BN_new();
     if (new_a == NULL ||
@@ -1056,15 +1034,14 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
   // If the size of the operands allow it, perform the optimized
   // RSAZ exponentiation. For further information see
   // crypto/bn/rsaz_exp.c and accompanying assembly modules.
-  if ((16 == a->top) && (16 == p->top) && (BN_num_bits(m) == 1024) &&
+  if ((16 == a->width) && (16 == p->width) && (BN_num_bits(m) == 1024) &&
       rsaz_avx2_eligible()) {
     if (!bn_wexpand(rr, 16)) {
       goto err;
     }
     RSAZ_1024_mod_exp_avx2(rr->d, a->d, p->d, m->d, mont->RR.d, mont->n0[0]);
-    rr->top = 16;
+    rr->width = 16;
     rr->neg = 0;
-    bn_correct_top(rr);
     ret = 1;
     goto err;
   }
@@ -1110,21 +1087,12 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
   // lay down tmp and am right after powers table
   tmp.d = (BN_ULONG *)(powerbuf + sizeof(m->d[0]) * top * numPowers);
   am.d = tmp.d + top;
-  tmp.top = am.top = 0;
+  tmp.width = am.width = 0;
   tmp.dmax = am.dmax = top;
   tmp.neg = am.neg = 0;
   tmp.flags = am.flags = BN_FLG_STATIC_DATA;
 
-// prepare a^0 in Montgomery domain
-// by Shay Gueron's suggestion
-  if (m->d[top - 1] & (((BN_ULONG)1) << (BN_BITS2 - 1))) {
-    // 2^(top*BN_BITS2) - m
-    tmp.d[0] = 0 - m->d[0];
-    for (i = 1; i < top; i++) {
-      tmp.d[i] = ~m->d[i];
-    }
-    tmp.top = top;
-  } else if (!BN_to_montgomery(&tmp, BN_value_one(), mont, ctx)) {
+  if (!bn_one_to_montgomery(&tmp, mont, ctx)) {
     goto err;
   }
 
@@ -1148,10 +1116,10 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
 
     // BN_to_montgomery can contaminate words above .top
     // [in BN_DEBUG[_DEBUG] build]...
-    for (i = am.top; i < top; i++) {
+    for (i = am.width; i < top; i++) {
       am.d[i] = 0;
     }
-    for (i = tmp.top; i < top; i++) {
+    for (i = tmp.width; i < top; i++) {
       tmp.d[i] = 0;
     }
 
@@ -1161,7 +1129,7 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
     }
 
     bn_scatter5(tmp.d, top, powerbuf, 0);
-    bn_scatter5(am.d, am.top, powerbuf, 1);
+    bn_scatter5(am.d, am.width, powerbuf, 1);
     bn_mul_mont(tmp.d, am.d, am.d, np, n0, top);
     bn_scatter5(tmp.d, top, powerbuf, 2);
 
@@ -1217,7 +1185,6 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
       }
     } else {
       const uint8_t *p_bytes = (const uint8_t *)p->d;
-      int max_bits = p->top * BN_BITS2;
       assert(bits < max_bits);
       // |p = 0| has been handled as a special case, so |max_bits| is at least
       // one word.
@@ -1229,7 +1196,7 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
       // here is the top bit, inclusive.
       if (bits - 4 >= max_bits - 8) {
         // Read five bits from |bits-4| through |bits|, inclusive.
-        wvalue = p_bytes[p->top * BN_BYTES - 1];
+        wvalue = p_bytes[p->width * BN_BYTES - 1];
         wvalue >>= (bits - 4) & 7;
         wvalue &= 0x1f;
         bits -= 5;
@@ -1248,8 +1215,7 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
     }
 
     ret = bn_from_montgomery(tmp.d, tmp.d, NULL, np, n0, top);
-    tmp.top = top;
-    bn_correct_top(&tmp);
+    tmp.width = top;
     if (ret) {
       if (!BN_copy(rr, &tmp)) {
         ret = 0;
@@ -1363,8 +1329,8 @@ int BN_mod_exp2_mont(BIGNUM *rr, const BIGNUM *a1, const BIGNUM *p1,
 
   // Allocate a montgomery context if it was not supplied by the caller.
   if (mont == NULL) {
-    new_mont = BN_MONT_CTX_new();
-    if (new_mont == NULL || !BN_MONT_CTX_set(new_mont, m, ctx)) {
+    new_mont = BN_MONT_CTX_new_for_modulus(m, ctx);
+    if (new_mont == NULL) {
       goto err;
     }
     mont = new_mont;

data/third_party/boringssl/crypto/fipsmodule/bn/gcd.c

@@ -114,121 +114,310 @@
 
 #include "internal.h"
 
-static BIGNUM *euclid(BIGNUM *a, BIGNUM *b) {
-  BIGNUM *t;
-  int shifts = 0;
 
-  // 0 <= b <= a
-  while (!BN_is_zero(b)) {
-    // 0 < b <= a
+static BN_ULONG word_is_odd_mask(BN_ULONG a) { return (BN_ULONG)0 - (a & 1); }
 
-    if (BN_is_odd(a)) {
-      if (BN_is_odd(b)) {
-        if (!BN_sub(a, a, b)) {
-          goto err;
-        }
-        if (!BN_rshift1(a, a)) {
-          goto err;
-        }
-        if (BN_cmp(a, b) < 0) {
-          t = a;
-          a = b;
-          b = t;
-        }
-      } else {
-        // a odd - b even
-        if (!BN_rshift1(b, b)) {
-          goto err;
-        }
-        if (BN_cmp(a, b) < 0) {
-          t = a;
-          a = b;
-          b = t;
-        }
-      }
-    } else {
-      // a is even
-      if (BN_is_odd(b)) {
-        if (!BN_rshift1(a, a)) {
-          goto err;
-        }
-        if (BN_cmp(a, b) < 0) {
-          t = a;
-          a = b;
-          b = t;
-        }
-      } else {
-        // a even - b even
-        if (!BN_rshift1(a, a)) {
-          goto err;
-        }
-        if (!BN_rshift1(b, b)) {
-          goto err;
-        }
-        shifts++;
-      }
-    }
-    // 0 <= b <= a
+static void maybe_rshift1_words(BN_ULONG *a, BN_ULONG mask, BN_ULONG *tmp,
+                                size_t num) {
+  bn_rshift1_words(tmp, a, num);
+  bn_select_words(a, mask, tmp, a, num);
+}
+
+static void maybe_rshift1_words_carry(BN_ULONG *a, BN_ULONG carry,
+                                      BN_ULONG mask, BN_ULONG *tmp,
+                                      size_t num) {
+  maybe_rshift1_words(a, mask, tmp, num);
+  if (num != 0) {
+    carry &= mask;
+    a[num - 1] |= carry << (BN_BITS2-1);
+  }
+}
+
+static BN_ULONG maybe_add_words(BN_ULONG *a, BN_ULONG mask, const BN_ULONG *b,
+                                BN_ULONG *tmp, size_t num) {
+  BN_ULONG carry = bn_add_words(tmp, a, b, num);
+  bn_select_words(a, mask, tmp, a, num);
+  return carry & mask;
+}
+
+static int bn_gcd_consttime(BIGNUM *r, unsigned *out_shift, const BIGNUM *x,
+                            const BIGNUM *y, BN_CTX *ctx) {
+  size_t width = x->width > y->width ? x->width : y->width;
+  if (width == 0) {
+    *out_shift = 0;
+    BN_zero(r);
+    return 1;
   }
 
-  if (shifts) {
-    if (!BN_lshift(a, a, shifts)) {
-      goto err;
-    }
+  // This is a constant-time implementation of Stein's algorithm (binary GCD).
+  int ret = 0;
+  BN_CTX_start(ctx);
+  BIGNUM *u = BN_CTX_get(ctx);
+  BIGNUM *v = BN_CTX_get(ctx);
+  BIGNUM *tmp = BN_CTX_get(ctx);
+  if (u == NULL || v == NULL || tmp == NULL ||
+      !BN_copy(u, x) ||
+      !BN_copy(v, y) ||
+      !bn_resize_words(u, width) ||
+      !bn_resize_words(v, width) ||
+      !bn_resize_words(tmp, width)) {
+    goto err;
+  }
+
+  // Each loop iteration halves at least one of |u| and |v|. Thus we need at
+  // most the combined bit width of inputs for at least one value to be zero.
+  unsigned x_bits = x->width * BN_BITS2, y_bits = y->width * BN_BITS2;
+  unsigned num_iters = x_bits + y_bits;
+  if (num_iters < x_bits) {
+    OPENSSL_PUT_ERROR(BN, BN_R_BIGNUM_TOO_LONG);
+    goto err;
   }
 
-  return a;
+  unsigned shift = 0;
+  for (unsigned i = 0; i < num_iters; i++) {
+    BN_ULONG both_odd = word_is_odd_mask(u->d[0]) & word_is_odd_mask(v->d[0]);
+
+    // If both |u| and |v| are odd, subtract the smaller from the larger.
+    BN_ULONG u_less_than_v =
+        (BN_ULONG)0 - bn_sub_words(tmp->d, u->d, v->d, width);
+    bn_select_words(u->d, both_odd & ~u_less_than_v, tmp->d, u->d, width);
+    bn_sub_words(tmp->d, v->d, u->d, width);
+    bn_select_words(v->d, both_odd & u_less_than_v, tmp->d, v->d, width);
+
+    // At least one of |u| and |v| is now even.
+    BN_ULONG u_is_odd = word_is_odd_mask(u->d[0]);
+    BN_ULONG v_is_odd = word_is_odd_mask(v->d[0]);
+    assert(!(u_is_odd & v_is_odd));
+
+    // If both are even, the final GCD gains a factor of two.
+    shift += 1 & (~u_is_odd & ~v_is_odd);
+
+    // Halve any which are even.
+    maybe_rshift1_words(u->d, ~u_is_odd, tmp->d, width);
+    maybe_rshift1_words(v->d, ~v_is_odd, tmp->d, width);
+  }
+
+  // One of |u| or |v| is zero at this point. The algorithm usually makes |u|
+  // zero, unless |y| was already zero on input. Fix this by combining the
+  // values.
+  assert(BN_is_zero(u) || BN_is_zero(v));
+  for (size_t i = 0; i < width; i++) {
+    v->d[i] |= u->d[i];
+  }
+
+  *out_shift = shift;
+  ret = bn_set_words(r, v->d, width);
 
 err:
-  return NULL;
+  BN_CTX_end(ctx);
+  return ret;
 }
 
-int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx) {
-  BIGNUM *a, *b, *t;
+int BN_gcd(BIGNUM *r, const BIGNUM *x, const BIGNUM *y, BN_CTX *ctx) {
+  unsigned shift;
+  return bn_gcd_consttime(r, &shift, x, y, ctx) &&
+         BN_lshift(r, r, shift);
+}
+
+int bn_is_relatively_prime(int *out_relatively_prime, const BIGNUM *x,
+                           const BIGNUM *y, BN_CTX *ctx) {
   int ret = 0;
+  BN_CTX_start(ctx);
+  unsigned shift;
+  BIGNUM *gcd = BN_CTX_get(ctx);
+  if (gcd == NULL ||
+      !bn_gcd_consttime(gcd, &shift, x, y, ctx)) {
+    goto err;
+  }
+
+  // Check that 2^|shift| * |gcd| is one.
+  if (gcd->width == 0) {
+    *out_relatively_prime = 0;
+  } else {
+    BN_ULONG mask = shift | (gcd->d[0] ^ 1);
+    for (int i = 1; i < gcd->width; i++) {
+      mask |= gcd->d[i];
+    }
+    *out_relatively_prime = mask == 0;
+  }
+  ret = 1;
+
+err:
+  BN_CTX_end(ctx);
+  return ret;
+}
 
+int bn_lcm_consttime(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx) {
   BN_CTX_start(ctx);
-  a = BN_CTX_get(ctx);
-  b = BN_CTX_get(ctx);
+  unsigned shift;
+  BIGNUM *gcd = BN_CTX_get(ctx);
+  int ret = gcd != NULL &&
+            bn_mul_consttime(r, a, b, ctx) &&
+            bn_gcd_consttime(gcd, &shift, a, b, ctx) &&
+            bn_div_consttime(r, NULL, r, gcd, ctx) &&
+            bn_rshift_secret_shift(r, r, shift, ctx);
+  BN_CTX_end(ctx);
+  return ret;
+}
 
-  if (a == NULL || b == NULL) {
-    goto err;
+int bn_mod_inverse_consttime(BIGNUM *r, int *out_no_inverse, const BIGNUM *a,
+                             const BIGNUM *n, BN_CTX *ctx) {
+  *out_no_inverse = 0;
+  if (BN_is_negative(a) || BN_ucmp(a, n) >= 0) {
+    OPENSSL_PUT_ERROR(BN, BN_R_INPUT_NOT_REDUCED);
+    return 0;
   }
-  if (BN_copy(a, in_a) == NULL) {
-    goto err;
+  if (BN_is_zero(a)) {
+    if (BN_is_one(n)) {
+      BN_zero(r);
+      return 1;
+    }
+    *out_no_inverse = 1;
+    OPENSSL_PUT_ERROR(BN, BN_R_NO_INVERSE);
+    return 0;
   }
-  if (BN_copy(b, in_b) == NULL) {
-    goto err;
+
+  // This is a constant-time implementation of the extended binary GCD
+  // algorithm. It is adapted from the Handbook of Applied Cryptography, section
+  // 14.4.3, algorithm 14.51, and modified to bound coefficients and avoid
+  // negative numbers.
+  //
+  // For more details and proof of correctness, see
+  // https://github.com/mit-plv/fiat-crypto/pull/333. In particular, see |step|
+  // and |mod_inverse_consttime| for the algorithm in Gallina and see
+  // |mod_inverse_consttime_spec| for the correctness result.
+
+  if (!BN_is_odd(a) && !BN_is_odd(n)) {
+    *out_no_inverse = 1;
+    OPENSSL_PUT_ERROR(BN, BN_R_NO_INVERSE);
+    return 0;
   }
 
-  a->neg = 0;
-  b->neg = 0;
+  // This function exists to compute the RSA private exponent, where |a| is one
+  // word. We'll thus use |a_width| when available.
+  size_t n_width = n->width, a_width = a->width;
+  if (a_width > n_width) {
+    a_width = n_width;
+  }
 
-  if (BN_cmp(a, b) < 0) {
-    t = a;
-    a = b;
-    b = t;
+  int ret = 0;
+  BN_CTX_start(ctx);
+  BIGNUM *u = BN_CTX_get(ctx);
+  BIGNUM *v = BN_CTX_get(ctx);
+  BIGNUM *A = BN_CTX_get(ctx);
+  BIGNUM *B = BN_CTX_get(ctx);
+  BIGNUM *C = BN_CTX_get(ctx);
+  BIGNUM *D = BN_CTX_get(ctx);
+  BIGNUM *tmp = BN_CTX_get(ctx);
+  BIGNUM *tmp2 = BN_CTX_get(ctx);
+  if (u == NULL || v == NULL || A == NULL || B == NULL || C == NULL ||
+      D == NULL || tmp == NULL || tmp2 == NULL ||
+      !BN_copy(u, a) ||
+      !BN_copy(v, n) ||
+      !BN_one(A) ||
+      !BN_one(D) ||
+      // For convenience, size |u| and |v| equivalently.
+      !bn_resize_words(u, n_width) ||
+      !bn_resize_words(v, n_width) ||
+      // |A| and |C| are bounded by |m|.
+      !bn_resize_words(A, n_width) ||
+      !bn_resize_words(C, n_width) ||
+      // |B| and |D| are bounded by |a|.
+      !bn_resize_words(B, a_width) ||
+      !bn_resize_words(D, a_width) ||
+      // |tmp| and |tmp2| may be used at either size.
+      !bn_resize_words(tmp, n_width) ||
+      !bn_resize_words(tmp2, n_width)) {
+    goto err;
   }
-  t = euclid(a, b);
-  if (t == NULL) {
+
+  // Each loop iteration halves at least one of |u| and |v|. Thus we need at
+  // most the combined bit width of inputs for at least one value to be zero.
+  unsigned a_bits = a_width * BN_BITS2, n_bits = n_width * BN_BITS2;
+  unsigned num_iters = a_bits + n_bits;
+  if (num_iters < a_bits) {
+    OPENSSL_PUT_ERROR(BN, BN_R_BIGNUM_TOO_LONG);
     goto err;
   }
 
-  if (BN_copy(r, t) == NULL) {
+  // Before and after each loop iteration, the following hold:
+  //
+  //   u = A*a - B*n
+  //   v = D*n - C*a
+  //   0 < u <= a
+  //   0 <= v <= n
+  //   0 <= A < n
+  //   0 <= B <= a
+  //   0 <= C < n
+  //   0 <= D <= a
+  //
+  // After each loop iteration, u and v only get smaller, and at least one of
+  // them shrinks by at least a factor of two.
+  for (unsigned i = 0; i < num_iters; i++) {
+    BN_ULONG both_odd = word_is_odd_mask(u->d[0]) & word_is_odd_mask(v->d[0]);
+
+    // If both |u| and |v| are odd, subtract the smaller from the larger.
+    BN_ULONG v_less_than_u =
+        (BN_ULONG)0 - bn_sub_words(tmp->d, v->d, u->d, n_width);
+    bn_select_words(v->d, both_odd & ~v_less_than_u, tmp->d, v->d, n_width);
+    bn_sub_words(tmp->d, u->d, v->d, n_width);
+    bn_select_words(u->d, both_odd & v_less_than_u, tmp->d, u->d, n_width);
+
+    // If we updated one of the values, update the corresponding coefficient.
+    BN_ULONG carry = bn_add_words(tmp->d, A->d, C->d, n_width);
+    carry -= bn_sub_words(tmp2->d, tmp->d, n->d, n_width);
+    bn_select_words(tmp->d, carry, tmp->d, tmp2->d, n_width);
+    bn_select_words(A->d, both_odd & v_less_than_u, tmp->d, A->d, n_width);
+    bn_select_words(C->d, both_odd & ~v_less_than_u, tmp->d, C->d, n_width);
+
+    bn_add_words(tmp->d, B->d, D->d, a_width);
+    bn_sub_words(tmp2->d, tmp->d, a->d, a_width);
+    bn_select_words(tmp->d, carry, tmp->d, tmp2->d, a_width);
+    bn_select_words(B->d, both_odd & v_less_than_u, tmp->d, B->d, a_width);
+    bn_select_words(D->d, both_odd & ~v_less_than_u, tmp->d, D->d, a_width);
+
+    // Our loop invariants hold at this point. Additionally, exactly one of |u|
+    // and |v| is now even.
+    BN_ULONG u_is_even = ~word_is_odd_mask(u->d[0]);
+    BN_ULONG v_is_even = ~word_is_odd_mask(v->d[0]);
+    assert(u_is_even != v_is_even);
+
+    // Halve the even one and adjust the corresponding coefficient.
+    maybe_rshift1_words(u->d, u_is_even, tmp->d, n_width);
+    BN_ULONG A_or_B_is_odd =
+        word_is_odd_mask(A->d[0]) | word_is_odd_mask(B->d[0]);
+    BN_ULONG A_carry =
+        maybe_add_words(A->d, A_or_B_is_odd & u_is_even, n->d, tmp->d, n_width);
+    BN_ULONG B_carry =
+        maybe_add_words(B->d, A_or_B_is_odd & u_is_even, a->d, tmp->d, a_width);
+    maybe_rshift1_words_carry(A->d, A_carry, u_is_even, tmp->d, n_width);
+    maybe_rshift1_words_carry(B->d, B_carry, u_is_even, tmp->d, a_width);
+
+    maybe_rshift1_words(v->d, v_is_even, tmp->d, n_width);
+    BN_ULONG C_or_D_is_odd =
+        word_is_odd_mask(C->d[0]) | word_is_odd_mask(D->d[0]);
+    BN_ULONG C_carry =
+        maybe_add_words(C->d, C_or_D_is_odd & v_is_even, n->d, tmp->d, n_width);
+    BN_ULONG D_carry =
+        maybe_add_words(D->d, C_or_D_is_odd & v_is_even, a->d, tmp->d, a_width);
+    maybe_rshift1_words_carry(C->d, C_carry, v_is_even, tmp->d, n_width);
+    maybe_rshift1_words_carry(D->d, D_carry, v_is_even, tmp->d, a_width);
+  }
+
+  assert(BN_is_zero(v));
+  if (!BN_is_one(u)) {
+    *out_no_inverse = 1;
+    OPENSSL_PUT_ERROR(BN, BN_R_NO_INVERSE);
     goto err;
   }
-  ret = 1;
+
+  ret = BN_copy(r, A) != NULL;
 
 err:
   BN_CTX_end(ctx);
   return ret;
 }
 
-// solves ax == 1 (mod n)
-static int bn_mod_inverse_general(BIGNUM *out, int *out_no_inverse,
-                                  const BIGNUM *a, const BIGNUM *n,
-                                  BN_CTX *ctx);
-
 int BN_mod_inverse_odd(BIGNUM *out, int *out_no_inverse, const BIGNUM *a,
                        const BIGNUM *n, BN_CTX *ctx) {
   *out_no_inverse = 0;
@@ -423,7 +612,7 @@ BIGNUM *BN_mod_inverse(BIGNUM *out, const BIGNUM *a, const BIGNUM *n,
 
   int no_inverse;
   if (!BN_is_odd(n)) {
-    if (!bn_mod_inverse_general(out, &no_inverse, a, n, ctx)) {
+    if (!bn_mod_inverse_consttime(out, &no_inverse, a, n, ctx)) {
       goto err;
     }
   } else if (!BN_mod_inverse_odd(out, &no_inverse, a, n, ctx)) {
@@ -469,139 +658,6 @@ err:
   return ret;
 }
 
-// bn_mod_inverse_general is the general inversion algorithm that works for
-// both even and odd |n|. It was specifically designed to contain fewer
-// branches that may leak sensitive information; see "New Branch Prediction
-// Vulnerabilities in OpenSSL and Necessary Software Countermeasures" by
-// Onur Acıçmez, Shay Gueron, and Jean-Pierre Seifert.
-static int bn_mod_inverse_general(BIGNUM *out, int *out_no_inverse,
-                                  const BIGNUM *a, const BIGNUM *n,
-                                  BN_CTX *ctx) {
-  BIGNUM *A, *B, *X, *Y, *M, *D, *T;
-  int ret = 0;
-  int sign;
-
-  *out_no_inverse = 0;
-
-  BN_CTX_start(ctx);
-  A = BN_CTX_get(ctx);
-  B = BN_CTX_get(ctx);
-  X = BN_CTX_get(ctx);
-  D = BN_CTX_get(ctx);
-  M = BN_CTX_get(ctx);
-  Y = BN_CTX_get(ctx);
-  T = BN_CTX_get(ctx);
-  if (T == NULL) {
-    goto err;
-  }
-
-  BIGNUM *R = out;
-
-  BN_zero(Y);
-  if (!BN_one(X) || BN_copy(B, a) == NULL || BN_copy(A, n) == NULL) {
-    goto err;
-  }
-  A->neg = 0;
-
-  sign = -1;
-  // From  B = a mod |n|,  A = |n|  it follows that
-  //
-  //      0 <= B < A,
-  //     -sign*X*a  ==  B   (mod |n|),
-  //      sign*Y*a  ==  A   (mod |n|).
-
-  while (!BN_is_zero(B)) {
-    BIGNUM *tmp;
-
-    //      0 < B < A,
-    // (*) -sign*X*a  ==  B   (mod |n|),
-    //      sign*Y*a  ==  A   (mod |n|)
-
-    // (D, M) := (A/B, A%B) ...
-    if (!BN_div(D, M, A, B, ctx)) {
-      goto err;
-    }
-
-    // Now
-    //      A = D*B + M;
-    // thus we have
-    // (**)  sign*Y*a  ==  D*B + M   (mod |n|).
-
-    tmp = A;  // keep the BIGNUM object, the value does not matter
-
-    // (A, B) := (B, A mod B) ...
-    A = B;
-    B = M;
-    // ... so we have  0 <= B < A  again
-
-    // Since the former  M  is now  B  and the former  B  is now  A,
-    // (**) translates into
-    //       sign*Y*a  ==  D*A + B    (mod |n|),
-    // i.e.
-    //       sign*Y*a - D*A  ==  B    (mod |n|).
-    // Similarly, (*) translates into
-    //      -sign*X*a  ==  A          (mod |n|).
-    //
-    // Thus,
-    //   sign*Y*a + D*sign*X*a  ==  B  (mod |n|),
-    // i.e.
-    //        sign*(Y + D*X)*a  ==  B  (mod |n|).
-    //
-    // So if we set  (X, Y, sign) := (Y + D*X, X, -sign),  we arrive back at
-    //      -sign*X*a  ==  B   (mod |n|),
-    //       sign*Y*a  ==  A   (mod |n|).
-    // Note that  X  and  Y  stay non-negative all the time.
-
-    if (!BN_mul(tmp, D, X, ctx)) {
-      goto err;
-    }
-    if (!BN_add(tmp, tmp, Y)) {
-      goto err;
-    }
-
-    M = Y;  // keep the BIGNUM object, the value does not matter
-    Y = X;
-    X = tmp;
-    sign = -sign;
-  }
-
-  if (!BN_is_one(A)) {
-    *out_no_inverse = 1;
-    OPENSSL_PUT_ERROR(BN, BN_R_NO_INVERSE);
-    goto err;
-  }
-
-  // The while loop (Euclid's algorithm) ends when
-  //      A == gcd(a,n);
-  // we have
-  //       sign*Y*a  ==  A  (mod |n|),
-  // where  Y  is non-negative.
-
-  if (sign < 0) {
-    if (!BN_sub(Y, n, Y)) {
-      goto err;
-    }
-  }
-  // Now  Y*a  ==  A  (mod |n|).
-
-  // Y*a == 1  (mod |n|)
-  if (!Y->neg && BN_ucmp(Y, n) < 0) {
-    if (!BN_copy(R, Y)) {
-      goto err;
-    }
-  } else {
-    if (!BN_nnmod(R, Y, n, ctx)) {
-      goto err;
-    }
-  }
-
-  ret = 1;
-
-err:
-  BN_CTX_end(ctx);
-  return ret;
-}
-
 int bn_mod_inverse_prime(BIGNUM *out, const BIGNUM *a, const BIGNUM *p,
                          BN_CTX *ctx, const BN_MONT_CTX *mont_p) {
   BN_CTX_start(ctx);