grpc 1.12.0 → 1.13.0.pre1

data/third_party/boringssl/crypto/fipsmodule/bn/bytes.c

@@ -77,7 +77,7 @@ BIGNUM *BN_bin2bn(const uint8_t *in, size_t len, BIGNUM *ret) {
   }
 
   if (len == 0) {
-    ret->top = 0;
+    ret->width = 0;
     return ret;
   }
 
@@ -93,7 +93,7 @@ BIGNUM *BN_bin2bn(const uint8_t *in, size_t len, BIGNUM *ret) {
   // |bn_wexpand| must check bounds on |num_words| to write it into
   // |ret->dmax|.
   assert(num_words <= INT_MAX);
-  ret->top = (int)num_words;
+  ret->width = (int)num_words;
   ret->neg = 0;
 
   while (len--) {
@@ -105,9 +105,6 @@ BIGNUM *BN_bin2bn(const uint8_t *in, size_t len, BIGNUM *ret) {
     }
   }
 
-  // need to call this due to clear byte at top if avoiding having the top bit
-  // set (-ve number)
-  bn_correct_top(ret);
   return ret;
 }
 
@@ -123,7 +120,7 @@ BIGNUM *BN_le2bn(const uint8_t *in, size_t len, BIGNUM *ret) {
   }
 
   if (len == 0) {
-    ret->top = 0;
+    ret->width = 0;
     ret->neg = 0;
     return ret;
   }
@@ -134,7 +131,7 @@ BIGNUM *BN_le2bn(const uint8_t *in, size_t len, BIGNUM *ret) {
     BN_free(bn);
     return NULL;
   }
-  ret->top = num_words;
+  ret->width = num_words;
 
   // Make sure the top bytes will be zeroed.
   ret->d[num_words - 1] = 0;
@@ -142,8 +139,6 @@ BIGNUM *BN_le2bn(const uint8_t *in, size_t len, BIGNUM *ret) {
   // We only support little-endian platforms, so we can simply memcpy the
   // internal representation.
   OPENSSL_memcpy(ret->d, in, len);
-
-  bn_correct_top(ret);
   return ret;
 }
 
@@ -159,88 +154,54 @@ size_t BN_bn2bin(const BIGNUM *in, uint8_t *out) {
   return n;
 }
 
+static int fits_in_bytes(const uint8_t *bytes, size_t num_bytes, size_t len) {
+  uint8_t mask = 0;
+  for (size_t i = len; i < num_bytes; i++) {
+    mask |= bytes[i];
+  }
+  return mask == 0;
+}
+
 int BN_bn2le_padded(uint8_t *out, size_t len, const BIGNUM *in) {
-  // If we don't have enough space, fail out.
-  size_t num_bytes = BN_num_bytes(in);
+  const uint8_t *bytes = (const uint8_t *)in->d;
+  size_t num_bytes = in->width * BN_BYTES;
   if (len < num_bytes) {
-    return 0;
+    if (!fits_in_bytes(bytes, num_bytes, len)) {
+      return 0;
+    }
+    num_bytes = len;
   }
 
   // We only support little-endian platforms, so we can simply memcpy into the
   // internal representation.
-  OPENSSL_memcpy(out, in->d, num_bytes);
-
+  OPENSSL_memcpy(out, bytes, num_bytes);
   // Pad out the rest of the buffer with zeroes.
   OPENSSL_memset(out + num_bytes, 0, len - num_bytes);
-
   return 1;
 }
 
-// constant_time_select_ulong returns |x| if |v| is 1 and |y| if |v| is 0. Its
-// behavior is undefined if |v| takes any other value.
-static BN_ULONG constant_time_select_ulong(int v, BN_ULONG x, BN_ULONG y) {
-  BN_ULONG mask = v;
-  mask--;
-
-  return (~mask & x) | (mask & y);
-}
-
-// constant_time_le_size_t returns 1 if |x| <= |y| and 0 otherwise. |x| and |y|
-// must not have their MSBs set.
-static int constant_time_le_size_t(size_t x, size_t y) {
-  return ((x - y - 1) >> (sizeof(size_t) * 8 - 1)) & 1;
-}
-
-// read_word_padded returns the |i|'th word of |in|, if it is not out of
-// bounds. Otherwise, it returns 0. It does so without branches on the size of
-// |in|, however it necessarily does not have the same memory access pattern. If
-// the access would be out of bounds, it reads the last word of |in|. |in| must
-// not be zero.
-static BN_ULONG read_word_padded(const BIGNUM *in, size_t i) {
-  // Read |in->d[i]| if valid. Otherwise, read the last word.
-  BN_ULONG l = in->d[constant_time_select_ulong(
-      constant_time_le_size_t(in->dmax, i), in->dmax - 1, i)];
-
-  // Clamp to zero if above |d->top|.
-  return constant_time_select_ulong(constant_time_le_size_t(in->top, i), 0, l);
-}
-
 int BN_bn2bin_padded(uint8_t *out, size_t len, const BIGNUM *in) {
-  // Special case for |in| = 0. Just branch as the probability is negligible.
-  if (BN_is_zero(in)) {
-    OPENSSL_memset(out, 0, len);
-    return 1;
-  }
-
-  // Check if the integer is too big. This case can exit early in non-constant
-  // time.
-  if ((size_t)in->top > (len + (BN_BYTES - 1)) / BN_BYTES) {
-    return 0;
-  }
-  if ((len % BN_BYTES) != 0) {
-    BN_ULONG l = read_word_padded(in, len / BN_BYTES);
-    if (l >> (8 * (len % BN_BYTES)) != 0) {
+  const uint8_t *bytes = (const uint8_t *)in->d;
+  size_t num_bytes = in->width * BN_BYTES;
+  if (len < num_bytes) {
+    if (!fits_in_bytes(bytes, num_bytes, len)) {
       return 0;
     }
+    num_bytes = len;
   }
 
-  // Write the bytes out one by one. Serialization is done without branching on
-  // the bits of |in| or on |in->top|, but if the routine would otherwise read
-  // out of bounds, the memory access pattern can't be fixed. However, for an
-  // RSA key of size a multiple of the word size, the probability of BN_BYTES
-  // leading zero octets is low.
-  //
-  // See Falko Stenzke, "Manger's Attack revisited", ICICS 2010.
-  size_t i = len;
-  while (i--) {
-    BN_ULONG l = read_word_padded(in, i / BN_BYTES);
-    *(out++) = (uint8_t)(l >> (8 * (i % BN_BYTES))) & 0xff;
+  // We only support little-endian platforms, so we can simply write the buffer
+  // in reverse.
+  for (size_t i = 0; i < num_bytes; i++) {
+    out[len - i - 1] = bytes[i];
   }
+  // Pad out the rest of the buffer with zeroes.
+  OPENSSL_memset(out, 0, len - num_bytes);
   return 1;
 }
 
 BN_ULONG BN_get_word(const BIGNUM *bn) {
-  switch (bn->top) {
+  switch (bn_minimal_width(bn)) {
     case 0:
       return 0;
     case 1:
@@ -251,7 +212,7 @@ BN_ULONG BN_get_word(const BIGNUM *bn) {
 }
 
 int BN_get_u64(const BIGNUM *bn, uint64_t *out) {
-  switch (bn->top) {
+  switch (bn_minimal_width(bn)) {
     case 0:
       *out = 0;
       return 1;

data/third_party/boringssl/crypto/fipsmodule/bn/cmp.c

@@ -63,33 +63,43 @@
 #include "../../internal.h"
 
 
-int BN_ucmp(const BIGNUM *a, const BIGNUM *b) {
-  int i;
-  BN_ULONG t1, t2, *ap, *bp;
-
-  i = a->top - b->top;
-  if (i != 0) {
-    return i;
+static int bn_cmp_words_consttime(const BN_ULONG *a, size_t a_len,
+                                  const BN_ULONG *b, size_t b_len) {
+  OPENSSL_COMPILE_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t),
+                         crypto_word_t_too_small);
+  int ret = 0;
+  // Process the common words in little-endian order.
+  size_t min = a_len < b_len ? a_len : b_len;
+  for (size_t i = 0; i < min; i++) {
+    crypto_word_t eq = constant_time_eq_w(a[i], b[i]);
+    crypto_word_t lt = constant_time_lt_w(a[i], b[i]);
+    ret =
+        constant_time_select_int(eq, ret, constant_time_select_int(lt, -1, 1));
   }
 
-  ap = a->d;
-  bp = b->d;
-  for (i = a->top - 1; i >= 0; i--) {
-    t1 = ap[i];
-    t2 = bp[i];
-    if (t1 != t2) {
-      return (t1 > t2) ? 1 : -1;
+  // If |a| or |b| has non-zero words beyond |min|, they take precedence.
+  if (a_len < b_len) {
+    crypto_word_t mask = 0;
+    for (size_t i = a_len; i < b_len; i++) {
+      mask |= b[i];
+    }
+    ret = constant_time_select_int(constant_time_is_zero_w(mask), ret, -1);
+  } else if (b_len < a_len) {
+    crypto_word_t mask = 0;
+    for (size_t i = b_len; i < a_len; i++) {
+      mask |= a[i];
     }
+    ret = constant_time_select_int(constant_time_is_zero_w(mask), ret, 1);
   }
 
-  return 0;
+  return ret;
 }
 
-int BN_cmp(const BIGNUM *a, const BIGNUM *b) {
-  int i;
-  int gt, lt;
-  BN_ULONG t1, t2;
+int BN_ucmp(const BIGNUM *a, const BIGNUM *b) {
+  return bn_cmp_words_consttime(a->d, a->width, b->d, b->width);
+}
 
+int BN_cmp(const BIGNUM *a, const BIGNUM *b) {
   if ((a == NULL) || (b == NULL)) {
     if (a != NULL) {
       return -1;
@@ -100,97 +110,25 @@ int BN_cmp(const BIGNUM *a, const BIGNUM *b) {
     }
   }
 
+  // We do not attempt to process the sign bit in constant time. Negative
+  // |BIGNUM|s should never occur in crypto, only calculators.
   if (a->neg != b->neg) {
     if (a->neg) {
       return -1;
     }
     return 1;
   }
-  if (a->neg == 0) {
-    gt = 1;
-    lt = -1;
-  } else {
-    gt = -1;
-    lt = 1;
-  }
-
-  if (a->top > b->top) {
-    return gt;
-  }
-  if (a->top < b->top) {
-    return lt;
-  }
-
-  for (i = a->top - 1; i >= 0; i--) {
-    t1 = a->d[i];
-    t2 = b->d[i];
-    if (t1 > t2) {
-      return gt;
-    } if (t1 < t2) {
-      return lt;
-    }
-  }
-
-  return 0;
-}
-
-int bn_cmp_words(const BN_ULONG *a, const BN_ULONG *b, int n) {
-  int i;
-  BN_ULONG aa, bb;
-
-  aa = a[n - 1];
-  bb = b[n - 1];
-  if (aa != bb) {
-    return (aa > bb) ? 1 : -1;
-  }
-
-  for (i = n - 2; i >= 0; i--) {
-    aa = a[i];
-    bb = b[i];
-    if (aa != bb) {
-      return (aa > bb) ? 1 : -1;
-    }
-  }
-  return 0;
-}
-
-int bn_cmp_part_words(const BN_ULONG *a, const BN_ULONG *b, int cl, int dl) {
-  int n, i;
-  n = cl - 1;
-
-  if (dl < 0) {
-    for (i = dl; i < 0; i++) {
-      if (b[n - i] != 0) {
-        return -1;  // a < b
-      }
-    }
-  }
-  if (dl > 0) {
-    for (i = dl; i > 0; i--) {
-      if (a[n + i] != 0) {
-        return 1;  // a > b
-      }
-    }
-  }
 
-  return bn_cmp_words(a, b, cl);
+  int ret = BN_ucmp(a, b);
+  return a->neg ? -ret : ret;
 }
 
 int bn_less_than_words(const BN_ULONG *a, const BN_ULONG *b, size_t len) {
-  OPENSSL_COMPILE_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t),
-                         crypto_word_t_too_small);
-  int ret = 0;
-  // Process the words in little-endian order.
-  for (size_t i = 0; i < len; i++) {
-    crypto_word_t eq = constant_time_eq_w(a[i], b[i]);
-    crypto_word_t lt = constant_time_lt_w(a[i], b[i]);
-    ret = constant_time_select_int(eq, ret, constant_time_select_int(lt, 1, 0));
-  }
-  return ret;
+  return bn_cmp_words_consttime(a, len, b, len) < 0;
 }
 
 int BN_abs_is_word(const BIGNUM *bn, BN_ULONG w) {
-  switch (bn->top) {
+  switch (bn_minimal_width(bn)) {
     case 1:
       return bn->d[0] == w;
     case 0:
@@ -205,14 +143,14 @@ int BN_cmp_word(const BIGNUM *a, BN_ULONG b) {
   BN_init(&b_bn);
 
   b_bn.d = &b;
-  b_bn.top = b > 0;
+  b_bn.width = b > 0;
   b_bn.dmax = 1;
   b_bn.flags = BN_FLG_STATIC_DATA;
   return BN_cmp(a, &b_bn);
 }
 
 int BN_is_zero(const BIGNUM *bn) {
-  return bn->top == 0;
+  return bn_minimal_width(bn) == 0;
 }
 
 int BN_is_one(const BIGNUM *bn) {
@@ -224,31 +162,39 @@ int BN_is_word(const BIGNUM *bn, BN_ULONG w) {
 }
 
 int BN_is_odd(const BIGNUM *bn) {
-  return bn->top > 0 && (bn->d[0] & 1) == 1;
+  return bn->width > 0 && (bn->d[0] & 1) == 1;
 }
 
 int BN_is_pow2(const BIGNUM *bn) {
-  if (bn->top == 0 || bn->neg) {
+  int width = bn_minimal_width(bn);
+  if (width == 0 || bn->neg) {
     return 0;
   }
 
-  for (int i = 0; i < bn->top - 1; i++) {
+  for (int i = 0; i < width - 1; i++) {
     if (bn->d[i] != 0) {
       return 0;
     }
   }
 
-  return 0 == (bn->d[bn->top-1] & (bn->d[bn->top-1] - 1));
+  return 0 == (bn->d[width-1] & (bn->d[width-1] - 1));
 }
 
 int BN_equal_consttime(const BIGNUM *a, const BIGNUM *b) {
-  if (a->top != b->top) {
-    return 0;
-  }
-
-  int limbs_are_equal =
-    CRYPTO_memcmp(a->d, b->d, (size_t)a->top * sizeof(a->d[0])) == 0;
-
-  return constant_time_select_int(constant_time_eq_int(a->neg, b->neg),
-                                  limbs_are_equal, 0);
+  BN_ULONG mask = 0;
+  // If |a| or |b| has more words than the other, all those words must be zero.
+  for (int i = a->width; i < b->width; i++) {
+    mask |= b->d[i];
+  }
+  for (int i = b->width; i < a->width; i++) {
+    mask |= a->d[i];
+  }
+  // Common words must match.
+  int min = a->width < b->width ? a->width : b->width;
+  for (int i = 0; i < min; i++) {
+    mask |= (a->d[i] ^ b->d[i]);
+  }
+  // The sign bit must match.
+  mask |= (a->neg ^ b->neg);
+  return mask == 0;
 }

data/third_party/boringssl/crypto/fipsmodule/bn/div.c

@@ -155,18 +155,18 @@ static inline void bn_div_rem_words(BN_ULONG *quotient_out, BN_ULONG *rem_out,
   //
   // These issues aren't specific to x86 and x86_64, so it might be worthwhile
   // to add more assembly language implementations.
-#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86) && defined(__GNUC__)
-  __asm__ volatile (
-    "divl %4"
-    : "=a"(*quotient_out), "=d"(*rem_out)
-    : "a"(n1), "d"(n0), "rm"(d0)
-    : "cc" );
-#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__GNUC__)
-  __asm__ volatile (
-    "divq %4"
-    : "=a"(*quotient_out), "=d"(*rem_out)
-    : "a"(n1), "d"(n0), "rm"(d0)
-    : "cc" );
+#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86) && \
+    (defined(__GNUC__) || defined(__clang__))
+  __asm__ volatile("divl %4"
+                   : "=a"(*quotient_out), "=d"(*rem_out)
+                   : "a"(n1), "d"(n0), "rm"(d0)
+                   : "cc");
+#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && \
+    (defined(__GNUC__) || defined(__clang__))
+  __asm__ volatile("divq %4"
+                   : "=a"(*quotient_out), "=d"(*rem_out)
+                   : "a"(n1), "d"(n0), "rm"(d0)
+                   : "cc");
 #else
 #if defined(BN_ULLONG)
   BN_ULLONG n = (((BN_ULLONG)n0) << BN_BITS2) | n1;
@@ -202,10 +202,16 @@ int BN_div(BIGNUM *quotient, BIGNUM *rem, const BIGNUM *numerator,
   BN_ULONG d0, d1;
   int num_n, div_n;
 
-  // Invalid zero-padding would have particularly bad consequences
-  // so don't just rely on bn_check_top() here
-  if ((numerator->top > 0 && numerator->d[numerator->top - 1] == 0) ||
-      (divisor->top > 0 && divisor->d[divisor->top - 1] == 0)) {
+  // This function relies on the historical minimal-width |BIGNUM| invariant.
+  // It is already not constant-time (constant-time reductions should use
+  // Montgomery logic), so we shrink all inputs and intermediate values to
+  // retain the previous behavior.
+
+  // Invalid zero-padding would have particularly bad consequences.
+  int numerator_width = bn_minimal_width(numerator);
+  int divisor_width = bn_minimal_width(divisor);
+  if ((numerator_width > 0 && numerator->d[numerator_width - 1] == 0) ||
+      (divisor_width > 0 && divisor->d[divisor_width - 1] == 0)) {
     OPENSSL_PUT_ERROR(BN, BN_R_NOT_INITIALIZED);
     return 0;
   }
@@ -234,46 +240,48 @@ int BN_div(BIGNUM *quotient, BIGNUM *rem, const BIGNUM *numerator,
   if (!BN_lshift(sdiv, divisor, norm_shift)) {
     goto err;
   }
+  bn_set_minimal_width(sdiv);
   sdiv->neg = 0;
   norm_shift += BN_BITS2;
   if (!BN_lshift(snum, numerator, norm_shift)) {
     goto err;
   }
+  bn_set_minimal_width(snum);
   snum->neg = 0;
 
   // Since we don't want to have special-case logic for the case where snum is
   // larger than sdiv, we pad snum with enough zeroes without changing its
   // value.
-  if (snum->top <= sdiv->top + 1) {
-    if (!bn_wexpand(snum, sdiv->top + 2)) {
+  if (snum->width <= sdiv->width + 1) {
+    if (!bn_wexpand(snum, sdiv->width + 2)) {
       goto err;
     }
-    for (int i = snum->top; i < sdiv->top + 2; i++) {
+    for (int i = snum->width; i < sdiv->width + 2; i++) {
       snum->d[i] = 0;
     }
-    snum->top = sdiv->top + 2;
+    snum->width = sdiv->width + 2;
   } else {
-    if (!bn_wexpand(snum, snum->top + 1)) {
+    if (!bn_wexpand(snum, snum->width + 1)) {
       goto err;
     }
-    snum->d[snum->top] = 0;
-    snum->top++;
+    snum->d[snum->width] = 0;
+    snum->width++;
   }
 
-  div_n = sdiv->top;
-  num_n = snum->top;
+  div_n = sdiv->width;
+  num_n = snum->width;
   loop = num_n - div_n;
   // Lets setup a 'window' into snum
   // This is the part that corresponds to the current
   // 'area' being divided
   wnum.neg = 0;
   wnum.d = &(snum->d[loop]);
-  wnum.top = div_n;
-  // only needed when BN_ucmp messes up the values between top and max
+  wnum.width = div_n;
+  // only needed when BN_ucmp messes up the values between width and max
   wnum.dmax = snum->dmax - loop;  // so we don't step out of bounds
 
   // Get the top 2 words of sdiv
-  // div_n=sdiv->top;
+  // div_n=sdiv->width;
   d0 = sdiv->d[div_n - 1];
   d1 = (div_n == 1) ? 0 : sdiv->d[div_n - 2];
 
@@ -285,7 +293,7 @@ int BN_div(BIGNUM *quotient, BIGNUM *rem, const BIGNUM *numerator,
   if (!bn_wexpand(res, loop + 1)) {
     goto err;
   }
-  res->top = loop - 1;
+  res->width = loop - 1;
   resp = &(res->d[loop - 1]);
 
   // space for temp
@@ -293,9 +301,9 @@ int BN_div(BIGNUM *quotient, BIGNUM *rem, const BIGNUM *numerator,
     goto err;
   }
 
-  // if res->top == 0 then clear the neg value otherwise decrease
+  // if res->width == 0 then clear the neg value otherwise decrease
   // the resp pointer
-  if (res->top == 0) {
+  if (res->width == 0) {
     res->neg = 0;
   } else {
     resp--;
@@ -371,7 +379,7 @@ int BN_div(BIGNUM *quotient, BIGNUM *rem, const BIGNUM *numerator,
     *resp = q;
   }
 
-  bn_correct_top(snum);
+  bn_set_minimal_width(snum);
 
   if (rem != NULL) {
     // Keep a copy of the neg flag in numerator because if |rem| == |numerator|
@@ -385,7 +393,7 @@ int BN_div(BIGNUM *quotient, BIGNUM *rem, const BIGNUM *numerator,
     }
   }
 
-  bn_correct_top(res);
+  bn_set_minimal_width(res);
   BN_CTX_end(ctx);
   return 1;
 
@@ -406,6 +414,78 @@ int BN_nnmod(BIGNUM *r, const BIGNUM *m, const BIGNUM *d, BN_CTX *ctx) {
   return (d->neg ? BN_sub : BN_add)(r, r, d);
 }
 
+// bn_mod_sub_words sets |r| to |a| - |b| (mod |m|), using |tmp| as scratch
+// space. Each array is |num| words long. |a| and |b| must be < |m|. Any pair of
+// |r|, |a|, and |b| may alias.
+static void bn_mod_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b,
+                             const BN_ULONG *m, BN_ULONG *tmp, size_t num) {
+  // r = a - b
+  BN_ULONG borrow = bn_sub_words(r, a, b, num);
+  // tmp = a - b + m
+  bn_add_words(tmp, r, m, num);
+  bn_select_words(r, 0 - borrow, tmp /* r < 0 */, r /* r >= 0 */, num);
+}
+
+// bn_mod_add_words sets |r| to |a| + |b| (mod |m|), using |tmp| as scratch
+// space. Each array is |num| words long. |a| and |b| must be < |m|. Any pair of
+// |r|, |a|, and |b| may alias.
+static void bn_mod_add_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b,
+                             const BN_ULONG *m, BN_ULONG *tmp, size_t num) {
+  // tmp = a + b. Note the result fits in |num|+1 words. We store the extra word
+  // in |carry|.
+  BN_ULONG carry = bn_add_words(tmp, a, b, num);
+  // r = a + b - m. We use |bn_sub_words| to perform the bulk of the
+  // subtraction, and then apply the borrow to |carry|.
+  carry -= bn_sub_words(r, tmp, m, num);
+  // |a| and |b| were both fully-reduced, so we know:
+  //
+  //   0 + 0 - m <= r < m + m - m
+  //          -m <= r < m
+  //
+  // If 0 <= |r| < |m|, |r| fits in |num| words and |carry| is zero. We then
+  // wish to select |r| as the answer. Otherwise -m <= r < 0 and we wish to
+  // return |r| + |m|, or |tmp|. |carry| must then be -1 or all ones. In both
+  // cases, |carry| is a suitable input to |bn_select_words|.
+  //
+  // Although |carry| may be one if |bn_add_words| returns one and
+  // |bn_sub_words| returns zero, this would give |r| > |m|, which violates are
+  // input assumptions.
+  assert(carry == 0 || carry == (BN_ULONG)-1);
+  bn_select_words(r, carry, tmp /* r < 0 */, r /* r >= 0 */, num);
+}
+
+static BIGNUM *bn_scratch_space_from_ctx(size_t width, BN_CTX *ctx) {
+  BIGNUM *ret = BN_CTX_get(ctx);
+  if (ret == NULL ||
+      !bn_wexpand(ret, width)) {
+    return NULL;
+  }
+  ret->neg = 0;
+  ret->width = width;
+  return ret;
+}
+
+// bn_resized_from_ctx returns |bn| with width at least |width| or NULL on
+// error. This is so it may be used with low-level "words" functions. If
+// necessary, it allocates a new |BIGNUM| with a lifetime of the current scope
+// in |ctx|, so the caller does not need to explicitly free it. |bn| must fit in
+// |width| words.
+static const BIGNUM *bn_resized_from_ctx(const BIGNUM *bn, size_t width,
+                                         BN_CTX *ctx) {
+  if ((size_t)bn->width >= width) {
+    // Any excess words must be zero.
+    assert(bn_fits_in_words(bn, width));
+    return bn;
+  }
+  BIGNUM *ret = bn_scratch_space_from_ctx(width, ctx);
+  if (ret == NULL ||
+      !BN_copy(ret, bn) ||
+      !bn_resize_words(ret, width)) {
+    return NULL;
+  }
+  return ret;
+}
+
 int BN_mod_add(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
                BN_CTX *ctx) {
   if (!BN_add(r, a, b)) {
@@ -416,13 +496,27 @@ int BN_mod_add(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
 
 int BN_mod_add_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
                      const BIGNUM *m) {
-  if (!BN_uadd(r, a, b)) {
-    return 0;
-  }
-  if (BN_ucmp(r, m) >= 0) {
-    return BN_usub(r, r, m);
+  BN_CTX *ctx = BN_CTX_new();
+  int ok = ctx != NULL &&
+           bn_mod_add_quick_ctx(r, a, b, m, ctx);
+  BN_CTX_free(ctx);
+  return ok;
+}
+
+int bn_mod_add_quick_ctx(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
+                         const BIGNUM *m, BN_CTX *ctx) {
+  BN_CTX_start(ctx);
+  a = bn_resized_from_ctx(a, m->width, ctx);
+  b = bn_resized_from_ctx(b, m->width, ctx);
+  BIGNUM *tmp = bn_scratch_space_from_ctx(m->width, ctx);
+  int ok = a != NULL && b != NULL && tmp != NULL &&
+           bn_wexpand(r, m->width);
+  if (ok) {
+    bn_mod_add_words(r->d, a->d, b->d, m->d, tmp->d, m->width);
+    r->width = m->width;
   }
-  return 1;
+  BN_CTX_end(ctx);
+  return ok;
 }
 
 int BN_mod_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
@@ -433,17 +527,29 @@ int BN_mod_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
   return BN_nnmod(r, r, m, ctx);
 }
 
-// BN_mod_sub variant that may be used if both  a  and  b  are non-negative
-// and less than  m
+int bn_mod_sub_quick_ctx(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
+                         const BIGNUM *m, BN_CTX *ctx) {
+  BN_CTX_start(ctx);
+  a = bn_resized_from_ctx(a, m->width, ctx);
+  b = bn_resized_from_ctx(b, m->width, ctx);
+  BIGNUM *tmp = bn_scratch_space_from_ctx(m->width, ctx);
+  int ok = a != NULL && b != NULL && tmp != NULL &&
+           bn_wexpand(r, m->width);
+  if (ok) {
+    bn_mod_sub_words(r->d, a->d, b->d, m->d, tmp->d, m->width);
+    r->width = m->width;
+  }
+  BN_CTX_end(ctx);
+  return ok;
+}
+
 int BN_mod_sub_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
                      const BIGNUM *m) {
-  if (!BN_sub(r, a, b)) {
-    return 0;
-  }
-  if (r->neg) {
-    return BN_add(r, r, m);
-  }
-  return 1;
+  BN_CTX *ctx = BN_CTX_new();
+  int ok = ctx != NULL &&
+           bn_mod_sub_quick_ctx(r, a, b, m, ctx);
+  BN_CTX_free(ctx);
+  return ok;
 }
 
 int BN_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
@@ -504,58 +610,33 @@ int BN_mod_lshift(BIGNUM *r, const BIGNUM *a, int n, const BIGNUM *m,
     abs_m->neg = 0;
   }
 
-  ret = BN_mod_lshift_quick(r, r, n, (abs_m ? abs_m : m));
+  ret = bn_mod_lshift_quick_ctx(r, r, n, (abs_m ? abs_m : m), ctx);
 
   BN_free(abs_m);
   return ret;
 }
 
-int BN_mod_lshift_quick(BIGNUM *r, const BIGNUM *a, int n, const BIGNUM *m) {
-  if (r != a) {
-    if (BN_copy(r, a) == NULL) {
-      return 0;
-    }
+int bn_mod_lshift_quick_ctx(BIGNUM *r, const BIGNUM *a, int n, const BIGNUM *m,
+                            BN_CTX *ctx) {
+  if (!BN_copy(r, a)) {
+    return 0;
   }
-
-  while (n > 0) {
-    int max_shift;
-
-    // 0 < r < m
-    max_shift = BN_num_bits(m) - BN_num_bits(r);
-    // max_shift >= 0
-
-    if (max_shift < 0) {
-      OPENSSL_PUT_ERROR(BN, BN_R_INPUT_NOT_REDUCED);
+  for (int i = 0; i < n; i++) {
+    if (!bn_mod_lshift1_quick_ctx(r, r, m, ctx)) {
       return 0;
     }
-
-    if (max_shift > n) {
-      max_shift = n;
-    }
-
-    if (max_shift) {
-      if (!BN_lshift(r, r, max_shift)) {
-        return 0;
-      }
-      n -= max_shift;
-    } else {
-      if (!BN_lshift1(r, r)) {
-        return 0;
-      }
-      --n;
-    }
-
-    // BN_num_bits(r) <= BN_num_bits(m)
-    if (BN_cmp(r, m) >= 0) {
-      if (!BN_sub(r, r, m)) {
-        return 0;
-      }
-    }
   }
-
   return 1;
 }
 
+int BN_mod_lshift_quick(BIGNUM *r, const BIGNUM *a, int n, const BIGNUM *m) {
+  BN_CTX *ctx = BN_CTX_new();
+  int ok = ctx != NULL &&
+           bn_mod_lshift_quick_ctx(r, a, n, m, ctx);
+  BN_CTX_free(ctx);
+  return ok;
+}
+
 int BN_mod_lshift1(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx) {
   if (!BN_lshift1(r, a)) {
     return 0;
@@ -564,15 +645,17 @@ int BN_mod_lshift1(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx) {
   return BN_nnmod(r, r, m, ctx);
 }
 
-int BN_mod_lshift1_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *m) {
-  if (!BN_lshift1(r, a)) {
-    return 0;
-  }
-  if (BN_cmp(r, m) >= 0) {
-    return BN_sub(r, r, m);
-  }
+int bn_mod_lshift1_quick_ctx(BIGNUM *r, const BIGNUM *a, const BIGNUM *m,
+                             BN_CTX *ctx) {
+  return bn_mod_add_quick_ctx(r, a, a, m, ctx);
+}
 
-  return 1;
+int BN_mod_lshift1_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *m) {
+  BN_CTX *ctx = BN_CTX_new();
+  int ok = ctx != NULL &&
+           bn_mod_lshift1_quick_ctx(r, a, m, ctx);
+  BN_CTX_free(ctx);
+  return ok;
 }
 
 BN_ULONG BN_div_word(BIGNUM *a, BN_ULONG w) {
@@ -584,7 +667,7 @@ BN_ULONG BN_div_word(BIGNUM *a, BN_ULONG w) {
     return (BN_ULONG) - 1;
   }
 
-  if (a->top == 0) {
+  if (a->width == 0) {
     return 0;
   }
 
@@ -595,7 +678,7 @@ BN_ULONG BN_div_word(BIGNUM *a, BN_ULONG w) {
     return (BN_ULONG) - 1;
   }
 
-  for (i = a->top - 1; i >= 0; i--) {
+  for (i = a->width - 1; i >= 0; i--) {
     BN_ULONG l = a->d[i];
     BN_ULONG d;
     BN_ULONG unused_rem;
@@ -604,20 +687,13 @@ BN_ULONG BN_div_word(BIGNUM *a, BN_ULONG w) {
     a->d[i] = d;
   }
 
-  if ((a->top > 0) && (a->d[a->top - 1] == 0)) {
-    a->top--;
-  }
-
-  if (a->top == 0) {
-    a->neg = 0;
-  }
-
+  bn_set_minimal_width(a);
   ret >>= j;
   return ret;
 }
 
 BN_ULONG BN_mod_word(const BIGNUM *a, BN_ULONG w) {
-#ifndef BN_ULLONG
+#ifndef BN_CAN_DIVIDE_ULLONG
   BN_ULONG ret = 0;
 #else
   BN_ULLONG ret = 0;
@@ -628,9 +704,9 @@ BN_ULONG BN_mod_word(const BIGNUM *a, BN_ULONG w) {
     return (BN_ULONG) -1;
   }
 
-#ifndef BN_ULLONG
-  // If |w| is too long and we don't have |BN_ULLONG| then we need to fall back
-  // to using |BN_div_word|.
+#ifndef BN_CAN_DIVIDE_ULLONG
+  // If |w| is too long and we don't have |BN_ULLONG| division then we need to
+  // fall back to using |BN_div_word|.
   if (w > ((BN_ULONG)1 << BN_BITS4)) {
     BIGNUM *tmp = BN_dup(a);
     if (tmp == NULL) {
@@ -642,8 +718,8 @@ BN_ULONG BN_mod_word(const BIGNUM *a, BN_ULONG w) {
   }
 #endif
 
-  for (i = a->top - 1; i >= 0; i--) {
-#ifndef BN_ULLONG
+  for (i = a->width - 1; i >= 0; i--) {
+#ifndef BN_CAN_DIVIDE_ULLONG
     ret = ((ret << BN_BITS4) | ((a->d[i] >> BN_BITS4) & BN_MASK2l)) % w;
     ret = ((ret << BN_BITS4) | (a->d[i] & BN_MASK2l)) % w;
 #else
@@ -654,7 +730,7 @@ BN_ULONG BN_mod_word(const BIGNUM *a, BN_ULONG w) {
 }
 
 int BN_mod_pow2(BIGNUM *r, const BIGNUM *a, size_t e) {
-  if (e == 0 || a->top == 0) {
+  if (e == 0 || a->width == 0) {
     BN_zero(r);
     return 1;
   }
@@ -662,7 +738,7 @@ int BN_mod_pow2(BIGNUM *r, const BIGNUM *a, size_t e) {
   size_t num_words = 1 + ((e - 1) / BN_BITS2);
 
   // If |a| definitely has less than |e| bits, just BN_copy.
-  if ((size_t) a->top < num_words) {
+  if ((size_t) a->width < num_words) {
     return BN_copy(r, a) != NULL;
   }
 
@@ -683,8 +759,8 @@ int BN_mod_pow2(BIGNUM *r, const BIGNUM *a, size_t e) {
 
   // Fill in the remaining fields of |r|.
   r->neg = a->neg;
-  r->top = (int) num_words;
-  bn_correct_top(r);
+  r->width = (int) num_words;
+  bn_set_minimal_width(r);
   return 1;
 }
 
@@ -706,27 +782,27 @@ int BN_nnmod_pow2(BIGNUM *r, const BIGNUM *a, size_t e) {
   }
 
   // Clear the upper words of |r|.
-  OPENSSL_memset(&r->d[r->top], 0, (num_words - r->top) * BN_BYTES);
+  OPENSSL_memset(&r->d[r->width], 0, (num_words - r->width) * BN_BYTES);
 
   // Set parameters of |r|.
   r->neg = 0;
-  r->top = (int) num_words;
+  r->width = (int) num_words;
 
   // Now, invert every word. The idea here is that we want to compute 2^e-|x|,
   // which is actually equivalent to the twos-complement representation of |x|
   // in |e| bits, which is -x = ~x + 1.
-  for (int i = 0; i < r->top; i++) {
+  for (int i = 0; i < r->width; i++) {
     r->d[i] = ~r->d[i];
   }
 
   // If our exponent doesn't span the top word, we have to mask the rest.
   size_t top_word_exponent = e % BN_BITS2;
   if (top_word_exponent != 0) {
-    r->d[r->top - 1] &= (((BN_ULONG) 1) << top_word_exponent) - 1;
+    r->d[r->width - 1] &= (((BN_ULONG) 1) << top_word_exponent) - 1;
   }
 
-  // Keep the correct_top invariant for BN_add.
-  bn_correct_top(r);
+  // Keep the minimal-width invariant for |BIGNUM|.
+  bn_set_minimal_width(r);
 
   // Finally, add one, for the reason described above.
   return BN_add(r, r, BN_value_one());