grpc-flamingo 1.11.0 → 1.15.0

data/third_party/boringssl/crypto/fipsmodule/modes/internal.h

@@ -66,38 +66,6 @@ extern "C" {
 #define STRICT_ALIGNMENT 0
 #endif
 
-#if defined(__GNUC__) && __GNUC__ >= 2
-static inline uint32_t CRYPTO_bswap4(uint32_t x) {
-  return __builtin_bswap32(x);
-}
-
-static inline uint64_t CRYPTO_bswap8(uint64_t x) {
-  return __builtin_bswap64(x);
-}
-#elif defined(_MSC_VER)
-OPENSSL_MSVC_PRAGMA(warning(push, 3))
-#include <intrin.h>
-OPENSSL_MSVC_PRAGMA(warning(pop))
-#pragma intrinsic(_byteswap_uint64, _byteswap_ulong)
-static inline uint32_t CRYPTO_bswap4(uint32_t x) {
-  return _byteswap_ulong(x);
-}
-
-static inline uint64_t CRYPTO_bswap8(uint64_t x) {
-  return _byteswap_uint64(x);
-}
-#else
-static inline uint32_t CRYPTO_bswap4(uint32_t x) {
-  x = (x >> 16) | (x << 16);
-  x = ((x & 0xff00ff00) >> 8) | ((x & 0x00ff00ff) << 8);
-  return x;
-}
-
-static inline uint64_t CRYPTO_bswap8(uint64_t x) {
-  return CRYPTO_bswap4(x >> 32) | (((uint64_t)CRYPTO_bswap4(x)) << 32);
-}
-#endif
-
 static inline uint32_t GETU32(const void *in) {
   uint32_t v;
   OPENSSL_memcpy(&v, in, sizeof(v));
@@ -281,6 +249,42 @@ OPENSSL_EXPORT void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, uint8_t *tag,
                                       size_t len);
 
 
+// CCM.
+
+typedef struct ccm128_context {
+  block128_f block;
+  ctr128_f ctr;
+  unsigned M, L;
+} CCM128_CONTEXT;
+
+// CRYPTO_ccm128_init initialises |ctx| to use |block| (typically AES) with the
+// specified |M| and |L| parameters. It returns one on success and zero if |M|
+// or |L| is invalid.
+int CRYPTO_ccm128_init(CCM128_CONTEXT *ctx, const void *key, block128_f block,
+                       ctr128_f ctr, unsigned M, unsigned L);
+
+// CRYPTO_ccm128_max_input returns the maximum input length accepted by |ctx|.
+size_t CRYPTO_ccm128_max_input(const CCM128_CONTEXT *ctx);
+
+// CRYPTO_ccm128_encrypt encrypts |len| bytes from |in| to |out| writing the tag
+// to |out_tag|. |key| must be the same key that was passed to
+// |CRYPTO_ccm128_init|. It returns one on success and zero otherwise.
+int CRYPTO_ccm128_encrypt(const CCM128_CONTEXT *ctx, const void *key,
+                          uint8_t *out, uint8_t *out_tag, size_t tag_len,
+                          const uint8_t *nonce, size_t nonce_len,
+                          const uint8_t *in, size_t len, const uint8_t *aad,
+                          size_t aad_len);
+
+// CRYPTO_ccm128_decrypt decrypts |len| bytes from |in| to |out|, writing the
+// expected tag to |out_tag|. |key| must be the same key that was passed to
+// |CRYPTO_ccm128_init|. It returns one on success and zero otherwise.
+int CRYPTO_ccm128_decrypt(const CCM128_CONTEXT *ctx, const void *key,
+                          uint8_t *out, uint8_t *out_tag, size_t tag_len,
+                          const uint8_t *nonce, size_t nonce_len,
+                          const uint8_t *in, size_t len, const uint8_t *aad,
+                          size_t aad_len);
+
+
 // CBC.
 
 // cbc128_f is the type of a function that performs CBC-mode encryption.

data/third_party/boringssl/crypto/fipsmodule/rand/ctrdrbg.c

@@ -74,11 +74,11 @@ static void ctr32_add(CTR_DRBG_STATE *drbg, uint32_t n) {
       CRYPTO_bswap4(CRYPTO_bswap4(drbg->counter.words[3]) + n);
 }
 
-static int CTR_DRBG_update(CTR_DRBG_STATE *drbg, const uint8_t *data,
+static int ctr_drbg_update(CTR_DRBG_STATE *drbg, const uint8_t *data,
                            size_t data_len) {
-  // Section 10.2.1.2. A value of |data_len| which less than
-  // |CTR_DRBG_ENTROPY_LEN| is permitted and acts the same as right-padding
-  // with zeros. This can save a copy.
+  // Per section 10.2.1.2, |data_len| must be |CTR_DRBG_ENTROPY_LEN|. Here, we
+  // allow shorter inputs and right-pad them with zeros. This is equivalent to
+  // the specified algorithm but saves a copy in |CTR_DRBG_generate|.
   if (data_len > CTR_DRBG_ENTROPY_LEN) {
     return 0;
   }
@@ -119,7 +119,7 @@ int CTR_DRBG_reseed(CTR_DRBG_STATE *drbg,
     entropy = entropy_copy;
   }
 
-  if (!CTR_DRBG_update(drbg, entropy, CTR_DRBG_ENTROPY_LEN)) {
+  if (!ctr_drbg_update(drbg, entropy, CTR_DRBG_ENTROPY_LEN)) {
     return 0;
   }
 
@@ -142,7 +142,7 @@ int CTR_DRBG_generate(CTR_DRBG_STATE *drbg, uint8_t *out, size_t out_len,
   }
 
   if (additional_data_len != 0 &&
-      !CTR_DRBG_update(drbg, additional_data, additional_data_len)) {
+      !ctr_drbg_update(drbg, additional_data, additional_data_len)) {
     return 0;
   }
 
@@ -187,7 +187,9 @@ int CTR_DRBG_generate(CTR_DRBG_STATE *drbg, uint8_t *out, size_t out_len,
     OPENSSL_memcpy(out, block, out_len);
   }
 
-  if (!CTR_DRBG_update(drbg, additional_data, additional_data_len)) {
+  // Right-padding |additional_data| in step 2.2 is handled implicitly by
+  // |ctr_drbg_update|, to save a copy.
+  if (!ctr_drbg_update(drbg, additional_data, additional_data_len)) {
     return 0;
   }
 

data/third_party/boringssl/crypto/fipsmodule/rsa/blinding.c

@@ -215,46 +215,22 @@ int BN_BLINDING_invert(BIGNUM *n, const BN_BLINDING *b, BN_MONT_CTX *mont,
 
 static int bn_blinding_create_param(BN_BLINDING *b, const BIGNUM *e,
                                     const BN_MONT_CTX *mont, BN_CTX *ctx) {
-  int retry_counter = 32;
-
-  do {
-    if (!BN_rand_range_ex(b->A, 1, &mont->N)) {
-      OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
-      return 0;
-    }
-
-    // |BN_from_montgomery| + |BN_mod_inverse_blinded| is equivalent to, but
-    // more efficient than, |BN_mod_inverse_blinded| + |BN_to_montgomery|.
-    if (!BN_from_montgomery(b->Ai, b->A, mont, ctx)) {
-      OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
-      return 0;
-    }
-
-    int no_inverse;
-    if (BN_mod_inverse_blinded(b->Ai, &no_inverse, b->Ai, mont, ctx)) {
-      break;
-    }
-
-    if (!no_inverse) {
-      OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
-      return 0;
-    }
-
-    // For reasonably-sized RSA keys, it should almost never be the case that a
-    // random value doesn't have an inverse.
-    if (retry_counter-- == 0) {
-      OPENSSL_PUT_ERROR(RSA, RSA_R_TOO_MANY_ITERATIONS);
-      return 0;
-    }
-    ERR_clear_error();
-  } while (1);
-
-  if (!BN_mod_exp_mont(b->A, b->A, e, &mont->N, ctx, mont)) {
-    OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
-    return 0;
-  }
-
-  if (!BN_to_montgomery(b->A, b->A, mont, ctx)) {
+  int no_inverse;
+  if (!BN_rand_range_ex(b->A, 1, &mont->N) ||
+      // Compute |b->A|^-1 in Montgomery form. Note |BN_from_montgomery| +
+      // |BN_mod_inverse_blinded| is equivalent to, but more efficient than,
+      // |BN_mod_inverse_blinded| + |BN_to_montgomery|.
+      //
+      // We do not retry if |b->A| has no inverse. Finding a non-invertible
+      // value of |b->A| is equivalent to factoring |mont->N|. There is
+      // negligible probability of stumbling on one at random.
+      !BN_from_montgomery(b->Ai, b->A, mont, ctx) ||
+      !BN_mod_inverse_blinded(b->Ai, &no_inverse, b->Ai, mont, ctx) ||
+      // TODO(davidben): |BN_mod_exp_mont| internally computes the result in
+      // Montgomery form. Save a pair of Montgomery reductions and a
+      // multiplication by returning that value directly.
+      !BN_mod_exp_mont(b->A, b->A, e, &mont->N, ctx, mont) ||
+      !BN_to_montgomery(b->A, b->A, mont, ctx)) {
     OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
     return 0;
   }

data/third_party/boringssl/crypto/fipsmodule/rsa/internal.h

@@ -114,15 +114,10 @@ int RSA_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
                           size_t len);
 
 
-// The following utility functions are exported for test purposes.
-
+// This constant is exported for test purposes.
 extern const BN_ULONG kBoringSSLRSASqrtTwo[];
 extern const size_t kBoringSSLRSASqrtTwoLen;
 
-// rsa_greater_than_pow2 returns one if |b| is greater than 2^|n| and zero
-// otherwise.
-int rsa_greater_than_pow2(const BIGNUM *b, int n);
-
 
 #if defined(__cplusplus)
 }  // extern C

data/third_party/boringssl/crypto/fipsmodule/rsa/rsa.c

@@ -132,17 +132,21 @@ void RSA_free(RSA *rsa) {
 
   CRYPTO_free_ex_data(g_rsa_ex_data_class_bss_get(), rsa, &rsa->ex_data);
 
-  BN_clear_free(rsa->n);
-  BN_clear_free(rsa->e);
-  BN_clear_free(rsa->d);
-  BN_clear_free(rsa->p);
-  BN_clear_free(rsa->q);
-  BN_clear_free(rsa->dmp1);
-  BN_clear_free(rsa->dmq1);
-  BN_clear_free(rsa->iqmp);
+  BN_free(rsa->n);
+  BN_free(rsa->e);
+  BN_free(rsa->d);
+  BN_free(rsa->p);
+  BN_free(rsa->q);
+  BN_free(rsa->dmp1);
+  BN_free(rsa->dmq1);
+  BN_free(rsa->iqmp);
   BN_MONT_CTX_free(rsa->mont_n);
   BN_MONT_CTX_free(rsa->mont_p);
   BN_MONT_CTX_free(rsa->mont_q);
+  BN_free(rsa->d_fixed);
+  BN_free(rsa->dmp1_fixed);
+  BN_free(rsa->dmq1_fixed);
+  BN_free(rsa->inv_small_mod_large_mont);
   for (u = 0; u < rsa->num_blindings; u++) {
     BN_BLINDING_free(rsa->blindings[u]);
   }
@@ -630,8 +634,25 @@ err:
   return ret;
 }
 
+static int check_mod_inverse(int *out_ok, const BIGNUM *a, const BIGNUM *ainv,
+                             const BIGNUM *m, int check_reduced, BN_CTX *ctx) {
+  BN_CTX_start(ctx);
+  BIGNUM *tmp = BN_CTX_get(ctx);
+  int ret = tmp != NULL &&
+            bn_mul_consttime(tmp, a, ainv, ctx) &&
+            bn_div_consttime(NULL, tmp, tmp, m, ctx);
+  if (ret) {
+    *out_ok = BN_is_one(tmp);
+    if (check_reduced && (BN_is_negative(ainv) || BN_cmp(ainv, m) >= 0)) {
+      *out_ok = 0;
+    }
+  }
+  BN_CTX_end(ctx);
+  return ret;
+}
+
 int RSA_check_key(const RSA *key) {
-  BIGNUM n, pm1, qm1, lcm, gcd, de, dmp1, dmq1, iqmp_times_q;
+  BIGNUM n, pm1, qm1, lcm, dmp1, dmq1, iqmp_times_q;
   BN_CTX *ctx;
   int ok = 0, has_crt_values;
 
@@ -666,26 +687,20 @@ int RSA_check_key(const RSA *key) {
   BN_init(&pm1);
   BN_init(&qm1);
   BN_init(&lcm);
-  BN_init(&gcd);
-  BN_init(&de);
   BN_init(&dmp1);
   BN_init(&dmq1);
   BN_init(&iqmp_times_q);
 
-  if (!BN_mul(&n, key->p, key->q, ctx) ||
+  int d_ok;
+  if (!bn_mul_consttime(&n, key->p, key->q, ctx) ||
       // lcm = lcm(p, q)
-      !BN_sub(&pm1, key->p, BN_value_one()) ||
-      !BN_sub(&qm1, key->q, BN_value_one()) ||
-      !BN_mul(&lcm, &pm1, &qm1, ctx) ||
-      !BN_gcd(&gcd, &pm1, &qm1, ctx)) {
-    OPENSSL_PUT_ERROR(RSA, ERR_LIB_BN);
-    goto out;
-  }
-
-  if (!BN_div(&lcm, NULL, &lcm, &gcd, ctx) ||
-      !BN_gcd(&gcd, &pm1, &qm1, ctx) ||
-      // de = d*e mod lcm(p, q).
-      !BN_mod_mul(&de, key->d, key->e, &lcm, ctx)) {
+      !bn_usub_consttime(&pm1, key->p, BN_value_one()) ||
+      !bn_usub_consttime(&qm1, key->q, BN_value_one()) ||
+      !bn_lcm_consttime(&lcm, &pm1, &qm1, ctx) ||
+      // Other implementations use the Euler totient rather than the Carmichael
+      // totient, so allow unreduced |key->d|.
+      !check_mod_inverse(&d_ok, key->e, key->d, &lcm,
+                         0 /* don't require reduced */, ctx)) {
     OPENSSL_PUT_ERROR(RSA, ERR_LIB_BN);
     goto out;
   }
@@ -695,11 +710,16 @@ int RSA_check_key(const RSA *key) {
     goto out;
   }
 
-  if (!BN_is_one(&de)) {
+  if (!d_ok) {
     OPENSSL_PUT_ERROR(RSA, RSA_R_D_E_NOT_CONGRUENT_TO_1);
     goto out;
   }
 
+  if (BN_is_negative(key->d) || BN_cmp(key->d, key->n) >= 0) {
+    OPENSSL_PUT_ERROR(RSA, RSA_R_D_OUT_OF_RANGE);
+    goto out;
+  }
+
   has_crt_values = key->dmp1 != NULL;
   if (has_crt_values != (key->dmq1 != NULL) ||
       has_crt_values != (key->iqmp != NULL)) {
@@ -708,20 +728,18 @@ int RSA_check_key(const RSA *key) {
   }
 
   if (has_crt_values) {
-    if (// dmp1 = d mod (p-1)
-        !BN_mod(&dmp1, key->d, &pm1, ctx) ||
-        // dmq1 = d mod (q-1)
-        !BN_mod(&dmq1, key->d, &qm1, ctx) ||
-        // iqmp = q^-1 mod p
-        !BN_mod_mul(&iqmp_times_q, key->iqmp, key->q, key->p, ctx)) {
+    int dmp1_ok, dmq1_ok, iqmp_ok;
+    if (!check_mod_inverse(&dmp1_ok, key->e, key->dmp1, &pm1,
+                           1 /* check reduced */, ctx) ||
+        !check_mod_inverse(&dmq1_ok, key->e, key->dmq1, &qm1,
+                           1 /* check reduced */, ctx) ||
+        !check_mod_inverse(&iqmp_ok, key->q, key->iqmp, key->p,
+                           1 /* check reduced */, ctx)) {
       OPENSSL_PUT_ERROR(RSA, ERR_LIB_BN);
       goto out;
     }
 
-    if (BN_cmp(&dmp1, key->dmp1) != 0 ||
-        BN_cmp(&dmq1, key->dmq1) != 0 ||
-        BN_cmp(key->iqmp, key->p) >= 0 ||
-        !BN_is_one(&iqmp_times_q)) {
+    if (!dmp1_ok || !dmq1_ok || !iqmp_ok) {
       OPENSSL_PUT_ERROR(RSA, RSA_R_CRT_VALUES_INCORRECT);
       goto out;
     }
@@ -734,8 +752,6 @@ out:
   BN_free(&pm1);
   BN_free(&qm1);
   BN_free(&lcm);
-  BN_free(&gcd);
-  BN_free(&de);
   BN_free(&dmp1);
   BN_free(&dmq1);
   BN_free(&iqmp_times_q);
@@ -760,8 +776,8 @@ static const BN_ULONG kSmallFactorsLimbs[] = {
 
 DEFINE_LOCAL_DATA(BIGNUM, g_small_factors) {
   out->d = (BN_ULONG *) kSmallFactorsLimbs;
-  out->top = OPENSSL_ARRAY_SIZE(kSmallFactorsLimbs);
-  out->dmax = out->top;
+  out->width = OPENSSL_ARRAY_SIZE(kSmallFactorsLimbs);
+  out->dmax = out->width;
   out->neg = 0;
   out->flags = BN_FLG_STATIC_DATA;
 }
@@ -852,6 +868,8 @@ int RSA_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
   return rsa_default_private_transform(rsa, out, in, len);
 }
 
+int RSA_flags(const RSA *rsa) { return rsa->flags; }
+
 int RSA_blinding_on(RSA *rsa, BN_CTX *ctx) {
   return 1;
 }

data/third_party/boringssl/crypto/fipsmodule/rsa/rsa_impl.c

@@ -109,6 +109,133 @@ static int check_modulus_and_exponent_sizes(const RSA *rsa) {
   return 1;
 }
 
+static int ensure_fixed_copy(BIGNUM **out, const BIGNUM *in, int width) {
+  if (*out != NULL) {
+    return 1;
+  }
+  BIGNUM *copy = BN_dup(in);
+  if (copy == NULL ||
+      !bn_resize_words(copy, width)) {
+    BN_free(copy);
+    return 0;
+  }
+  *out = copy;
+  return 1;
+}
+
+// freeze_private_key finishes initializing |rsa|'s private key components.
+// After this function has returned, |rsa| may not be changed. This is needed
+// because |RSA| is a public struct and, additionally, OpenSSL 1.1.0 opaquified
+// it wrong (see https://github.com/openssl/openssl/issues/5158).
+static int freeze_private_key(RSA *rsa, BN_CTX *ctx) {
+  CRYPTO_MUTEX_lock_read(&rsa->lock);
+  int frozen = rsa->private_key_frozen;
+  CRYPTO_MUTEX_unlock_read(&rsa->lock);
+  if (frozen) {
+    return 1;
+  }
+
+  int ret = 0;
+  CRYPTO_MUTEX_lock_write(&rsa->lock);
+  if (rsa->private_key_frozen) {
+    ret = 1;
+    goto err;
+  }
+
+  // Pre-compute various intermediate values, as well as copies of private
+  // exponents with correct widths. Note that other threads may concurrently
+  // read from |rsa->n|, |rsa->e|, etc., so any fixes must be in separate
+  // copies. We use |mont_n->N|, |mont_p->N|, and |mont_q->N| as copies of |n|,
+  // |p|, and |q| with the correct minimal widths.
+
+  if (rsa->mont_n == NULL) {
+    rsa->mont_n = BN_MONT_CTX_new_for_modulus(rsa->n, ctx);
+    if (rsa->mont_n == NULL) {
+      goto err;
+    }
+  }
+  const BIGNUM *n_fixed = &rsa->mont_n->N;
+
+  // The only public upper-bound of |rsa->d| is the bit length of |rsa->n|. The
+  // ASN.1 serialization of RSA private keys unfortunately leaks the byte length
+  // of |rsa->d|, but normalize it so we only leak it once, rather than per
+  // operation.
+  if (rsa->d != NULL &&
+      !ensure_fixed_copy(&rsa->d_fixed, rsa->d, n_fixed->width)) {
+    goto err;
+  }
+
+  if (rsa->p != NULL && rsa->q != NULL) {
+    if (rsa->mont_p == NULL) {
+      rsa->mont_p = BN_MONT_CTX_new_for_modulus(rsa->p, ctx);
+      if (rsa->mont_p == NULL) {
+        goto err;
+      }
+    }
+    const BIGNUM *p_fixed = &rsa->mont_p->N;
+
+    if (rsa->mont_q == NULL) {
+      rsa->mont_q = BN_MONT_CTX_new_for_modulus(rsa->q, ctx);
+      if (rsa->mont_q == NULL) {
+        goto err;
+      }
+    }
+    const BIGNUM *q_fixed = &rsa->mont_q->N;
+
+    if (rsa->dmp1 != NULL && rsa->dmq1 != NULL) {
+      // Key generation relies on this function to compute |iqmp|.
+      if (rsa->iqmp == NULL) {
+        BIGNUM *iqmp = BN_new();
+        if (iqmp == NULL ||
+            !bn_mod_inverse_secret_prime(iqmp, rsa->q, rsa->p, ctx,
+                                         rsa->mont_p)) {
+          BN_free(iqmp);
+          goto err;
+        }
+        rsa->iqmp = iqmp;
+      }
+
+      // CRT components are only publicly bounded by their corresponding
+      // moduli's bit lengths. |rsa->iqmp| is unused outside of this one-time
+      // setup, so we do not compute a fixed-width version of it.
+      if (!ensure_fixed_copy(&rsa->dmp1_fixed, rsa->dmp1, p_fixed->width) ||
+          !ensure_fixed_copy(&rsa->dmq1_fixed, rsa->dmq1, q_fixed->width)) {
+        goto err;
+      }
+
+      // Compute |inv_small_mod_large_mont|. Note that it is always modulo the
+      // larger prime, independent of what is stored in |rsa->iqmp|.
+      if (rsa->inv_small_mod_large_mont == NULL) {
+        BIGNUM *inv_small_mod_large_mont = BN_new();
+        int ok;
+        if (BN_cmp(rsa->p, rsa->q) < 0) {
+          ok = inv_small_mod_large_mont != NULL &&
+               bn_mod_inverse_secret_prime(inv_small_mod_large_mont, rsa->p,
+                                           rsa->q, ctx, rsa->mont_q) &&
+               BN_to_montgomery(inv_small_mod_large_mont,
+                                inv_small_mod_large_mont, rsa->mont_q, ctx);
+        } else {
+          ok = inv_small_mod_large_mont != NULL &&
+               BN_to_montgomery(inv_small_mod_large_mont, rsa->iqmp,
+                                rsa->mont_p, ctx);
+        }
+        if (!ok) {
+          BN_free(inv_small_mod_large_mont);
+          goto err;
+        }
+        rsa->inv_small_mod_large_mont = inv_small_mod_large_mont;
+      }
+    }
+  }
+
+  rsa->private_key_frozen = 1;
+  ret = 1;
+
+err:
+  CRYPTO_MUTEX_unlock_write(&rsa->lock);
+  return ret;
+}
+
 size_t rsa_default_size(const RSA *rsa) {
   return BN_num_bytes(rsa->n);
 }
@@ -181,7 +308,7 @@ int RSA_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
   }
 
   if (!BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx) ||
-      !BN_mod_exp_mont(result, f, rsa->e, rsa->n, ctx, rsa->mont_n)) {
+      !BN_mod_exp_mont(result, f, rsa->e, &rsa->mont_n->N, ctx, rsa->mont_n)) {
     goto err;
   }
 
@@ -487,7 +614,7 @@ int RSA_verify_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
   }
 
   if (!BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx) ||
-      !BN_mod_exp_mont(result, f, rsa->e, rsa->n, ctx, rsa->mont_n)) {
+      !BN_mod_exp_mont(result, f, rsa->e, &rsa->mont_n->N, ctx, rsa->mont_n)) {
     goto err;
   }
 
@@ -560,7 +687,7 @@ int rsa_default_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
     goto err;
   }
 
-  if (!BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx)) {
+  if (!freeze_private_key(rsa, ctx)) {
     OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
     goto err;
   }
@@ -592,7 +719,7 @@ int rsa_default_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
     if (!mod_exp(result, f, rsa, ctx)) {
       goto err;
     }
-  } else if (!BN_mod_exp_mont_consttime(result, f, rsa->d, rsa->n, ctx,
+  } else if (!BN_mod_exp_mont_consttime(result, f, rsa->d_fixed, rsa->n, ctx,
                                         rsa->mont_n)) {
     goto err;
   }
@@ -622,6 +749,12 @@ int rsa_default_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
     goto err;
   }
 
+  // The computation should have left |result| as a maximally-wide number, so
+  // that it and serializing does not leak information about the magnitude of
+  // the result.
+  //
+  // See Falko Stenzke, "Manger's Attack revisited", ICICS 2010.
+  assert(result->width == rsa->mont_n->N.width);
   if (!BN_bn2bin_padded(out, len, result)) {
     OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
     goto err;
@@ -641,6 +774,37 @@ err:
   return ret;
 }
 
+// mod_montgomery sets |r| to |I| mod |p|. |I| must already be fully reduced
+// modulo |p| times |q|. It returns one on success and zero on error.
+static int mod_montgomery(BIGNUM *r, const BIGNUM *I, const BIGNUM *p,
+                          const BN_MONT_CTX *mont_p, const BIGNUM *q,
+                          BN_CTX *ctx) {
+  // Reducing in constant-time with Montgomery reduction requires I <= p * R. We
+  // have I < p * q, so this follows if q < R. In particular, this always holds
+  // if p and q are the same size, which is true for any RSA keys we or anyone
+  // sane generates. For other keys, we fall back to |BN_mod|.
+  if (!bn_less_than_montgomery_R(q, mont_p)) {
+    return BN_mod(r, I, p, ctx);
+  }
+
+  if (// Reduce mod p with Montgomery reduction. This computes I * R^-1 mod p.
+      !BN_from_montgomery(r, I, mont_p, ctx) ||
+      // Multiply by R^2 and do another Montgomery reduction to compute
+      // I * R^-1 * R^2 * R^-1 = I mod p.
+      !BN_to_montgomery(r, r, mont_p, ctx)) {
+    return 0;
+  }
+
+  // By precomputing R^3 mod p (normally |BN_MONT_CTX| only uses R^2 mod p) and
+  // adjusting the API for |BN_mod_exp_mont_consttime|, we could instead compute
+  // I * R mod p here and save a reduction per prime. But this would require
+  // changing the RSAZ code and may not be worth it. Note that the RSAZ code
+  // uses a different radix, so it uses R' = 2^1044. There we'd actually want
+  // R^2 * R', and would futher benefit from a precomputed R'^2. It currently
+  // converts |mont_p->RR| to R'^2.
+  return 1;
+}
+
 static int mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx) {
   assert(ctx != NULL);
 
@@ -653,82 +817,67 @@ static int mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx) {
   assert(rsa->dmq1 != NULL);
   assert(rsa->iqmp != NULL);
 
-  BIGNUM *r1, *m1, *vrfy;
+  BIGNUM *r1, *m1;
   int ret = 0;
 
   BN_CTX_start(ctx);
   r1 = BN_CTX_get(ctx);
   m1 = BN_CTX_get(ctx);
-  vrfy = BN_CTX_get(ctx);
   if (r1 == NULL ||
-      m1 == NULL ||
-      vrfy == NULL) {
-    goto err;
-  }
-
-  if (!BN_MONT_CTX_set_locked(&rsa->mont_p, &rsa->lock, rsa->p, ctx) ||
-      !BN_MONT_CTX_set_locked(&rsa->mont_q, &rsa->lock, rsa->q, ctx)) {
+      m1 == NULL) {
     goto err;
   }
 
-  if (!BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx)) {
+  if (!freeze_private_key(rsa, ctx)) {
     goto err;
   }
 
-  // compute I mod q
-  if (!BN_mod(r1, I, rsa->q, ctx)) {
-    goto err;
-  }
-
-  // compute r1^dmq1 mod q
-  if (!BN_mod_exp_mont_consttime(m1, r1, rsa->dmq1, rsa->q, ctx, rsa->mont_q)) {
-    goto err;
-  }
-
-  // compute I mod p
-  if (!BN_mod(r1, I, rsa->p, ctx)) {
-    goto err;
-  }
-
-  // compute r1^dmp1 mod p
-  if (!BN_mod_exp_mont_consttime(r0, r1, rsa->dmp1, rsa->p, ctx, rsa->mont_p)) {
-    goto err;
-  }
-
-  if (!BN_sub(r0, r0, m1)) {
-    goto err;
-  }
-  // This will help stop the size of r0 increasing, which does
-  // affect the multiply if it optimised for a power of 2 size
-  if (BN_is_negative(r0)) {
-    if (!BN_add(r0, r0, rsa->p)) {
-      goto err;
-    }
-  }
-
-  if (!BN_mul(r1, r0, rsa->iqmp, ctx)) {
-    goto err;
-  }
-
-  if (!BN_mod(r0, r1, rsa->p, ctx)) {
-    goto err;
-  }
-
-  // If p < q it is occasionally possible for the correction of
-  // adding 'p' if r0 is negative above to leave the result still
-  // negative. This can break the private key operations: the following
-  // second correction should *always* correct this rare occurrence.
-  // This will *never* happen with OpenSSL generated keys because
-  // they ensure p > q [steve]
-  if (BN_is_negative(r0)) {
-    if (!BN_add(r0, r0, rsa->p)) {
-      goto err;
-    }
-  }
-  if (!BN_mul(r1, r0, rsa->q, ctx)) {
-    goto err;
-  }
-  if (!BN_add(r0, r1, m1)) {
+  // Implementing RSA with CRT in constant-time is sensitive to which prime is
+  // larger. Canonicalize fields so that |p| is the larger prime.
+  const BIGNUM *dmp1 = rsa->dmp1_fixed, *dmq1 = rsa->dmq1_fixed;
+  const BN_MONT_CTX *mont_p = rsa->mont_p, *mont_q = rsa->mont_q;
+  if (BN_cmp(rsa->p, rsa->q) < 0) {
+    mont_p = rsa->mont_q;
+    mont_q = rsa->mont_p;
+    dmp1 = rsa->dmq1_fixed;
+    dmq1 = rsa->dmp1_fixed;
+  }
+
+  // Use the minimal-width versions of |n|, |p|, and |q|. Either works, but if
+  // someone gives us non-minimal values, these will be slightly more efficient
+  // on the non-Montgomery operations.
+  const BIGNUM *n = &rsa->mont_n->N;
+  const BIGNUM *p = &mont_p->N;
+  const BIGNUM *q = &mont_q->N;
+
+  // This is a pre-condition for |mod_montgomery|. It was already checked by the
+  // caller.
+  assert(BN_ucmp(I, n) < 0);
+
+  if (// |m1| is the result modulo |q|.
+      !mod_montgomery(r1, I, q, mont_q, p, ctx) ||
+      !BN_mod_exp_mont_consttime(m1, r1, dmq1, q, ctx, mont_q) ||
+      // |r0| is the result modulo |p|.
+      !mod_montgomery(r1, I, p, mont_p, q, ctx) ||
+      !BN_mod_exp_mont_consttime(r0, r1, dmp1, p, ctx, mont_p) ||
+      // Compute r0 = r0 - m1 mod p. |p| is the larger prime, so |m1| is already
+      // fully reduced mod |p|.
+      !bn_mod_sub_consttime(r0, r0, m1, p, ctx) ||
+      // r0 = r0 * iqmp mod p. We use Montgomery multiplication to compute this
+      // in constant time. |inv_small_mod_large_mont| is in Montgomery form and
+      // r0 is not, so the result is taken out of Montgomery form.
+      !BN_mod_mul_montgomery(r0, r0, rsa->inv_small_mod_large_mont, mont_p,
+                             ctx) ||
+      // r0 = r0 * q + m1 gives the final result. Reducing modulo q gives m1, so
+      // it is correct mod p. Reducing modulo p gives (r0-m1)*iqmp*q + m1 = r0,
+      // so it is correct mod q. Finally, the result is bounded by [m1, n + m1),
+      // and the result is at least |m1|, so this must be the unique answer in
+      // [0, n).
+      !bn_mul_consttime(r0, r0, q, ctx) ||
+      !bn_uadd_consttime(r0, r0, m1) ||
+      // The result should be bounded by |n|, but fixed-width operations may
+      // bound the width slightly higher, so fix it.
+      !bn_resize_words(r0, n->width)) {
     goto err;
   }
 
@@ -775,24 +924,20 @@ const BN_ULONG kBoringSSLRSASqrtTwo[] = {
 };
 const size_t kBoringSSLRSASqrtTwoLen = OPENSSL_ARRAY_SIZE(kBoringSSLRSASqrtTwo);
 
-int rsa_greater_than_pow2(const BIGNUM *b, int n) {
-  if (BN_is_negative(b) || n == INT_MAX) {
-    return 0;
-  }
-
-  int b_bits = BN_num_bits(b);
-  return b_bits > n + 1 || (b_bits == n + 1 && !BN_is_pow2(b));
-}
-
 // generate_prime sets |out| to a prime with length |bits| such that |out|-1 is
 // relatively prime to |e|. If |p| is non-NULL, |out| will also not be close to
-// |p|.
+// |p|. |sqrt2| must be ⌊2^(bits-1)×√2⌋ (or a slightly overestimate for large
+// sizes), and |pow2_bits_100| must be 2^(bits-100).
 static int generate_prime(BIGNUM *out, int bits, const BIGNUM *e,
-                          const BIGNUM *p, BN_CTX *ctx, BN_GENCB *cb) {
+                          const BIGNUM *p, const BIGNUM *sqrt2,
+                          const BIGNUM *pow2_bits_100, BN_CTX *ctx,
+                          BN_GENCB *cb) {
   if (bits < 128 || (bits % BN_BITS2) != 0) {
     OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
     return 0;
   }
+  assert(BN_is_pow2(pow2_bits_100));
+  assert(BN_is_bit_set(pow2_bits_100, bits - 100));
 
   // See FIPS 186-4 appendix B.3.3, steps 4 and 5. Note |bits| here is nlen/2.
 
@@ -823,57 +968,45 @@ static int generate_prime(BIGNUM *out, int bits, const BIGNUM *e,
 
     if (p != NULL) {
       // If |p| and |out| are too close, try again (step 5.4).
-      if (!BN_sub(tmp, out, p)) {
+      if (!bn_abs_sub_consttime(tmp, out, p, ctx)) {
         goto err;
       }
-      BN_set_negative(tmp, 0);
-      if (!rsa_greater_than_pow2(tmp, bits - 100)) {
+      if (BN_cmp(tmp, pow2_bits_100) <= 0) {
         continue;
       }
     }
 
-    // If out < 2^(bits-1)×√2, try again (steps 4.4 and 5.5).
-    //
-    // We check the most significant words, so we retry if ⌊out/2^k⌋ <= ⌊b/2^k⌋,
-    // where b = 2^(bits-1)×√2 and k = max(0, bits - 1536). For key sizes up to
-    // 3072 (bits = 1536), k = 0, so we are testing that ⌊out⌋ <= ⌊b⌋. out is an
-    // integer and b is not, so this is equivalent to out < b. That is, the
-    // comparison is exact for FIPS key sizes.
+    // If out < 2^(bits-1)×√2, try again (steps 4.4 and 5.5). This is equivalent
+    // to out <= ⌊2^(bits-1)×√2⌋, or out <= sqrt2 for FIPS key sizes.
     //
     // For larger keys, the comparison is approximate, leaning towards
     // retrying. That is, we reject a negligible fraction of primes that are
     // within the FIPS bound, but we will never accept a prime outside the
-    // bound, ensuring the resulting RSA key is the right size. Specifically, if
-    // the FIPS bound holds, we have ⌊out/2^k⌋ < out/2^k < b/2^k. This implies
-    // ⌊out/2^k⌋ <= ⌊b/2^k⌋. That is, the FIPS bound implies our bound and so we
-    // are slightly tighter.
-    size_t out_len = (size_t)out->top;
-    assert(out_len == (size_t)bits / BN_BITS2);
-    size_t to_check = kBoringSSLRSASqrtTwoLen;
-    if (to_check > out_len) {
-      to_check = out_len;
-    }
-    if (!bn_less_than_words(
-            kBoringSSLRSASqrtTwo + kBoringSSLRSASqrtTwoLen - to_check,
-            out->d + out_len - to_check, to_check)) {
+    // bound, ensuring the resulting RSA key is the right size.
+    if (BN_cmp(out, sqrt2) <= 0) {
       continue;
     }
 
-    // Check gcd(out-1, e) is one (steps 4.5 and 5.6).
-    if (!BN_sub(tmp, out, BN_value_one()) ||
-        !BN_gcd(tmp, tmp, e, ctx)) {
-      goto err;
-    }
-    if (BN_is_one(tmp)) {
-      // Test |out| for primality (steps 4.5.1 and 5.6.1).
-      int is_probable_prime;
-      if (!BN_primality_test(&is_probable_prime, out, BN_prime_checks, ctx, 1,
-                             cb)) {
+    // RSA key generation's bottleneck is discarding composites. If it fails
+    // trial division, do not bother computing a GCD or performing Rabin-Miller.
+    if (!bn_odd_number_is_obviously_composite(out)) {
+      // Check gcd(out-1, e) is one (steps 4.5 and 5.6).
+      int relatively_prime;
+      if (!BN_sub(tmp, out, BN_value_one()) ||
+          !bn_is_relatively_prime(&relatively_prime, tmp, e, ctx)) {
         goto err;
       }
-      if (is_probable_prime) {
-        ret = 1;
-        goto err;
+      if (relatively_prime) {
+        // Test |out| for primality (steps 4.5.1 and 5.6.1).
+        int is_probable_prime;
+        if (!BN_primality_test(&is_probable_prime, out, BN_prime_checks, ctx, 0,
+                               cb)) {
+          goto err;
+        }
+        if (is_probable_prime) {
+          ret = 1;
+          goto err;
+        }
       }
     }
 
@@ -909,7 +1042,19 @@ int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb) {
     return 0;
   }
 
+  // Reject excessively large public exponents. Windows CryptoAPI and Go don't
+  // support values larger than 32 bits, so match their limits for generating
+  // keys. (|check_modulus_and_exponent_sizes| uses a slightly more conservative
+  // value, but we don't need to support generating such keys.)
+  // https://github.com/golang/go/issues/3161
+  // https://msdn.microsoft.com/en-us/library/aa387685(VS.85).aspx
+  if (BN_num_bits(e_value) > 32) {
+    OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_E_VALUE);
+    return 0;
+  }
+
   int ret = 0;
+  int prime_bits = bits / 2;
   BN_CTX *ctx = BN_CTX_new();
   if (ctx == NULL) {
     goto bn_err;
@@ -918,8 +1063,13 @@ int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb) {
   BIGNUM *totient = BN_CTX_get(ctx);
   BIGNUM *pm1 = BN_CTX_get(ctx);
   BIGNUM *qm1 = BN_CTX_get(ctx);
-  BIGNUM *gcd = BN_CTX_get(ctx);
-  if (totient == NULL || pm1 == NULL || qm1 == NULL || gcd == NULL) {
+  BIGNUM *sqrt2 = BN_CTX_get(ctx);
+  BIGNUM *pow2_prime_bits_100 = BN_CTX_get(ctx);
+  BIGNUM *pow2_prime_bits = BN_CTX_get(ctx);
+  if (totient == NULL || pm1 == NULL || qm1 == NULL || sqrt2 == NULL ||
+      pow2_prime_bits_100 == NULL || pow2_prime_bits == NULL ||
+      !BN_set_bit(pow2_prime_bits_100, prime_bits - 100) ||
+      !BN_set_bit(pow2_prime_bits, prime_bits)) {
     goto bn_err;
   }
 
@@ -930,8 +1080,7 @@ int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb) {
       !ensure_bignum(&rsa->p) ||
       !ensure_bignum(&rsa->q) ||
       !ensure_bignum(&rsa->dmp1) ||
-      !ensure_bignum(&rsa->dmq1) ||
-      !ensure_bignum(&rsa->iqmp)) {
+      !ensure_bignum(&rsa->dmq1)) {
     goto bn_err;
   }
 
@@ -939,13 +1088,36 @@ int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb) {
     goto bn_err;
   }
 
-  int prime_bits = bits / 2;
+  // Compute sqrt2 >= ⌊2^(prime_bits-1)×√2⌋.
+  if (!bn_set_words(sqrt2, kBoringSSLRSASqrtTwo, kBoringSSLRSASqrtTwoLen)) {
+    goto bn_err;
+  }
+  int sqrt2_bits = kBoringSSLRSASqrtTwoLen * BN_BITS2;
+  assert(sqrt2_bits == (int)BN_num_bits(sqrt2));
+  if (sqrt2_bits > prime_bits) {
+    // For key sizes up to 3072 (prime_bits = 1536), this is exactly
+    // ⌊2^(prime_bits-1)×√2⌋.
+    if (!BN_rshift(sqrt2, sqrt2, sqrt2_bits - prime_bits)) {
+      goto bn_err;
+    }
+  } else if (prime_bits > sqrt2_bits) {
+    // For key sizes beyond 3072, this is approximate. We err towards retrying
+    // to ensure our key is the right size and round up.
+    if (!BN_add_word(sqrt2, 1) ||
+        !BN_lshift(sqrt2, sqrt2, prime_bits - sqrt2_bits)) {
+      goto bn_err;
+    }
+  }
+  assert(prime_bits == (int)BN_num_bits(sqrt2));
+
   do {
     // Generate p and q, each of size |prime_bits|, using the steps outlined in
     // appendix FIPS 186-4 appendix B.3.3.
-    if (!generate_prime(rsa->p, prime_bits, rsa->e, NULL, ctx, cb) ||
+    if (!generate_prime(rsa->p, prime_bits, rsa->e, NULL, sqrt2,
+                        pow2_prime_bits_100, ctx, cb) ||
         !BN_GENCB_call(cb, 3, 0) ||
-        !generate_prime(rsa->q, prime_bits, rsa->e, rsa->p, ctx, cb) ||
+        !generate_prime(rsa->q, prime_bits, rsa->e, rsa->p, sqrt2,
+                        pow2_prime_bits_100, ctx, cb) ||
         !BN_GENCB_call(cb, 3, 1)) {
       goto bn_err;
     }
@@ -963,27 +1135,27 @@ int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb) {
     // q-1. However, we do operations with Chinese Remainder Theorem, so we only
     // use d (mod p-1) and d (mod q-1) as exponents. Using a minimal totient
     // does not affect those two values.
-    if (!BN_sub(pm1, rsa->p, BN_value_one()) ||
-        !BN_sub(qm1, rsa->q, BN_value_one()) ||
-        !BN_mul(totient, pm1, qm1, ctx) ||
-        !BN_gcd(gcd, pm1, qm1, ctx) ||
-        !BN_div(totient, NULL, totient, gcd, ctx) ||
-        !BN_mod_inverse(rsa->d, rsa->e, totient, ctx)) {
+    int no_inverse;
+    if (!bn_usub_consttime(pm1, rsa->p, BN_value_one()) ||
+        !bn_usub_consttime(qm1, rsa->q, BN_value_one()) ||
+        !bn_lcm_consttime(totient, pm1, qm1, ctx) ||
+        !bn_mod_inverse_consttime(rsa->d, &no_inverse, rsa->e, totient, ctx)) {
       goto bn_err;
     }
 
-    // Check that |rsa->d| > 2^|prime_bits| and try again if it fails. See
-    // appendix B.3.1's guidance on values for d.
-  } while (!rsa_greater_than_pow2(rsa->d, prime_bits));
+    // Retry if |rsa->d| <= 2^|prime_bits|. See appendix B.3.1's guidance on
+    // values for d.
+  } while (BN_cmp(rsa->d, pow2_prime_bits) <= 0);
 
   if (// Calculate n.
-      !BN_mul(rsa->n, rsa->p, rsa->q, ctx) ||
+      !bn_mul_consttime(rsa->n, rsa->p, rsa->q, ctx) ||
       // Calculate d mod (p-1).
-      !BN_mod(rsa->dmp1, rsa->d, pm1, ctx) ||
+      !bn_div_consttime(NULL, rsa->dmp1, rsa->d, pm1, ctx) ||
       // Calculate d mod (q-1)
-      !BN_mod(rsa->dmq1, rsa->d, qm1, ctx)) {
+      !bn_div_consttime(NULL, rsa->dmq1, rsa->d, qm1, ctx)) {
     goto bn_err;
   }
+  bn_set_minimal_width(rsa->n);
 
   // Sanity-check that |rsa->n| has the specified size. This is implied by
   // |generate_prime|'s bounds.
@@ -992,13 +1164,9 @@ int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb) {
     goto err;
   }
 
-  // Calculate inverse of q mod p. Note that although RSA key generation is far
-  // from constant-time, |bn_mod_inverse_secret_prime| uses the same modular
-  // exponentation logic as in RSA private key operations and, if the RSAZ-1024
-  // code is enabled, will be optimized for common RSA prime sizes.
-  if (!BN_MONT_CTX_set_locked(&rsa->mont_p, &rsa->lock, rsa->p, ctx) ||
-      !bn_mod_inverse_secret_prime(rsa->iqmp, rsa->q, rsa->p, ctx,
-                                   rsa->mont_p)) {
+  // Call |freeze_private_key| to compute the inverse of q mod p, by way of
+  // |rsa->mont_p|.
+  if (!freeze_private_key(rsa, ctx)) {
     goto bn_err;
   }
 
@@ -1047,5 +1215,4 @@ DEFINE_METHOD_FUNCTION(RSA_METHOD, RSA_default_method) {
   // |rsa_default_*| implementation.
   OPENSSL_memset(out, 0, sizeof(RSA_METHOD));
   out->common.is_static = 1;
-  out->flags = RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE;
 }