grpc 1.4.5 → 1.6.0.pre1

data/third_party/boringssl/crypto/rand/urandom.c

@@ -12,16 +12,26 @@
  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
 
+#if !defined(_GNU_SOURCE)
+#define _GNU_SOURCE  /* needed for syscall() on Linux. */
+#endif
+
 #include <openssl/rand.h>
 
-#if !defined(OPENSSL_WINDOWS) && !defined(BORINGSSL_UNSAFE_FUZZER_MODE)
+#if !defined(OPENSSL_WINDOWS) && !defined(OPENSSL_FUCHSIA) && \
+    !defined(BORINGSSL_UNSAFE_DETERMINISTIC_MODE)
 
 #include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 
+#if defined(OPENSSL_LINUX)
+#include <sys/syscall.h>
+#endif
+
 #include <openssl/thread.h>
 #include <openssl/mem.h>
 
@@ -29,6 +39,43 @@
 #include "../internal.h"
 
 
+#if defined(OPENSSL_LINUX)
+
+#if defined(OPENSSL_X86_64)
+#define EXPECTED_SYS_getrandom 318
+#elif defined(OPENSSL_X86)
+#define EXPECTED_SYS_getrandom 355
+#elif defined(OPENSSL_AARCH64)
+#define EXPECTED_SYS_getrandom 278
+#elif defined(OPENSSL_ARM)
+#define EXPECTED_SYS_getrandom 384
+#elif defined(OPENSSL_PPC64LE)
+#define EXPECTED_SYS_getrandom 359
+#endif
+
+#if defined(EXPECTED_SYS_getrandom)
+#define USE_SYS_getrandom
+
+#if defined(SYS_getrandom)
+
+#if SYS_getrandom != EXPECTED_SYS_getrandom
+#error "system call number for getrandom is not the expected value"
+#endif
+
+#else  /* SYS_getrandom */
+
+#define SYS_getrandom EXPECTED_SYS_getrandom
+
+#endif  /* SYS_getrandom */
+
+#endif /* EXPECTED_SYS_getrandom */
+
+#if !defined(GRND_NONBLOCK)
+#define GRND_NONBLOCK 1
+#endif
+
+#endif  /* OPENSSL_LINUX */
+
 /* This file implements a PRNG by reading from /dev/urandom, optionally with a
  * buffer, which is unsafe across |fork|. */
 
@@ -44,12 +91,16 @@ struct rand_buffer {
 /* requested_lock is used to protect the |*_requested| variables. */
 static struct CRYPTO_STATIC_MUTEX requested_lock = CRYPTO_STATIC_MUTEX_INIT;
 
-/* urandom_fd_requested is set by |RAND_set_urandom_fd|.  It's protected by
+/* The following constants are magic values of |urandom_fd|. */
+static const int kUnset = -2;
+static const int kHaveGetrandom = -3;
+
+/* urandom_fd_requested is set by |RAND_set_urandom_fd|. It's protected by
  * |requested_lock|. */
-static int urandom_fd_requested = -2;
+static int urandom_fd_requested = -2 /* kUnset */;
 
 /* urandom_fd is a file descriptor to /dev/urandom. It's protected by |once|. */
-static int urandom_fd = -2;
+static int urandom_fd = -2 /* kUnset */;
 
 /* urandom_buffering_requested is set by |RAND_enable_fork_unsafe_buffering|.
  * It's protected by |requested_lock|. */
@@ -71,7 +122,32 @@ static void init_once(void) {
   int fd = urandom_fd_requested;
   CRYPTO_STATIC_MUTEX_unlock_read(&requested_lock);
 
-  if (fd == -2) {
+#if defined(USE_SYS_getrandom)
+  uint8_t dummy;
+  long getrandom_ret =
+      syscall(SYS_getrandom, &dummy, sizeof(dummy), GRND_NONBLOCK);
+
+  if (getrandom_ret == 1) {
+    urandom_fd = kHaveGetrandom;
+    return;
+  } else if (getrandom_ret == -1 && errno == EAGAIN) {
+    fprintf(stderr,
+            "getrandom indicates that the entropy pool has not been "
+            "initialized. Rather than continue with poor entropy, this process "
+            "will block until entropy is available.\n");
+    do {
+      getrandom_ret =
+          syscall(SYS_getrandom, &dummy, sizeof(dummy), 0 /* no flags */);
+    } while (getrandom_ret == -1 && errno == EINTR);
+
+    if (getrandom_ret == 1) {
+      urandom_fd = kHaveGetrandom;
+      return;
+    }
+  }
+#endif  /* USE_SYS_getrandom */
+
+  if (fd == kUnset) {
     do {
       fd = open("/dev/urandom", O_RDONLY);
     } while (fd == -1 && errno == EINTR);
@@ -107,7 +183,9 @@ void RAND_set_urandom_fd(int fd) {
   CRYPTO_STATIC_MUTEX_unlock_write(&requested_lock);
 
   CRYPTO_once(&once, init_once);
-  if (urandom_fd != fd) {
+  if (urandom_fd == kHaveGetrandom) {
+    close(fd);
+  } else if (urandom_fd != fd) {
     abort();  // Already initialized.
   }
 }
@@ -119,7 +197,7 @@ void RAND_enable_fork_unsafe_buffering(int fd) {
       abort();
     }
   } else {
-    fd = -2;
+    fd = kUnset;
   }
 
   CRYPTO_STATIC_MUTEX_lock_write(&requested_lock);
@@ -128,8 +206,16 @@ void RAND_enable_fork_unsafe_buffering(int fd) {
   CRYPTO_STATIC_MUTEX_unlock_write(&requested_lock);
 
   CRYPTO_once(&once, init_once);
-  if (urandom_buffering != 1 || (fd >= 0 && urandom_fd != fd)) {
-    abort();  // Already initialized.
+  if (urandom_buffering != 1) {
+    abort();  // Already initialized
+  }
+
+  if (fd >= 0) {
+    if (urandom_fd == kHaveGetrandom) {
+      close(fd);
+    } else if (urandom_fd != fd) {
+      abort();  // Already initialized.
+    }
   }
 }
 
@@ -144,7 +230,7 @@ static struct rand_buffer *get_thread_local_buffer(void) {
   if (buf == NULL) {
     return NULL;
   }
-  buf->used = BUF_SIZE;  /* To trigger a |read_full| on first use. */
+  buf->used = BUF_SIZE;  /* To trigger a |fill_with_entropy| on first use. */
   if (!CRYPTO_set_thread_local(OPENSSL_THREAD_LOCAL_URANDOM_BUF, buf,
                                OPENSSL_free)) {
     OPENSSL_free(buf);
@@ -154,15 +240,42 @@ static struct rand_buffer *get_thread_local_buffer(void) {
   return buf;
 }
 
-/* read_full reads exactly |len| bytes from |fd| into |out| and returns 1. In
- * the case of an error it returns 0. */
-static char read_full(int fd, uint8_t *out, size_t len) {
-  ssize_t r;
+#if defined(USE_SYS_getrandom) && defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+void __msan_unpoison(void *, size_t);
+#endif
+#endif
 
+/* fill_with_entropy writes |len| bytes of entropy into |out|. It returns one
+ * on success and zero on error. */
+static char fill_with_entropy(uint8_t *out, size_t len) {
   while (len > 0) {
-    do {
-      r = read(fd, out, len);
-    } while (r == -1 && errno == EINTR);
+    ssize_t r;
+
+    if (urandom_fd == kHaveGetrandom) {
+#if defined(USE_SYS_getrandom)
+      do {
+        r = syscall(SYS_getrandom, out, len, 0 /* no flags */);
+      } while (r == -1 && errno == EINTR);
+
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+      if (r > 0) {
+        /* MSAN doesn't recognise |syscall| and thus doesn't notice that we
+         * have initialised the output buffer. */
+        __msan_unpoison(out, r);
+      }
+#endif /* memory_sanitizer */
+#endif /*__has_feature */
+
+#else /* USE_SYS_getrandom */
+      abort();
+#endif
+    } else {
+      do {
+        r = read(urandom_fd, out, len);
+      } while (r == -1 && errno == EINTR);
+    }
 
     if (r <= 0) {
       return 0;
@@ -181,12 +294,12 @@ static void read_from_buffer(struct rand_buffer *buf,
   size_t remaining = BUF_SIZE - buf->used;
 
   while (requested > remaining) {
-    memcpy(out, &buf->rand[buf->used], remaining);
+    OPENSSL_memcpy(out, &buf->rand[buf->used], remaining);
     buf->used += remaining;
     out += remaining;
     requested -= remaining;
 
-    if (!read_full(urandom_fd, buf->rand, BUF_SIZE)) {
+    if (!fill_with_entropy(buf->rand, BUF_SIZE)) {
       abort();
       return;
     }
@@ -194,7 +307,7 @@ static void read_from_buffer(struct rand_buffer *buf,
     remaining = BUF_SIZE;
   }
 
-  memcpy(out, &buf->rand[buf->used], requested);
+  OPENSSL_memcpy(out, &buf->rand[buf->used], requested);
   buf->used += requested;
 }
 
@@ -213,9 +326,10 @@ void CRYPTO_sysrand(uint8_t *out, size_t requested) {
     }
   }
 
-  if (!read_full(urandom_fd, out, requested)) {
+  if (!fill_with_entropy(out, requested)) {
     abort();
   }
 }
 
-#endif  /* !OPENSSL_WINDOWS && !BORINGSSL_UNSAFE_FUZZER_MODE */
+#endif /* !OPENSSL_WINDOWS && !defined(OPENSSL_FUCHSIA) && \
+          !BORINGSSL_UNSAFE_DETERMINISTIC_MODE */

data/third_party/boringssl/crypto/rand/windows.c

@@ -14,7 +14,7 @@
 
 #include <openssl/rand.h>
 
-#if defined(OPENSSL_WINDOWS) && !defined(BORINGSSL_UNSAFE_FUZZER_MODE)
+#if defined(OPENSSL_WINDOWS) && !defined(BORINGSSL_UNSAFE_DETERMINISTIC_MODE)
 
 #include <limits.h>
 #include <stdlib.h>
@@ -50,4 +50,4 @@ void CRYPTO_sysrand(uint8_t *out, size_t requested) {
   return;
 }
 
-#endif  /* OPENSSL_WINDOWS && !BORINGSSL_UNSAFE_FUZZER_MODE */
+#endif  /* OPENSSL_WINDOWS && !BORINGSSL_UNSAFE_DETERMINISTIC_MODE */

data/third_party/boringssl/crypto/rsa/blinding.c

@@ -115,6 +115,7 @@
 #include <openssl/err.h>
 
 #include "internal.h"
+#include "../internal.h"
 
 
 #define BN_BLINDING_COUNTER 32
@@ -134,7 +135,7 @@ BN_BLINDING *BN_BLINDING_new(void) {
     OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
     return NULL;
   }
-  memset(ret, 0, sizeof(BN_BLINDING));
+  OPENSSL_memset(ret, 0, sizeof(BN_BLINDING));
 
   ret->A = BN_new();
   if (ret->A == NULL) {

data/third_party/boringssl/crypto/rsa/padding.c

@@ -92,10 +92,10 @@ int RSA_padding_add_PKCS1_type_1(uint8_t *to, unsigned to_len,
 
   /* pad out with 0xff data */
   j = to_len - 3 - from_len;
-  memset(p, 0xff, j);
+  OPENSSL_memset(p, 0xff, j);
   p += j;
   *(p++) = 0;
-  memcpy(p, from, from_len);
+  OPENSSL_memcpy(p, from, from_len);
   return 1;
 }
 
@@ -146,7 +146,7 @@ int RSA_padding_check_PKCS1_type_1(uint8_t *to, unsigned to_len,
     OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
     return -1;
   }
-  memcpy(to, p, j);
+  OPENSSL_memcpy(to, p, j);
 
   return j;
 }
@@ -188,7 +188,7 @@ int RSA_padding_add_PKCS1_type_2(uint8_t *to, unsigned to_len,
 
   *(p++) = 0;
 
-  memcpy(p, from, from_len);
+  OPENSSL_memcpy(p, from, from_len);
   return 1;
 }
 
@@ -254,7 +254,7 @@ int RSA_padding_check_PKCS1_type_2(uint8_t *to, unsigned to_len,
     return -1;
   }
 
-  memcpy(to, &from[zero_index], msg_len);
+  OPENSSL_memcpy(to, &from[zero_index], msg_len);
   return (int)msg_len;
 }
 
@@ -270,7 +270,7 @@ int RSA_padding_add_none(uint8_t *to, unsigned to_len, const uint8_t *from,
     return 0;
   }
 
-  memcpy(to, from, from_len);
+  OPENSSL_memcpy(to, from, from_len);
   return 1;
 }
 
@@ -305,7 +305,7 @@ static int PKCS1_MGF1(uint8_t *out, size_t len, const uint8_t *seed,
       if (!EVP_DigestFinal_ex(&ctx, digest, NULL)) {
         goto err;
       }
-      memcpy(out, digest, len);
+      OPENSSL_memcpy(out, digest, len);
       len = 0;
     }
   }
@@ -358,9 +358,9 @@ int RSA_padding_add_PKCS1_OAEP_mgf1(uint8_t *to, unsigned to_len,
   if (!EVP_Digest(param, param_len, db, NULL, md, NULL)) {
     return 0;
   }
-  memset(db + mdlen, 0, emlen - from_len - 2 * mdlen - 1);
+  OPENSSL_memset(db + mdlen, 0, emlen - from_len - 2 * mdlen - 1);
   db[emlen - from_len - mdlen - 1] = 0x01;
-  memcpy(db + emlen - from_len - mdlen, from, from_len);
+  OPENSSL_memcpy(db + emlen - from_len - mdlen, from, from_len);
   if (!RAND_bytes(seed, mdlen)) {
     return 0;
   }
@@ -471,7 +471,7 @@ int RSA_padding_check_PKCS1_OAEP_mgf1(uint8_t *to, unsigned to_len,
     OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
     mlen = -1;
   } else {
-    memcpy(to, db + one_index, mlen);
+    OPENSSL_memcpy(to, db + one_index, mlen);
   }
 
   OPENSSL_free(db);
@@ -579,7 +579,7 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const uint8_t *mHash,
   if (!EVP_DigestFinal_ex(&ctx, H_, NULL)) {
     goto err;
   }
-  if (memcmp(H_, H, hLen)) {
+  if (OPENSSL_memcmp(H_, H, hLen)) {
     OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_SIGNATURE);
     ret = 0;
   } else {

data/third_party/boringssl/crypto/rsa/rsa.c

@@ -82,7 +82,7 @@ RSA *RSA_new_method(const ENGINE *engine) {
     return NULL;
   }
 
-  memset(rsa, 0, sizeof(RSA));
+  OPENSSL_memset(rsa, 0, sizeof(RSA));
 
   if (engine) {
     rsa->meth = ENGINE_get_RSA_method(engine);
@@ -446,8 +446,8 @@ int RSA_add_pkcs1_prefix(uint8_t **out_msg, size_t *out_msg_len,
       return 0;
     }
 
-    memcpy(signed_msg, prefix, prefix_len);
-    memcpy(signed_msg + prefix_len, msg, msg_len);
+    OPENSSL_memcpy(signed_msg, prefix, prefix_len);
+    OPENSSL_memcpy(signed_msg + prefix_len, msg, msg_len);
 
     *out_msg = signed_msg;
     *out_msg_len = signed_msg_len;
@@ -532,7 +532,7 @@ int RSA_verify(int hash_nid, const uint8_t *msg, size_t msg_len,
     goto out;
   }
 
-  if (len != signed_msg_len || memcmp(buf, signed_msg, len) != 0) {
+  if (len != signed_msg_len || OPENSSL_memcmp(buf, signed_msg, len) != 0) {
     OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_SIGNATURE);
     goto out;
   }

data/third_party/boringssl/crypto/rsa/rsa_asn1.c

@@ -66,6 +66,7 @@
 
 #include "internal.h"
 #include "../bytestring/internal.h"
+#include "../internal.h"
 
 
 static int parse_integer_buggy(CBS *cbs, BIGNUM **out, int buggy) {
@@ -183,7 +184,7 @@ static RSA_additional_prime *rsa_parse_additional_prime(CBS *cbs) {
     OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
     return 0;
   }
-  memset(ret, 0, sizeof(RSA_additional_prime));
+  OPENSSL_memset(ret, 0, sizeof(RSA_additional_prime));
 
   CBS child;
   if (!CBS_get_asn1(cbs, &child, CBS_ASN1_SEQUENCE) ||
@@ -280,6 +281,11 @@ RSA *RSA_parse_private_key(CBS *cbs) {
     goto err;
   }
 
+  if (!RSA_check_key(ret)) {
+    OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_RSA_PARAMETERS);
+    goto err;
+  }
+
   BN_CTX_free(ctx);
   BN_free(product_of_primes_so_far);
   return ret;

data/third_party/boringssl/crypto/rsa/rsa_impl.c

@@ -65,6 +65,7 @@
 #include <openssl/thread.h>
 
 #include "internal.h"
+#include "../bn/internal.h"
 #include "../internal.h"
 
 
@@ -263,7 +264,7 @@ static BN_BLINDING *rsa_blinding_get(RSA *rsa, unsigned *index_used,
   if (new_blindings == NULL) {
     goto err1;
   }
-  memcpy(new_blindings, rsa->blindings,
+  OPENSSL_memcpy(new_blindings, rsa->blindings,
          sizeof(BN_BLINDING *) * rsa->num_blindings);
   new_blindings[rsa->num_blindings] = ret;
 
@@ -271,7 +272,7 @@ static BN_BLINDING *rsa_blinding_get(RSA *rsa, unsigned *index_used,
   if (new_blindings_inuse == NULL) {
     goto err2;
   }
-  memcpy(new_blindings_inuse, rsa->blindings_inuse, rsa->num_blindings);
+  OPENSSL_memcpy(new_blindings_inuse, rsa->blindings_inuse, rsa->num_blindings);
   new_blindings_inuse[rsa->num_blindings] = 1;
   *index_used = rsa->num_blindings;
 
@@ -590,17 +591,9 @@ int rsa_default_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
     if (!mod_exp(result, f, rsa, ctx)) {
       goto err;
     }
-  } else {
-    BIGNUM local_d;
-    BIGNUM *d = NULL;
-
-    BN_init(&local_d);
-    d = &local_d;
-    BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
-
-    if (!BN_mod_exp_mont_consttime(result, f, d, rsa->n, ctx, rsa->mont_n)) {
-      goto err;
-    }
+  } else if (!BN_mod_exp_mont_consttime(result, f, rsa->d, rsa->n, ctx,
+                                        rsa->mont_n)) {
+    goto err;
   }
 
   /* Verify the result to protect against fault attacks as described in the
@@ -658,8 +651,6 @@ static int mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx) {
   assert(rsa->iqmp != NULL);
 
   BIGNUM *r1, *m1, *vrfy;
-  BIGNUM local_dmp1, local_dmq1, local_c, local_r1;
-  BIGNUM *dmp1, *dmq1, *c, *pr1;
   int ret = 0;
   size_t i, num_additional_primes = 0;
 
@@ -677,23 +668,9 @@ static int mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx) {
     goto err;
   }
 
-  {
-    BIGNUM local_p, local_q;
-    BIGNUM *p = NULL, *q = NULL;
-
-    /* Make sure BN_mod in Montgomery initialization uses BN_FLG_CONSTTIME. */
-    BN_init(&local_p);
-    p = &local_p;
-    BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME);
-
-    BN_init(&local_q);
-    q = &local_q;
-    BN_with_flags(q, rsa->q, BN_FLG_CONSTTIME);
-
-    if (!BN_MONT_CTX_set_locked(&rsa->mont_p, &rsa->lock, p, ctx) ||
-        !BN_MONT_CTX_set_locked(&rsa->mont_q, &rsa->lock, q, ctx)) {
-      goto err;
-    }
+  if (!BN_MONT_CTX_set_locked(&rsa->mont_p, &rsa->lock, rsa->p, ctx) ||
+      !BN_MONT_CTX_set_locked(&rsa->mont_q, &rsa->lock, rsa->q, ctx)) {
+    goto err;
   }
 
   if (!BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx)) {
@@ -701,30 +678,22 @@ static int mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx) {
   }
 
   /* compute I mod q */
-  c = &local_c;
-  BN_with_flags(c, I, BN_FLG_CONSTTIME);
-  if (!BN_mod(r1, c, rsa->q, ctx)) {
+  if (!BN_mod(r1, I, rsa->q, ctx)) {
     goto err;
   }
 
   /* compute r1^dmq1 mod q */
-  dmq1 = &local_dmq1;
-  BN_with_flags(dmq1, rsa->dmq1, BN_FLG_CONSTTIME);
-  if (!BN_mod_exp_mont_consttime(m1, r1, dmq1, rsa->q, ctx, rsa->mont_q)) {
+  if (!BN_mod_exp_mont_consttime(m1, r1, rsa->dmq1, rsa->q, ctx, rsa->mont_q)) {
     goto err;
   }
 
   /* compute I mod p */
-  c = &local_c;
-  BN_with_flags(c, I, BN_FLG_CONSTTIME);
-  if (!BN_mod(r1, c, rsa->p, ctx)) {
+  if (!BN_mod(r1, I, rsa->p, ctx)) {
     goto err;
   }
 
   /* compute r1^dmp1 mod p */
-  dmp1 = &local_dmp1;
-  BN_with_flags(dmp1, rsa->dmp1, BN_FLG_CONSTTIME);
-  if (!BN_mod_exp_mont_consttime(r0, r1, dmp1, rsa->p, ctx, rsa->mont_p)) {
+  if (!BN_mod_exp_mont_consttime(r0, r1, rsa->dmp1, rsa->p, ctx, rsa->mont_p)) {
     goto err;
   }
 
@@ -743,11 +712,7 @@ static int mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx) {
     goto err;
   }
 
-  /* Turn BN_FLG_CONSTTIME flag on before division operation */
-  pr1 = &local_r1;
-  BN_with_flags(pr1, r1, BN_FLG_CONSTTIME);
-
-  if (!BN_mod(r0, pr1, rsa->p, ctx)) {
+  if (!BN_mod(r0, r1, rsa->p, ctx)) {
     goto err;
   }
 
@@ -771,30 +736,23 @@ static int mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx) {
 
   for (i = 0; i < num_additional_primes; i++) {
     /* multi-prime RSA. */
-    BIGNUM local_exp, local_prime;
-    BIGNUM *exp = &local_exp, *prime = &local_prime;
     RSA_additional_prime *ap =
         sk_RSA_additional_prime_value(rsa->additional_primes, i);
 
-    BN_with_flags(exp, ap->exp, BN_FLG_CONSTTIME);
-    BN_with_flags(prime, ap->prime, BN_FLG_CONSTTIME);
-
     /* c will already point to a BIGNUM with the correct flags. */
-    if (!BN_mod(r1, c, prime, ctx)) {
+    if (!BN_mod(r1, I, ap->prime, ctx)) {
       goto err;
     }
 
-    if (!BN_MONT_CTX_set_locked(&ap->mont, &rsa->lock, prime, ctx) ||
-        !BN_mod_exp_mont_consttime(m1, r1, exp, prime, ctx, ap->mont)) {
+    if (!BN_MONT_CTX_set_locked(&ap->mont, &rsa->lock, ap->prime, ctx) ||
+        !BN_mod_exp_mont_consttime(m1, r1, ap->exp, ap->prime, ctx, ap->mont)) {
       goto err;
     }
 
-    BN_set_flags(m1, BN_FLG_CONSTTIME);
-
     if (!BN_sub(m1, m1, r0) ||
         !BN_mul(m1, m1, ap->coeff, ctx) ||
-        !BN_mod(m1, m1, prime, ctx) ||
-        (BN_is_negative(m1) && !BN_add(m1, m1, prime)) ||
+        !BN_mod(m1, m1, ap->prime, ctx) ||
+        (BN_is_negative(m1) && !BN_add(m1, m1, ap->prime)) ||
         !BN_mul(m1, m1, ap->r, ctx) ||
         !BN_add(r0, r0, m1)) {
       goto err;
@@ -811,8 +769,6 @@ err:
 int rsa_default_multi_prime_keygen(RSA *rsa, int bits, int num_primes,
                                    BIGNUM *e_value, BN_GENCB *cb) {
   BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *r3 = NULL, *tmp;
-  BIGNUM local_r0, local_d, local_p;
-  BIGNUM *pr0, *d, *p;
   int prime_bits, ok = -1, n = 0, i, j;
   BN_CTX *ctx = NULL;
   STACK_OF(RSA_additional_prime) *additional_primes = NULL;
@@ -848,7 +804,7 @@ int rsa_default_multi_prime_keygen(RSA *rsa, int bits, int num_primes,
     if (ap == NULL) {
       goto err;
     }
-    memset(ap, 0, sizeof(RSA_additional_prime));
+    OPENSSL_memset(ap, 0, sizeof(RSA_additional_prime));
     ap->prime = BN_new();
     ap->exp = BN_new();
     ap->coeff = BN_new();
@@ -1041,31 +997,27 @@ int rsa_default_multi_prime_keygen(RSA *rsa, int bits, int num_primes,
       goto err;
     }
   }
-  pr0 = &local_r0;
-  BN_with_flags(pr0, r0, BN_FLG_CONSTTIME);
-  if (!BN_mod_inverse(rsa->d, rsa->e, pr0, ctx)) {
+  if (!BN_mod_inverse(rsa->d, rsa->e, r0, ctx)) {
     goto err; /* d */
   }
 
-  /* set up d for correct BN_FLG_CONSTTIME flag */
-  d = &local_d;
-  BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
-
   /* calculate d mod (p-1) */
-  if (!BN_mod(rsa->dmp1, d, r1, ctx)) {
+  if (!BN_mod(rsa->dmp1, rsa->d, r1, ctx)) {
     goto err;
   }
 
   /* calculate d mod (q-1) */
-  if (!BN_mod(rsa->dmq1, d, r2, ctx)) {
+  if (!BN_mod(rsa->dmq1, rsa->d, r2, ctx)) {
     goto err;
   }
 
-  /* calculate inverse of q mod p */
-  p = &local_p;
-  BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME);
-
-  if (!BN_mod_inverse(rsa->iqmp, rsa->q, p, ctx)) {
+  /* Calculate inverse of q mod p. Note that although RSA key generation is far
+   * from constant-time, |bn_mod_inverse_secret_prime| uses the same modular
+   * exponentation logic as in RSA private key operations and, if the RSAZ-1024
+   * code is enabled, will be optimized for common RSA prime sizes. */
+  if (!BN_MONT_CTX_set_locked(&rsa->mont_p, &rsa->lock, rsa->p, ctx) ||
+      !bn_mod_inverse_secret_prime(rsa->iqmp, rsa->q, rsa->p, ctx,
+                                   rsa->mont_p)) {
     goto err;
   }
 
@@ -1074,15 +1026,24 @@ int rsa_default_multi_prime_keygen(RSA *rsa, int bits, int num_primes,
         sk_RSA_additional_prime_value(additional_primes, i - 2);
     if (!BN_sub(ap->exp, ap->prime, BN_value_one()) ||
         !BN_mod(ap->exp, rsa->d, ap->exp, ctx) ||
-        !BN_mod_inverse(ap->coeff, ap->r, ap->prime, ctx)) {
+        !BN_MONT_CTX_set_locked(&ap->mont, &rsa->lock, ap->prime, ctx) ||
+        !bn_mod_inverse_secret_prime(ap->coeff, ap->r, ap->prime, ctx,
+                                     ap->mont)) {
       goto err;
     }
   }
 
-  ok = 1;
   rsa->additional_primes = additional_primes;
   additional_primes = NULL;
 
+  /* The key generation process is complex and thus error-prone. It could be
+   * disastrous to generate and then use a bad key so double-check that the key
+   * makes sense. */
+  ok = RSA_check_key(rsa);
+  if (!ok) {
+    OPENSSL_PUT_ERROR(RSA, RSA_R_INTERNAL_ERROR);
+  }
+
 err:
   if (ok == -1) {
     OPENSSL_PUT_ERROR(RSA, ERR_LIB_BN);