grpc 1.56.2 → 1.60.0

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/montgomery_inv.c

@@ -159,27 +159,63 @@ static uint64_t bn_neg_inv_mod_r_u64(uint64_t n) {
   return v;
 }
 
-int bn_mod_exp_base_2_consttime(BIGNUM *r, unsigned p, const BIGNUM *n,
-                                BN_CTX *ctx) {
-  assert(!BN_is_zero(n));
-  assert(!BN_is_negative(n));
-  assert(BN_is_odd(n));
+int bn_mont_ctx_set_RR_consttime(BN_MONT_CTX *mont, BN_CTX *ctx) {
+  assert(!BN_is_zero(&mont->N));
+  assert(!BN_is_negative(&mont->N));
+  assert(BN_is_odd(&mont->N));
+  assert(bn_minimal_width(&mont->N) == mont->N.width);
 
-  BN_zero(r);
-
-  unsigned n_bits = BN_num_bits(n);
+  unsigned n_bits = BN_num_bits(&mont->N);
   assert(n_bits != 0);
-  assert(p > n_bits);
   if (n_bits == 1) {
-    return 1;
+    BN_zero(&mont->RR);
+    return bn_resize_words(&mont->RR, mont->N.width);
   }
 
-  // Set |r| to the larger power of two smaller than |n|, then shift with
-  // reductions the rest of the way.
-  if (!BN_set_bit(r, n_bits - 1) ||
-      !bn_mod_lshift_consttime(r, r, p - (n_bits - 1), n, ctx)) {
+  unsigned lgBigR = mont->N.width * BN_BITS2;
+  assert(lgBigR >= n_bits);
+
+  // RR is R, or 2^lgBigR, in the Montgomery domain. We can compute 2 in the
+  // Montgomery domain, 2R or 2^(lgBigR+1), and then use Montgomery
+  // square-and-multiply to exponentiate.
+  //
+  // The multiply steps take 2^n R to 2^(n+1) R. It is faster to double
+  // the value instead. The square steps take 2^n R to 2^(2n) R. This is
+  // equivalent to doubling n times. When n is below some threshold, doubling is
+  // faster. When above, squaring is faster.
+  //
+  // We double to this threshold, then switch to Montgomery squaring. From
+  // benchmarking various 32-bit and 64-bit architectures, the word count seems
+  // to work well as a threshold. (Doubling scales linearly and Montgomery
+  // reduction scales quadratically, so the threshold should scale roughly
+  // linearly.)
+  unsigned threshold = mont->N.width;
+  unsigned iters;
+  for (iters = 0; iters < sizeof(lgBigR) * 8; iters++) {
+    if ((lgBigR >> iters) <= threshold) {
+      break;
+    }
+  }
+
+  // Compute 2^(lgBigR >> iters) R, or 2^((lgBigR >> iters) + lgBigR), by
+  // doubling. The first n_bits - 1 doubles can be skipped because we don't need
+  // to reduce.
+  if (!BN_set_bit(&mont->RR, n_bits - 1) ||
+      !bn_mod_lshift_consttime(&mont->RR, &mont->RR,
+                               (lgBigR >> iters) + lgBigR - (n_bits - 1),
+                               &mont->N, ctx)) {
     return 0;
   }
 
-  return 1;
+  for (unsigned i = iters - 1; i < iters; i--) {
+    if (!BN_mod_mul_montgomery(&mont->RR, &mont->RR, &mont->RR, mont, ctx)) {
+      return 0;
+    }
+    if ((lgBigR & (1u << i)) != 0 &&
+        !bn_mod_lshift1_consttime(&mont->RR, &mont->RR, &mont->N, ctx)) {
+      return 0;
+    }
+  }
+
+  return bn_resize_words(&mont->RR, mont->N.width);
 }

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/rsaz_exp.c

@@ -24,13 +24,13 @@
 #include "../../internal.h"
 
 
-// one is 1 in RSAZ's representation.
-alignas(64) static const BN_ULONG one[40] = {
+// rsaz_one is 1 in RSAZ's representation.
+alignas(64) static const BN_ULONG rsaz_one[40] = {
     1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-// two80 is 2^80 in RSAZ's representation. Note RSAZ uses base 2^29, so this is
+// rsaz_two80 is 2^80 in RSAZ's representation. Note RSAZ uses base 2^29, so this is
 // 2^(29*2 + 22) = 2^80, not 2^(64*2 + 22).
-alignas(64) static const BN_ULONG two80[40] = {
+alignas(64) static const BN_ULONG rsaz_two80[40] = {
     0, 0, 1 << 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0,       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
@@ -64,12 +64,12 @@ void RSAZ_1024_mod_exp_avx2(BN_ULONG result_norm[16],
   // giving R = 2^(36*29) = 2^1044.
   rsaz_1024_mul_avx2(R2, R2, R2, m, k0);
   // R2 = 2^2048 * 2^2048 / 2^1044 = 2^3052
-  rsaz_1024_mul_avx2(R2, R2, two80, m, k0);
+  rsaz_1024_mul_avx2(R2, R2, rsaz_two80, m, k0);
   // R2 = 2^3052 * 2^80 / 2^1044 = 2^2088 = (2^1044)^2
 
   // table[0] = 1
   // table[1] = a_inv^1
-  rsaz_1024_mul_avx2(result, R2, one, m, k0);
+  rsaz_1024_mul_avx2(result, R2, rsaz_one, m, k0);
   rsaz_1024_mul_avx2(a_inv, a_inv, R2, m, k0);
   rsaz_1024_scatter5_avx2(table_s, result, 0);
   rsaz_1024_scatter5_avx2(table_s, a_inv, 1);
@@ -125,7 +125,7 @@ void RSAZ_1024_mod_exp_avx2(BN_ULONG result_norm[16],
   rsaz_1024_mul_avx2(result, result, a_inv, m, k0);
 
   // Convert from Montgomery.
-  rsaz_1024_mul_avx2(result, result, one, m, k0);
+  rsaz_1024_mul_avx2(result, result, rsaz_one, m, k0);
 
   rsaz_1024_red2norm_avx2(result_norm, result);
   BN_ULONG scratch[16];

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/delocate.h

@@ -27,9 +27,8 @@
   type *name##_bss_get(void) __attribute__((const));
 // For FIPS builds we require that CRYPTO_ONCE_INIT be zero.
 #define DEFINE_STATIC_ONCE(name) DEFINE_BSS_GET(CRYPTO_once_t, name)
-// For FIPS builds we require that CRYPTO_STATIC_MUTEX_INIT be zero.
-#define DEFINE_STATIC_MUTEX(name) \
-  DEFINE_BSS_GET(struct CRYPTO_STATIC_MUTEX, name)
+// For FIPS builds we require that CRYPTO_MUTEX_INIT be zero.
+#define DEFINE_STATIC_MUTEX(name) DEFINE_BSS_GET(CRYPTO_MUTEX, name)
 // For FIPS builds we require that CRYPTO_EX_DATA_CLASS_INIT be zero.
 #define DEFINE_STATIC_EX_DATA_CLASS(name) \
   DEFINE_BSS_GET(CRYPTO_EX_DATA_CLASS, name)
@@ -40,9 +39,9 @@
 #define DEFINE_STATIC_ONCE(name)                \
   static CRYPTO_once_t name = CRYPTO_ONCE_INIT; \
   static CRYPTO_once_t *name##_bss_get(void) { return &name; }
-#define DEFINE_STATIC_MUTEX(name)                                    \
-  static struct CRYPTO_STATIC_MUTEX name = CRYPTO_STATIC_MUTEX_INIT; \
-  static struct CRYPTO_STATIC_MUTEX *name##_bss_get(void) { return &name; }
+#define DEFINE_STATIC_MUTEX(name)               \
+  static CRYPTO_MUTEX name = CRYPTO_MUTEX_INIT; \
+  static CRYPTO_MUTEX *name##_bss_get(void) { return &name; }
 #define DEFINE_STATIC_EX_DATA_CLASS(name)                       \
   static CRYPTO_EX_DATA_CLASS name = CRYPTO_EX_DATA_CLASS_INIT; \
   static CRYPTO_EX_DATA_CLASS *name##_bss_get(void) { return &name; }

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/check.c

@@ -57,12 +57,40 @@
 #include <openssl/dh.h>
 
 #include <openssl/bn.h>
+#include <openssl/err.h>
 
 #include "internal.h"
 
 
+int dh_check_params_fast(const DH *dh) {
+  // Most operations scale with p and q.
+  if (BN_is_negative(dh->p) || !BN_is_odd(dh->p) ||
+      BN_num_bits(dh->p) > OPENSSL_DH_MAX_MODULUS_BITS) {
+    OPENSSL_PUT_ERROR(DH, DH_R_INVALID_PARAMETERS);
+    return 0;
+  }
+
+  // q must be bounded by p.
+  if (dh->q != NULL && (BN_is_negative(dh->q) || BN_ucmp(dh->q, dh->p) > 0)) {
+    OPENSSL_PUT_ERROR(DH, DH_R_INVALID_PARAMETERS);
+    return 0;
+  }
+
+  // g must be an element of p's multiplicative group.
+  if (BN_is_negative(dh->g) || BN_is_zero(dh->g) ||
+      BN_ucmp(dh->g, dh->p) >= 0) {
+    OPENSSL_PUT_ERROR(DH, DH_R_INVALID_PARAMETERS);
+    return 0;
+  }
+
+  return 1;
+}
+
 int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *out_flags) {
   *out_flags = 0;
+  if (!dh_check_params_fast(dh)) {
+    return 0;
+  }
 
   BN_CTX *ctx = BN_CTX_new();
   if (ctx == NULL) {
@@ -73,17 +101,14 @@ int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *out_flags) {
   int ok = 0;
 
   // Check |pub_key| is greater than 1.
-  BIGNUM *tmp = BN_CTX_get(ctx);
-  if (tmp == NULL ||
-      !BN_set_word(tmp, 1)) {
-    goto err;
-  }
-  if (BN_cmp(pub_key, tmp) <= 0) {
+  if (BN_cmp(pub_key, BN_value_one()) <= 0) {
     *out_flags |= DH_CHECK_PUBKEY_TOO_SMALL;
   }
 
   // Check |pub_key| is less than |dh->p| - 1.
-  if (!BN_copy(tmp, dh->p) ||
+  BIGNUM *tmp = BN_CTX_get(ctx);
+  if (tmp == NULL ||
+      !BN_copy(tmp, dh->p) ||
       !BN_sub_word(tmp, 1)) {
     goto err;
   }
@@ -113,6 +138,11 @@ err:
 
 
 int DH_check(const DH *dh, int *out_flags) {
+  *out_flags = 0;
+  if (!dh_check_params_fast(dh)) {
+    return 0;
+  }
+
   // Check that p is a safe prime and if g is 2, 3 or 5, check that it is a
   // suitable generator where:
   //   for 2, p mod 24 == 11
@@ -124,7 +154,6 @@ int DH_check(const DH *dh, int *out_flags) {
   BN_ULONG l;
   BIGNUM *t1 = NULL, *t2 = NULL;
 
-  *out_flags = 0;
   ctx = BN_CTX_new();
   if (ctx == NULL) {
     goto err;

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/dh.c

@@ -70,8 +70,6 @@
 #include "internal.h"
 
 
-#define OPENSSL_DH_MAX_MODULUS_BITS 10000
-
 DH *DH_new(void) {
   DH *dh = OPENSSL_malloc(sizeof(DH));
   if (dh == NULL) {
@@ -191,15 +189,14 @@ int DH_set_length(DH *dh, unsigned priv_length) {
 int DH_generate_key(DH *dh) {
   boringssl_ensure_ffdh_self_test();
 
+  if (!dh_check_params_fast(dh)) {
+    return 0;
+  }
+
   int ok = 0;
   int generate_new_key = 0;
   BN_CTX *ctx = NULL;
-  BIGNUM *pub_key = NULL, *priv_key = NULL;
-
-  if (BN_num_bits(dh->p) > OPENSSL_DH_MAX_MODULUS_BITS) {
-    OPENSSL_PUT_ERROR(DH, DH_R_MODULUS_TOO_LARGE);
-    goto err;
-  }
+  BIGNUM *pub_key = NULL, *priv_key = NULL, *priv_key_limit = NULL;
 
   ctx = BN_CTX_new();
   if (ctx == NULL) {
@@ -232,22 +229,44 @@ int DH_generate_key(DH *dh) {
 
   if (generate_new_key) {
     if (dh->q) {
-      if (!BN_rand_range_ex(priv_key, 2, dh->q)) {
+      // Section 5.6.1.1.4 of SP 800-56A Rev3 generates a private key uniformly
+      // from [1, min(2^N-1, q-1)].
+      //
+      // Although SP 800-56A Rev3 now permits a private key length N,
+      // |dh->priv_length| historically was ignored when q is available. We
+      // continue to ignore it and interpret such a configuration as N = len(q).
+      if (!BN_rand_range_ex(priv_key, 1, dh->q)) {
         goto err;
       }
     } else {
-      // secret exponent length
-      unsigned priv_bits = dh->priv_length;
-      if (priv_bits == 0) {
-        const unsigned p_bits = BN_num_bits(dh->p);
-        if (p_bits == 0) {
+      // If q is unspecified, we expect p to be a safe prime, with g generating
+      // the (p-1)/2 subgroup. So, we use q = (p-1)/2. (If g generates a smaller
+      // prime-order subgroup, q will still divide (p-1)/2.)
+      //
+      // We set N from |dh->priv_length|. Section 5.6.1.1.4 of SP 800-56A Rev3
+      // says to reject N > len(q), or N > num_bits(p) - 1. However, this logic
+      // originally aligned with PKCS#3, which allows num_bits(p). Instead, we
+      // clamp |dh->priv_length| before invoking the algorithm.
+
+      // Compute M = min(2^N, q).
+      priv_key_limit = BN_new();
+      if (priv_key_limit == NULL) {
+        goto err;
+      }
+      if (dh->priv_length == 0 || dh->priv_length >= BN_num_bits(dh->p) - 1) {
+        // M = q = (p - 1) / 2.
+        if (!BN_rshift1(priv_key_limit, dh->p)) {
+          goto err;
+        }
+      } else {
+        // M = 2^N.
+        if (!BN_set_bit(priv_key_limit, dh->priv_length)) {
           goto err;
         }
-
-        priv_bits = p_bits - 1;
       }
 
-      if (!BN_rand(priv_key, priv_bits, BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ANY)) {
+      // Choose a private key uniformly from [1, M-1].
+      if (!BN_rand_range_ex(priv_key, 1, priv_key_limit)) {
         goto err;
       }
     }
@@ -273,14 +292,14 @@ err:
   if (dh->priv_key == NULL) {
     BN_free(priv_key);
   }
+  BN_free(priv_key_limit);
   BN_CTX_free(ctx);
   return ok;
 }
 
 static int dh_compute_key(DH *dh, BIGNUM *out_shared_key,
                           const BIGNUM *peers_key, BN_CTX *ctx) {
-  if (BN_num_bits(dh->p) > OPENSSL_DH_MAX_MODULUS_BITS) {
-    OPENSSL_PUT_ERROR(DH, DH_R_MODULUS_TOO_LARGE);
+  if (!dh_check_params_fast(dh)) {
     return 0;
   }
 

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/internal.h

@@ -19,11 +19,15 @@
 
 #include <openssl/thread.h>
 
+#include "../../internal.h"
+
 #if defined(__cplusplus)
 extern "C" {
 #endif
 
 
+#define OPENSSL_DH_MAX_MODULUS_BITS 10000
+
 struct dh_st {
   BIGNUM *p;
   BIGNUM *g;
@@ -42,6 +46,11 @@ struct dh_st {
   CRYPTO_refcount_t references;
 };
 
+// dh_check_params_fast checks basic invariants on |dh|'s domain parameters. It
+// does not check that |dh| forms a valid group, only that the sizes are within
+// DoS bounds.
+int dh_check_params_fast(const DH *dh);
+
 // dh_compute_key_padded_no_self_test does the same as |DH_compute_key_padded|,
 // but doesn't try to run the self-test first. This is for use in the self tests
 // themselves, to prevent an infinite loop.

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/builtin_curves.h

@@ -0,0 +1,277 @@
+/* Copyright (c) 2023, Google Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+// This file is generated by make_tables.go.
+
+// P-224
+OPENSSL_UNUSED static const uint64_t kP224FieldN0 = 0xffffffffffffffff;
+OPENSSL_UNUSED static const uint64_t kP224OrderN0 = 0xd6e242706a1fc2eb;
+#if defined(OPENSSL_64_BIT)
+OPENSSL_UNUSED static const uint64_t kP224Field[] = {
+    0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff,
+    0x00000000ffffffff};
+OPENSSL_UNUSED static const uint64_t kP224Order[] = {
+    0x13dd29455c5c2a3d, 0xffff16a2e0b8f03e, 0xffffffffffffffff,
+    0x00000000ffffffff};
+OPENSSL_UNUSED static const uint64_t kP224B[] = {
+    0x270b39432355ffb4, 0x5044b0b7d7bfd8ba, 0x0c04b3abf5413256,
+    0x00000000b4050a85};
+OPENSSL_UNUSED static const uint64_t kP224GX[] = {
+    0x343280d6115c1d21, 0x4a03c1d356c21122, 0x6bb4bf7f321390b9,
+    0x00000000b70e0cbd};
+OPENSSL_UNUSED static const uint64_t kP224GY[] = {
+    0x44d5819985007e34, 0xcd4375a05a074764, 0xb5f723fb4c22dfe6,
+    0x00000000bd376388};
+OPENSSL_UNUSED static const uint64_t kP224FieldR[] = {
+    0xffffffff00000000, 0xffffffffffffffff, 0x0000000000000000,
+    0x0000000000000000};
+OPENSSL_UNUSED static const uint64_t kP224FieldRR[] = {
+    0xffffffff00000001, 0xffffffff00000000, 0xfffffffe00000000,
+    0x00000000ffffffff};
+OPENSSL_UNUSED static const uint64_t kP224OrderRR[] = {
+    0x29947a695f517d15, 0xabc8ff5931d63f4b, 0x6ad15f7cd9714856,
+    0x00000000b1e97961};
+OPENSSL_UNUSED static const uint64_t kP224MontB[] = {
+    0xe768cdf663c059cd, 0x107ac2f3ccf01310, 0x3dceba98c8528151,
+    0x000000007fc02f93};
+OPENSSL_UNUSED static const uint64_t kP224MontGX[] = {
+    0xbc9052266d0a4aea, 0x852597366018bfaa, 0x6dd3af9bf96bec05,
+    0x00000000a21b5e60};
+OPENSSL_UNUSED static const uint64_t kP224MontGY[] = {
+    0x2edca1e5eff3ede8, 0xf8cd672b05335a6b, 0xaea9c5ae03dfe878,
+    0x00000000614786f1};
+#elif defined(OPENSSL_32_BIT)
+OPENSSL_UNUSED static const uint32_t kP224Field[] = {
+    0x00000001, 0x00000000, 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff,
+    0xffffffff};
+OPENSSL_UNUSED static const uint32_t kP224Order[] = {
+    0x5c5c2a3d, 0x13dd2945, 0xe0b8f03e, 0xffff16a2, 0xffffffff, 0xffffffff,
+    0xffffffff};
+OPENSSL_UNUSED static const uint32_t kP224B[] = {
+    0x2355ffb4, 0x270b3943, 0xd7bfd8ba, 0x5044b0b7, 0xf5413256, 0x0c04b3ab,
+    0xb4050a85};
+OPENSSL_UNUSED static const uint32_t kP224GX[] = {
+    0x115c1d21, 0x343280d6, 0x56c21122, 0x4a03c1d3, 0x321390b9, 0x6bb4bf7f,
+    0xb70e0cbd};
+OPENSSL_UNUSED static const uint32_t kP224GY[] = {
+    0x85007e34, 0x44d58199, 0x5a074764, 0xcd4375a0, 0x4c22dfe6, 0xb5f723fb,
+    0xbd376388};
+OPENSSL_UNUSED static const uint32_t kP224FieldR[] = {
+    0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000};
+OPENSSL_UNUSED static const uint32_t kP224FieldRR[] = {
+    0x00000001, 0x00000000, 0x00000000, 0xfffffffe, 0xffffffff, 0xffffffff,
+    0x00000000};
+OPENSSL_UNUSED static const uint32_t kP224OrderRR[] = {
+    0x3ad01289, 0x6bdaae6c, 0x97a54552, 0x6ad09d91, 0xb1e97961, 0x1822bc47,
+    0xd4baa4cf};
+OPENSSL_UNUSED static const uint32_t kP224MontB[] = {
+    0xe768cdf7, 0xccf01310, 0x743b1cc0, 0xc8528150, 0x3dceba98, 0x7fc02f93,
+    0x9c3fa633};
+OPENSSL_UNUSED static const uint32_t kP224MontGX[] = {
+    0xbc905227, 0x6018bfaa, 0xf22fe220, 0xf96bec04, 0x6dd3af9b, 0xa21b5e60,
+    0x92f5b516};
+OPENSSL_UNUSED static const uint32_t kP224MontGY[] = {
+    0x2edca1e6, 0x05335a6b, 0xe8c15513, 0x03dfe878, 0xaea9c5ae, 0x614786f1,
+    0x100c1218};
+#else
+#error "unknown word size"
+#endif
+
+// P-256
+OPENSSL_UNUSED static const uint64_t kP256FieldN0 = 0x0000000000000001;
+OPENSSL_UNUSED static const uint64_t kP256OrderN0 = 0xccd1c8aaee00bc4f;
+#if defined(OPENSSL_64_BIT)
+OPENSSL_UNUSED static const uint64_t kP256Field[] = {
+    0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000,
+    0xffffffff00000001};
+OPENSSL_UNUSED static const uint64_t kP256Order[] = {
+    0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff,
+    0xffffffff00000000};
+OPENSSL_UNUSED static const uint64_t kP256FieldR[] = {
+    0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff,
+    0x00000000fffffffe};
+OPENSSL_UNUSED static const uint64_t kP256FieldRR[] = {
+    0x0000000000000003, 0xfffffffbffffffff, 0xfffffffffffffffe,
+    0x00000004fffffffd};
+OPENSSL_UNUSED static const uint64_t kP256OrderRR[] = {
+    0x83244c95be79eea2, 0x4699799c49bd6fa6, 0x2845b2392b6bec59,
+    0x66e12d94f3d95620};
+OPENSSL_UNUSED static const uint64_t kP256MontB[] = {
+    0xd89cdf6229c4bddf, 0xacf005cd78843090, 0xe5a220abf7212ed6,
+    0xdc30061d04874834};
+OPENSSL_UNUSED static const uint64_t kP256MontGX[] = {
+    0x79e730d418a9143c, 0x75ba95fc5fedb601, 0x79fb732b77622510,
+    0x18905f76a53755c6};
+OPENSSL_UNUSED static const uint64_t kP256MontGY[] = {
+    0xddf25357ce95560a, 0x8b4ab8e4ba19e45c, 0xd2e88688dd21f325,
+    0x8571ff1825885d85};
+#elif defined(OPENSSL_32_BIT)
+OPENSSL_UNUSED static const uint32_t kP256Field[] = {
+    0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000,
+    0x00000001, 0xffffffff};
+OPENSSL_UNUSED static const uint32_t kP256Order[] = {
+    0xfc632551, 0xf3b9cac2, 0xa7179e84, 0xbce6faad, 0xffffffff, 0xffffffff,
+    0x00000000, 0xffffffff};
+OPENSSL_UNUSED static const uint32_t kP256FieldR[] = {
+    0x00000001, 0x00000000, 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff,
+    0xfffffffe, 0x00000000};
+OPENSSL_UNUSED static const uint32_t kP256FieldRR[] = {
+    0x00000003, 0x00000000, 0xffffffff, 0xfffffffb, 0xfffffffe, 0xffffffff,
+    0xfffffffd, 0x00000004};
+OPENSSL_UNUSED static const uint32_t kP256OrderRR[] = {
+    0xbe79eea2, 0x83244c95, 0x49bd6fa6, 0x4699799c, 0x2b6bec59, 0x2845b239,
+    0xf3d95620, 0x66e12d94};
+OPENSSL_UNUSED static const uint32_t kP256MontB[] = {
+    0x29c4bddf, 0xd89cdf62, 0x78843090, 0xacf005cd, 0xf7212ed6, 0xe5a220ab,
+    0x04874834, 0xdc30061d};
+OPENSSL_UNUSED static const uint32_t kP256MontGX[] = {
+    0x18a9143c, 0x79e730d4, 0x5fedb601, 0x75ba95fc, 0x77622510, 0x79fb732b,
+    0xa53755c6, 0x18905f76};
+OPENSSL_UNUSED static const uint32_t kP256MontGY[] = {
+    0xce95560a, 0xddf25357, 0xba19e45c, 0x8b4ab8e4, 0xdd21f325, 0xd2e88688,
+    0x25885d85, 0x8571ff18};
+#else
+#error "unknown word size"
+#endif
+
+// P-384
+OPENSSL_UNUSED static const uint64_t kP384FieldN0 = 0x0000000100000001;
+OPENSSL_UNUSED static const uint64_t kP384OrderN0 = 0x6ed46089e88fdc45;
+#if defined(OPENSSL_64_BIT)
+OPENSSL_UNUSED static const uint64_t kP384Field[] = {
+    0x00000000ffffffff, 0xffffffff00000000, 0xfffffffffffffffe,
+    0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff};
+OPENSSL_UNUSED static const uint64_t kP384Order[] = {
+    0xecec196accc52973, 0x581a0db248b0a77a, 0xc7634d81f4372ddf,
+    0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff};
+OPENSSL_UNUSED static const uint64_t kP384FieldR[] = {
+    0xffffffff00000001, 0x00000000ffffffff, 0x0000000000000001,
+    0x0000000000000000, 0x0000000000000000, 0x0000000000000000};
+OPENSSL_UNUSED static const uint64_t kP384FieldRR[] = {
+    0xfffffffe00000001, 0x0000000200000000, 0xfffffffe00000000,
+    0x0000000200000000, 0x0000000000000001, 0x0000000000000000};
+OPENSSL_UNUSED static const uint64_t kP384OrderRR[] = {
+    0x2d319b2419b409a9, 0xff3d81e5df1aa419, 0xbc3e483afcb82947,
+    0xd40d49174aab1cc5, 0x3fb05b7a28266895, 0x0c84ee012b39bf21};
+OPENSSL_UNUSED static const uint64_t kP384MontB[] = {
+    0x081188719d412dcc, 0xf729add87a4c32ec, 0x77f2209b1920022e,
+    0xe3374bee94938ae2, 0xb62b21f41f022094, 0xcd08114b604fbff9};
+OPENSSL_UNUSED static const uint64_t kP384MontGX[] = {
+    0x3dd0756649c0b528, 0x20e378e2a0d6ce38, 0x879c3afc541b4d6e,
+    0x6454868459a30eff, 0x812ff723614ede2b, 0x4d3aadc2299e1513};
+OPENSSL_UNUSED static const uint64_t kP384MontGY[] = {
+    0x23043dad4b03a4fe, 0xa1bfa8bf7bb4a9ac, 0x8bade7562e83b050,
+    0xc6c3521968f4ffd9, 0xdd8002263969a840, 0x2b78abc25a15c5e9};
+#elif defined(OPENSSL_32_BIT)
+OPENSSL_UNUSED static const uint32_t kP384Field[] = {
+    0xffffffff, 0x00000000, 0x00000000, 0xffffffff, 0xfffffffe, 0xffffffff,
+    0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff};
+OPENSSL_UNUSED static const uint32_t kP384Order[] = {
+    0xccc52973, 0xecec196a, 0x48b0a77a, 0x581a0db2, 0xf4372ddf, 0xc7634d81,
+    0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff};
+OPENSSL_UNUSED static const uint32_t kP384FieldR[] = {
+    0x00000001, 0xffffffff, 0xffffffff, 0x00000000, 0x00000001, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000};
+OPENSSL_UNUSED static const uint32_t kP384FieldRR[] = {
+    0x00000001, 0xfffffffe, 0x00000000, 0x00000002, 0x00000000, 0xfffffffe,
+    0x00000000, 0x00000002, 0x00000001, 0x00000000, 0x00000000, 0x00000000};
+OPENSSL_UNUSED static const uint32_t kP384OrderRR[] = {
+    0x19b409a9, 0x2d319b24, 0xdf1aa419, 0xff3d81e5, 0xfcb82947, 0xbc3e483a,
+    0x4aab1cc5, 0xd40d4917, 0x28266895, 0x3fb05b7a, 0x2b39bf21, 0x0c84ee01};
+OPENSSL_UNUSED static const uint32_t kP384MontB[] = {
+    0x9d412dcc, 0x08118871, 0x7a4c32ec, 0xf729add8, 0x1920022e, 0x77f2209b,
+    0x94938ae2, 0xe3374bee, 0x1f022094, 0xb62b21f4, 0x604fbff9, 0xcd08114b};
+OPENSSL_UNUSED static const uint32_t kP384MontGX[] = {
+    0x49c0b528, 0x3dd07566, 0xa0d6ce38, 0x20e378e2, 0x541b4d6e, 0x879c3afc,
+    0x59a30eff, 0x64548684, 0x614ede2b, 0x812ff723, 0x299e1513, 0x4d3aadc2};
+OPENSSL_UNUSED static const uint32_t kP384MontGY[] = {
+    0x4b03a4fe, 0x23043dad, 0x7bb4a9ac, 0xa1bfa8bf, 0x2e83b050, 0x8bade756,
+    0x68f4ffd9, 0xc6c35219, 0x3969a840, 0xdd800226, 0x5a15c5e9, 0x2b78abc2};
+#else
+#error "unknown word size"
+#endif
+
+// P-521
+OPENSSL_UNUSED static const uint64_t kP521FieldN0 = 0x0000000000000001;
+OPENSSL_UNUSED static const uint64_t kP521OrderN0 = 0x1d2f5ccd79a995c7;
+#if defined(OPENSSL_64_BIT)
+OPENSSL_UNUSED static const uint64_t kP521Field[] = {
+    0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff,
+    0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff,
+    0xffffffffffffffff, 0xffffffffffffffff, 0x00000000000001ff};
+OPENSSL_UNUSED static const uint64_t kP521Order[] = {
+    0xbb6fb71e91386409, 0x3bb5c9b8899c47ae, 0x7fcc0148f709a5d0,
+    0x51868783bf2f966b, 0xfffffffffffffffa, 0xffffffffffffffff,
+    0xffffffffffffffff, 0xffffffffffffffff, 0x00000000000001ff};
+OPENSSL_UNUSED static const uint64_t kP521FieldR[] = {
+    0x0080000000000000, 0x0000000000000000, 0x0000000000000000,
+    0x0000000000000000, 0x0000000000000000, 0x0000000000000000,
+    0x0000000000000000, 0x0000000000000000, 0x0000000000000000};
+OPENSSL_UNUSED static const uint64_t kP521FieldRR[] = {
+    0x0000000000000000, 0x0000400000000000, 0x0000000000000000,
+    0x0000000000000000, 0x0000000000000000, 0x0000000000000000,
+    0x0000000000000000, 0x0000000000000000, 0x0000000000000000};
+OPENSSL_UNUSED static const uint64_t kP521OrderRR[] = {
+    0x137cd04dcf15dd04, 0xf707badce5547ea3, 0x12a78d38794573ff,
+    0xd3721ef557f75e06, 0xdd6e23d82e49c7db, 0xcff3d142b7756e3e,
+    0x5bcc6d61a8e567bc, 0x2d8e03d1492d0d45, 0x000000000000003d};
+OPENSSL_UNUSED static const uint64_t kP521MontB[] = {
+    0x8014654fae586387, 0x78f7a28fea35a81f, 0x839ab9efc41e961a,
+    0xbd8b29605e9dd8df, 0xf0ab0c9ca8f63f49, 0xf9dc5a44c8c77884,
+    0x77516d392dccd98a, 0x0fc94d10d05b42a0, 0x000000000000004d};
+OPENSSL_UNUSED static const uint64_t kP521MontGX[] = {
+    0xb331a16381adc101, 0x4dfcbf3f18e172de, 0x6f19a459e0c2b521,
+    0x947f0ee093d17fd4, 0xdd50a5af3bf7f3ac, 0x90fc1457b035a69e,
+    0x214e32409c829fda, 0xe6cf1f65b311cada, 0x0000000000000074};
+OPENSSL_UNUSED static const uint64_t kP521MontGY[] = {
+    0x28460e4a5a9e268e, 0x20445f4a3b4fe8b3, 0xb09a9e3843513961,
+    0x2062a85c809fd683, 0x164bf7394caf7a13, 0x340bd7de8b939f33,
+    0xeccc7aa224abcda2, 0x022e452fda163e8d, 0x00000000000001e0};
+#elif defined(OPENSSL_32_BIT)
+OPENSSL_UNUSED static const uint32_t kP521Field[] = {
+    0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
+    0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
+    0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x000001ff};
+OPENSSL_UNUSED static const uint32_t kP521Order[] = {
+    0x91386409, 0xbb6fb71e, 0x899c47ae, 0x3bb5c9b8, 0xf709a5d0, 0x7fcc0148,
+    0xbf2f966b, 0x51868783, 0xfffffffa, 0xffffffff, 0xffffffff, 0xffffffff,
+    0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x000001ff};
+OPENSSL_UNUSED static const uint32_t kP521FieldR[] = {
+    0x00800000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000};
+OPENSSL_UNUSED static const uint32_t kP521FieldRR[] = {
+    0x00000000, 0x00004000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+    0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000};
+OPENSSL_UNUSED static const uint32_t kP521OrderRR[] = {
+    0x61c64ca7, 0x1163115a, 0x4374a642, 0x18354a56, 0x0791d9dc, 0x5d4dd6d3,
+    0xd3402705, 0x4fb35b72, 0xb7756e3a, 0xcff3d142, 0xa8e567bc, 0x5bcc6d61,
+    0x492d0d45, 0x2d8e03d1, 0x8c44383d, 0x5b5a3afe, 0x0000019a};
+OPENSSL_UNUSED static const uint32_t kP521MontB[] = {
+    0x8014654f, 0xea35a81f, 0x78f7a28f, 0xc41e961a, 0x839ab9ef, 0x5e9dd8df,
+    0xbd8b2960, 0xa8f63f49, 0xf0ab0c9c, 0xc8c77884, 0xf9dc5a44, 0x2dccd98a,
+    0x77516d39, 0xd05b42a0, 0x0fc94d10, 0xb0c70e4d, 0x0000015c};
+OPENSSL_UNUSED static const uint32_t kP521MontGX[] = {
+    0xb331a163, 0x18e172de, 0x4dfcbf3f, 0xe0c2b521, 0x6f19a459, 0x93d17fd4,
+    0x947f0ee0, 0x3bf7f3ac, 0xdd50a5af, 0xb035a69e, 0x90fc1457, 0x9c829fda,
+    0x214e3240, 0xb311cada, 0xe6cf1f65, 0x5b820274, 0x00000103};
+OPENSSL_UNUSED static const uint32_t kP521MontGY[] = {
+    0x28460e4a, 0x3b4fe8b3, 0x20445f4a, 0x43513961, 0xb09a9e38, 0x809fd683,
+    0x2062a85c, 0x4caf7a13, 0x164bf739, 0x8b939f33, 0x340bd7de, 0x24abcda2,
+    0xeccc7aa2, 0xda163e8d, 0x022e452f, 0x3c4d1de0, 0x000000b5};
+#else
+#error "unknown word size"
+#endif