grpc 1.24.0 → 1.25.0

data/third_party/boringssl/crypto/fipsmodule/ec/p256-x86_64.c

@@ -23,6 +23,7 @@
 #include <string.h>
 
 #include <openssl/bn.h>
+#include <openssl/cpu.h>
 #include <openssl/crypto.h>
 #include <openssl/err.h>
 
@@ -47,7 +48,8 @@ static const BN_ULONG ONE[P256_LIMBS] = {
 // Precomputed tables for the default generator
 #include "p256-x86_64-table.h"
 
-// Recode window to a signed digit, see util-64.c for details
+// Recode window to a signed digit, see |ec_GFp_nistp_recode_scalar_bits| in
+// util.c for details
 static unsigned booth_recode_w5(unsigned in) {
   unsigned s, d;
 
@@ -197,19 +199,13 @@ static void ecp_nistz256_mod_inverse_mont(BN_ULONG r[P256_LIMBS],
   ecp_nistz256_mul_mont(r, res, in);
 }
 
-// ecp_nistz256_bignum_to_field_elem copies the contents of |in| to |out| and
-// returns one if it fits. Otherwise it returns zero.
-static int ecp_nistz256_bignum_to_field_elem(BN_ULONG out[P256_LIMBS],
-                                             const BIGNUM *in) {
-  return bn_copy_words(out, P256_LIMBS, in);
-}
-
 // r = p * p_scalar
-static int ecp_nistz256_windowed_mul(const EC_GROUP *group, P256_POINT *r,
-                                     const EC_POINT *p,
-                                     const EC_SCALAR *p_scalar) {
+static void ecp_nistz256_windowed_mul(const EC_GROUP *group, P256_POINT *r,
+                                      const EC_RAW_POINT *p,
+                                      const EC_SCALAR *p_scalar) {
   assert(p != NULL);
   assert(p_scalar != NULL);
+  assert(group->field.width == P256_LIMBS);
 
   static const unsigned kWindowSize = 5;
   static const unsigned kMask = (1 << (5 /* kWindowSize */ + 1)) - 1;
@@ -226,13 +222,10 @@ static int ecp_nistz256_windowed_mul(const EC_GROUP *group, P256_POINT *r,
   // not stored. All other values are actually stored with an offset of -1 in
   // table.
   P256_POINT *row = table;
-
-  if (!ecp_nistz256_bignum_to_field_elem(row[1 - 1].X, &p->X) ||
-      !ecp_nistz256_bignum_to_field_elem(row[1 - 1].Y, &p->Y) ||
-      !ecp_nistz256_bignum_to_field_elem(row[1 - 1].Z, &p->Z)) {
-    OPENSSL_PUT_ERROR(EC, EC_R_COORDINATES_OUT_OF_RANGE);
-    return 0;
-  }
+  assert(group->field.width == P256_LIMBS);
+  OPENSSL_memcpy(row[1 - 1].X, p->X.words, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(row[1 - 1].Y, p->Y.words, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(row[1 - 1].Z, p->Z.words, P256_LIMBS * sizeof(BN_ULONG));
 
   ecp_nistz256_point_double(&row[2 - 1], &row[1 - 1]);
   ecp_nistz256_point_add(&row[3 - 1], &row[2 - 1], &row[1 - 1]);
@@ -296,110 +289,160 @@ static int ecp_nistz256_windowed_mul(const EC_GROUP *group, P256_POINT *r,
   copy_conditional(h.Y, tmp, wvalue & 1);
 
   ecp_nistz256_point_add(r, r, &h);
-
-  return 1;
 }
 
-static int ecp_nistz256_points_mul(const EC_GROUP *group, EC_POINT *r,
-                                   const EC_SCALAR *g_scalar,
-                                   const EC_POINT *p_,
-                                   const EC_SCALAR *p_scalar, BN_CTX *ctx) {
-  assert((p_ != NULL) == (p_scalar != NULL));
+typedef union {
+  P256_POINT p;
+  P256_POINT_AFFINE a;
+} p256_point_union_t;
 
+static unsigned calc_first_wvalue(unsigned *index, const uint8_t p_str[33]) {
   static const unsigned kWindowSize = 7;
   static const unsigned kMask = (1 << (7 /* kWindowSize */ + 1)) - 1;
+  *index = kWindowSize;
 
-  alignas(32) union {
-    P256_POINT p;
-    P256_POINT_AFFINE a;
-  } t, p;
+  unsigned wvalue = (p_str[0] << 1) & kMask;
+  return booth_recode_w7(wvalue);
+}
+
+static unsigned calc_wvalue(unsigned *index, const uint8_t p_str[33]) {
+  static const unsigned kWindowSize = 7;
+  static const unsigned kMask = (1 << (7 /* kWindowSize */ + 1)) - 1;
 
-  if (g_scalar != NULL) {
-    uint8_t p_str[33];
-    OPENSSL_memcpy(p_str, g_scalar->bytes, 32);
-    p_str[32] = 0;
+  const unsigned off = (*index - 1) / 8;
+  unsigned wvalue = p_str[off] | p_str[off + 1] << 8;
+  wvalue = (wvalue >> ((*index - 1) % 8)) & kMask;
+  *index += kWindowSize;
 
-    // First window
-    unsigned wvalue = (p_str[0] << 1) & kMask;
-    unsigned index = kWindowSize;
+  return booth_recode_w7(wvalue);
+}
 
-    wvalue = booth_recode_w7(wvalue);
+static void ecp_nistz256_point_mul(const EC_GROUP *group, EC_RAW_POINT *r,
+                                   const EC_RAW_POINT *p,
+                                   const EC_SCALAR *scalar) {
+  alignas(32) P256_POINT out;
+  ecp_nistz256_windowed_mul(group, &out, p, scalar);
 
-    const PRECOMP256_ROW *const precomputed_table =
-        (const PRECOMP256_ROW *)ecp_nistz256_precomputed;
-    ecp_nistz256_select_w7(&p.a, precomputed_table[0], wvalue >> 1);
+  assert(group->field.width == P256_LIMBS);
+  OPENSSL_memcpy(r->X.words, out.X, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Y.words, out.Y, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Z.words, out.Z, P256_LIMBS * sizeof(BN_ULONG));
+}
 
-    ecp_nistz256_neg(p.p.Z, p.p.Y);
-    copy_conditional(p.p.Y, p.p.Z, wvalue & 1);
+static void ecp_nistz256_point_mul_base(const EC_GROUP *group, EC_RAW_POINT *r,
+                                        const EC_SCALAR *scalar) {
+  alignas(32) p256_point_union_t t, p;
 
-    // Convert |p| from affine to Jacobian coordinates. We set Z to zero if |p|
-    // is infinity and |ONE| otherwise. |p| was computed from the table, so it
-    // is infinity iff |wvalue >> 1| is zero.
-    OPENSSL_memset(p.p.Z, 0, sizeof(p.p.Z));
-    copy_conditional(p.p.Z, ONE, is_not_zero(wvalue >> 1));
+  uint8_t p_str[33];
+  OPENSSL_memcpy(p_str, scalar->bytes, 32);
+  p_str[32] = 0;
 
-    for (int i = 1; i < 37; i++) {
-      unsigned off = (index - 1) / 8;
-      wvalue = p_str[off] | p_str[off + 1] << 8;
-      wvalue = (wvalue >> ((index - 1) % 8)) & kMask;
-      index += kWindowSize;
+  // First window
+  unsigned index = 0;
+  unsigned wvalue = calc_first_wvalue(&index, p_str);
 
-      wvalue = booth_recode_w7(wvalue);
+  ecp_nistz256_select_w7(&p.a, ecp_nistz256_precomputed[0], wvalue >> 1);
+  ecp_nistz256_neg(p.p.Z, p.p.Y);
+  copy_conditional(p.p.Y, p.p.Z, wvalue & 1);
 
-      ecp_nistz256_select_w7(&t.a, precomputed_table[i], wvalue >> 1);
+  // Convert |p| from affine to Jacobian coordinates. We set Z to zero if |p|
+  // is infinity and |ONE| otherwise. |p| was computed from the table, so it
+  // is infinity iff |wvalue >> 1| is zero.
+  OPENSSL_memset(p.p.Z, 0, sizeof(p.p.Z));
+  copy_conditional(p.p.Z, ONE, is_not_zero(wvalue >> 1));
 
-      ecp_nistz256_neg(t.p.Z, t.a.Y);
-      copy_conditional(t.a.Y, t.p.Z, wvalue & 1);
+  for (int i = 1; i < 37; i++) {
+    wvalue = calc_wvalue(&index, p_str);
 
-      ecp_nistz256_point_add_affine(&p.p, &p.p, &t.a);
-    }
+    ecp_nistz256_select_w7(&t.a, ecp_nistz256_precomputed[i], wvalue >> 1);
+
+    ecp_nistz256_neg(t.p.Z, t.a.Y);
+    copy_conditional(t.a.Y, t.p.Z, wvalue & 1);
+
+    // Note |ecp_nistz256_point_add_affine| does not work if |p.p| and |t.a|
+    // are the same non-infinity point.
+    ecp_nistz256_point_add_affine(&p.p, &p.p, &t.a);
   }
 
-  const int p_is_infinity = g_scalar == NULL;
-  if (p_scalar != NULL) {
-    P256_POINT *out = &t.p;
-    if (p_is_infinity) {
-      out = &p.p;
-    }
+  assert(group->field.width == P256_LIMBS);
+  OPENSSL_memcpy(r->X.words, p.p.X, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Y.words, p.p.Y, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Z.words, p.p.Z, P256_LIMBS * sizeof(BN_ULONG));
+}
 
-    if (!ecp_nistz256_windowed_mul(group, out, p_, p_scalar)) {
-      return 0;
-    }
+static void ecp_nistz256_points_mul_public(const EC_GROUP *group,
+                                           EC_RAW_POINT *r,
+                                           const EC_SCALAR *g_scalar,
+                                           const EC_RAW_POINT *p_,
+                                           const EC_SCALAR *p_scalar) {
+  assert(p_ != NULL && p_scalar != NULL && g_scalar != NULL);
 
-    if (!p_is_infinity) {
-      ecp_nistz256_point_add(&p.p, &p.p, out);
-    }
+  alignas(32) p256_point_union_t t, p;
+  uint8_t p_str[33];
+  OPENSSL_memcpy(p_str, g_scalar->bytes, 32);
+  p_str[32] = 0;
+
+  // First window
+  unsigned index = 0;
+  unsigned wvalue = calc_first_wvalue(&index, p_str);
+
+  // Convert |p| from affine to Jacobian coordinates. We set Z to zero if |p|
+  // is infinity and |ONE| otherwise. |p| was computed from the table, so it
+  // is infinity iff |wvalue >> 1| is zero.
+  if ((wvalue >> 1) != 0) {
+    OPENSSL_memcpy(&p.a, &ecp_nistz256_precomputed[0][(wvalue >> 1) - 1],
+                   sizeof(p.a));
+    OPENSSL_memcpy(&p.p.Z, ONE, sizeof(p.p.Z));
+  } else {
+    OPENSSL_memset(&p.a, 0, sizeof(p.a));
+    OPENSSL_memset(p.p.Z, 0, sizeof(p.p.Z));
   }
 
-  // Not constant-time, but we're only operating on the public output.
-  if (!bn_set_words(&r->X, p.p.X, P256_LIMBS) ||
-      !bn_set_words(&r->Y, p.p.Y, P256_LIMBS) ||
-      !bn_set_words(&r->Z, p.p.Z, P256_LIMBS)) {
-    return 0;
+  if ((wvalue & 1) == 1) {
+    ecp_nistz256_neg(p.p.Y, p.p.Y);
   }
 
-  return 1;
-}
+  for (int i = 1; i < 37; i++) {
+    wvalue = calc_wvalue(&index, p_str);
 
-static int ecp_nistz256_get_affine(const EC_GROUP *group, const EC_POINT *point,
-                                   BIGNUM *x, BIGNUM *y, BN_CTX *ctx) {
-  BN_ULONG z_inv2[P256_LIMBS];
-  BN_ULONG z_inv3[P256_LIMBS];
-  BN_ULONG point_x[P256_LIMBS], point_y[P256_LIMBS], point_z[P256_LIMBS];
+    if ((wvalue >> 1) == 0) {
+      continue;
+    }
 
-  if (EC_POINT_is_at_infinity(group, point)) {
-    OPENSSL_PUT_ERROR(EC, EC_R_POINT_AT_INFINITY);
-    return 0;
+    OPENSSL_memcpy(&t.a, &ecp_nistz256_precomputed[i][(wvalue >> 1) - 1],
+                   sizeof(p.a));
+
+    if ((wvalue & 1) == 1) {
+      ecp_nistz256_neg(t.a.Y, t.a.Y);
+    }
+
+    // Note |ecp_nistz256_point_add_affine| does not work if |p.p| and |t.a|
+    // are the same non-infinity point, so it is important that we compute the
+    // |g_scalar| term before the |p_scalar| term.
+    ecp_nistz256_point_add_affine(&p.p, &p.p, &t.a);
   }
 
-  if (!ecp_nistz256_bignum_to_field_elem(point_x, &point->X) ||
-      !ecp_nistz256_bignum_to_field_elem(point_y, &point->Y) ||
-      !ecp_nistz256_bignum_to_field_elem(point_z, &point->Z)) {
-    OPENSSL_PUT_ERROR(EC, EC_R_COORDINATES_OUT_OF_RANGE);
+  ecp_nistz256_windowed_mul(group, &t.p, p_, p_scalar);
+  ecp_nistz256_point_add(&p.p, &p.p, &t.p);
+
+  assert(group->field.width == P256_LIMBS);
+  OPENSSL_memcpy(r->X.words, p.p.X, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Y.words, p.p.Y, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Z.words, p.p.Z, P256_LIMBS * sizeof(BN_ULONG));
+}
+
+static int ecp_nistz256_get_affine(const EC_GROUP *group,
+                                   const EC_RAW_POINT *point, EC_FELEM *x,
+                                   EC_FELEM *y) {
+  if (ec_GFp_simple_is_at_infinity(group, point)) {
+    OPENSSL_PUT_ERROR(EC, EC_R_POINT_AT_INFINITY);
     return 0;
   }
 
-  ecp_nistz256_mod_inverse_mont(z_inv3, point_z);
+  BN_ULONG z_inv2[P256_LIMBS];
+  BN_ULONG z_inv3[P256_LIMBS];
+  assert(group->field.width == P256_LIMBS);
+  ecp_nistz256_mod_inverse_mont(z_inv3, point->Z.words);
   ecp_nistz256_sqr_mont(z_inv2, z_inv3);
 
   // Instead of using |ecp_nistz256_from_mont| to convert the |x| coordinate
@@ -409,39 +452,200 @@ static int ecp_nistz256_get_affine(const EC_GROUP *group, const EC_POINT *point,
   ecp_nistz256_from_mont(z_inv2, z_inv2);
 
   if (x != NULL) {
-    BN_ULONG x_aff[P256_LIMBS];
-    ecp_nistz256_mul_mont(x_aff, z_inv2, point_x);
-    if (!bn_set_words(x, x_aff, P256_LIMBS)) {
-      OPENSSL_PUT_ERROR(EC, ERR_R_MALLOC_FAILURE);
-      return 0;
-    }
+    ecp_nistz256_mul_mont(x->words, z_inv2, point->X.words);
   }
 
   if (y != NULL) {
-    BN_ULONG y_aff[P256_LIMBS];
     ecp_nistz256_mul_mont(z_inv3, z_inv3, z_inv2);
-    ecp_nistz256_mul_mont(y_aff, z_inv3, point_y);
-    if (!bn_set_words(y, y_aff, P256_LIMBS)) {
-      OPENSSL_PUT_ERROR(EC, ERR_R_MALLOC_FAILURE);
-      return 0;
-    }
+    ecp_nistz256_mul_mont(y->words, z_inv3, point->Y.words);
   }
 
   return 1;
 }
 
+static void ecp_nistz256_add(const EC_GROUP *group, EC_RAW_POINT *r,
+                             const EC_RAW_POINT *a_, const EC_RAW_POINT *b_) {
+  P256_POINT a, b;
+  OPENSSL_memcpy(a.X, a_->X.words, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(a.Y, a_->Y.words, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(a.Z, a_->Z.words, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(b.X, b_->X.words, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(b.Y, b_->Y.words, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(b.Z, b_->Z.words, P256_LIMBS * sizeof(BN_ULONG));
+  ecp_nistz256_point_add(&a, &a, &b);
+  OPENSSL_memcpy(r->X.words, a.X, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Y.words, a.Y, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Z.words, a.Z, P256_LIMBS * sizeof(BN_ULONG));
+}
+
+static void ecp_nistz256_dbl(const EC_GROUP *group, EC_RAW_POINT *r,
+                             const EC_RAW_POINT *a_) {
+  P256_POINT a;
+  OPENSSL_memcpy(a.X, a_->X.words, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(a.Y, a_->Y.words, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(a.Z, a_->Z.words, P256_LIMBS * sizeof(BN_ULONG));
+  ecp_nistz256_point_double(&a, &a);
+  OPENSSL_memcpy(r->X.words, a.X, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Y.words, a.Y, P256_LIMBS * sizeof(BN_ULONG));
+  OPENSSL_memcpy(r->Z.words, a.Z, P256_LIMBS * sizeof(BN_ULONG));
+}
+
+static void ecp_nistz256_inv_mod_ord(const EC_GROUP *group, EC_SCALAR *out,
+                                     const EC_SCALAR *in) {
+  // table[i] stores a power of |in| corresponding to the matching enum value.
+  enum {
+    // The following indices specify the power in binary.
+    i_1 = 0,
+    i_10,
+    i_11,
+    i_101,
+    i_111,
+    i_1010,
+    i_1111,
+    i_10101,
+    i_101010,
+    i_101111,
+    // The following indices specify 2^N-1, or N ones in a row.
+    i_x6,
+    i_x8,
+    i_x16,
+    i_x32
+  };
+  BN_ULONG table[15][P256_LIMBS];
+
+  // https://briansmith.org/ecc-inversion-addition-chains-01#p256_scalar_inversion
+  //
+  // Even though this code path spares 12 squarings, 4.5%, and 13
+  // multiplications, 25%, the overall sign operation is not that much faster,
+  // not more that 2%. Most of the performance of this function comes from the
+  // scalar operations.
+
+  // Pre-calculate powers.
+  OPENSSL_memcpy(table[i_1], in->words, P256_LIMBS * sizeof(BN_ULONG));
+
+  ecp_nistz256_ord_sqr_mont(table[i_10], table[i_1], 1);
+
+  ecp_nistz256_ord_mul_mont(table[i_11], table[i_1], table[i_10]);
+
+  ecp_nistz256_ord_mul_mont(table[i_101], table[i_11], table[i_10]);
+
+  ecp_nistz256_ord_mul_mont(table[i_111], table[i_101], table[i_10]);
+
+  ecp_nistz256_ord_sqr_mont(table[i_1010], table[i_101], 1);
+
+  ecp_nistz256_ord_mul_mont(table[i_1111], table[i_1010], table[i_101]);
+
+  ecp_nistz256_ord_sqr_mont(table[i_10101], table[i_1010], 1);
+  ecp_nistz256_ord_mul_mont(table[i_10101], table[i_10101], table[i_1]);
+
+  ecp_nistz256_ord_sqr_mont(table[i_101010], table[i_10101], 1);
+
+  ecp_nistz256_ord_mul_mont(table[i_101111], table[i_101010], table[i_101]);
+
+  ecp_nistz256_ord_mul_mont(table[i_x6], table[i_101010], table[i_10101]);
+
+  ecp_nistz256_ord_sqr_mont(table[i_x8], table[i_x6], 2);
+  ecp_nistz256_ord_mul_mont(table[i_x8], table[i_x8], table[i_11]);
+
+  ecp_nistz256_ord_sqr_mont(table[i_x16], table[i_x8], 8);
+  ecp_nistz256_ord_mul_mont(table[i_x16], table[i_x16], table[i_x8]);
+
+  ecp_nistz256_ord_sqr_mont(table[i_x32], table[i_x16], 16);
+  ecp_nistz256_ord_mul_mont(table[i_x32], table[i_x32], table[i_x16]);
+
+  // Compute |in| raised to the order-2.
+  ecp_nistz256_ord_sqr_mont(out->words, table[i_x32], 64);
+  ecp_nistz256_ord_mul_mont(out->words, out->words, table[i_x32]);
+  static const struct {
+    uint8_t p, i;
+  } kChain[27] = {{32, i_x32},    {6, i_101111}, {5, i_111},    {4, i_11},
+                  {5, i_1111},    {5, i_10101},  {4, i_101},    {3, i_101},
+                  {3, i_101},     {5, i_111},    {9, i_101111}, {6, i_1111},
+                  {2, i_1},       {5, i_1},      {6, i_1111},   {5, i_111},
+                  {4, i_111},     {5, i_111},    {5, i_101},    {3, i_11},
+                  {10, i_101111}, {2, i_11},     {5, i_11},     {5, i_11},
+                  {3, i_1},       {7, i_10101},  {6, i_1111}};
+  for (size_t i = 0; i < OPENSSL_ARRAY_SIZE(kChain); i++) {
+    ecp_nistz256_ord_sqr_mont(out->words, out->words, kChain[i].p);
+    ecp_nistz256_ord_mul_mont(out->words, out->words, table[kChain[i].i]);
+  }
+}
+
+static int ecp_nistz256_mont_inv_mod_ord_vartime(const EC_GROUP *group,
+                                                 EC_SCALAR *out,
+                                                 const EC_SCALAR *in) {
+  if ((OPENSSL_ia32cap_get()[1] & (1 << 28)) == 0) {
+    // No AVX support; fallback to generic code.
+    return ec_GFp_simple_mont_inv_mod_ord_vartime(group, out, in);
+  }
+
+  assert(group->order.width == P256_LIMBS);
+  if (!beeu_mod_inverse_vartime(out->words, in->words, group->order.d)) {
+    return 0;
+  }
+
+  // The result should be returned in the Montgomery domain.
+  ec_scalar_to_montgomery(group, out, out);
+  return 1;
+}
+
+static int ecp_nistz256_cmp_x_coordinate(const EC_GROUP *group,
+                                         const EC_RAW_POINT *p,
+                                         const EC_SCALAR *r) {
+  if (ec_GFp_simple_is_at_infinity(group, p)) {
+    return 0;
+  }
+
+  assert(group->order.width == P256_LIMBS);
+  assert(group->field.width == P256_LIMBS);
+
+  // We wish to compare X/Z^2 with r. This is equivalent to comparing X with
+  // r*Z^2. Note that X and Z are represented in Montgomery form, while r is
+  // not.
+  BN_ULONG r_Z2[P256_LIMBS], Z2_mont[P256_LIMBS], X[P256_LIMBS];
+  ecp_nistz256_mul_mont(Z2_mont, p->Z.words, p->Z.words);
+  ecp_nistz256_mul_mont(r_Z2, r->words, Z2_mont);
+  ecp_nistz256_from_mont(X, p->X.words);
+
+  if (OPENSSL_memcmp(r_Z2, X, sizeof(r_Z2)) == 0) {
+    return 1;
+  }
+
+  // During signing the x coefficient is reduced modulo the group order.
+  // Therefore there is a small possibility, less than 1/2^128, that group_order
+  // < p.x < P. in that case we need not only to compare against |r| but also to
+  // compare against r+group_order.
+  if (bn_less_than_words(r->words, group->field_minus_order.words,
+                         P256_LIMBS)) {
+    // We can ignore the carry because: r + group_order < p < 2^256.
+    bn_add_words(r_Z2, r->words, group->order.d, P256_LIMBS);
+    ecp_nistz256_mul_mont(r_Z2, r_Z2, Z2_mont);
+    if (OPENSSL_memcmp(r_Z2, X, sizeof(r_Z2)) == 0) {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
 DEFINE_METHOD_FUNCTION(EC_METHOD, EC_GFp_nistz256_method) {
   out->group_init = ec_GFp_mont_group_init;
   out->group_finish = ec_GFp_mont_group_finish;
   out->group_set_curve = ec_GFp_mont_group_set_curve;
   out->point_get_affine_coordinates = ecp_nistz256_get_affine;
-  out->mul = ecp_nistz256_points_mul;
-  out->mul_public = ecp_nistz256_points_mul;
-  out->field_mul = ec_GFp_mont_field_mul;
-  out->field_sqr = ec_GFp_mont_field_sqr;
-  out->field_encode = ec_GFp_mont_field_encode;
-  out->field_decode = ec_GFp_mont_field_decode;
-};
+  out->add = ecp_nistz256_add;
+  out->dbl = ecp_nistz256_dbl;
+  out->mul = ecp_nistz256_point_mul;
+  out->mul_base = ecp_nistz256_point_mul_base;
+  out->mul_public = ecp_nistz256_points_mul_public;
+  out->felem_mul = ec_GFp_mont_felem_mul;
+  out->felem_sqr = ec_GFp_mont_felem_sqr;
+  out->bignum_to_felem = ec_GFp_mont_bignum_to_felem;
+  out->felem_to_bignum = ec_GFp_mont_felem_to_bignum;
+  out->scalar_inv_montgomery = ecp_nistz256_inv_mod_ord;
+  out->scalar_inv_montgomery_vartime = ecp_nistz256_mont_inv_mod_ord_vartime;
+  out->cmp_x_coordinate = ecp_nistz256_cmp_x_coordinate;
+}
 
 #endif /* !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && \
           !defined(OPENSSL_SMALL) */