grpc 1.55.0 → 1.56.0.pre3

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/gcm.c

@@ -62,9 +62,9 @@
 static const size_t kSizeTWithoutLower4Bits = (size_t) -16;
 
 
-#define GCM_MUL(ctx, Xi) gcm_gmult_nohw((ctx)->Xi.u, (ctx)->gcm_key.Htable)
+#define GCM_MUL(ctx, Xi) gcm_gmult_nohw((ctx)->Xi, (ctx)->gcm_key.Htable)
 #define GHASH(ctx, in, len) \
-  gcm_ghash_nohw((ctx)->Xi.u, (ctx)->gcm_key.Htable, in, len)
+  gcm_ghash_nohw((ctx)->Xi, (ctx)->gcm_key.Htable, in, len)
 // GHASH_CHUNK is "stride parameter" missioned to mitigate cache
 // trashing effect. In other words idea is to hash data while it's
 // still in L1 cache after encryption pass...
@@ -126,23 +126,23 @@ void gcm_init_ssse3(u128 Htable[16], const uint64_t H[2]) {
 
 #ifdef GCM_FUNCREF
 #undef GCM_MUL
-#define GCM_MUL(ctx, Xi) (*gcm_gmult_p)((ctx)->Xi.u, (ctx)->gcm_key.Htable)
+#define GCM_MUL(ctx, Xi) (*gcm_gmult_p)((ctx)->Xi, (ctx)->gcm_key.Htable)
 #undef GHASH
 #define GHASH(ctx, in, len) \
-  (*gcm_ghash_p)((ctx)->Xi.u, (ctx)->gcm_key.Htable, in, len)
+  (*gcm_ghash_p)((ctx)->Xi, (ctx)->gcm_key.Htable, in, len)
 #endif  // GCM_FUNCREF
 
 #if defined(HW_GCM) && defined(OPENSSL_X86_64)
 static size_t hw_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
                              const AES_KEY *key, uint8_t ivec[16],
-                             uint64_t *Xi) {
-  return aesni_gcm_encrypt(in, out, len, key, ivec, Xi);
+                             uint8_t Xi[16], const u128 Htable[16]) {
+  return aesni_gcm_encrypt(in, out, len, key, ivec, Htable, Xi);
 }
 
 static size_t hw_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
                              const AES_KEY *key, uint8_t ivec[16],
-                             uint64_t *Xi) {
-  return aesni_gcm_decrypt(in, out, len, key, ivec, Xi);
+                             uint8_t Xi[16], const u128 Htable[16]) {
+  return aesni_gcm_decrypt(in, out, len, key, ivec, Htable, Xi);
 }
 #endif  // HW_GCM && X86_64
 
@@ -150,38 +150,36 @@ static size_t hw_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
 
 static size_t hw_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
                              const AES_KEY *key, uint8_t ivec[16],
-                             uint64_t *Xi) {
+                             uint8_t Xi[16], const u128 Htable[16]) {
   const size_t len_blocks = len & kSizeTWithoutLower4Bits;
   if (!len_blocks) {
     return 0;
   }
-  aes_gcm_enc_kernel(in, len_blocks * 8, out, Xi, ivec, key);
+  aes_gcm_enc_kernel(in, len_blocks * 8, out, Xi, ivec, key, Htable);
   return len_blocks;
 }
 
 static size_t hw_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
                              const AES_KEY *key, uint8_t ivec[16],
-                             uint64_t *Xi) {
+                             uint8_t Xi[16], const u128 Htable[16]) {
   const size_t len_blocks = len & kSizeTWithoutLower4Bits;
   if (!len_blocks) {
     return 0;
   }
-  aes_gcm_dec_kernel(in, len_blocks * 8, out, Xi, ivec, key);
+  aes_gcm_dec_kernel(in, len_blocks * 8, out, Xi, ivec, key, Htable);
   return len_blocks;
 }
 
 #endif  // HW_GCM && AARCH64
 
 void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
-                       u128 *out_key, u128 out_table[16], int *out_is_avx,
+                       u128 out_table[16], int *out_is_avx,
                        const uint8_t gcm_key[16]) {
   *out_is_avx = 0;
 
-  // H is stored in host byte order.
+  // H is passed to |gcm_init_*| as a pair of byte-swapped, 64-bit values.
   uint64_t H[2] = {CRYPTO_load_u64_be(gcm_key),
                    CRYPTO_load_u64_be(gcm_key + 8)};
-  out_key->hi = H[0];
-  out_key->lo = H[1];
 
 #if defined(GHASH_ASM_X86_64)
   if (crypto_gcm_clmul_enabled()) {
@@ -247,89 +245,88 @@ void CRYPTO_gcm128_init_key(GCM128_KEY *gcm_key, const AES_KEY *aes_key,
   (*block)(ghash_key, ghash_key, aes_key);
 
   int is_avx;
-  CRYPTO_ghash_init(&gcm_key->gmult, &gcm_key->ghash, &gcm_key->H,
-                    gcm_key->Htable, &is_avx, ghash_key);
+  CRYPTO_ghash_init(&gcm_key->gmult, &gcm_key->ghash, gcm_key->Htable, &is_avx,
+                    ghash_key);
 
 #if defined(OPENSSL_AARCH64) && !defined(OPENSSL_NO_ASM)
-    gcm_key->use_hw_gcm_crypt = (gcm_pmull_capable() && block_is_hwaes) ? 1 :
-      0;
+  gcm_key->use_hw_gcm_crypt = (gcm_pmull_capable() && block_is_hwaes) ? 1 : 0;
 #else
-    gcm_key->use_hw_gcm_crypt = (is_avx && block_is_hwaes) ? 1 : 0;
+  gcm_key->use_hw_gcm_crypt = (is_avx && block_is_hwaes) ? 1 : 0;
 #endif
 }
 
 void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const AES_KEY *key,
                          const uint8_t *iv, size_t len) {
 #ifdef GCM_FUNCREF
-  void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
+  void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) =
       ctx->gcm_key.gmult;
 #endif
 
-  ctx->Yi.u[0] = 0;
-  ctx->Yi.u[1] = 0;
-  ctx->Xi.u[0] = 0;
-  ctx->Xi.u[1] = 0;
-  ctx->len.u[0] = 0;  // AAD length
-  ctx->len.u[1] = 0;  // message length
+  OPENSSL_memset(&ctx->Yi, 0, sizeof(ctx->Yi));
+  OPENSSL_memset(&ctx->Xi, 0, sizeof(ctx->Xi));
+  ctx->len.aad = 0;
+  ctx->len.msg = 0;
   ctx->ares = 0;
   ctx->mres = 0;
 
   uint32_t ctr;
   if (len == 12) {
-    OPENSSL_memcpy(ctx->Yi.c, iv, 12);
-    ctx->Yi.c[15] = 1;
+    OPENSSL_memcpy(ctx->Yi, iv, 12);
+    ctx->Yi[15] = 1;
     ctr = 1;
   } else {
     uint64_t len0 = len;
 
     while (len >= 16) {
-      for (size_t i = 0; i < 16; ++i) {
-        ctx->Yi.c[i] ^= iv[i];
-      }
+      CRYPTO_xor16(ctx->Yi, ctx->Yi, iv);
       GCM_MUL(ctx, Yi);
       iv += 16;
       len -= 16;
     }
     if (len) {
       for (size_t i = 0; i < len; ++i) {
-        ctx->Yi.c[i] ^= iv[i];
+        ctx->Yi[i] ^= iv[i];
       }
       GCM_MUL(ctx, Yi);
     }
-    len0 <<= 3;
-    ctx->Yi.u[1] ^= CRYPTO_bswap8(len0);
+
+    uint8_t len_block[16];
+    OPENSSL_memset(len_block, 0, 8);
+    CRYPTO_store_u64_be(len_block + 8, len0 << 3);
+    CRYPTO_xor16(ctx->Yi, ctx->Yi, len_block);
 
     GCM_MUL(ctx, Yi);
-    ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
+    ctr = CRYPTO_load_u32_be(ctx->Yi + 12);
   }
 
-  (*ctx->gcm_key.block)(ctx->Yi.c, ctx->EK0.c, key);
+  (*ctx->gcm_key.block)(ctx->Yi, ctx->EK0, key);
   ++ctr;
-  ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
+  CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
 }
 
 int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad, size_t len) {
 #ifdef GCM_FUNCREF
-  void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
+  void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) =
       ctx->gcm_key.gmult;
-  void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+  void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
                       size_t len) = ctx->gcm_key.ghash;
 #endif
 
-  if (ctx->len.u[1]) {
+  if (ctx->len.msg != 0) {
+    // The caller must have finished the AAD before providing other input.
     return 0;
   }
 
-  uint64_t alen = ctx->len.u[0] + len;
+  uint64_t alen = ctx->len.aad + len;
   if (alen > (UINT64_C(1) << 61) || (sizeof(len) == 8 && alen < len)) {
     return 0;
   }
-  ctx->len.u[0] = alen;
+  ctx->len.aad = alen;
 
   unsigned n = ctx->ares;
   if (n) {
     while (n && len) {
-      ctx->Xi.c[n] ^= *(aad++);
+      ctx->Xi[n] ^= *(aad++);
       --len;
       n = (n + 1) % 16;
     }
@@ -353,7 +350,7 @@ int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad, size_t len) {
   if (len != 0) {
     n = (unsigned int)len;
     for (size_t i = 0; i < len; ++i) {
-      ctx->Xi.c[i] ^= aad[i];
+      ctx->Xi[i] ^= aad[i];
     }
   }
 
@@ -365,18 +362,18 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
                           const uint8_t *in, uint8_t *out, size_t len) {
   block128_f block = ctx->gcm_key.block;
 #ifdef GCM_FUNCREF
-  void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
+  void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) =
       ctx->gcm_key.gmult;
-  void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+  void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
                       size_t len) = ctx->gcm_key.ghash;
 #endif
 
-  uint64_t mlen = ctx->len.u[1] + len;
+  uint64_t mlen = ctx->len.msg + len;
   if (mlen > ((UINT64_C(1) << 36) - 32) ||
       (sizeof(len) == 8 && mlen < len)) {
     return 0;
   }
-  ctx->len.u[1] = mlen;
+  ctx->len.msg = mlen;
 
   if (ctx->ares) {
     // First call to encrypt finalizes GHASH(AAD)
@@ -387,7 +384,7 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
   unsigned n = ctx->mres;
   if (n) {
     while (n && len) {
-      ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
+      ctx->Xi[n] ^= *(out++) = *(in++) ^ ctx->EKi[n];
       --len;
       n = (n + 1) % 16;
     }
@@ -399,19 +396,15 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
     }
   }
 
-  uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
+  uint32_t ctr = CRYPTO_load_u32_be(ctx->Yi + 12);
   while (len >= GHASH_CHUNK) {
     size_t j = GHASH_CHUNK;
 
     while (j) {
-      (*block)(ctx->Yi.c, ctx->EKi.c, key);
+      (*block)(ctx->Yi, ctx->EKi, key);
       ++ctr;
-      ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
-      for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
-        CRYPTO_store_word_le(out + i,
-                             CRYPTO_load_word_le(in + i) ^
-                                 ctx->EKi.t[i / sizeof(crypto_word_t)]);
-      }
+      CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
+      CRYPTO_xor16(out, in, ctx->EKi);
       out += 16;
       in += 16;
       j -= 16;
@@ -422,14 +415,10 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
   size_t len_blocks = len & kSizeTWithoutLower4Bits;
   if (len_blocks != 0) {
     while (len >= 16) {
-      (*block)(ctx->Yi.c, ctx->EKi.c, key);
+      (*block)(ctx->Yi, ctx->EKi, key);
       ++ctr;
-      ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
-      for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
-        CRYPTO_store_word_le(out + i,
-                             CRYPTO_load_word_le(in + i) ^
-                                 ctx->EKi.t[i / sizeof(crypto_word_t)]);
-      }
+      CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
+      CRYPTO_xor16(out, in, ctx->EKi);
       out += 16;
       in += 16;
       len -= 16;
@@ -437,11 +426,11 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
     GHASH(ctx, out - len_blocks, len_blocks);
   }
   if (len) {
-    (*block)(ctx->Yi.c, ctx->EKi.c, key);
+    (*block)(ctx->Yi, ctx->EKi, key);
     ++ctr;
-    ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
+    CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
     while (len--) {
-      ctx->Xi.c[n] ^= out[n] = in[n] ^ ctx->EKi.c[n];
+      ctx->Xi[n] ^= out[n] = in[n] ^ ctx->EKi[n];
       ++n;
     }
   }
@@ -455,18 +444,18 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
                           size_t len) {
   block128_f block = ctx->gcm_key.block;
 #ifdef GCM_FUNCREF
-  void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
+  void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) =
       ctx->gcm_key.gmult;
-  void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+  void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
                       size_t len) = ctx->gcm_key.ghash;
 #endif
 
-  uint64_t mlen = ctx->len.u[1] + len;
+  uint64_t mlen = ctx->len.msg + len;
   if (mlen > ((UINT64_C(1) << 36) - 32) ||
       (sizeof(len) == 8 && mlen < len)) {
     return 0;
   }
-  ctx->len.u[1] = mlen;
+  ctx->len.msg = mlen;
 
   if (ctx->ares) {
     // First call to decrypt finalizes GHASH(AAD)
@@ -478,8 +467,8 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
   if (n) {
     while (n && len) {
       uint8_t c = *(in++);
-      *(out++) = c ^ ctx->EKi.c[n];
-      ctx->Xi.c[n] ^= c;
+      *(out++) = c ^ ctx->EKi[n];
+      ctx->Xi[n] ^= c;
       --len;
       n = (n + 1) % 16;
     }
@@ -491,20 +480,16 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
     }
   }
 
-  uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
+  uint32_t ctr = CRYPTO_load_u32_be(ctx->Yi + 12);
   while (len >= GHASH_CHUNK) {
     size_t j = GHASH_CHUNK;
 
     GHASH(ctx, in, GHASH_CHUNK);
     while (j) {
-      (*block)(ctx->Yi.c, ctx->EKi.c, key);
+      (*block)(ctx->Yi, ctx->EKi, key);
       ++ctr;
-      ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
-      for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
-        CRYPTO_store_word_le(out + i,
-                             CRYPTO_load_word_le(in + i) ^
-                                 ctx->EKi.t[i / sizeof(crypto_word_t)]);
-      }
+      CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
+      CRYPTO_xor16(out, in, ctx->EKi);
       out += 16;
       in += 16;
       j -= 16;
@@ -515,27 +500,23 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
   if (len_blocks != 0) {
     GHASH(ctx, in, len_blocks);
     while (len >= 16) {
-      (*block)(ctx->Yi.c, ctx->EKi.c, key);
+      (*block)(ctx->Yi, ctx->EKi, key);
       ++ctr;
-      ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
-      for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
-        CRYPTO_store_word_le(out + i,
-                             CRYPTO_load_word_le(in + i) ^
-                                 ctx->EKi.t[i / sizeof(crypto_word_t)]);
-      }
+      CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
+      CRYPTO_xor16(out, in, ctx->EKi);
       out += 16;
       in += 16;
       len -= 16;
     }
   }
   if (len) {
-    (*block)(ctx->Yi.c, ctx->EKi.c, key);
+    (*block)(ctx->Yi, ctx->EKi, key);
     ++ctr;
-    ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
+    CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
     while (len--) {
       uint8_t c = in[n];
-      ctx->Xi.c[n] ^= c;
-      out[n] = c ^ ctx->EKi.c[n];
+      ctx->Xi[n] ^= c;
+      out[n] = c ^ ctx->EKi[n];
       ++n;
     }
   }
@@ -548,18 +529,18 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
                                 const uint8_t *in, uint8_t *out, size_t len,
                                 ctr128_f stream) {
 #ifdef GCM_FUNCREF
-  void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
+  void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) =
       ctx->gcm_key.gmult;
-  void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+  void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
                       size_t len) = ctx->gcm_key.ghash;
 #endif
 
-  uint64_t mlen = ctx->len.u[1] + len;
+  uint64_t mlen = ctx->len.msg + len;
   if (mlen > ((UINT64_C(1) << 36) - 32) ||
       (sizeof(len) == 8 && mlen < len)) {
     return 0;
   }
-  ctx->len.u[1] = mlen;
+  ctx->len.msg = mlen;
 
   if (ctx->ares) {
     // First call to encrypt finalizes GHASH(AAD)
@@ -570,7 +551,7 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
   unsigned n = ctx->mres;
   if (n) {
     while (n && len) {
-      ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
+      ctx->Xi[n] ^= *(out++) = *(in++) ^ ctx->EKi[n];
       --len;
       n = (n + 1) % 16;
     }
@@ -587,18 +568,19 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
   if (ctx->gcm_key.use_hw_gcm_crypt && len > 0) {
     // |hw_gcm_encrypt| may not process all the input given to it. It may
     // not process *any* of its input if it is deemed too small.
-    size_t bulk = hw_gcm_encrypt(in, out, len, key, ctx->Yi.c, ctx->Xi.u);
+    size_t bulk = hw_gcm_encrypt(in, out, len, key, ctx->Yi, ctx->Xi,
+                                 ctx->gcm_key.Htable);
     in += bulk;
     out += bulk;
     len -= bulk;
   }
 #endif
 
-  uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
+  uint32_t ctr = CRYPTO_load_u32_be(ctx->Yi + 12);
   while (len >= GHASH_CHUNK) {
-    (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
+    (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi);
     ctr += GHASH_CHUNK / 16;
-    ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
+    CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
     GHASH(ctx, out, GHASH_CHUNK);
     out += GHASH_CHUNK;
     in += GHASH_CHUNK;
@@ -608,20 +590,20 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
   if (len_blocks != 0) {
     size_t j = len_blocks / 16;
 
-    (*stream)(in, out, j, key, ctx->Yi.c);
+    (*stream)(in, out, j, key, ctx->Yi);
     ctr += (unsigned int)j;
-    ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
+    CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
     in += len_blocks;
     len -= len_blocks;
     GHASH(ctx, out, len_blocks);
     out += len_blocks;
   }
   if (len) {
-    (*ctx->gcm_key.block)(ctx->Yi.c, ctx->EKi.c, key);
+    (*ctx->gcm_key.block)(ctx->Yi, ctx->EKi, key);
     ++ctr;
-    ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
+    CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
     while (len--) {
-      ctx->Xi.c[n] ^= out[n] = in[n] ^ ctx->EKi.c[n];
+      ctx->Xi[n] ^= out[n] = in[n] ^ ctx->EKi[n];
       ++n;
     }
   }
@@ -634,18 +616,18 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
                                 const uint8_t *in, uint8_t *out, size_t len,
                                 ctr128_f stream) {
 #ifdef GCM_FUNCREF
-  void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
+  void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) =
       ctx->gcm_key.gmult;
-  void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+  void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
                       size_t len) = ctx->gcm_key.ghash;
 #endif
 
-  uint64_t mlen = ctx->len.u[1] + len;
+  uint64_t mlen = ctx->len.msg + len;
   if (mlen > ((UINT64_C(1) << 36) - 32) ||
       (sizeof(len) == 8 && mlen < len)) {
     return 0;
   }
-  ctx->len.u[1] = mlen;
+  ctx->len.msg = mlen;
 
   if (ctx->ares) {
     // First call to decrypt finalizes GHASH(AAD)
@@ -657,8 +639,8 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
   if (n) {
     while (n && len) {
       uint8_t c = *(in++);
-      *(out++) = c ^ ctx->EKi.c[n];
-      ctx->Xi.c[n] ^= c;
+      *(out++) = c ^ ctx->EKi[n];
+      ctx->Xi[n] ^= c;
       --len;
       n = (n + 1) % 16;
     }
@@ -675,19 +657,20 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
   if (ctx->gcm_key.use_hw_gcm_crypt && len > 0) {
     // |hw_gcm_decrypt| may not process all the input given to it. It may
     // not process *any* of its input if it is deemed too small.
-    size_t bulk = hw_gcm_decrypt(in, out, len, key, ctx->Yi.c, ctx->Xi.u);
+    size_t bulk = hw_gcm_decrypt(in, out, len, key, ctx->Yi, ctx->Xi,
+                                 ctx->gcm_key.Htable);
     in += bulk;
     out += bulk;
     len -= bulk;
   }
 #endif
 
-  uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
+  uint32_t ctr = CRYPTO_load_u32_be(ctx->Yi + 12);
   while (len >= GHASH_CHUNK) {
     GHASH(ctx, in, GHASH_CHUNK);
-    (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
+    (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi);
     ctr += GHASH_CHUNK / 16;
-    ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
+    CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
     out += GHASH_CHUNK;
     in += GHASH_CHUNK;
     len -= GHASH_CHUNK;
@@ -697,21 +680,21 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
     size_t j = len_blocks / 16;
 
     GHASH(ctx, in, len_blocks);
-    (*stream)(in, out, j, key, ctx->Yi.c);
+    (*stream)(in, out, j, key, ctx->Yi);
     ctr += (unsigned int)j;
-    ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
+    CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
     out += len_blocks;
     in += len_blocks;
     len -= len_blocks;
   }
   if (len) {
-    (*ctx->gcm_key.block)(ctx->Yi.c, ctx->EKi.c, key);
+    (*ctx->gcm_key.block)(ctx->Yi, ctx->EKi, key);
     ++ctr;
-    ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
+    CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
     while (len--) {
       uint8_t c = in[n];
-      ctx->Xi.c[n] ^= c;
-      out[n] = c ^ ctx->EKi.c[n];
+      ctx->Xi[n] ^= c;
+      out[n] = c ^ ctx->EKi[n];
       ++n;
     }
   }
@@ -722,7 +705,7 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
 
 int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag, size_t len) {
 #ifdef GCM_FUNCREF
-  void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
+  void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) =
       ctx->gcm_key.gmult;
 #endif
 
@@ -730,15 +713,15 @@ int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag, size_t len) {
     GCM_MUL(ctx, Xi);
   }
 
-  ctx->Xi.u[0] ^= CRYPTO_bswap8(ctx->len.u[0] << 3);
-  ctx->Xi.u[1] ^= CRYPTO_bswap8(ctx->len.u[1] << 3);
+  uint8_t len_block[16];
+  CRYPTO_store_u64_be(len_block, ctx->len.aad << 3);
+  CRYPTO_store_u64_be(len_block + 8, ctx->len.msg << 3);
+  CRYPTO_xor16(ctx->Xi, ctx->Xi, len_block);
   GCM_MUL(ctx, Xi);
-
-  ctx->Xi.u[0] ^= ctx->EK0.u[0];
-  ctx->Xi.u[1] ^= ctx->EK0.u[1];
+  CRYPTO_xor16(ctx->Xi, ctx->Xi, ctx->EK0);
 
   if (tag && len <= sizeof(ctx->Xi)) {
-    return CRYPTO_memcmp(ctx->Xi.c, tag, len) == 0;
+    return CRYPTO_memcmp(ctx->Xi, tag, len) == 0;
   } else {
     return 0;
   }
@@ -746,8 +729,7 @@ int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag, size_t len) {
 
 void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, unsigned char *tag, size_t len) {
   CRYPTO_gcm128_finish(ctx, NULL, 0);
-  OPENSSL_memcpy(tag, ctx->Xi.c,
-                 len <= sizeof(ctx->Xi.c) ? len : sizeof(ctx->Xi.c));
+  OPENSSL_memcpy(tag, ctx->Xi, len <= sizeof(ctx->Xi) ? len : sizeof(ctx->Xi));
 }
 
 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/gcm_nohw.c

@@ -274,31 +274,29 @@ static void gcm_polyval_nohw(uint64_t Xi[2], const u128 *H) {
   Xi[1] = r3;
 }
 
-void gcm_gmult_nohw(uint64_t Xi[2], const u128 Htable[16]) {
+void gcm_gmult_nohw(uint8_t Xi[16], const u128 Htable[16]) {
   uint64_t swapped[2];
-  swapped[0] = CRYPTO_bswap8(Xi[1]);
-  swapped[1] = CRYPTO_bswap8(Xi[0]);
+  swapped[0] = CRYPTO_load_u64_be(Xi + 8);
+  swapped[1] = CRYPTO_load_u64_be(Xi);
   gcm_polyval_nohw(swapped, &Htable[0]);
-  Xi[0] = CRYPTO_bswap8(swapped[1]);
-  Xi[1] = CRYPTO_bswap8(swapped[0]);
+  CRYPTO_store_u64_be(Xi, swapped[1]);
+  CRYPTO_store_u64_be(Xi + 8, swapped[0]);
 }
 
-void gcm_ghash_nohw(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+void gcm_ghash_nohw(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
                     size_t len) {
   uint64_t swapped[2];
-  swapped[0] = CRYPTO_bswap8(Xi[1]);
-  swapped[1] = CRYPTO_bswap8(Xi[0]);
+  swapped[0] = CRYPTO_load_u64_be(Xi + 8);
+  swapped[1] = CRYPTO_load_u64_be(Xi);
 
   while (len >= 16) {
-    uint64_t block[2];
-    OPENSSL_memcpy(block, inp, 16);
-    swapped[0] ^= CRYPTO_bswap8(block[1]);
-    swapped[1] ^= CRYPTO_bswap8(block[0]);
+    swapped[0] ^= CRYPTO_load_u64_be(inp + 8);
+    swapped[1] ^= CRYPTO_load_u64_be(inp);
     gcm_polyval_nohw(swapped, &Htable[0]);
     inp += 16;
     len -= 16;
   }
 
-  Xi[0] = CRYPTO_bswap8(swapped[1]);
-  Xi[1] = CRYPTO_bswap8(swapped[0]);
+  CRYPTO_store_u64_be(Xi, swapped[1]);
+  CRYPTO_store_u64_be(Xi + 8, swapped[0]);
 }