grpc 1.15.0 → 1.30.0

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/gcm_nohw.c

@@ -0,0 +1,304 @@
+/* Copyright (c) 2019, Google Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+#include <openssl/base.h>
+
+#include "../../internal.h"
+#include "internal.h"
+
+#if !defined(BORINGSSL_HAS_UINT128) && defined(OPENSSL_SSE2)
+#include <emmintrin.h>
+#endif
+
+
+// This file contains a constant-time implementation of GHASH based on the notes
+// in https://bearssl.org/constanttime.html#ghash-for-gcm and the reduction
+// algorithm described in
+// https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf.
+//
+// Unlike the BearSSL notes, we use uint128_t in the 64-bit implementation. Our
+// primary compilers (clang, clang-cl, and gcc) all support it. MSVC will run
+// the 32-bit implementation, but we can use its intrinsics if necessary.
+
+#if defined(BORINGSSL_HAS_UINT128)
+
+static void gcm_mul64_nohw(uint64_t *out_lo, uint64_t *out_hi, uint64_t a,
+                           uint64_t b) {
+  // One term every four bits means the largest term is 64/4 = 16, which barely
+  // overflows into the next term. Using one term every five bits would cost 25
+  // multiplications instead of 16. It is faster to mask off the bottom four
+  // bits of |a|, giving a largest term of 60/4 = 15, and apply the bottom bits
+  // separately.
+  uint64_t a0 = a & UINT64_C(0x1111111111111110);
+  uint64_t a1 = a & UINT64_C(0x2222222222222220);
+  uint64_t a2 = a & UINT64_C(0x4444444444444440);
+  uint64_t a3 = a & UINT64_C(0x8888888888888880);
+
+  uint64_t b0 = b & UINT64_C(0x1111111111111111);
+  uint64_t b1 = b & UINT64_C(0x2222222222222222);
+  uint64_t b2 = b & UINT64_C(0x4444444444444444);
+  uint64_t b3 = b & UINT64_C(0x8888888888888888);
+
+  uint128_t c0 = (a0 * (uint128_t)b0) ^ (a1 * (uint128_t)b3) ^
+                 (a2 * (uint128_t)b2) ^ (a3 * (uint128_t)b1);
+  uint128_t c1 = (a0 * (uint128_t)b1) ^ (a1 * (uint128_t)b0) ^
+                 (a2 * (uint128_t)b3) ^ (a3 * (uint128_t)b2);
+  uint128_t c2 = (a0 * (uint128_t)b2) ^ (a1 * (uint128_t)b1) ^
+                 (a2 * (uint128_t)b0) ^ (a3 * (uint128_t)b3);
+  uint128_t c3 = (a0 * (uint128_t)b3) ^ (a1 * (uint128_t)b2) ^
+                 (a2 * (uint128_t)b1) ^ (a3 * (uint128_t)b0);
+
+  // Multiply the bottom four bits of |a| with |b|.
+  uint64_t a0_mask = UINT64_C(0) - (a & 1);
+  uint64_t a1_mask = UINT64_C(0) - ((a >> 1) & 1);
+  uint64_t a2_mask = UINT64_C(0) - ((a >> 2) & 1);
+  uint64_t a3_mask = UINT64_C(0) - ((a >> 3) & 1);
+  uint128_t extra = (a0_mask & b) ^ ((uint128_t)(a1_mask & b) << 1) ^
+                    ((uint128_t)(a2_mask & b) << 2) ^
+                    ((uint128_t)(a3_mask & b) << 3);
+
+  *out_lo = (((uint64_t)c0) & UINT64_C(0x1111111111111111)) ^
+            (((uint64_t)c1) & UINT64_C(0x2222222222222222)) ^
+            (((uint64_t)c2) & UINT64_C(0x4444444444444444)) ^
+            (((uint64_t)c3) & UINT64_C(0x8888888888888888)) ^ ((uint64_t)extra);
+  *out_hi = (((uint64_t)(c0 >> 64)) & UINT64_C(0x1111111111111111)) ^
+            (((uint64_t)(c1 >> 64)) & UINT64_C(0x2222222222222222)) ^
+            (((uint64_t)(c2 >> 64)) & UINT64_C(0x4444444444444444)) ^
+            (((uint64_t)(c3 >> 64)) & UINT64_C(0x8888888888888888)) ^
+            ((uint64_t)(extra >> 64));
+}
+
+#elif defined(OPENSSL_SSE2)
+
+static __m128i gcm_mul32_nohw(uint32_t a, uint32_t b) {
+  // One term every four bits means the largest term is 32/4 = 8, which does not
+  // overflow into the next term.
+  __m128i aa = _mm_setr_epi32(a, 0, a, 0);
+  __m128i bb = _mm_setr_epi32(b, 0, b, 0);
+
+  __m128i a0a0 =
+      _mm_and_si128(aa, _mm_setr_epi32(0x11111111, 0, 0x11111111, 0));
+  __m128i a2a2 =
+      _mm_and_si128(aa, _mm_setr_epi32(0x44444444, 0, 0x44444444, 0));
+  __m128i b0b1 =
+      _mm_and_si128(bb, _mm_setr_epi32(0x11111111, 0, 0x22222222, 0));
+  __m128i b2b3 =
+      _mm_and_si128(bb, _mm_setr_epi32(0x44444444, 0, 0x88888888, 0));
+
+  __m128i c0c1 =
+      _mm_xor_si128(_mm_mul_epu32(a0a0, b0b1), _mm_mul_epu32(a2a2, b2b3));
+  __m128i c2c3 =
+      _mm_xor_si128(_mm_mul_epu32(a2a2, b0b1), _mm_mul_epu32(a0a0, b2b3));
+
+  __m128i a1a1 =
+      _mm_and_si128(aa, _mm_setr_epi32(0x22222222, 0, 0x22222222, 0));
+  __m128i a3a3 =
+      _mm_and_si128(aa, _mm_setr_epi32(0x88888888, 0, 0x88888888, 0));
+  __m128i b3b0 =
+      _mm_and_si128(bb, _mm_setr_epi32(0x88888888, 0, 0x11111111, 0));
+  __m128i b1b2 =
+      _mm_and_si128(bb, _mm_setr_epi32(0x22222222, 0, 0x44444444, 0));
+
+  c0c1 = _mm_xor_si128(c0c1, _mm_mul_epu32(a1a1, b3b0));
+  c0c1 = _mm_xor_si128(c0c1, _mm_mul_epu32(a3a3, b1b2));
+  c2c3 = _mm_xor_si128(c2c3, _mm_mul_epu32(a3a3, b3b0));
+  c2c3 = _mm_xor_si128(c2c3, _mm_mul_epu32(a1a1, b1b2));
+
+  c0c1 = _mm_and_si128(
+      c0c1, _mm_setr_epi32(0x11111111, 0x11111111, 0x22222222, 0x22222222));
+  c2c3 = _mm_and_si128(
+      c2c3, _mm_setr_epi32(0x44444444, 0x44444444, 0x88888888, 0x88888888));
+
+  c0c1 = _mm_xor_si128(c0c1, c2c3);
+  // c0 ^= c1
+  c0c1 = _mm_xor_si128(c0c1, _mm_srli_si128(c0c1, 8));
+  return c0c1;
+}
+
+static void gcm_mul64_nohw(uint64_t *out_lo, uint64_t *out_hi, uint64_t a,
+                           uint64_t b) {
+  uint32_t a0 = a & 0xffffffff;
+  uint32_t a1 = a >> 32;
+  uint32_t b0 = b & 0xffffffff;
+  uint32_t b1 = b >> 32;
+  // Karatsuba multiplication.
+  __m128i lo = gcm_mul32_nohw(a0, b0);
+  __m128i hi = gcm_mul32_nohw(a1, b1);
+  __m128i mid = gcm_mul32_nohw(a0 ^ a1, b0 ^ b1);
+  mid = _mm_xor_si128(mid, lo);
+  mid = _mm_xor_si128(mid, hi);
+  __m128i ret = _mm_unpacklo_epi64(lo, hi);
+  mid = _mm_slli_si128(mid, 4);
+  mid = _mm_and_si128(mid, _mm_setr_epi32(0, 0xffffffff, 0xffffffff, 0));
+  ret = _mm_xor_si128(ret, mid);
+  memcpy(out_lo, &ret, 8);
+  memcpy(out_hi, ((char*)&ret) + 8, 8);
+}
+
+#else  // !BORINGSSL_HAS_UINT128 && !OPENSSL_SSE2
+
+static uint64_t gcm_mul32_nohw(uint32_t a, uint32_t b) {
+  // One term every four bits means the largest term is 32/4 = 8, which does not
+  // overflow into the next term.
+  uint32_t a0 = a & 0x11111111;
+  uint32_t a1 = a & 0x22222222;
+  uint32_t a2 = a & 0x44444444;
+  uint32_t a3 = a & 0x88888888;
+
+  uint32_t b0 = b & 0x11111111;
+  uint32_t b1 = b & 0x22222222;
+  uint32_t b2 = b & 0x44444444;
+  uint32_t b3 = b & 0x88888888;
+
+  uint64_t c0 = (a0 * (uint64_t)b0) ^ (a1 * (uint64_t)b3) ^
+                (a2 * (uint64_t)b2) ^ (a3 * (uint64_t)b1);
+  uint64_t c1 = (a0 * (uint64_t)b1) ^ (a1 * (uint64_t)b0) ^
+                (a2 * (uint64_t)b3) ^ (a3 * (uint64_t)b2);
+  uint64_t c2 = (a0 * (uint64_t)b2) ^ (a1 * (uint64_t)b1) ^
+                (a2 * (uint64_t)b0) ^ (a3 * (uint64_t)b3);
+  uint64_t c3 = (a0 * (uint64_t)b3) ^ (a1 * (uint64_t)b2) ^
+                (a2 * (uint64_t)b1) ^ (a3 * (uint64_t)b0);
+
+  return (c0 & UINT64_C(0x1111111111111111)) |
+         (c1 & UINT64_C(0x2222222222222222)) |
+         (c2 & UINT64_C(0x4444444444444444)) |
+         (c3 & UINT64_C(0x8888888888888888));
+}
+
+static void gcm_mul64_nohw(uint64_t *out_lo, uint64_t *out_hi, uint64_t a,
+                           uint64_t b) {
+  uint32_t a0 = a & 0xffffffff;
+  uint32_t a1 = a >> 32;
+  uint32_t b0 = b & 0xffffffff;
+  uint32_t b1 = b >> 32;
+  // Karatsuba multiplication.
+  uint64_t lo = gcm_mul32_nohw(a0, b0);
+  uint64_t hi = gcm_mul32_nohw(a1, b1);
+  uint64_t mid = gcm_mul32_nohw(a0 ^ a1, b0 ^ b1) ^ lo ^ hi;
+  *out_lo = lo ^ (mid << 32);
+  *out_hi = hi ^ (mid >> 32);
+}
+
+#endif  // BORINGSSL_HAS_UINT128
+
+void gcm_init_nohw(u128 Htable[16], const uint64_t Xi[2]) {
+  // We implement GHASH in terms of POLYVAL, as described in RFC8452. This
+  // avoids a shift by 1 in the multiplication, needed to account for bit
+  // reversal losing a bit after multiplication, that is,
+  // rev128(X) * rev128(Y) = rev255(X*Y).
+  //
+  // Per Appendix A, we run mulX_POLYVAL. Note this is the same transformation
+  // applied by |gcm_init_clmul|, etc. Note |Xi| has already been byteswapped.
+  //
+  // See also slide 16 of
+  // https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf
+  Htable[0].lo = Xi[1];
+  Htable[0].hi = Xi[0];
+
+  uint64_t carry = Htable[0].hi >> 63;
+  carry = 0u - carry;
+
+  Htable[0].hi <<= 1;
+  Htable[0].hi |= Htable[0].lo >> 63;
+  Htable[0].lo <<= 1;
+
+  // The irreducible polynomial is 1 + x^121 + x^126 + x^127 + x^128, so we
+  // conditionally add 0xc200...0001.
+  Htable[0].lo ^= carry & 1;
+  Htable[0].hi ^= carry & UINT64_C(0xc200000000000000);
+
+  // This implementation does not use the rest of |Htable|.
+}
+
+static void gcm_polyval_nohw(uint64_t Xi[2], const u128 *H) {
+  // Karatsuba multiplication. The product of |Xi| and |H| is stored in |r0|
+  // through |r3|. Note there is no byte or bit reversal because we are
+  // evaluating POLYVAL.
+  uint64_t r0, r1;
+  gcm_mul64_nohw(&r0, &r1, Xi[0], H->lo);
+  uint64_t r2, r3;
+  gcm_mul64_nohw(&r2, &r3, Xi[1], H->hi);
+  uint64_t mid0, mid1;
+  gcm_mul64_nohw(&mid0, &mid1, Xi[0] ^ Xi[1], H->hi ^ H->lo);
+  mid0 ^= r0 ^ r2;
+  mid1 ^= r1 ^ r3;
+  r2 ^= mid1;
+  r1 ^= mid0;
+
+  // Now we multiply our 256-bit result by x^-128 and reduce. |r2| and
+  // |r3| shifts into position and we must multiply |r0| and |r1| by x^-128. We
+  // have:
+  //
+  //       1 = x^121 + x^126 + x^127 + x^128
+  //  x^-128 = x^-7 + x^-2 + x^-1 + 1
+  //
+  // This is the GHASH reduction step, but with bits flowing in reverse.
+
+  // The x^-7, x^-2, and x^-1 terms shift bits past x^0, which would require
+  // another reduction steps. Instead, we gather the excess bits, incorporate
+  // them into |r0| and |r1| and reduce once. See slides 17-19
+  // of https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf.
+  r1 ^= (r0 << 63) ^ (r0 << 62) ^ (r0 << 57);
+
+  // 1
+  r2 ^= r0;
+  r3 ^= r1;
+
+  // x^-1
+  r2 ^= r0 >> 1;
+  r2 ^= r1 << 63;
+  r3 ^= r1 >> 1;
+
+  // x^-2
+  r2 ^= r0 >> 2;
+  r2 ^= r1 << 62;
+  r3 ^= r1 >> 2;
+
+  // x^-7
+  r2 ^= r0 >> 7;
+  r2 ^= r1 << 57;
+  r3 ^= r1 >> 7;
+
+  Xi[0] = r2;
+  Xi[1] = r3;
+}
+
+void gcm_gmult_nohw(uint64_t Xi[2], const u128 Htable[16]) {
+  uint64_t swapped[2];
+  swapped[0] = CRYPTO_bswap8(Xi[1]);
+  swapped[1] = CRYPTO_bswap8(Xi[0]);
+  gcm_polyval_nohw(swapped, &Htable[0]);
+  Xi[0] = CRYPTO_bswap8(swapped[1]);
+  Xi[1] = CRYPTO_bswap8(swapped[0]);
+}
+
+void gcm_ghash_nohw(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                    size_t len) {
+  uint64_t swapped[2];
+  swapped[0] = CRYPTO_bswap8(Xi[1]);
+  swapped[1] = CRYPTO_bswap8(Xi[0]);
+
+  while (len >= 16) {
+    uint64_t block[2];
+    OPENSSL_memcpy(block, inp, 16);
+    swapped[0] ^= CRYPTO_bswap8(block[1]);
+    swapped[1] ^= CRYPTO_bswap8(block[0]);
+    gcm_polyval_nohw(swapped, &Htable[0]);
+    inp += 16;
+    len -= 16;
+  }
+
+  Xi[0] = CRYPTO_bswap8(swapped[1]);
+  Xi[1] = CRYPTO_bswap8(swapped[0]);
+}

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/internal.h

@@ -0,0 +1,441 @@
+/* ====================================================================
+ * Copyright (c) 2008 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ==================================================================== */
+
+#ifndef OPENSSL_HEADER_MODES_INTERNAL_H
+#define OPENSSL_HEADER_MODES_INTERNAL_H
+
+#include <openssl/base.h>
+
+#include <openssl/aes.h>
+#include <openssl/cpu.h>
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "../../internal.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+
+static inline uint32_t GETU32(const void *in) {
+  uint32_t v;
+  OPENSSL_memcpy(&v, in, sizeof(v));
+  return CRYPTO_bswap4(v);
+}
+
+static inline void PUTU32(void *out, uint32_t v) {
+  v = CRYPTO_bswap4(v);
+  OPENSSL_memcpy(out, &v, sizeof(v));
+}
+
+static inline size_t load_word_le(const void *in) {
+  size_t v;
+  OPENSSL_memcpy(&v, in, sizeof(v));
+  return v;
+}
+
+static inline void store_word_le(void *out, size_t v) {
+  OPENSSL_memcpy(out, &v, sizeof(v));
+}
+
+// block128_f is the type of an AES block cipher implementation.
+//
+// Unlike upstream OpenSSL, it and the other functions in this file hard-code
+// |AES_KEY|. It is undefined in C to call a function pointer with anything
+// other than the original type. Thus we either must match |block128_f| to the
+// type signature of |AES_encrypt| and friends or pass in |void*| wrapper
+// functions.
+//
+// These functions are called exclusively with AES, so we use the former.
+typedef void (*block128_f)(const uint8_t in[16], uint8_t out[16],
+                           const AES_KEY *key);
+
+
+// CTR.
+
+// ctr128_f is the type of a function that performs CTR-mode encryption.
+typedef void (*ctr128_f)(const uint8_t *in, uint8_t *out, size_t blocks,
+                         const AES_KEY *key, const uint8_t ivec[16]);
+
+// CRYPTO_ctr128_encrypt encrypts (or decrypts, it's the same in CTR mode)
+// |len| bytes from |in| to |out| using |block| in counter mode. There's no
+// requirement that |len| be a multiple of any value and any partial blocks are
+// stored in |ecount_buf| and |*num|, which must be zeroed before the initial
+// call. The counter is a 128-bit, big-endian value in |ivec| and is
+// incremented by this function.
+void CRYPTO_ctr128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16],
+                           uint8_t ecount_buf[16], unsigned *num,
+                           block128_f block);
+
+// CRYPTO_ctr128_encrypt_ctr32 acts like |CRYPTO_ctr128_encrypt| but takes
+// |ctr|, a function that performs CTR mode but only deals with the lower 32
+// bits of the counter. This is useful when |ctr| can be an optimised
+// function.
+void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len,
+                                 const AES_KEY *key, uint8_t ivec[16],
+                                 uint8_t ecount_buf[16], unsigned *num,
+                                 ctr128_f ctr);
+
+
+// GCM.
+//
+// This API differs from the upstream API slightly. The |GCM128_CONTEXT| does
+// not have a |key| pointer that points to the key as upstream's version does.
+// Instead, every function takes a |key| parameter. This way |GCM128_CONTEXT|
+// can be safely copied. Additionally, |gcm_key| is split into a separate
+// struct.
+
+typedef struct { uint64_t hi,lo; } u128;
+
+// gmult_func multiplies |Xi| by the GCM key and writes the result back to
+// |Xi|.
+typedef void (*gmult_func)(uint64_t Xi[2], const u128 Htable[16]);
+
+// ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from
+// |inp|. The result is written back to |Xi| and the |len| argument must be a
+// multiple of 16.
+typedef void (*ghash_func)(uint64_t Xi[2], const u128 Htable[16],
+                           const uint8_t *inp, size_t len);
+
+typedef struct gcm128_key_st {
+  // Note the MOVBE-based, x86-64, GHASH assembly requires |H| and |Htable| to
+  // be the first two elements of this struct. Additionally, some assembly
+  // routines require a 16-byte-aligned |Htable| when hashing data, but not
+  // initialization. |GCM128_KEY| is not itself aligned to simplify embedding in
+  // |EVP_AEAD_CTX|, but |Htable|'s offset must be a multiple of 16.
+  u128 H;
+  u128 Htable[16];
+  gmult_func gmult;
+  ghash_func ghash;
+
+  block128_f block;
+
+  // use_aesni_gcm_crypt is true if this context should use the assembly
+  // functions |aesni_gcm_encrypt| and |aesni_gcm_decrypt| to process data.
+  unsigned use_aesni_gcm_crypt:1;
+} GCM128_KEY;
+
+// GCM128_CONTEXT contains state for a single GCM operation. The structure
+// should be zero-initialized before use.
+typedef struct {
+  // The following 5 names follow names in GCM specification
+  union {
+    uint64_t u[2];
+    uint32_t d[4];
+    uint8_t c[16];
+    size_t t[16 / sizeof(size_t)];
+  } Yi, EKi, EK0, len, Xi;
+
+  // Note that the order of |Xi| and |gcm_key| is fixed by the MOVBE-based,
+  // x86-64, GHASH assembly. Additionally, some assembly routines require
+  // |gcm_key| to be 16-byte aligned. |GCM128_KEY| is not itself aligned to
+  // simplify embedding in |EVP_AEAD_CTX|.
+  alignas(16) GCM128_KEY gcm_key;
+
+  unsigned mres, ares;
+} GCM128_CONTEXT;
+
+#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
+// crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is
+// used.
+int crypto_gcm_clmul_enabled(void);
+#endif
+
+// CRYPTO_ghash_init writes a precomputed table of powers of |gcm_key| to
+// |out_table| and sets |*out_mult| and |*out_hash| to (potentially hardware
+// accelerated) functions for performing operations in the GHASH field. If the
+// AVX implementation was used |*out_is_avx| will be true.
+void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
+                       u128 *out_key, u128 out_table[16], int *out_is_avx,
+                       const uint8_t gcm_key[16]);
+
+// CRYPTO_gcm128_init_key initialises |gcm_key| to use |block| (typically AES)
+// with the given key. |block_is_hwaes| is one if |block| is |aes_hw_encrypt|.
+OPENSSL_EXPORT void CRYPTO_gcm128_init_key(GCM128_KEY *gcm_key,
+                                           const AES_KEY *key, block128_f block,
+                                           int block_is_hwaes);
+
+// CRYPTO_gcm128_setiv sets the IV (nonce) for |ctx|. The |key| must be the
+// same key that was passed to |CRYPTO_gcm128_init|.
+OPENSSL_EXPORT void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const AES_KEY *key,
+                                        const uint8_t *iv, size_t iv_len);
+
+// CRYPTO_gcm128_aad sets the authenticated data for an instance of GCM.
+// This must be called before and data is encrypted. It returns one on success
+// and zero otherwise.
+OPENSSL_EXPORT int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad,
+                                     size_t len);
+
+// CRYPTO_gcm128_encrypt encrypts |len| bytes from |in| to |out|. The |key|
+// must be the same key that was passed to |CRYPTO_gcm128_init|. It returns one
+// on success and zero otherwise.
+OPENSSL_EXPORT int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
+                                         const AES_KEY *key, const uint8_t *in,
+                                         uint8_t *out, size_t len);
+
+// CRYPTO_gcm128_decrypt decrypts |len| bytes from |in| to |out|. The |key|
+// must be the same key that was passed to |CRYPTO_gcm128_init|. It returns one
+// on success and zero otherwise.
+OPENSSL_EXPORT int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
+                                         const AES_KEY *key, const uint8_t *in,
+                                         uint8_t *out, size_t len);
+
+// CRYPTO_gcm128_encrypt_ctr32 encrypts |len| bytes from |in| to |out| using
+// a CTR function that only handles the bottom 32 bits of the nonce, like
+// |CRYPTO_ctr128_encrypt_ctr32|. The |key| must be the same key that was
+// passed to |CRYPTO_gcm128_init|. It returns one on success and zero
+// otherwise.
+OPENSSL_EXPORT int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
+                                               const AES_KEY *key,
+                                               const uint8_t *in, uint8_t *out,
+                                               size_t len, ctr128_f stream);
+
+// CRYPTO_gcm128_decrypt_ctr32 decrypts |len| bytes from |in| to |out| using
+// a CTR function that only handles the bottom 32 bits of the nonce, like
+// |CRYPTO_ctr128_encrypt_ctr32|. The |key| must be the same key that was
+// passed to |CRYPTO_gcm128_init|. It returns one on success and zero
+// otherwise.
+OPENSSL_EXPORT int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
+                                               const AES_KEY *key,
+                                               const uint8_t *in, uint8_t *out,
+                                               size_t len, ctr128_f stream);
+
+// CRYPTO_gcm128_finish calculates the authenticator and compares it against
+// |len| bytes of |tag|. It returns one on success and zero otherwise.
+OPENSSL_EXPORT int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag,
+                                        size_t len);
+
+// CRYPTO_gcm128_tag calculates the authenticator and copies it into |tag|.
+// The minimum of |len| and 16 bytes are copied into |tag|.
+OPENSSL_EXPORT void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, uint8_t *tag,
+                                      size_t len);
+
+
+// GCM assembly.
+
+void gcm_init_nohw(u128 Htable[16], const uint64_t H[2]);
+void gcm_gmult_nohw(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_nohw(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                    size_t len);
+
+#if !defined(OPENSSL_NO_ASM)
+
+#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
+#define GCM_FUNCREF
+void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_clmul(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_clmul(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                     size_t len);
+
+OPENSSL_INLINE char gcm_ssse3_capable(void) {
+  return (OPENSSL_ia32cap_get()[1] & (1 << (41 - 32))) != 0;
+}
+
+// |gcm_gmult_ssse3| and |gcm_ghash_ssse3| require |Htable| to be
+// 16-byte-aligned, but |gcm_init_ssse3| does not.
+void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in,
+                     size_t len);
+
+#if defined(OPENSSL_X86_64)
+#define GHASH_ASM_X86_64
+void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_avx(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_avx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in,
+                   size_t len);
+
+#define AESNI_GCM
+size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                         const AES_KEY *key, uint8_t ivec[16], uint64_t *Xi);
+size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
+                         const AES_KEY *key, uint8_t ivec[16], uint64_t *Xi);
+#endif  // OPENSSL_X86_64
+
+#if defined(OPENSSL_X86)
+#define GHASH_ASM_X86
+#endif  // OPENSSL_X86
+
+#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
+#define GHASH_ASM_ARM
+#define GCM_FUNCREF
+
+OPENSSL_INLINE int gcm_pmull_capable(void) {
+  return CRYPTO_is_ARMv8_PMULL_capable();
+}
+
+void gcm_init_v8(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_v8(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_v8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                  size_t len);
+
+OPENSSL_INLINE int gcm_neon_capable(void) { return CRYPTO_is_NEON_capable(); }
+
+void gcm_init_neon(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_neon(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_neon(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                    size_t len);
+
+#elif defined(OPENSSL_PPC64LE)
+#define GHASH_ASM_PPC64LE
+#define GCM_FUNCREF
+void gcm_init_p8(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_p8(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_p8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                  size_t len);
+#endif
+#endif  // OPENSSL_NO_ASM
+
+
+// CBC.
+
+// cbc128_f is the type of a function that performs CBC-mode encryption.
+typedef void (*cbc128_f)(const uint8_t *in, uint8_t *out, size_t len,
+                         const AES_KEY *key, uint8_t ivec[16], int enc);
+
+// CRYPTO_cbc128_encrypt encrypts |len| bytes from |in| to |out| using the
+// given IV and block cipher in CBC mode. The input need not be a multiple of
+// 128 bits long, but the output will round up to the nearest 128 bit multiple,
+// zero padding the input if needed. The IV will be updated on return.
+void CRYPTO_cbc128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16],
+                           block128_f block);
+
+// CRYPTO_cbc128_decrypt decrypts |len| bytes from |in| to |out| using the
+// given IV and block cipher in CBC mode. If |len| is not a multiple of 128
+// bits then only that many bytes will be written, but a multiple of 128 bits
+// is always read from |in|. The IV will be updated on return.
+void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16],
+                           block128_f block);
+
+
+// OFB.
+
+// CRYPTO_ofb128_encrypt encrypts (or decrypts, it's the same with OFB mode)
+// |len| bytes from |in| to |out| using |block| in OFB mode. There's no
+// requirement that |len| be a multiple of any value and any partial blocks are
+// stored in |ivec| and |*num|, the latter must be zero before the initial
+// call.
+void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16], unsigned *num,
+                           block128_f block);
+
+
+// CFB.
+
+// CRYPTO_cfb128_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
+// from |in| to |out| using |block| in CFB mode. There's no requirement that
+// |len| be a multiple of any value and any partial blocks are stored in |ivec|
+// and |*num|, the latter must be zero before the initial call.
+void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16], unsigned *num,
+                           int enc, block128_f block);
+
+// CRYPTO_cfb128_8_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
+// from |in| to |out| using |block| in CFB-8 mode. Prior to the first call
+// |num| should be set to zero.
+void CRYPTO_cfb128_8_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                             const AES_KEY *key, uint8_t ivec[16],
+                             unsigned *num, int enc, block128_f block);
+
+// CRYPTO_cfb128_1_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
+// from |in| to |out| using |block| in CFB-1 mode. Prior to the first call
+// |num| should be set to zero.
+void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits,
+                             const AES_KEY *key, uint8_t ivec[16],
+                             unsigned *num, int enc, block128_f block);
+
+size_t CRYPTO_cts128_encrypt_block(const uint8_t *in, uint8_t *out, size_t len,
+                                   const AES_KEY *key, uint8_t ivec[16],
+                                   block128_f block);
+
+
+// POLYVAL.
+//
+// POLYVAL is a polynomial authenticator that operates over a field very
+// similar to the one that GHASH uses. See
+// https://tools.ietf.org/html/draft-irtf-cfrg-gcmsiv-02#section-3.
+
+typedef union {
+  uint64_t u[2];
+  uint8_t c[16];
+} polyval_block;
+
+struct polyval_ctx {
+  // Note that the order of |S|, |H| and |Htable| is fixed by the MOVBE-based,
+  // x86-64, GHASH assembly. Additionally, some assembly routines require
+  // |Htable| to be 16-byte aligned.
+  polyval_block S;
+  u128 H;
+  alignas(16) u128 Htable[16];
+  gmult_func gmult;
+  ghash_func ghash;
+};
+
+// CRYPTO_POLYVAL_init initialises |ctx| using |key|.
+void CRYPTO_POLYVAL_init(struct polyval_ctx *ctx, const uint8_t key[16]);
+
+// CRYPTO_POLYVAL_update_blocks updates the accumulator in |ctx| given the
+// blocks from |in|. Only a whole number of blocks can be processed so |in_len|
+// must be a multiple of 16.
+void CRYPTO_POLYVAL_update_blocks(struct polyval_ctx *ctx, const uint8_t *in,
+                                  size_t in_len);
+
+// CRYPTO_POLYVAL_finish writes the accumulator from |ctx| to |out|.
+void CRYPTO_POLYVAL_finish(const struct polyval_ctx *ctx, uint8_t out[16]);
+
+
+#if defined(__cplusplus)
+}  // extern C
+#endif
+
+#endif  // OPENSSL_HEADER_MODES_INTERNAL_H