grpc 1.24.0 → 1.25.0.pre1

data/third_party/boringssl/crypto/fipsmodule/bn/random.c

@@ -108,10 +108,10 @@
 
 #include <openssl/bn.h>
 
+#include <limits.h>
 #include <string.h>
 
 #include <openssl/err.h>
-#include <openssl/mem.h>
 #include <openssl/rand.h>
 #include <openssl/type_check.h>
 
@@ -121,9 +121,6 @@
 
 
 int BN_rand(BIGNUM *rnd, int bits, int top, int bottom) {
-  uint8_t *buf = NULL;
-  int ret = 0, bit, bytes, mask;
-
   if (rnd == NULL) {
     return 0;
   }
@@ -144,48 +141,41 @@ int BN_rand(BIGNUM *rnd, int bits, int top, int bottom) {
     return 1;
   }
 
-  bytes = (bits + 7) / 8;
-  bit = (bits - 1) % 8;
-  mask = 0xff << (bit + 1);
-
-  buf = OPENSSL_malloc(bytes);
-  if (buf == NULL) {
-    OPENSSL_PUT_ERROR(BN, ERR_R_MALLOC_FAILURE);
-    goto err;
+  if (bits > INT_MAX - (BN_BITS2 - 1)) {
+    OPENSSL_PUT_ERROR(BN, BN_R_BIGNUM_TOO_LONG);
+    return 0;
   }
 
-  // Make a random number and set the top and bottom bits.
-  RAND_bytes(buf, bytes);
+  int words = (bits + BN_BITS2 - 1) / BN_BITS2;
+  int bit = (bits - 1) % BN_BITS2;
+  const BN_ULONG kOne = 1;
+  const BN_ULONG kThree = 3;
+  BN_ULONG mask = bit < BN_BITS2 - 1 ? (kOne << (bit + 1)) - 1 : BN_MASK2;
+  if (!bn_wexpand(rnd, words)) {
+    return 0;
+  }
 
+  RAND_bytes((uint8_t *)rnd->d, words * sizeof(BN_ULONG));
+  rnd->d[words - 1] &= mask;
   if (top != BN_RAND_TOP_ANY) {
     if (top == BN_RAND_TOP_TWO && bits > 1) {
       if (bit == 0) {
-        buf[0] = 1;
-        buf[1] |= 0x80;
+        rnd->d[words - 1] |= 1;
+        rnd->d[words - 2] |= kOne << (BN_BITS2 - 1);
       } else {
-        buf[0] |= (3 << (bit - 1));
+        rnd->d[words - 1] |= kThree << (bit - 1);
       }
     } else {
-      buf[0] |= (1 << bit);
+      rnd->d[words - 1] |= kOne << bit;
     }
   }
-
-  buf[0] &= ~mask;
-
-  // Set the bottom bit if requested,
-  if (bottom == BN_RAND_BOTTOM_ODD)  {
-    buf[bytes - 1] |= 1;
-  }
-
-  if (!BN_bin2bn(buf, bytes, rnd)) {
-    goto err;
+  if (bottom == BN_RAND_BOTTOM_ODD) {
+    rnd->d[0] |= 1;
   }
 
-  ret = 1;
-
-err:
-  OPENSSL_free(buf);
-  return ret;
+  rnd->neg = 0;
+  rnd->width = words;
+  return 1;
 }
 
 int BN_pseudo_rand(BIGNUM *rnd, int bits, int top, int bottom) {
@@ -205,8 +195,8 @@ static crypto_word_t bn_less_than_word_mask(const BN_ULONG *a, size_t len,
   }
 
   // |a| < |b| iff a[1..len-1] are all zero and a[0] < b.
-  OPENSSL_COMPILE_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t),
-                         crypto_word_t_too_small);
+  OPENSSL_STATIC_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t),
+                        "crypto_word_t is too small");
   crypto_word_t mask = 0;
   for (size_t i = 1; i < len; i++) {
     mask |= a[i];

data/third_party/boringssl/crypto/fipsmodule/bn/rsaz_exp.c

@@ -12,27 +12,16 @@
  * (2) University of Haifa, Israel
  */
 
-#include <openssl/base.h>
-
-#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64)
-
 #include "rsaz_exp.h"
 
+#if defined(RSAZ_ENABLED)
+
 #include <openssl/mem.h>
 
+#include "internal.h"
 #include "../../internal.h"
 
 
-// See crypto/bn/asm/rsaz-avx2.pl for further details.
-void rsaz_1024_norm2red_avx2(void *red, const void *norm);
-void rsaz_1024_mul_avx2(void *ret, const void *a, const void *b, const void *n,
-                        BN_ULONG k);
-void rsaz_1024_sqr_avx2(void *ret, const void *a, const void *n, BN_ULONG k,
-                        int cnt);
-void rsaz_1024_scatter5_avx2(void *tbl, const void *val, int i);
-void rsaz_1024_gather5_avx2(void *val, const void *tbl, int i);
-void rsaz_1024_red2norm_avx2(void *norm, const void *red);
-
 // one is 1 in RSAZ's representation.
 alignas(64) static const BN_ULONG one[40] = {
     1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -44,19 +33,25 @@ alignas(64) static const BN_ULONG two80[40] = {
     0, 0, 0,       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
 void RSAZ_1024_mod_exp_avx2(BN_ULONG result_norm[16],
-	const BN_ULONG base_norm[16], const BN_ULONG exponent[16],
-	const BN_ULONG m_norm[16], const BN_ULONG RR[16], BN_ULONG k0) {
-  alignas(64) uint8_t storage[(320 * 3) + (32 * 9 * 16)];  // 5.5KB
-  unsigned char *a_inv, *m, *result, *table_s = storage + (320 * 3),
-                                     *R2 = table_s;  // borrow
+                            const BN_ULONG base_norm[16],
+                            const BN_ULONG exponent[16],
+                            const BN_ULONG m_norm[16], const BN_ULONG RR[16],
+                            BN_ULONG k0,
+                            BN_ULONG storage[MOD_EXP_CTIME_STORAGE_LEN]) {
+  OPENSSL_STATIC_ASSERT(MOD_EXP_CTIME_MIN_CACHE_LINE_WIDTH % 64 == 0,
+                        "MOD_EXP_CTIME_MIN_CACHE_LINE_WIDTH is too small");
+  assert((uintptr_t)storage % 64 == 0);
+
+  BN_ULONG *a_inv, *m, *result, *table_s = storage + 40 * 3, *R2 = table_s;
+  // Note |R2| aliases |table_s|.
   if (((((uintptr_t)storage & 4095) + 320) >> 12) != 0) {
     result = storage;
-    a_inv = storage + 320;
-    m = storage + (320 * 2);  // should not cross page
+    a_inv = storage + 40;
+    m = storage + 40 * 2;  // should not cross page
   } else {
     m = storage;  // should not cross page
-    result = storage + 320;
-    a_inv = storage + (320 * 2);
+    result = storage + 40;
+    a_inv = storage + 40 * 2;
   }
 
   rsaz_1024_norm2red_avx2(m, m_norm);
@@ -225,7 +220,7 @@ void RSAZ_1024_mod_exp_avx2(BN_ULONG result_norm[16],
 
   rsaz_1024_red2norm_avx2(result_norm, result);
 
-  OPENSSL_cleanse(storage, sizeof(storage));
+  OPENSSL_cleanse(storage, MOD_EXP_CTIME_STORAGE_LEN * sizeof(BN_ULONG));
 }
 
-#endif  // OPENSSL_X86_64
+#endif  // RSAZ_ENABLED

data/third_party/boringssl/crypto/fipsmodule/bn/rsaz_exp.h

@@ -16,18 +16,89 @@
 #define OPENSSL_HEADER_BN_RSAZ_EXP_H
 
 #include <openssl/bn.h>
+#include <openssl/cpu.h>
+
+#include "internal.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64)
+#define RSAZ_ENABLED
+
 
 // RSAZ_1024_mod_exp_avx2 sets |result| to |base_norm| raised to |exponent|
 // modulo |m_norm|. |base_norm| must be fully-reduced and |exponent| must have
 // the high bit set (it is 1024 bits wide). |RR| and |k0| must be |RR| and |n0|,
-// respectively, extracted from |m_norm|'s |BN_MONT_CTX|.
+// respectively, extracted from |m_norm|'s |BN_MONT_CTX|. |storage_words| is a
+// temporary buffer that must be aligned to |MOD_EXP_CTIME_MIN_CACHE_LINE_WIDTH|
+// bytes.
 void RSAZ_1024_mod_exp_avx2(BN_ULONG result[16], const BN_ULONG base_norm[16],
                             const BN_ULONG exponent[16],
                             const BN_ULONG m_norm[16], const BN_ULONG RR[16],
-                            BN_ULONG k0);
+                            BN_ULONG k0,
+                            BN_ULONG storage_words[MOD_EXP_CTIME_STORAGE_LEN]);
+
+OPENSSL_INLINE int rsaz_avx2_capable(void) {
+  const uint32_t *cap = OPENSSL_ia32cap_get();
+  return (cap[2] & (1 << 5)) != 0;  // AVX2
+}
+
+OPENSSL_INLINE int rsaz_avx2_preferred(void) {
+  const uint32_t *cap = OPENSSL_ia32cap_get();
+  static const uint32_t kBMI2AndADX = (1 << 8) | (1 << 19);
+  if ((cap[2] & kBMI2AndADX) == kBMI2AndADX) {
+    // If BMI2 and ADX are available, x86_64-mont5.pl is faster.
+    return 0;
+  }
+  return (cap[2] & (1 << 5)) != 0;  // AVX2
+}
+
+
+// Assembly functions.
+
+// RSAZ represents 1024-bit integers using unsaturated 29-bit limbs stored in
+// 64-bit integers. This requires 36 limbs but padded up to 40.
+//
+// See crypto/bn/asm/rsaz-avx2.pl for further details.
+
+// rsaz_1024_norm2red_avx2 converts |norm| from |BIGNUM| to RSAZ representation
+// and writes the result to |red|.
+void rsaz_1024_norm2red_avx2(BN_ULONG red[40], const BN_ULONG norm[16]);
+
+// rsaz_1024_mul_avx2 computes |a| * |b| mod |n| and writes the result to |ret|.
+// Inputs and outputs are in Montgomery form, using RSAZ's representation. |k|
+// is -|n|^-1 mod 2^64 or |n0| from |BN_MONT_CTX|.
+void rsaz_1024_mul_avx2(BN_ULONG ret[40], const BN_ULONG a[40],
+                        const BN_ULONG b[40], const BN_ULONG n[40], BN_ULONG k);
+
+// rsaz_1024_mul_avx2 computes |a|^(2*|count|) mod |n| and writes the result to
+// |ret|. Inputs and outputs are in Montgomery form, using RSAZ's
+// representation. |k| is -|n|^-1 mod 2^64 or |n0| from |BN_MONT_CTX|.
+void rsaz_1024_sqr_avx2(BN_ULONG ret[40], const BN_ULONG a[40],
+                        const BN_ULONG n[40], BN_ULONG k, int count);
+
+// rsaz_1024_scatter5_avx2 stores |val| at index |i| of |tbl|. |i| must be
+// positive and at most 31. Note the table only uses 18 |BN_ULONG|s per entry
+// instead of 40. It packs two 29-bit limbs into each |BN_ULONG| and only stores
+// 36 limbs rather than the padded 40.
+void rsaz_1024_scatter5_avx2(BN_ULONG tbl[32 * 18], const BN_ULONG val[40],
+                             int i);
+
+// rsaz_1024_gather5_avx2 loads index |i| of |tbl| and writes it to |val|.
+void rsaz_1024_gather5_avx2(BN_ULONG val[40], const BN_ULONG tbl[32 * 18],
+                            int i);
+
+// rsaz_1024_red2norm_avx2 converts |red| from RSAZ to |BIGNUM| representation
+// and writes the result to |norm|.
+void rsaz_1024_red2norm_avx2(BN_ULONG norm[16], const BN_ULONG red[40]);
+
+
+#endif  // !OPENSSL_NO_ASM && OPENSSL_X86_64
 
-// rsaz_avx2_eligible returns one if |RSAZ_1024_mod_exp_avx2| should be used and
-// zero otherwise.
-int rsaz_avx2_eligible(void);
+#if defined(__cplusplus)
+}  // extern "C"
+#endif
 
 #endif  // OPENSSL_HEADER_BN_RSAZ_EXP_H

data/third_party/boringssl/crypto/fipsmodule/bn/shift.c

@@ -133,8 +133,8 @@ int BN_lshift1(BIGNUM *r, const BIGNUM *a) {
   return 1;
 }
 
-static void bn_rshift_words(BN_ULONG *r, const BN_ULONG *a, unsigned shift,
-                            size_t num) {
+void bn_rshift_words(BN_ULONG *r, const BN_ULONG *a, unsigned shift,
+                     size_t num) {
   unsigned shift_bits = shift % BN_BITS2;
   size_t shift_words = shift / BN_BITS2;
   if (shift_words >= num) {
@@ -296,15 +296,15 @@ int BN_mask_bits(BIGNUM *a, int n) {
 }
 
 static int bn_count_low_zero_bits_word(BN_ULONG l) {
-  OPENSSL_COMPILE_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t),
-                         crypto_word_t_too_small);
-  OPENSSL_COMPILE_ASSERT(sizeof(int) <= sizeof(crypto_word_t),
-                         crypto_word_t_too_small_2);
-  OPENSSL_COMPILE_ASSERT(BN_BITS2 == sizeof(BN_ULONG) * 8,
-                         bn_ulong_has_padding_bits);
+  OPENSSL_STATIC_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t),
+                        "crypto_word_t is too small");
+  OPENSSL_STATIC_ASSERT(sizeof(int) <= sizeof(crypto_word_t),
+                        "crypto_word_t is too small");
+  OPENSSL_STATIC_ASSERT(BN_BITS2 == sizeof(BN_ULONG) * 8,
+                        "BN_ULONG has padding bits");
   // C has very bizarre rules for types smaller than an int.
-  OPENSSL_COMPILE_ASSERT(sizeof(BN_ULONG) >= sizeof(int),
-                         bn_ulong_is_promoted_to_int);
+  OPENSSL_STATIC_ASSERT(sizeof(BN_ULONG) >= sizeof(int),
+                        "BN_ULONG gets promoted to int");
 
   crypto_word_t mask;
   int bits = 0;
@@ -342,10 +342,10 @@ static int bn_count_low_zero_bits_word(BN_ULONG l) {
 }
 
 int BN_count_low_zero_bits(const BIGNUM *bn) {
-  OPENSSL_COMPILE_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t),
-                         crypto_word_t_too_small);
-  OPENSSL_COMPILE_ASSERT(sizeof(int) <= sizeof(crypto_word_t),
-                         crypto_word_t_too_small_2);
+  OPENSSL_STATIC_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t),
+                        "crypto_word_t is too small");
+  OPENSSL_STATIC_ASSERT(sizeof(int) <= sizeof(crypto_word_t),
+                        "crypto_word_t is too small");
 
   int ret = 0;
   crypto_word_t saw_nonzero = 0;

data/third_party/boringssl/crypto/fipsmodule/cipher/cipher.c

@@ -125,9 +125,10 @@ int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in) {
   return 1;
 }
 
-void EVP_CIPHER_CTX_reset(EVP_CIPHER_CTX *ctx) {
+int EVP_CIPHER_CTX_reset(EVP_CIPHER_CTX *ctx) {
   EVP_CIPHER_CTX_cleanup(ctx);
   EVP_CIPHER_CTX_init(ctx);
+  return 1;
 }
 
 int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
@@ -191,7 +192,7 @@ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
 
       case EVP_CIPH_CFB_MODE:
         ctx->num = 0;
-        // fall-through
+        OPENSSL_FALLTHROUGH;
 
       case EVP_CIPH_CBC_MODE:
         assert(EVP_CIPHER_CTX_iv_length(ctx) <= sizeof(ctx->iv));
@@ -496,6 +497,10 @@ int EVP_CIPHER_CTX_nid(const EVP_CIPHER_CTX *ctx) {
   return ctx->cipher->nid;
 }
 
+int EVP_CIPHER_CTX_encrypting(const EVP_CIPHER_CTX *ctx) {
+  return ctx->encrypt;
+}
+
 unsigned EVP_CIPHER_CTX_block_size(const EVP_CIPHER_CTX *ctx) {
   return ctx->cipher->block_size;
 }

data/third_party/boringssl/crypto/fipsmodule/cipher/e_aes.c

@@ -46,6 +46,7 @@
  * OF THE POSSIBILITY OF SUCH DAMAGE.
  * ==================================================================== */
 
+#include <assert.h>
 #include <string.h>
 
 #include <openssl/aead.h>
@@ -63,13 +64,52 @@
 #include "../modes/internal.h"
 #include "../delocate.h"
 
-#if defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
-#include <openssl/arm_arch.h>
-#endif
-
 
+OPENSSL_MSVC_PRAGMA(warning(push))
 OPENSSL_MSVC_PRAGMA(warning(disable: 4702))  // Unreachable code.
 
+#if defined(BSAES)
+static void vpaes_ctr32_encrypt_blocks_with_bsaes(const uint8_t *in,
+                                                  uint8_t *out, size_t blocks,
+                                                  const AES_KEY *key,
+                                                  const uint8_t ivec[16]) {
+  // |bsaes_ctr32_encrypt_blocks| is faster than |vpaes_ctr32_encrypt_blocks|,
+  // but it takes at least one full 8-block batch to amortize the conversion.
+  if (blocks < 8) {
+    vpaes_ctr32_encrypt_blocks(in, out, blocks, key, ivec);
+    return;
+  }
+
+  size_t bsaes_blocks = blocks;
+  if (bsaes_blocks % 8 < 6) {
+    // |bsaes_ctr32_encrypt_blocks| internally works in 8-block batches. If the
+    // final batch is too small (under six blocks), it is faster to loop over
+    // |vpaes_encrypt|. Round |bsaes_blocks| down to a multiple of 8.
+    bsaes_blocks -= bsaes_blocks % 8;
+  }
+
+  AES_KEY bsaes;
+  vpaes_encrypt_key_to_bsaes(&bsaes, key);
+  bsaes_ctr32_encrypt_blocks(in, out, bsaes_blocks, &bsaes, ivec);
+  OPENSSL_cleanse(&bsaes, sizeof(bsaes));
+
+  in += 16 * bsaes_blocks;
+  out += 16 * bsaes_blocks;
+  blocks -= bsaes_blocks;
+
+  union {
+    uint32_t u32[4];
+    uint8_t u8[16];
+  } new_ivec;
+  memcpy(new_ivec.u8, ivec, 16);
+  uint32_t ctr = CRYPTO_bswap4(new_ivec.u32[3]) + bsaes_blocks;
+  new_ivec.u32[3] = CRYPTO_bswap4(ctr);
+
+  // Finish any remaining blocks with |vpaes_ctr32_encrypt_blocks|.
+  vpaes_ctr32_encrypt_blocks(in, out, blocks, key, new_ivec.u8);
+}
+#endif  // BSAES
+
 typedef struct {
   union {
     double align;
@@ -83,13 +123,13 @@ typedef struct {
 } EVP_AES_KEY;
 
 typedef struct {
+  GCM128_CONTEXT gcm;
   union {
     double align;
     AES_KEY ks;
   } ks;         // AES key schedule to use
   int key_set;  // Set if key initialised
   int iv_set;   // Set if an iv is set
-  GCM128_CONTEXT gcm;
   uint8_t *iv;  // Temporary IV store
   int ivlen;         // IV length
   int taglen;
@@ -97,129 +137,6 @@ typedef struct {
   ctr128_f ctr;
 } EVP_AES_GCM_CTX;
 
-#if !defined(OPENSSL_NO_ASM) && \
-    (defined(OPENSSL_X86_64) || defined(OPENSSL_X86))
-#define VPAES
-static char vpaes_capable(void) {
-  return (OPENSSL_ia32cap_P[1] & (1 << (41 - 32))) != 0;
-}
-
-#if defined(OPENSSL_X86_64)
-#define BSAES
-static char bsaes_capable(void) {
-  return vpaes_capable();
-}
-#endif
-
-#elif !defined(OPENSSL_NO_ASM) && \
-    (defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64))
-
-#if defined(OPENSSL_ARM) && __ARM_MAX_ARCH__ >= 7
-#define BSAES
-static char bsaes_capable(void) {
-  return CRYPTO_is_NEON_capable();
-}
-#endif
-
-#endif
-
-
-#if defined(BSAES)
-// On platforms where BSAES gets defined (just above), then these functions are
-// provided by asm.
-void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
-                       const AES_KEY *key, uint8_t ivec[16], int enc);
-void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len,
-                                const AES_KEY *key, const uint8_t ivec[16]);
-#else
-static char bsaes_capable(void) {
-  return 0;
-}
-
-// On other platforms, bsaes_capable() will always return false and so the
-// following will never be called.
-static void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
-                              const AES_KEY *key, uint8_t ivec[16], int enc) {
-  abort();
-}
-
-static void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
-                                       size_t len, const AES_KEY *key,
-                                       const uint8_t ivec[16]) {
-  abort();
-}
-#endif
-
-#if defined(VPAES)
-// On platforms where VPAES gets defined (just above), then these functions are
-// provided by asm.
-int vpaes_set_encrypt_key(const uint8_t *userKey, int bits, AES_KEY *key);
-int vpaes_set_decrypt_key(const uint8_t *userKey, int bits, AES_KEY *key);
-
-void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
-void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
-
-void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
-                       const AES_KEY *key, uint8_t *ivec, int enc);
-#else
-static char vpaes_capable(void) {
-  return 0;
-}
-
-// On other platforms, vpaes_capable() will always return false and so the
-// following will never be called.
-static int vpaes_set_encrypt_key(const uint8_t *userKey, int bits,
-                                 AES_KEY *key) {
-  abort();
-}
-static int vpaes_set_decrypt_key(const uint8_t *userKey, int bits,
-                                 AES_KEY *key) {
-  abort();
-}
-static void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {
-  abort();
-}
-static void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {
-  abort();
-}
-static void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
-                              const AES_KEY *key, uint8_t *ivec, int enc) {
-  abort();
-}
-#endif
-
-#if !defined(OPENSSL_NO_ASM) && \
-    (defined(OPENSSL_X86_64) || defined(OPENSSL_X86))
-int aesni_set_encrypt_key(const uint8_t *userKey, int bits, AES_KEY *key);
-int aesni_set_decrypt_key(const uint8_t *userKey, int bits, AES_KEY *key);
-
-void aesni_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
-void aesni_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
-
-void aesni_ecb_encrypt(const uint8_t *in, uint8_t *out, size_t length,
-                       const AES_KEY *key, int enc);
-void aesni_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
-                       const AES_KEY *key, uint8_t *ivec, int enc);
-
-#else
-
-// On other platforms, aesni_capable() will always return false and so the
-// following will never be called.
-static void aesni_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {
-  abort();
-}
-static int aesni_set_encrypt_key(const uint8_t *userKey, int bits,
-                                 AES_KEY *key) {
-  abort();
-}
-static void aesni_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
-                                       size_t blocks, const void *key,
-                                       const uint8_t *ivec) {
-  abort();
-}
-
-#endif
-
 static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key,
                         const uint8_t *iv, int enc) {
   int ret, mode;
@@ -229,49 +146,74 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key,
   if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) {
     if (hwaes_capable()) {
       ret = aes_hw_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
-      dat->block = (block128_f)aes_hw_decrypt;
+      dat->block = aes_hw_decrypt;
       dat->stream.cbc = NULL;
       if (mode == EVP_CIPH_CBC_MODE) {
-        dat->stream.cbc = (cbc128_f)aes_hw_cbc_encrypt;
+        dat->stream.cbc = aes_hw_cbc_encrypt;
       }
     } else if (bsaes_capable() && mode == EVP_CIPH_CBC_MODE) {
-      ret = AES_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
-      dat->block = (block128_f)AES_decrypt;
-      dat->stream.cbc = (cbc128_f)bsaes_cbc_encrypt;
+      assert(vpaes_capable());
+      ret = vpaes_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
+      if (ret == 0) {
+        vpaes_decrypt_key_to_bsaes(&dat->ks.ks, &dat->ks.ks);
+      }
+      // If |dat->stream.cbc| is provided, |dat->block| is never used.
+      dat->block = NULL;
+      dat->stream.cbc = bsaes_cbc_encrypt;
     } else if (vpaes_capable()) {
       ret = vpaes_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
-      dat->block = (block128_f)vpaes_decrypt;
-      dat->stream.cbc =
-          mode == EVP_CIPH_CBC_MODE ? (cbc128_f)vpaes_cbc_encrypt : NULL;
+      dat->block = vpaes_decrypt;
+      dat->stream.cbc = NULL;
+#if defined(VPAES_CBC)
+      if (mode == EVP_CIPH_CBC_MODE) {
+        dat->stream.cbc = vpaes_cbc_encrypt;
+      }
+#endif
     } else {
-      ret = AES_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
-      dat->block = (block128_f)AES_decrypt;
-      dat->stream.cbc =
-          mode == EVP_CIPH_CBC_MODE ? (cbc128_f)AES_cbc_encrypt : NULL;
+      ret = aes_nohw_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
+      dat->block = aes_nohw_decrypt;
+      dat->stream.cbc = NULL;
+#if defined(AES_NOHW_CBC)
+      if (mode == EVP_CIPH_CBC_MODE) {
+        dat->stream.cbc = aes_nohw_cbc_encrypt;
+      }
+#endif
     }
   } else if (hwaes_capable()) {
     ret = aes_hw_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
-    dat->block = (block128_f)aes_hw_encrypt;
+    dat->block = aes_hw_encrypt;
     dat->stream.cbc = NULL;
     if (mode == EVP_CIPH_CBC_MODE) {
-      dat->stream.cbc = (cbc128_f)aes_hw_cbc_encrypt;
+      dat->stream.cbc = aes_hw_cbc_encrypt;
     } else if (mode == EVP_CIPH_CTR_MODE) {
-      dat->stream.ctr = (ctr128_f)aes_hw_ctr32_encrypt_blocks;
+      dat->stream.ctr = aes_hw_ctr32_encrypt_blocks;
     }
-  } else if (bsaes_capable() && mode == EVP_CIPH_CTR_MODE) {
-    ret = AES_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
-    dat->block = (block128_f)AES_encrypt;
-    dat->stream.ctr = (ctr128_f)bsaes_ctr32_encrypt_blocks;
   } else if (vpaes_capable()) {
     ret = vpaes_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
-    dat->block = (block128_f)vpaes_encrypt;
-    dat->stream.cbc =
-        mode == EVP_CIPH_CBC_MODE ? (cbc128_f)vpaes_cbc_encrypt : NULL;
+    dat->block = vpaes_encrypt;
+    dat->stream.cbc = NULL;
+#if defined(VPAES_CBC)
+    if (mode == EVP_CIPH_CBC_MODE) {
+      dat->stream.cbc = vpaes_cbc_encrypt;
+    }
+#endif
+    if (mode == EVP_CIPH_CTR_MODE) {
+#if defined(BSAES)
+      assert(bsaes_capable());
+      dat->stream.ctr = vpaes_ctr32_encrypt_blocks_with_bsaes;
+#elif defined(VPAES_CTR32)
+      dat->stream.ctr = vpaes_ctr32_encrypt_blocks;
+#endif
+    }
   } else {
-    ret = AES_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
-    dat->block = (block128_f)AES_encrypt;
-    dat->stream.cbc =
-        mode == EVP_CIPH_CBC_MODE ? (cbc128_f)AES_cbc_encrypt : NULL;
+    ret = aes_nohw_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks);
+    dat->block = aes_nohw_encrypt;
+    dat->stream.cbc = NULL;
+#if defined(AES_NOHW_CBC)
+    if (mode == EVP_CIPH_CBC_MODE) {
+      dat->stream.cbc = aes_nohw_cbc_encrypt;
+    }
+#endif
   }
 
   if (ret < 0) {
@@ -287,11 +229,11 @@ static int aes_cbc_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in,
   EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
 
   if (dat->stream.cbc) {
-    (*dat->stream.cbc)(in, out, len, &dat->ks, ctx->iv, ctx->encrypt);
+    (*dat->stream.cbc)(in, out, len, &dat->ks.ks, ctx->iv, ctx->encrypt);
   } else if (ctx->encrypt) {
-    CRYPTO_cbc128_encrypt(in, out, len, &dat->ks, ctx->iv, dat->block);
+    CRYPTO_cbc128_encrypt(in, out, len, &dat->ks.ks, ctx->iv, dat->block);
   } else {
-    CRYPTO_cbc128_decrypt(in, out, len, &dat->ks, ctx->iv, dat->block);
+    CRYPTO_cbc128_decrypt(in, out, len, &dat->ks.ks, ctx->iv, dat->block);
   }
 
   return 1;
@@ -308,7 +250,7 @@ static int aes_ecb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in,
 
   len -= bl;
   for (size_t i = 0; i <= len; i += bl) {
-    (*dat->block)(in + i, out + i, &dat->ks);
+    (*dat->block)(in + i, out + i, &dat->ks.ks);
   }
 
   return 1;
@@ -319,11 +261,11 @@ static int aes_ctr_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in,
   EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
 
   if (dat->stream.ctr) {
-    CRYPTO_ctr128_encrypt_ctr32(in, out, len, &dat->ks, ctx->iv, ctx->buf,
+    CRYPTO_ctr128_encrypt_ctr32(in, out, len, &dat->ks.ks, ctx->iv, ctx->buf,
                                 &ctx->num, dat->stream.ctr);
   } else {
-    CRYPTO_ctr128_encrypt(in, out, len, &dat->ks, ctx->iv, ctx->buf, &ctx->num,
-                          dat->block);
+    CRYPTO_ctr128_encrypt(in, out, len, &dat->ks.ks, ctx->iv, ctx->buf,
+                          &ctx->num, dat->block);
   }
   return 1;
 }
@@ -332,78 +274,91 @@ static int aes_ofb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in,
                           size_t len) {
   EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
 
-  CRYPTO_ofb128_encrypt(in, out, len, &dat->ks, ctx->iv, &ctx->num, dat->block);
+  CRYPTO_ofb128_encrypt(in, out, len, &dat->ks.ks, ctx->iv, &ctx->num,
+                        dat->block);
   return 1;
 }
 
-static char aesni_capable(void);
-
-ctr128_f aes_ctr_set_key(AES_KEY *aes_key, GCM128_CONTEXT *gcm_ctx,
+ctr128_f aes_ctr_set_key(AES_KEY *aes_key, GCM128_KEY *gcm_key,
                          block128_f *out_block, const uint8_t *key,
                          size_t key_bytes) {
-  if (aesni_capable()) {
-    aesni_set_encrypt_key(key, key_bytes * 8, aes_key);
-    if (gcm_ctx != NULL) {
-      CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)aesni_encrypt, 1);
-    }
-    if (out_block) {
-      *out_block = (block128_f) aesni_encrypt;
-    }
-    return (ctr128_f)aesni_ctr32_encrypt_blocks;
-  }
-
   if (hwaes_capable()) {
     aes_hw_set_encrypt_key(key, key_bytes * 8, aes_key);
-    if (gcm_ctx != NULL) {
-      CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)aes_hw_encrypt, 0);
-    }
-    if (out_block) {
-      *out_block = (block128_f) aes_hw_encrypt;
-    }
-    return (ctr128_f)aes_hw_ctr32_encrypt_blocks;
-  }
-
-  if (bsaes_capable()) {
-    AES_set_encrypt_key(key, key_bytes * 8, aes_key);
-    if (gcm_ctx != NULL) {
-      CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)AES_encrypt, 0);
+    if (gcm_key != NULL) {
+      CRYPTO_gcm128_init_key(gcm_key, aes_key, aes_hw_encrypt, 1);
     }
     if (out_block) {
-      *out_block = (block128_f) AES_encrypt;
+      *out_block = aes_hw_encrypt;
     }
-    return (ctr128_f)bsaes_ctr32_encrypt_blocks;
+    return aes_hw_ctr32_encrypt_blocks;
   }
 
   if (vpaes_capable()) {
     vpaes_set_encrypt_key(key, key_bytes * 8, aes_key);
     if (out_block) {
-      *out_block = (block128_f) vpaes_encrypt;
+      *out_block = vpaes_encrypt;
     }
-    if (gcm_ctx != NULL) {
-      CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)vpaes_encrypt, 0);
+    if (gcm_key != NULL) {
+      CRYPTO_gcm128_init_key(gcm_key, aes_key, vpaes_encrypt, 0);
     }
+#if defined(BSAES)
+    assert(bsaes_capable());
+    return vpaes_ctr32_encrypt_blocks_with_bsaes;
+#elif defined(VPAES_CTR32)
+    return vpaes_ctr32_encrypt_blocks;
+#else
     return NULL;
+#endif
   }
 
-  AES_set_encrypt_key(key, key_bytes * 8, aes_key);
-  if (gcm_ctx != NULL) {
-    CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)AES_encrypt, 0);
+  aes_nohw_set_encrypt_key(key, key_bytes * 8, aes_key);
+  if (gcm_key != NULL) {
+    CRYPTO_gcm128_init_key(gcm_key, aes_key, aes_nohw_encrypt, 0);
   }
   if (out_block) {
-    *out_block = (block128_f) AES_encrypt;
+    *out_block = aes_nohw_encrypt;
   }
   return NULL;
 }
 
+#if defined(OPENSSL_32_BIT)
+#define EVP_AES_GCM_CTX_PADDING (4+8)
+#else
+#define EVP_AES_GCM_CTX_PADDING 8
+#endif
+
+static EVP_AES_GCM_CTX *aes_gcm_from_cipher_ctx(EVP_CIPHER_CTX *ctx) {
+#if defined(__GNUC__) || defined(__clang__)
+  OPENSSL_STATIC_ASSERT(
+      alignof(EVP_AES_GCM_CTX) <= 16,
+      "EVP_AES_GCM_CTX needs more alignment than this function provides");
+#endif
+
+  // |malloc| guarantees up to 4-byte alignment on 32-bit and 8-byte alignment
+  // on 64-bit systems, so we need to adjust to reach 16-byte alignment.
+  assert(ctx->cipher->ctx_size ==
+         sizeof(EVP_AES_GCM_CTX) + EVP_AES_GCM_CTX_PADDING);
+
+  char *ptr = ctx->cipher_data;
+#if defined(OPENSSL_32_BIT)
+  assert((uintptr_t)ptr % 4 == 0);
+  ptr += (uintptr_t)ptr & 4;
+#endif
+  assert((uintptr_t)ptr % 8 == 0);
+  ptr += (uintptr_t)ptr & 8;
+  return (EVP_AES_GCM_CTX *)ptr;
+}
+
 static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key,
                             const uint8_t *iv, int enc) {
-  EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
+  EVP_AES_GCM_CTX *gctx = aes_gcm_from_cipher_ctx(ctx);
   if (!iv && !key) {
     return 1;
   }
   if (key) {
-    gctx->ctr =
-        aes_ctr_set_key(&gctx->ks.ks, &gctx->gcm, NULL, key, ctx->key_len);
+    OPENSSL_memset(&gctx->gcm, 0, sizeof(gctx->gcm));
+    gctx->ctr = aes_ctr_set_key(&gctx->ks.ks, &gctx->gcm.gcm_key, NULL, key,
+                                ctx->key_len);
     // If we have an iv can set it directly, otherwise use saved IV.
     if (iv == NULL && gctx->iv_set) {
       iv = gctx->iv;
@@ -427,7 +382,7 @@ static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key,
 }
 
 static void aes_gcm_cleanup(EVP_CIPHER_CTX *c) {
-  EVP_AES_GCM_CTX *gctx = c->cipher_data;
+  EVP_AES_GCM_CTX *gctx = aes_gcm_from_cipher_ctx(c);
   OPENSSL_cleanse(&gctx->gcm, sizeof(gctx->gcm));
   if (gctx->iv != c->iv) {
     OPENSSL_free(gctx->iv);
@@ -451,7 +406,7 @@ static void ctr64_inc(uint8_t *counter) {
 }
 
 static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) {
-  EVP_AES_GCM_CTX *gctx = c->cipher_data;
+  EVP_AES_GCM_CTX *gctx = aes_gcm_from_cipher_ctx(c);
   switch (type) {
     case EVP_CTRL_INIT:
       gctx->key_set = 0;
@@ -462,7 +417,7 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) {
       gctx->iv_gen = 0;
       return 1;
 
-    case EVP_CTRL_GCM_SET_IVLEN:
+    case EVP_CTRL_AEAD_SET_IVLEN:
       if (arg <= 0) {
         return 0;
       }
@@ -480,7 +435,7 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) {
       gctx->ivlen = arg;
       return 1;
 
-    case EVP_CTRL_GCM_SET_TAG:
+    case EVP_CTRL_AEAD_SET_TAG:
       if (arg <= 0 || arg > 16 || c->encrypt) {
         return 0;
       }
@@ -488,14 +443,14 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) {
       gctx->taglen = arg;
       return 1;
 
-    case EVP_CTRL_GCM_GET_TAG:
+    case EVP_CTRL_AEAD_GET_TAG:
       if (arg <= 0 || arg > 16 || !c->encrypt || gctx->taglen < 0) {
         return 0;
       }
       OPENSSL_memcpy(ptr, c->buf, arg);
       return 1;
 
-    case EVP_CTRL_GCM_SET_IV_FIXED:
+    case EVP_CTRL_AEAD_SET_IV_FIXED:
       // Special case: -1 length restores whole IV
       if (arg == -1) {
         OPENSSL_memcpy(gctx->iv, ptr, gctx->ivlen);
@@ -543,7 +498,10 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) {
 
     case EVP_CTRL_COPY: {
       EVP_CIPHER_CTX *out = ptr;
-      EVP_AES_GCM_CTX *gctx_out = out->cipher_data;
+      EVP_AES_GCM_CTX *gctx_out = aes_gcm_from_cipher_ctx(out);
+      // |EVP_CIPHER_CTX_copy| copies this generically, but we must redo it in
+      // case |out->cipher_data| and |in->cipher_data| are differently aligned.
+      OPENSSL_memcpy(gctx_out, gctx, sizeof(EVP_AES_GCM_CTX));
       if (gctx->iv == c->iv) {
         gctx_out->iv = out->iv;
       } else {
@@ -563,7 +521,7 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) {
 
 static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in,
                           size_t len) {
-  EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
+  EVP_AES_GCM_CTX *gctx = aes_gcm_from_cipher_ctx(ctx);
 
   // If not set up, return error
   if (!gctx->key_set) {
@@ -677,8 +635,8 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_128_gcm_generic) {
   out->block_size = 1;
   out->key_len = 16;
   out->iv_len = 12;
-  out->ctx_size = sizeof(EVP_AES_GCM_CTX);
-  out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV |
+  out->ctx_size = sizeof(EVP_AES_GCM_CTX) + EVP_AES_GCM_CTX_PADDING;
+  out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | EVP_CIPH_CUSTOM_COPY |
                EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT |
                EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER;
   out->init = aes_gcm_init_key;
@@ -725,6 +683,19 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_192_ecb_generic) {
   out->cipher = aes_ecb_cipher;
 }
 
+DEFINE_LOCAL_DATA(EVP_CIPHER, aes_192_ofb_generic) {
+  memset(out, 0, sizeof(EVP_CIPHER));
+
+  out->nid = NID_aes_192_ofb128;
+  out->block_size = 1;
+  out->key_len = 24;
+  out->iv_len = 16;
+  out->ctx_size = sizeof(EVP_AES_KEY);
+  out->flags = EVP_CIPH_OFB_MODE;
+  out->init = aes_init_key;
+  out->cipher = aes_ofb_cipher;
+}
+
 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_192_gcm_generic) {
   memset(out, 0, sizeof(EVP_CIPHER));
 
@@ -732,8 +703,8 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_192_gcm_generic) {
   out->block_size = 1;
   out->key_len = 24;
   out->iv_len = 12;
-  out->ctx_size = sizeof(EVP_AES_GCM_CTX);
-  out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV |
+  out->ctx_size = sizeof(EVP_AES_GCM_CTX) + EVP_AES_GCM_CTX_PADDING;
+  out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | EVP_CIPH_CUSTOM_COPY |
                EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT |
                EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER;
   out->init = aes_gcm_init_key;
@@ -800,8 +771,8 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_256_gcm_generic) {
   out->block_size = 1;
   out->key_len = 32;
   out->iv_len = 12;
-  out->ctx_size = sizeof(EVP_AES_GCM_CTX);
-  out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV |
+  out->ctx_size = sizeof(EVP_AES_GCM_CTX) + EVP_AES_GCM_CTX_PADDING;
+  out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | EVP_CIPH_CUSTOM_COPY |
                EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT |
                EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER;
   out->init = aes_gcm_init_key;
@@ -810,126 +781,22 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_256_gcm_generic) {
   out->ctrl = aes_gcm_ctrl;
 }
 
-#if !defined(OPENSSL_NO_ASM) && \
-    (defined(OPENSSL_X86_64) || defined(OPENSSL_X86))
-
-// AES-NI section.
+#if defined(HWAES_ECB)
 
-static char aesni_capable(void) {
-  return (OPENSSL_ia32cap_P[1] & (1 << (57 - 32))) != 0;
-}
-
-static int aesni_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key,
-                          const uint8_t *iv, int enc) {
-  int ret, mode;
-  EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
-
-  mode = ctx->cipher->flags & EVP_CIPH_MODE_MASK;
-  if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) {
-    ret = aesni_set_decrypt_key(key, ctx->key_len * 8, ctx->cipher_data);
-    dat->block = (block128_f)aesni_decrypt;
-    dat->stream.cbc =
-        mode == EVP_CIPH_CBC_MODE ? (cbc128_f)aesni_cbc_encrypt : NULL;
-  } else {
-    ret = aesni_set_encrypt_key(key, ctx->key_len * 8, ctx->cipher_data);
-    dat->block = (block128_f)aesni_encrypt;
-    if (mode == EVP_CIPH_CBC_MODE) {
-      dat->stream.cbc = (cbc128_f)aesni_cbc_encrypt;
-    } else if (mode == EVP_CIPH_CTR_MODE) {
-      dat->stream.ctr = (ctr128_f)aesni_ctr32_encrypt_blocks;
-    } else {
-      dat->stream.cbc = NULL;
-    }
-  }
-
-  if (ret < 0) {
-    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_AES_KEY_SETUP_FAILED);
-    return 0;
-  }
-
-  return 1;
-}
-
-static int aesni_cbc_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out,
-                            const uint8_t *in, size_t len) {
-  aesni_cbc_encrypt(in, out, len, ctx->cipher_data, ctx->iv, ctx->encrypt);
-
-  return 1;
-}
-
-static int aesni_ecb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out,
-                            const uint8_t *in, size_t len) {
+static int aes_hw_ecb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out,
+                             const uint8_t *in, size_t len) {
   size_t bl = ctx->cipher->block_size;
 
   if (len < bl) {
     return 1;
   }
 
-  aesni_ecb_encrypt(in, out, len, ctx->cipher_data, ctx->encrypt);
+  aes_hw_ecb_encrypt(in, out, len, ctx->cipher_data, ctx->encrypt);
 
   return 1;
 }
 
-static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key,
-                              const uint8_t *iv, int enc) {
-  EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
-  if (!iv && !key) {
-    return 1;
-  }
-  if (key) {
-    aesni_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
-    CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f)aesni_encrypt, 1);
-    gctx->ctr = (ctr128_f)aesni_ctr32_encrypt_blocks;
-    // If we have an iv can set it directly, otherwise use
-    // saved IV.
-    if (iv == NULL && gctx->iv_set) {
-      iv = gctx->iv;
-    }
-    if (iv) {
-      CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen);
-      gctx->iv_set = 1;
-    }
-    gctx->key_set = 1;
-  } else {
-    // If key set use IV, otherwise copy
-    if (gctx->key_set) {
-      CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen);
-    } else {
-      OPENSSL_memcpy(gctx->iv, iv, gctx->ivlen);
-    }
-    gctx->iv_set = 1;
-    gctx->iv_gen = 0;
-  }
-  return 1;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_cbc) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_128_cbc;
-  out->block_size = 16;
-  out->key_len = 16;
-  out->iv_len = 16;
-  out->ctx_size = sizeof(EVP_AES_KEY);
-  out->flags = EVP_CIPH_CBC_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aesni_cbc_cipher;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ctr) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_128_ctr;
-  out->block_size = 1;
-  out->key_len = 16;
-  out->iv_len = 16;
-  out->ctx_size = sizeof(EVP_AES_KEY);
-  out->flags = EVP_CIPH_CTR_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aes_ctr_cipher;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ecb) {
+DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_128_ecb) {
   memset(out, 0, sizeof(EVP_CIPHER));
 
   out->nid = NID_aes_128_ecb;
@@ -937,67 +804,11 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ecb) {
   out->key_len = 16;
   out->ctx_size = sizeof(EVP_AES_KEY);
   out->flags = EVP_CIPH_ECB_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aesni_ecb_cipher;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ofb) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_128_ofb128;
-  out->block_size = 1;
-  out->key_len = 16;
-  out->iv_len = 16;
-  out->ctx_size = sizeof(EVP_AES_KEY);
-  out->flags = EVP_CIPH_OFB_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aes_ofb_cipher;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_gcm) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_128_gcm;
-  out->block_size = 1;
-  out->key_len = 16;
-  out->iv_len = 12;
-  out->ctx_size = sizeof(EVP_AES_GCM_CTX);
-  out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV |
-               EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT |
-               EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER;
-  out->init = aesni_gcm_init_key;
-  out->cipher = aes_gcm_cipher;
-  out->cleanup = aes_gcm_cleanup;
-  out->ctrl = aes_gcm_ctrl;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_cbc) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_192_cbc;
-  out->block_size = 16;
-  out->key_len = 24;
-  out->iv_len = 16;
-  out->ctx_size = sizeof(EVP_AES_KEY);
-  out->flags = EVP_CIPH_CBC_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aesni_cbc_cipher;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ctr) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_192_ctr;
-  out->block_size = 1;
-  out->key_len = 24;
-  out->iv_len = 16;
-  out->ctx_size = sizeof(EVP_AES_KEY);
-  out->flags = EVP_CIPH_CTR_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aes_ctr_cipher;
+  out->init = aes_init_key;
+  out->cipher = aes_hw_ecb_cipher;
 }
 
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ecb) {
+DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_192_ecb) {
   memset(out, 0, sizeof(EVP_CIPHER));
 
   out->nid = NID_aes_192_ecb;
@@ -1005,54 +816,11 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ecb) {
   out->key_len = 24;
   out->ctx_size = sizeof(EVP_AES_KEY);
   out->flags = EVP_CIPH_ECB_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aesni_ecb_cipher;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_gcm) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_192_gcm;
-  out->block_size = 1;
-  out->key_len = 24;
-  out->iv_len = 12;
-  out->ctx_size = sizeof(EVP_AES_GCM_CTX);
-  out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV |
-               EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT |
-               EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER;
-  out->init = aesni_gcm_init_key;
-  out->cipher = aes_gcm_cipher;
-  out->cleanup = aes_gcm_cleanup;
-  out->ctrl = aes_gcm_ctrl;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_cbc) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_256_cbc;
-  out->block_size = 16;
-  out->key_len = 32;
-  out->iv_len = 16;
-  out->ctx_size = sizeof(EVP_AES_KEY);
-  out->flags = EVP_CIPH_CBC_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aesni_cbc_cipher;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ctr) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_256_ctr;
-  out->block_size = 1;
-  out->key_len = 32;
-  out->iv_len = 16;
-  out->ctx_size = sizeof(EVP_AES_KEY);
-  out->flags = EVP_CIPH_CTR_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aes_ctr_cipher;
+  out->init = aes_init_key;
+  out->cipher = aes_hw_ecb_cipher;
 }
 
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ecb) {
+DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_256_ecb) {
   memset(out, 0, sizeof(EVP_CIPHER));
 
   out->nid = NID_aes_256_ecb;
@@ -1060,80 +828,51 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ecb) {
   out->key_len = 32;
   out->ctx_size = sizeof(EVP_AES_KEY);
   out->flags = EVP_CIPH_ECB_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aesni_ecb_cipher;
-}
-
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ofb) {
-  memset(out, 0, sizeof(EVP_CIPHER));
-
-  out->nid = NID_aes_256_ofb128;
-  out->block_size = 1;
-  out->key_len = 32;
-  out->iv_len = 16;
-  out->ctx_size = sizeof(EVP_AES_KEY);
-  out->flags = EVP_CIPH_OFB_MODE;
-  out->init = aesni_init_key;
-  out->cipher = aes_ofb_cipher;
+  out->init = aes_init_key;
+  out->cipher = aes_hw_ecb_cipher;
 }
 
-DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_gcm) {
-  memset(out, 0, sizeof(EVP_CIPHER));
+#define EVP_ECB_CIPHER_FUNCTION(keybits)            \
+  const EVP_CIPHER *EVP_aes_##keybits##_ecb(void) { \
+    if (hwaes_capable()) {                          \
+      return aes_hw_##keybits##_ecb();              \
+    }                                               \
+    return aes_##keybits##_ecb_generic();           \
+  }
 
-  out->nid = NID_aes_256_gcm;
-  out->block_size = 1;
-  out->key_len = 32;
-  out->iv_len = 12;
-  out->ctx_size = sizeof(EVP_AES_GCM_CTX);
-  out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV |
-               EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT |
-               EVP_CIPH_CTRL_INIT | EVP_CIPH_CUSTOM_COPY |
-               EVP_CIPH_FLAG_AEAD_CIPHER;
-  out->init = aesni_gcm_init_key;
-  out->cipher = aes_gcm_cipher;
-  out->cleanup = aes_gcm_cleanup;
-  out->ctrl = aes_gcm_ctrl;
-}
+#else
 
-#define EVP_CIPHER_FUNCTION(keybits, mode)             \
-  const EVP_CIPHER *EVP_aes_##keybits##_##mode(void) { \
-    if (aesni_capable()) {                             \
-      return aesni_##keybits##_##mode();               \
-    } else {                                           \
-      return aes_##keybits##_##mode##_generic();       \
-    }                                                  \
+#define EVP_ECB_CIPHER_FUNCTION(keybits)            \
+  const EVP_CIPHER *EVP_aes_##keybits##_ecb(void) { \
+    return aes_##keybits##_ecb_generic();           \
   }
 
-#else  // ^^^  OPENSSL_X86_64 || OPENSSL_X86
-
-static char aesni_capable(void) {
-  return 0;
-}
+#endif  // HWAES_ECB
 
 #define EVP_CIPHER_FUNCTION(keybits, mode)             \
   const EVP_CIPHER *EVP_aes_##keybits##_##mode(void) { \
     return aes_##keybits##_##mode##_generic();         \
   }
 
-#endif
-
 EVP_CIPHER_FUNCTION(128, cbc)
 EVP_CIPHER_FUNCTION(128, ctr)
-EVP_CIPHER_FUNCTION(128, ecb)
 EVP_CIPHER_FUNCTION(128, ofb)
 EVP_CIPHER_FUNCTION(128, gcm)
 
 EVP_CIPHER_FUNCTION(192, cbc)
 EVP_CIPHER_FUNCTION(192, ctr)
-EVP_CIPHER_FUNCTION(192, ecb)
+EVP_CIPHER_FUNCTION(192, ofb)
 EVP_CIPHER_FUNCTION(192, gcm)
 
 EVP_CIPHER_FUNCTION(256, cbc)
 EVP_CIPHER_FUNCTION(256, ctr)
-EVP_CIPHER_FUNCTION(256, ecb)
 EVP_CIPHER_FUNCTION(256, ofb)
 EVP_CIPHER_FUNCTION(256, gcm)
 
+EVP_ECB_CIPHER_FUNCTION(128)
+EVP_ECB_CIPHER_FUNCTION(192)
+EVP_ECB_CIPHER_FUNCTION(256)
+
 
 #define EVP_AEAD_AES_GCM_TAG_LEN 16
 
@@ -1142,21 +881,16 @@ struct aead_aes_gcm_ctx {
     double align;
     AES_KEY ks;
   } ks;
-  GCM128_CONTEXT gcm;
+  GCM128_KEY gcm_key;
   ctr128_f ctr;
 };
 
-struct aead_aes_gcm_tls12_ctx {
-  struct aead_aes_gcm_ctx gcm_ctx;
-  uint64_t min_next_nonce;
-};
-
 static int aead_aes_gcm_init_impl(struct aead_aes_gcm_ctx *gcm_ctx,
                                   size_t *out_tag_len, const uint8_t *key,
                                   size_t key_len, size_t tag_len) {
   const size_t key_bits = key_len * 8;
 
-  if (key_bits != 128 && key_bits != 256) {
+  if (key_bits != 128 && key_bits != 192 && key_bits != 256) {
     OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_KEY_LENGTH);
     return 0;  // EVP_AEAD_CTX_init should catch this.
   }
@@ -1171,34 +905,35 @@ static int aead_aes_gcm_init_impl(struct aead_aes_gcm_ctx *gcm_ctx,
   }
 
   gcm_ctx->ctr =
-      aes_ctr_set_key(&gcm_ctx->ks.ks, &gcm_ctx->gcm, NULL, key, key_len);
+      aes_ctr_set_key(&gcm_ctx->ks.ks, &gcm_ctx->gcm_key, NULL, key, key_len);
   *out_tag_len = tag_len;
   return 1;
 }
 
+OPENSSL_STATIC_ASSERT(sizeof(((EVP_AEAD_CTX *)NULL)->state) >=
+                          sizeof(struct aead_aes_gcm_ctx),
+                      "AEAD state is too small");
+#if defined(__GNUC__) || defined(__clang__)
+OPENSSL_STATIC_ASSERT(alignof(union evp_aead_ctx_st_state) >=
+                          alignof(struct aead_aes_gcm_ctx),
+                      "AEAD state has insufficient alignment");
+#endif
+
 static int aead_aes_gcm_init(EVP_AEAD_CTX *ctx, const uint8_t *key,
                              size_t key_len, size_t requested_tag_len) {
-  struct aead_aes_gcm_ctx *gcm_ctx;
-  gcm_ctx = OPENSSL_malloc(sizeof(struct aead_aes_gcm_ctx));
-  if (gcm_ctx == NULL) {
-    return 0;
-  }
+  struct aead_aes_gcm_ctx *gcm_ctx = (struct aead_aes_gcm_ctx *) &ctx->state;
 
   size_t actual_tag_len;
   if (!aead_aes_gcm_init_impl(gcm_ctx, &actual_tag_len, key, key_len,
                               requested_tag_len)) {
-    OPENSSL_free(gcm_ctx);
     return 0;
   }
 
-  ctx->aead_state = gcm_ctx;
   ctx->tag_len = actual_tag_len;
   return 1;
 }
 
-static void aead_aes_gcm_cleanup(EVP_AEAD_CTX *ctx) {
-  OPENSSL_free(ctx->aead_state);
-}
+static void aead_aes_gcm_cleanup(EVP_AEAD_CTX *ctx) {}
 
 static int aead_aes_gcm_seal_scatter(const EVP_AEAD_CTX *ctx, uint8_t *out,
                                      uint8_t *out_tag, size_t *out_tag_len,
@@ -1208,8 +943,7 @@ static int aead_aes_gcm_seal_scatter(const EVP_AEAD_CTX *ctx, uint8_t *out,
                                      const uint8_t *extra_in,
                                      size_t extra_in_len,
                                      const uint8_t *ad, size_t ad_len) {
-  const struct aead_aes_gcm_ctx *gcm_ctx = ctx->aead_state;
-  GCM128_CONTEXT gcm;
+  struct aead_aes_gcm_ctx *gcm_ctx = (struct aead_aes_gcm_ctx *) &ctx->state;
 
   if (extra_in_len + ctx->tag_len < ctx->tag_len) {
     OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TOO_LARGE);
@@ -1226,7 +960,9 @@ static int aead_aes_gcm_seal_scatter(const EVP_AEAD_CTX *ctx, uint8_t *out,
 
   const AES_KEY *key = &gcm_ctx->ks.ks;
 
-  OPENSSL_memcpy(&gcm, &gcm_ctx->gcm, sizeof(gcm));
+  GCM128_CONTEXT gcm;
+  OPENSSL_memset(&gcm, 0, sizeof(gcm));
+  OPENSSL_memcpy(&gcm.gcm_key, &gcm_ctx->gcm_key, sizeof(gcm.gcm_key));
   CRYPTO_gcm128_setiv(&gcm, key, nonce, nonce_len);
 
   if (ad_len > 0 && !CRYPTO_gcm128_aad(&gcm, ad, ad_len)) {
@@ -1268,9 +1004,8 @@ static int aead_aes_gcm_open_gather(const EVP_AEAD_CTX *ctx, uint8_t *out,
                                     const uint8_t *in, size_t in_len,
                                     const uint8_t *in_tag, size_t in_tag_len,
                                     const uint8_t *ad, size_t ad_len) {
-  const struct aead_aes_gcm_ctx *gcm_ctx = ctx->aead_state;
+  struct aead_aes_gcm_ctx *gcm_ctx = (struct aead_aes_gcm_ctx *) &ctx->state;
   uint8_t tag[EVP_AEAD_AES_GCM_TAG_LEN];
-  GCM128_CONTEXT gcm;
 
   if (nonce_len == 0) {
     OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_INVALID_NONCE_SIZE);
@@ -1284,7 +1019,9 @@ static int aead_aes_gcm_open_gather(const EVP_AEAD_CTX *ctx, uint8_t *out,
 
   const AES_KEY *key = &gcm_ctx->ks.ks;
 
-  OPENSSL_memcpy(&gcm, &gcm_ctx->gcm, sizeof(gcm));
+  GCM128_CONTEXT gcm;
+  OPENSSL_memset(&gcm, 0, sizeof(gcm));
+  OPENSSL_memcpy(&gcm.gcm_key, &gcm_ctx->gcm_key, sizeof(gcm.gcm_key));
   CRYPTO_gcm128_setiv(&gcm, key, nonce, nonce_len);
 
   if (!CRYPTO_gcm128_aad(&gcm, ad, ad_len)) {
@@ -1326,6 +1063,21 @@ DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_128_gcm) {
   out->open_gather = aead_aes_gcm_open_gather;
 }
 
+DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_192_gcm) {
+  memset(out, 0, sizeof(EVP_AEAD));
+
+  out->key_len = 24;
+  out->nonce_len = 12;
+  out->overhead = EVP_AEAD_AES_GCM_TAG_LEN;
+  out->max_tag_len = EVP_AEAD_AES_GCM_TAG_LEN;
+  out->seal_scatter_supports_extra_in = 1;
+
+  out->init = aead_aes_gcm_init;
+  out->cleanup = aead_aes_gcm_cleanup;
+  out->seal_scatter = aead_aes_gcm_seal_scatter;
+  out->open_gather = aead_aes_gcm_open_gather;
+}
+
 DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_256_gcm) {
   memset(out, 0, sizeof(EVP_AEAD));
 
@@ -1341,38 +1093,45 @@ DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_256_gcm) {
   out->open_gather = aead_aes_gcm_open_gather;
 }
 
+struct aead_aes_gcm_tls12_ctx {
+  struct aead_aes_gcm_ctx gcm_ctx;
+  uint64_t min_next_nonce;
+};
+
+OPENSSL_STATIC_ASSERT(sizeof(((EVP_AEAD_CTX *)NULL)->state) >=
+                          sizeof(struct aead_aes_gcm_tls12_ctx),
+                      "AEAD state is too small");
+#if defined(__GNUC__) || defined(__clang__)
+OPENSSL_STATIC_ASSERT(alignof(union evp_aead_ctx_st_state) >=
+                          alignof(struct aead_aes_gcm_tls12_ctx),
+                      "AEAD state has insufficient alignment");
+#endif
+
 static int aead_aes_gcm_tls12_init(EVP_AEAD_CTX *ctx, const uint8_t *key,
                                    size_t key_len, size_t requested_tag_len) {
-  struct aead_aes_gcm_tls12_ctx *gcm_ctx;
-  gcm_ctx = OPENSSL_malloc(sizeof(struct aead_aes_gcm_tls12_ctx));
-  if (gcm_ctx == NULL) {
-    return 0;
-  }
+  struct aead_aes_gcm_tls12_ctx *gcm_ctx =
+      (struct aead_aes_gcm_tls12_ctx *) &ctx->state;
 
   gcm_ctx->min_next_nonce = 0;
 
   size_t actual_tag_len;
   if (!aead_aes_gcm_init_impl(&gcm_ctx->gcm_ctx, &actual_tag_len, key, key_len,
                               requested_tag_len)) {
-    OPENSSL_free(gcm_ctx);
     return 0;
   }
 
-  ctx->aead_state = gcm_ctx;
   ctx->tag_len = actual_tag_len;
   return 1;
 }
 
-static void aead_aes_gcm_tls12_cleanup(EVP_AEAD_CTX *ctx) {
-  OPENSSL_free(ctx->aead_state);
-}
-
 static int aead_aes_gcm_tls12_seal_scatter(
     const EVP_AEAD_CTX *ctx, uint8_t *out, uint8_t *out_tag,
     size_t *out_tag_len, size_t max_out_tag_len, const uint8_t *nonce,
     size_t nonce_len, const uint8_t *in, size_t in_len, const uint8_t *extra_in,
     size_t extra_in_len, const uint8_t *ad, size_t ad_len) {
-  struct aead_aes_gcm_tls12_ctx *gcm_ctx = ctx->aead_state;
+  struct aead_aes_gcm_tls12_ctx *gcm_ctx =
+      (struct aead_aes_gcm_tls12_ctx *) &ctx->state;
+
   if (nonce_len != 12) {
     OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_UNSUPPORTED_NONCE_SIZE);
     return 0;
@@ -1406,7 +1165,7 @@ DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_128_gcm_tls12) {
   out->seal_scatter_supports_extra_in = 1;
 
   out->init = aead_aes_gcm_tls12_init;
-  out->cleanup = aead_aes_gcm_tls12_cleanup;
+  out->cleanup = aead_aes_gcm_cleanup;
   out->seal_scatter = aead_aes_gcm_tls12_seal_scatter;
   out->open_gather = aead_aes_gcm_open_gather;
 }
@@ -1421,17 +1180,125 @@ DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_256_gcm_tls12) {
   out->seal_scatter_supports_extra_in = 1;
 
   out->init = aead_aes_gcm_tls12_init;
-  out->cleanup = aead_aes_gcm_tls12_cleanup;
+  out->cleanup = aead_aes_gcm_cleanup;
   out->seal_scatter = aead_aes_gcm_tls12_seal_scatter;
   out->open_gather = aead_aes_gcm_open_gather;
 }
 
+struct aead_aes_gcm_tls13_ctx {
+  struct aead_aes_gcm_ctx gcm_ctx;
+  uint64_t min_next_nonce;
+  uint64_t mask;
+  uint8_t first;
+};
+
+OPENSSL_STATIC_ASSERT(sizeof(((EVP_AEAD_CTX *)NULL)->state) >=
+                          sizeof(struct aead_aes_gcm_tls13_ctx),
+                      "AEAD state is too small");
+#if defined(__GNUC__) || defined(__clang__)
+OPENSSL_STATIC_ASSERT(alignof(union evp_aead_ctx_st_state) >=
+                          alignof(struct aead_aes_gcm_tls13_ctx),
+                      "AEAD state has insufficient alignment");
+#endif
+
+static int aead_aes_gcm_tls13_init(EVP_AEAD_CTX *ctx, const uint8_t *key,
+                                   size_t key_len, size_t requested_tag_len) {
+  struct aead_aes_gcm_tls13_ctx *gcm_ctx =
+      (struct aead_aes_gcm_tls13_ctx *) &ctx->state;
+
+  gcm_ctx->min_next_nonce = 0;
+  gcm_ctx->first = 1;
+
+  size_t actual_tag_len;
+  if (!aead_aes_gcm_init_impl(&gcm_ctx->gcm_ctx, &actual_tag_len, key, key_len,
+                              requested_tag_len)) {
+    return 0;
+  }
+
+  ctx->tag_len = actual_tag_len;
+  return 1;
+}
+
+static int aead_aes_gcm_tls13_seal_scatter(
+    const EVP_AEAD_CTX *ctx, uint8_t *out, uint8_t *out_tag,
+    size_t *out_tag_len, size_t max_out_tag_len, const uint8_t *nonce,
+    size_t nonce_len, const uint8_t *in, size_t in_len, const uint8_t *extra_in,
+    size_t extra_in_len, const uint8_t *ad, size_t ad_len) {
+  struct aead_aes_gcm_tls13_ctx *gcm_ctx =
+      (struct aead_aes_gcm_tls13_ctx *) &ctx->state;
+
+  if (nonce_len != 12) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_UNSUPPORTED_NONCE_SIZE);
+    return 0;
+  }
+
+  // The given nonces must be strictly monotonically increasing. See
+  // https://tools.ietf.org/html/rfc8446#section-5.3 for details of the TLS 1.3
+  // nonce construction.
+  uint64_t given_counter;
+  OPENSSL_memcpy(&given_counter, nonce + nonce_len - sizeof(given_counter),
+                 sizeof(given_counter));
+  given_counter = CRYPTO_bswap8(given_counter);
+
+  if (gcm_ctx->first) {
+    // In the first call the sequence number will be zero and therefore the
+    // given nonce will be 0 ^ mask = mask.
+    gcm_ctx->mask = given_counter;
+    gcm_ctx->first = 0;
+  }
+  given_counter ^= gcm_ctx->mask;
+
+  if (given_counter == UINT64_MAX ||
+      given_counter < gcm_ctx->min_next_nonce) {
+    OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_INVALID_NONCE);
+    return 0;
+  }
+
+  gcm_ctx->min_next_nonce = given_counter + 1;
+
+  return aead_aes_gcm_seal_scatter(ctx, out, out_tag, out_tag_len,
+                                   max_out_tag_len, nonce, nonce_len, in,
+                                   in_len, extra_in, extra_in_len, ad, ad_len);
+}
+
+DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_128_gcm_tls13) {
+  memset(out, 0, sizeof(EVP_AEAD));
+
+  out->key_len = 16;
+  out->nonce_len = 12;
+  out->overhead = EVP_AEAD_AES_GCM_TAG_LEN;
+  out->max_tag_len = EVP_AEAD_AES_GCM_TAG_LEN;
+  out->seal_scatter_supports_extra_in = 1;
+
+  out->init = aead_aes_gcm_tls13_init;
+  out->cleanup = aead_aes_gcm_cleanup;
+  out->seal_scatter = aead_aes_gcm_tls13_seal_scatter;
+  out->open_gather = aead_aes_gcm_open_gather;
+}
+
+DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_256_gcm_tls13) {
+  memset(out, 0, sizeof(EVP_AEAD));
+
+  out->key_len = 32;
+  out->nonce_len = 12;
+  out->overhead = EVP_AEAD_AES_GCM_TAG_LEN;
+  out->max_tag_len = EVP_AEAD_AES_GCM_TAG_LEN;
+  out->seal_scatter_supports_extra_in = 1;
+
+  out->init = aead_aes_gcm_tls13_init;
+  out->cleanup = aead_aes_gcm_cleanup;
+  out->seal_scatter = aead_aes_gcm_tls13_seal_scatter;
+  out->open_gather = aead_aes_gcm_open_gather;
+}
+
 int EVP_has_aes_hardware(void) {
 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
-  return aesni_capable() && crypto_gcm_clmul_enabled();
+  return hwaes_capable() && crypto_gcm_clmul_enabled();
 #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
   return hwaes_capable() && CRYPTO_is_ARMv8_PMULL_capable();
 #else
   return 0;
 #endif
 }
+
+OPENSSL_MSVC_PRAGMA(warning(pop))