grpc 1.64.3 → 1.71.0

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/internal.h

@@ -1,62 +1,85 @@
-/* Copyright (c) 2017, Google Inc.
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
- * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
- * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+// Copyright 2017 The BoringSSL Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 #ifndef OPENSSL_HEADER_AES_INTERNAL_H
 #define OPENSSL_HEADER_AES_INTERNAL_H
 
 #include <stdlib.h>
 
+#include <openssl/aes.h>
+
 #include "../../internal.h"
 
-#if defined(__cplusplus)
 extern "C" {
-#endif
 
 
+// block128_f is the type of an AES block cipher implementation.
+//
+// Unlike upstream OpenSSL, it and the other functions in this file hard-code
+// |AES_KEY|. It is undefined in C to call a function pointer with anything
+// other than the original type. Thus we either must match |block128_f| to the
+// type signature of |AES_encrypt| and friends or pass in |void*| wrapper
+// functions.
+//
+// These functions are called exclusively with AES, so we use the former.
+typedef void (*block128_f)(const uint8_t in[16], uint8_t out[16],
+                           const AES_KEY *key);
+
+// ctr128_f is the type of a function that performs CTR-mode encryption.
+typedef void (*ctr128_f)(const uint8_t *in, uint8_t *out, size_t blocks,
+                         const AES_KEY *key, const uint8_t ivec[16]);
+
+// aes_ctr_set_key initialises |*aes_key| using |key_bytes| bytes from |key|,
+// where |key_bytes| must either be 16, 24 or 32. If not NULL, |*out_block| is
+// set to a function that encrypts single blocks. If not NULL, |*out_is_hwaes|
+// is set to whether the hardware AES implementation was used. It returns a
+// function for optimised CTR-mode.
+ctr128_f aes_ctr_set_key(AES_KEY *aes_key, int *out_is_hwaes,
+                         block128_f *out_block, const uint8_t *key,
+                         size_t key_bytes);
+
+
+// AES implementations.
+
 #if !defined(OPENSSL_NO_ASM)
 
 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
 #define HWAES
 #define HWAES_ECB
 
-OPENSSL_INLINE int hwaes_capable(void) { return CRYPTO_is_AESNI_capable(); }
+inline int hwaes_capable(void) { return CRYPTO_is_AESNI_capable(); }
 
 #define VPAES
-#if defined(OPENSSL_X86_64)
-#define VPAES_CTR32
-#endif
 #define VPAES_CBC
-OPENSSL_INLINE int vpaes_capable(void) { return CRYPTO_is_SSSE3_capable(); }
+inline int vpaes_capable(void) { return CRYPTO_is_SSSE3_capable(); }
 
 #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
 #define HWAES
 
-OPENSSL_INLINE int hwaes_capable(void) { return CRYPTO_is_ARMv8_AES_capable(); }
+inline int hwaes_capable(void) { return CRYPTO_is_ARMv8_AES_capable(); }
 
 #if defined(OPENSSL_ARM)
 #define BSAES
 #define VPAES
-#define VPAES_CTR32
-OPENSSL_INLINE int bsaes_capable(void) { return CRYPTO_is_NEON_capable(); }
-OPENSSL_INLINE int vpaes_capable(void) { return CRYPTO_is_NEON_capable(); }
+inline int bsaes_capable(void) { return CRYPTO_is_NEON_capable(); }
+inline int vpaes_capable(void) { return CRYPTO_is_NEON_capable(); }
 #endif
 
 #if defined(OPENSSL_AARCH64)
 #define VPAES
 #define VPAES_CBC
-#define VPAES_CTR32
-OPENSSL_INLINE int vpaes_capable(void) { return CRYPTO_is_NEON_capable(); }
+inline int vpaes_capable(void) { return CRYPTO_is_NEON_capable(); }
 #endif
 
 #endif
@@ -66,52 +89,75 @@ OPENSSL_INLINE int vpaes_capable(void) { return CRYPTO_is_NEON_capable(); }
 
 #if defined(HWAES)
 
-int aes_hw_set_encrypt_key(const uint8_t *user_key, const int bits,
-                           AES_KEY *key);
-int aes_hw_set_decrypt_key(const uint8_t *user_key, const int bits,
-                           AES_KEY *key);
+int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits, AES_KEY *key);
+int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits, AES_KEY *key);
 void aes_hw_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
 void aes_hw_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
 void aes_hw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
-                        const AES_KEY *key, uint8_t *ivec, const int enc);
+                        const AES_KEY *key, uint8_t *ivec, int enc);
 void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len,
                                  const AES_KEY *key, const uint8_t ivec[16]);
 
+#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
+// On x86 and x86_64, |aes_hw_set_decrypt_key| is implemented in terms of
+// |aes_hw_set_encrypt_key| and a conversion function.
+void aes_hw_encrypt_key_to_decrypt_key(AES_KEY *key);
+
+// There are two variants of this function, one which uses aeskeygenassist
+// ("base") and one which uses aesenclast + pshufb ("alt"). aesenclast is
+// overall faster but is slower on some older processors. It doesn't use AVX,
+// but AVX is used as a proxy to detecting this. See
+// https://groups.google.com/g/mailing.openssl.dev/c/OuFXwW4NfO8/m/7d2ZXVjkxVkJ
+//
+// TODO(davidben): It is unclear if the aeskeygenassist version is still
+// worthwhile. However, the aesenclast version requires SSSE3. SSSE3 long
+// predates AES-NI, but it's not clear if AES-NI implies SSSE3. In OpenSSL, the
+// CCM AES-NI assembly seems to assume it does.
+inline int aes_hw_set_encrypt_key_alt_capable(void) {
+  return hwaes_capable() && CRYPTO_is_SSSE3_capable();
+}
+inline int aes_hw_set_encrypt_key_alt_preferred(void) {
+  return hwaes_capable() && CRYPTO_is_AVX_capable();
+}
+int aes_hw_set_encrypt_key_base(const uint8_t *user_key, int bits,
+                                AES_KEY *key);
+int aes_hw_set_encrypt_key_alt(const uint8_t *user_key, int bits, AES_KEY *key);
+#endif  // OPENSSL_X86 || OPENSSL_X86_64
+
 #else
 
 // If HWAES isn't defined then we provide dummy functions for each of the hwaes
 // functions.
-OPENSSL_INLINE int hwaes_capable(void) { return 0; }
+inline int hwaes_capable(void) { return 0; }
 
-OPENSSL_INLINE int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits,
-                                          AES_KEY *key) {
+inline int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits,
+                                  AES_KEY *key) {
   abort();
 }
 
-OPENSSL_INLINE int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits,
-                                          AES_KEY *key) {
+inline int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits,
+                                  AES_KEY *key) {
   abort();
 }
 
-OPENSSL_INLINE void aes_hw_encrypt(const uint8_t *in, uint8_t *out,
-                                   const AES_KEY *key) {
+inline void aes_hw_encrypt(const uint8_t *in, uint8_t *out,
+                           const AES_KEY *key) {
   abort();
 }
 
-OPENSSL_INLINE void aes_hw_decrypt(const uint8_t *in, uint8_t *out,
-                                   const AES_KEY *key) {
+inline void aes_hw_decrypt(const uint8_t *in, uint8_t *out,
+                           const AES_KEY *key) {
   abort();
 }
 
-OPENSSL_INLINE void aes_hw_cbc_encrypt(const uint8_t *in, uint8_t *out,
-                                       size_t length, const AES_KEY *key,
-                                       uint8_t *ivec, int enc) {
+inline void aes_hw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
+                               const AES_KEY *key, uint8_t *ivec, int enc) {
   abort();
 }
 
-OPENSSL_INLINE void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
-                                                size_t len, const AES_KEY *key,
-                                                const uint8_t ivec[16]) {
+inline void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
+                                        size_t len, const AES_KEY *key,
+                                        const uint8_t ivec[16]) {
   abort();
 }
 
@@ -120,7 +166,7 @@ OPENSSL_INLINE void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
 
 #if defined(HWAES_ECB)
 void aes_hw_ecb_encrypt(const uint8_t *in, uint8_t *out, size_t length,
-                        const AES_KEY *key, const int enc);
+                        const AES_KEY *key, int enc);
 #endif  // HWAES_ECB
 
 
@@ -133,30 +179,32 @@ void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len,
 // VPAES to BSAES conversions are available on all BSAES platforms.
 void vpaes_encrypt_key_to_bsaes(AES_KEY *out_bsaes, const AES_KEY *vpaes);
 void vpaes_decrypt_key_to_bsaes(AES_KEY *out_bsaes, const AES_KEY *vpaes);
+void vpaes_ctr32_encrypt_blocks_with_bsaes(const uint8_t *in, uint8_t *out,
+                                           size_t blocks, const AES_KEY *key,
+                                           const uint8_t ivec[16]);
 #else
-OPENSSL_INLINE char bsaes_capable(void) { return 0; }
+inline int bsaes_capable(void) { return 0; }
 
 // On other platforms, bsaes_capable() will always return false and so the
 // following will never be called.
-OPENSSL_INLINE void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out,
-                                      size_t length, const AES_KEY *key,
-                                      uint8_t ivec[16], int enc) {
+inline void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
+                              const AES_KEY *key, uint8_t ivec[16], int enc) {
   abort();
 }
 
-OPENSSL_INLINE void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
-                                               size_t len, const AES_KEY *key,
-                                               const uint8_t ivec[16]) {
+inline void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
+                                       size_t len, const AES_KEY *key,
+                                       const uint8_t ivec[16]) {
   abort();
 }
 
-OPENSSL_INLINE void vpaes_encrypt_key_to_bsaes(AES_KEY *out_bsaes,
-                                               const AES_KEY *vpaes) {
+inline void vpaes_encrypt_key_to_bsaes(AES_KEY *out_bsaes,
+                                       const AES_KEY *vpaes) {
   abort();
 }
 
-OPENSSL_INLINE void vpaes_decrypt_key_to_bsaes(AES_KEY *out_bsaes,
-                                               const AES_KEY *vpaes) {
+inline void vpaes_decrypt_key_to_bsaes(AES_KEY *out_bsaes,
+                                       const AES_KEY *vpaes) {
   abort();
 }
 #endif  // !BSAES
@@ -175,34 +223,34 @@ void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
 void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
                        const AES_KEY *key, uint8_t *ivec, int enc);
 #endif
-#if defined(VPAES_CTR32)
 void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len,
                                 const AES_KEY *key, const uint8_t ivec[16]);
-#endif
 #else
-OPENSSL_INLINE char vpaes_capable(void) { return 0; }
+inline int vpaes_capable(void) { return 0; }
 
 // On other platforms, vpaes_capable() will always return false and so the
 // following will never be called.
-OPENSSL_INLINE int vpaes_set_encrypt_key(const uint8_t *userKey, int bits,
-                                         AES_KEY *key) {
+inline int vpaes_set_encrypt_key(const uint8_t *userKey, int bits,
+                                 AES_KEY *key) {
+  abort();
+}
+inline int vpaes_set_decrypt_key(const uint8_t *userKey, int bits,
+                                 AES_KEY *key) {
   abort();
 }
-OPENSSL_INLINE int vpaes_set_decrypt_key(const uint8_t *userKey, int bits,
-                                         AES_KEY *key) {
+inline void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {
   abort();
 }
-OPENSSL_INLINE void vpaes_encrypt(const uint8_t *in, uint8_t *out,
-                                  const AES_KEY *key) {
+inline void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {
   abort();
 }
-OPENSSL_INLINE void vpaes_decrypt(const uint8_t *in, uint8_t *out,
-                                  const AES_KEY *key) {
+inline void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
+                              const AES_KEY *key, uint8_t *ivec, int enc) {
   abort();
 }
-OPENSSL_INLINE void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out,
-                                      size_t length, const AES_KEY *key,
-                                      uint8_t *ivec, int enc) {
+inline void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
+                                       size_t len, const AES_KEY *key,
+                                       const uint8_t ivec[16]) {
   abort();
 }
 #endif  // !VPAES
@@ -218,11 +266,338 @@ void aes_nohw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
                                    size_t blocks, const AES_KEY *key,
                                    const uint8_t ivec[16]);
 void aes_nohw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len,
-                          const AES_KEY *key, uint8_t *ivec, const int enc);
+                          const AES_KEY *key, uint8_t *ivec, int enc);
+
+// Modes
+
+inline void CRYPTO_xor16(uint8_t out[16], const uint8_t a[16],
+                         const uint8_t b[16]) {
+  // TODO(davidben): Ideally we'd leave this to the compiler, which could use
+  // vector registers, etc. But the compiler doesn't know that |in| and |out|
+  // cannot partially alias. |restrict| is slightly two strict (we allow exact
+  // aliasing), but perhaps in-place could be a separate function?
+  static_assert(16 % sizeof(crypto_word_t) == 0,
+                "block cannot be evenly divided into words");
+  for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
+    CRYPTO_store_word_le(
+        out + i, CRYPTO_load_word_le(a + i) ^ CRYPTO_load_word_le(b + i));
+  }
+}
 
 
-#if defined(__cplusplus)
-}  // extern C
+// CTR.
+
+// CRYPTO_ctr128_encrypt_ctr32 encrypts (or decrypts, it's the same in CTR mode)
+// |len| bytes from |in| to |out| using |block| in counter mode. There's no
+// requirement that |len| be a multiple of any value and any partial blocks are
+// stored in |ecount_buf| and |*num|, which must be zeroed before the initial
+// call. The counter is a 128-bit, big-endian value in |ivec| and is
+// incremented by this function. If the counter overflows, it wraps around.
+// |ctr| must be a function that performs CTR mode but only deals with the lower
+// 32 bits of the counter.
+void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len,
+                                 const AES_KEY *key, uint8_t ivec[16],
+                                 uint8_t ecount_buf[16], unsigned *num,
+                                 ctr128_f ctr);
+
+
+// GCM.
+//
+// This API differs from the upstream API slightly. The |GCM128_CONTEXT| does
+// not have a |key| pointer that points to the key as upstream's version does.
+// Instead, every function takes a |key| parameter. This way |GCM128_CONTEXT|
+// can be safely copied. Additionally, |gcm_key| is split into a separate
+// struct.
+
+// gcm_impl_t specifies an assembly implementation of AES-GCM.
+enum gcm_impl_t {
+  gcm_separate = 0,  // No combined AES-GCM, but may have AES-CTR and GHASH.
+  gcm_x86_aesni,
+  gcm_x86_vaes_avx2,
+  gcm_x86_vaes_avx10_512,
+  gcm_arm64_aes,
+};
+
+typedef struct { uint64_t hi,lo; } u128;
+
+// gmult_func multiplies |Xi| by the GCM key and writes the result back to
+// |Xi|.
+typedef void (*gmult_func)(uint8_t Xi[16], const u128 Htable[16]);
+
+// ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from
+// |inp|. The result is written back to |Xi| and the |len| argument must be a
+// multiple of 16.
+typedef void (*ghash_func)(uint8_t Xi[16], const u128 Htable[16],
+                           const uint8_t *inp, size_t len);
+
+typedef struct gcm128_key_st {
+  u128 Htable[16];
+  gmult_func gmult;
+  ghash_func ghash;
+  AES_KEY aes;
+
+  ctr128_f ctr;
+  block128_f block;
+  enum gcm_impl_t impl;
+} GCM128_KEY;
+
+// GCM128_CONTEXT contains state for a single GCM operation. The structure
+// should be zero-initialized before use.
+typedef struct {
+  // The following 5 names follow names in GCM specification
+  uint8_t Yi[16];
+  uint8_t EKi[16];
+  uint8_t EK0[16];
+  struct {
+    uint64_t aad;
+    uint64_t msg;
+  } len;
+  uint8_t Xi[16];
+  unsigned mres, ares;
+} GCM128_CONTEXT;
+
+#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
+// crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is
+// used.
+int crypto_gcm_clmul_enabled(void);
+#endif
+
+// CRYPTO_ghash_init writes a precomputed table of powers of |gcm_key| to
+// |out_table| and sets |*out_mult| and |*out_hash| to (potentially hardware
+// accelerated) functions for performing operations in the GHASH field.
+void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
+                       u128 out_table[16], const uint8_t gcm_key[16]);
+
+// CRYPTO_gcm128_init_aes_key initialises |gcm_key| to with AES key |key|.
+void CRYPTO_gcm128_init_aes_key(GCM128_KEY *gcm_key, const uint8_t *key,
+                                size_t key_bytes);
+
+// CRYPTO_gcm128_init_ctx initializes |ctx| to encrypt with |key| and |iv|.
+void CRYPTO_gcm128_init_ctx(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                            const uint8_t *iv, size_t iv_len);
+
+// CRYPTO_gcm128_aad adds to the authenticated data for an instance of GCM.
+// This must be called before and data is encrypted. |key| must be the same
+// value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on success
+// and zero otherwise.
+int CRYPTO_gcm128_aad(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                      const uint8_t *aad, size_t aad_len);
+
+// CRYPTO_gcm128_encrypt encrypts |len| bytes from |in| to |out|. |key| must be
+// the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on
+// success and zero otherwise.
+int CRYPTO_gcm128_encrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                          const uint8_t *in, uint8_t *out, size_t len);
+
+// CRYPTO_gcm128_decrypt decrypts |len| bytes from |in| to |out|. |key| must be
+// the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on
+// success and zero otherwise.
+int CRYPTO_gcm128_decrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                          const uint8_t *in, uint8_t *out, size_t len);
+
+// CRYPTO_gcm128_finish calculates the authenticator and compares it against
+// |len| bytes of |tag|. |key| must be the same value that was passed to
+// |CRYPTO_gcm128_init_ctx|. It returns one on success and zero otherwise.
+int CRYPTO_gcm128_finish(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                         const uint8_t *tag, size_t len);
+
+// CRYPTO_gcm128_tag calculates the authenticator and copies it into |tag|.
+// The minimum of |len| and 16 bytes are copied into |tag|. |key| must be the
+// same value that was passed to |CRYPTO_gcm128_init_ctx|.
+void CRYPTO_gcm128_tag(const GCM128_KEY *key, GCM128_CONTEXT *ctx, uint8_t *tag,
+                       size_t len);
+
+
+// GCM assembly.
+
+void gcm_init_nohw(u128 Htable[16], const uint64_t H[2]);
+void gcm_gmult_nohw(uint8_t Xi[16], const u128 Htable[16]);
+void gcm_ghash_nohw(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
+                    size_t len);
+
+#if !defined(OPENSSL_NO_ASM)
+
+#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
+#define GCM_FUNCREF
+void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_clmul(uint8_t Xi[16], const u128 Htable[16]);
+void gcm_ghash_clmul(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
+                     size_t len);
+
+void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_ssse3(uint8_t Xi[16], const u128 Htable[16]);
+void gcm_ghash_ssse3(uint8_t Xi[16], const u128 Htable[16], const uint8_t *in,
+                     size_t len);
+
+#if defined(OPENSSL_X86_64)
+#define GHASH_ASM_X86_64
+void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_avx(uint8_t Xi[16], const u128 Htable[16]);
+void gcm_ghash_avx(uint8_t Xi[16], const u128 Htable[16], const uint8_t *in,
+                   size_t len);
+
+#define HW_GCM
+size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                         const AES_KEY *key, uint8_t ivec[16],
+                         const u128 Htable[16], uint8_t Xi[16]);
+size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
+                         const AES_KEY *key, uint8_t ivec[16],
+                         const u128 Htable[16], uint8_t Xi[16]);
+
+void gcm_init_vpclmulqdq_avx2(u128 Htable[16], const uint64_t H[2]);
+void gcm_gmult_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16]);
+void gcm_ghash_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16],
+                               const uint8_t *in, size_t len);
+void aes_gcm_enc_update_vaes_avx2(const uint8_t *in, uint8_t *out, size_t len,
+                                  const AES_KEY *key, const uint8_t ivec[16],
+                                  const u128 Htable[16], uint8_t Xi[16]);
+void aes_gcm_dec_update_vaes_avx2(const uint8_t *in, uint8_t *out, size_t len,
+                                  const AES_KEY *key, const uint8_t ivec[16],
+                                  const u128 Htable[16], uint8_t Xi[16]);
+
+void gcm_init_vpclmulqdq_avx10_512(u128 Htable[16], const uint64_t H[2]);
+void gcm_gmult_vpclmulqdq_avx10(uint8_t Xi[16], const u128 Htable[16]);
+void gcm_ghash_vpclmulqdq_avx10_512(uint8_t Xi[16], const u128 Htable[16],
+                                    const uint8_t *in, size_t len);
+void aes_gcm_enc_update_vaes_avx10_512(const uint8_t *in, uint8_t *out,
+                                       size_t len, const AES_KEY *key,
+                                       const uint8_t ivec[16],
+                                       const u128 Htable[16], uint8_t Xi[16]);
+void aes_gcm_dec_update_vaes_avx10_512(const uint8_t *in, uint8_t *out,
+                                       size_t len, const AES_KEY *key,
+                                       const uint8_t ivec[16],
+                                       const u128 Htable[16], uint8_t Xi[16]);
+
+#endif  // OPENSSL_X86_64
+
+#if defined(OPENSSL_X86)
+#define GHASH_ASM_X86
+#endif  // OPENSSL_X86
+
+#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
+
+#define GHASH_ASM_ARM
+#define GCM_FUNCREF
+
+inline int gcm_pmull_capable(void) { return CRYPTO_is_ARMv8_PMULL_capable(); }
+
+void gcm_init_v8(u128 Htable[16], const uint64_t H[2]);
+void gcm_gmult_v8(uint8_t Xi[16], const u128 Htable[16]);
+void gcm_ghash_v8(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
+                  size_t len);
+
+inline int gcm_neon_capable(void) { return CRYPTO_is_NEON_capable(); }
+
+void gcm_init_neon(u128 Htable[16], const uint64_t H[2]);
+void gcm_gmult_neon(uint8_t Xi[16], const u128 Htable[16]);
+void gcm_ghash_neon(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
+                    size_t len);
+
+#if defined(OPENSSL_AARCH64)
+#define HW_GCM
+// These functions are defined in aesv8-gcm-armv8.pl.
+void aes_gcm_enc_kernel(const uint8_t *in, uint64_t in_bits, void *out,
+                        void *Xi, uint8_t *ivec, const AES_KEY *key,
+                        const u128 Htable[16]);
+void aes_gcm_dec_kernel(const uint8_t *in, uint64_t in_bits, void *out,
+                        void *Xi, uint8_t *ivec, const AES_KEY *key,
+                        const u128 Htable[16]);
 #endif
 
+#endif
+#endif  // OPENSSL_NO_ASM
+
+
+// CBC.
+
+// cbc128_f is the type of a function that performs CBC-mode encryption.
+typedef void (*cbc128_f)(const uint8_t *in, uint8_t *out, size_t len,
+                         const AES_KEY *key, uint8_t ivec[16], int enc);
+
+// CRYPTO_cbc128_encrypt encrypts |len| bytes from |in| to |out| using the
+// given IV and block cipher in CBC mode. The input need not be a multiple of
+// 128 bits long, but the output will round up to the nearest 128 bit multiple,
+// zero padding the input if needed. The IV will be updated on return.
+void CRYPTO_cbc128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16],
+                           block128_f block);
+
+// CRYPTO_cbc128_decrypt decrypts |len| bytes from |in| to |out| using the
+// given IV and block cipher in CBC mode. If |len| is not a multiple of 128
+// bits then only that many bytes will be written, but a multiple of 128 bits
+// is always read from |in|. The IV will be updated on return.
+void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16],
+                           block128_f block);
+
+
+// OFB.
+
+// CRYPTO_ofb128_encrypt encrypts (or decrypts, it's the same with OFB mode)
+// |len| bytes from |in| to |out| using |block| in OFB mode. There's no
+// requirement that |len| be a multiple of any value and any partial blocks are
+// stored in |ivec| and |*num|, the latter must be zero before the initial
+// call.
+void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16], unsigned *num,
+                           block128_f block);
+
+
+// CFB.
+
+// CRYPTO_cfb128_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
+// from |in| to |out| using |block| in CFB mode. There's no requirement that
+// |len| be a multiple of any value and any partial blocks are stored in |ivec|
+// and |*num|, the latter must be zero before the initial call.
+void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16], unsigned *num,
+                           int enc, block128_f block);
+
+// CRYPTO_cfb128_8_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
+// from |in| to |out| using |block| in CFB-8 mode. Prior to the first call
+// |num| should be set to zero.
+void CRYPTO_cfb128_8_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                             const AES_KEY *key, uint8_t ivec[16],
+                             unsigned *num, int enc, block128_f block);
+
+// CRYPTO_cfb128_1_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
+// from |in| to |out| using |block| in CFB-1 mode. Prior to the first call
+// |num| should be set to zero.
+void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits,
+                             const AES_KEY *key, uint8_t ivec[16],
+                             unsigned *num, int enc, block128_f block);
+
+size_t CRYPTO_cts128_encrypt_block(const uint8_t *in, uint8_t *out, size_t len,
+                                   const AES_KEY *key, uint8_t ivec[16],
+                                   block128_f block);
+
+
+// POLYVAL.
+//
+// POLYVAL is a polynomial authenticator that operates over a field very
+// similar to the one that GHASH uses. See
+// https://www.rfc-editor.org/rfc/rfc8452.html#section-3.
+
+struct polyval_ctx {
+  uint8_t S[16];
+  u128 Htable[16];
+  gmult_func gmult;
+  ghash_func ghash;
+};
+
+// CRYPTO_POLYVAL_init initialises |ctx| using |key|.
+void CRYPTO_POLYVAL_init(struct polyval_ctx *ctx, const uint8_t key[16]);
+
+// CRYPTO_POLYVAL_update_blocks updates the accumulator in |ctx| given the
+// blocks from |in|. Only a whole number of blocks can be processed so |in_len|
+// must be a multiple of 16.
+void CRYPTO_POLYVAL_update_blocks(struct polyval_ctx *ctx, const uint8_t *in,
+                                  size_t in_len);
+
+// CRYPTO_POLYVAL_finish writes the accumulator from |ctx| to |out|.
+void CRYPTO_POLYVAL_finish(const struct polyval_ctx *ctx, uint8_t out[16]);
+
+
+}  // extern C
+
 #endif  // OPENSSL_HEADER_AES_INTERNAL_H