grpc 1.24.0 → 1.25.0.pre1

data/third_party/boringssl/crypto/fipsmodule/modes/internal.h

@@ -51,6 +51,10 @@
 
 #include <openssl/base.h>
 
+#include <openssl/aes.h>
+#include <openssl/cpu.h>
+
+#include <stdlib.h>
 #include <string.h>
 
 #include "../../internal.h"
@@ -60,12 +64,6 @@ extern "C" {
 #endif
 
 
-#define STRICT_ALIGNMENT 1
-#if defined(OPENSSL_X86_64) || defined(OPENSSL_X86) || defined(OPENSSL_AARCH64)
-#undef STRICT_ALIGNMENT
-#define STRICT_ALIGNMENT 0
-#endif
-
 static inline uint32_t GETU32(const void *in) {
   uint32_t v;
   OPENSSL_memcpy(&v, in, sizeof(v));
@@ -87,62 +85,24 @@ static inline void store_word_le(void *out, size_t v) {
   OPENSSL_memcpy(out, &v, sizeof(v));
 }
 
-// block128_f is the type of a 128-bit, block cipher.
+// block128_f is the type of an AES block cipher implementation.
+//
+// Unlike upstream OpenSSL, it and the other functions in this file hard-code
+// |AES_KEY|. It is undefined in C to call a function pointer with anything
+// other than the original type. Thus we either must match |block128_f| to the
+// type signature of |AES_encrypt| and friends or pass in |void*| wrapper
+// functions.
+//
+// These functions are called exclusively with AES, so we use the former.
 typedef void (*block128_f)(const uint8_t in[16], uint8_t out[16],
-                           const void *key);
-
-// GCM definitions
-typedef struct { uint64_t hi,lo; } u128;
-
-// gmult_func multiplies |Xi| by the GCM key and writes the result back to
-// |Xi|.
-typedef void (*gmult_func)(uint64_t Xi[2], const u128 Htable[16]);
-
-// ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from
-// |inp|. The result is written back to |Xi| and the |len| argument must be a
-// multiple of 16.
-typedef void (*ghash_func)(uint64_t Xi[2], const u128 Htable[16],
-                           const uint8_t *inp, size_t len);
-
-// This differs from upstream's |gcm128_context| in that it does not have the
-// |key| pointer, in order to make it |memcpy|-friendly. Rather the key is
-// passed into each call that needs it.
-struct gcm128_context {
-  // Following 6 names follow names in GCM specification
-  union {
-    uint64_t u[2];
-    uint32_t d[4];
-    uint8_t c[16];
-    size_t t[16 / sizeof(size_t)];
-  } Yi, EKi, EK0, len, Xi;
-
-  // Note that the order of |Xi|, |H| and |Htable| is fixed by the MOVBE-based,
-  // x86-64, GHASH assembly.
-  u128 H;
-  u128 Htable[16];
-  gmult_func gmult;
-  ghash_func ghash;
-
-  unsigned int mres, ares;
-  block128_f block;
-
-  // use_aesni_gcm_crypt is true if this context should use the assembly
-  // functions |aesni_gcm_encrypt| and |aesni_gcm_decrypt| to process data.
-  unsigned use_aesni_gcm_crypt:1;
-};
-
-#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
-// crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is
-// used.
-int crypto_gcm_clmul_enabled(void);
-#endif
+                           const AES_KEY *key);
 
 
 // CTR.
 
 // ctr128_f is the type of a function that performs CTR-mode encryption.
 typedef void (*ctr128_f)(const uint8_t *in, uint8_t *out, size_t blocks,
-                         const void *key, const uint8_t ivec[16]);
+                         const AES_KEY *key, const uint8_t ivec[16]);
 
 // CRYPTO_ctr128_encrypt encrypts (or decrypts, it's the same in CTR mode)
 // |len| bytes from |in| to |out| using |block| in counter mode. There's no
@@ -151,7 +111,7 @@ typedef void (*ctr128_f)(const uint8_t *in, uint8_t *out, size_t blocks,
 // call. The counter is a 128-bit, big-endian value in |ivec| and is
 // incremented by this function.
 void CRYPTO_ctr128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
-                           const void *key, uint8_t ivec[16],
+                           const AES_KEY *key, uint8_t ivec[16],
                            uint8_t ecount_buf[16], unsigned *num,
                            block128_f block);
 
@@ -160,25 +120,74 @@ void CRYPTO_ctr128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
 // bits of the counter. This is useful when |ctr| can be an optimised
 // function.
 void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len,
-                                 const void *key, uint8_t ivec[16],
+                                 const AES_KEY *key, uint8_t ivec[16],
                                  uint8_t ecount_buf[16], unsigned *num,
                                  ctr128_f ctr);
 
-#if !defined(OPENSSL_NO_ASM) && \
-    (defined(OPENSSL_X86) || defined(OPENSSL_X86_64))
-void aesni_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t blocks,
-                                const void *key, const uint8_t *ivec);
-#endif
-
 
 // GCM.
 //
 // This API differs from the upstream API slightly. The |GCM128_CONTEXT| does
 // not have a |key| pointer that points to the key as upstream's version does.
 // Instead, every function takes a |key| parameter. This way |GCM128_CONTEXT|
-// can be safely copied.
+// can be safely copied. Additionally, |gcm_key| is split into a separate
+// struct.
+
+typedef struct { uint64_t hi,lo; } u128;
+
+// gmult_func multiplies |Xi| by the GCM key and writes the result back to
+// |Xi|.
+typedef void (*gmult_func)(uint64_t Xi[2], const u128 Htable[16]);
+
+// ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from
+// |inp|. The result is written back to |Xi| and the |len| argument must be a
+// multiple of 16.
+typedef void (*ghash_func)(uint64_t Xi[2], const u128 Htable[16],
+                           const uint8_t *inp, size_t len);
+
+typedef struct gcm128_key_st {
+  // Note the MOVBE-based, x86-64, GHASH assembly requires |H| and |Htable| to
+  // be the first two elements of this struct. Additionally, some assembly
+  // routines require a 16-byte-aligned |Htable| when hashing data, but not
+  // initialization. |GCM128_KEY| is not itself aligned to simplify embedding in
+  // |EVP_AEAD_CTX|, but |Htable|'s offset must be a multiple of 16.
+  u128 H;
+  u128 Htable[16];
+  gmult_func gmult;
+  ghash_func ghash;
+
+  block128_f block;
+
+  // use_aesni_gcm_crypt is true if this context should use the assembly
+  // functions |aesni_gcm_encrypt| and |aesni_gcm_decrypt| to process data.
+  unsigned use_aesni_gcm_crypt:1;
+} GCM128_KEY;
+
+// GCM128_CONTEXT contains state for a single GCM operation. The structure
+// should be zero-initialized before use.
+typedef struct {
+  // The following 5 names follow names in GCM specification
+  union {
+    uint64_t u[2];
+    uint32_t d[4];
+    uint8_t c[16];
+    size_t t[16 / sizeof(size_t)];
+  } Yi, EKi, EK0, len, Xi;
+
+  // Note that the order of |Xi| and |gcm_key| is fixed by the MOVBE-based,
+  // x86-64, GHASH assembly. Additionally, some assembly routines require
+  // |gcm_key| to be 16-byte aligned. |GCM128_KEY| is not itself aligned to
+  // simplify embedding in |EVP_AEAD_CTX|.
+  alignas(16) GCM128_KEY gcm_key;
 
-typedef struct gcm128_context GCM128_CONTEXT;
+  unsigned mres, ares;
+} GCM128_CONTEXT;
+
+#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
+// crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is
+// used.
+int crypto_gcm_clmul_enabled(void);
+#endif
 
 // CRYPTO_ghash_init writes a precomputed table of powers of |gcm_key| to
 // |out_table| and sets |*out_mult| and |*out_hash| to (potentially hardware
@@ -186,16 +195,17 @@ typedef struct gcm128_context GCM128_CONTEXT;
 // AVX implementation was used |*out_is_avx| will be true.
 void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
                        u128 *out_key, u128 out_table[16], int *out_is_avx,
-                       const uint8_t *gcm_key);
+                       const uint8_t gcm_key[16]);
 
-// CRYPTO_gcm128_init initialises |ctx| to use |block| (typically AES) with
-// the given key. |is_aesni_encrypt| is one if |block| is |aesni_encrypt|.
-OPENSSL_EXPORT void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx, const void *key,
-                                       block128_f block, int is_aesni_encrypt);
+// CRYPTO_gcm128_init_key initialises |gcm_key| to use |block| (typically AES)
+// with the given key. |block_is_hwaes| is one if |block| is |aes_hw_encrypt|.
+OPENSSL_EXPORT void CRYPTO_gcm128_init_key(GCM128_KEY *gcm_key,
+                                           const AES_KEY *key, block128_f block,
+                                           int block_is_hwaes);
 
 // CRYPTO_gcm128_setiv sets the IV (nonce) for |ctx|. The |key| must be the
 // same key that was passed to |CRYPTO_gcm128_init|.
-OPENSSL_EXPORT void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const void *key,
+OPENSSL_EXPORT void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const AES_KEY *key,
                                         const uint8_t *iv, size_t iv_len);
 
 // CRYPTO_gcm128_aad sets the authenticated data for an instance of GCM.
@@ -207,16 +217,16 @@ OPENSSL_EXPORT int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad,
 // CRYPTO_gcm128_encrypt encrypts |len| bytes from |in| to |out|. The |key|
 // must be the same key that was passed to |CRYPTO_gcm128_init|. It returns one
 // on success and zero otherwise.
-OPENSSL_EXPORT int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const void *key,
-                                         const uint8_t *in, uint8_t *out,
-                                         size_t len);
+OPENSSL_EXPORT int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
+                                         const AES_KEY *key, const uint8_t *in,
+                                         uint8_t *out, size_t len);
 
 // CRYPTO_gcm128_decrypt decrypts |len| bytes from |in| to |out|. The |key|
 // must be the same key that was passed to |CRYPTO_gcm128_init|. It returns one
 // on success and zero otherwise.
-OPENSSL_EXPORT int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const void *key,
-                                         const uint8_t *in, uint8_t *out,
-                                         size_t len);
+OPENSSL_EXPORT int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
+                                         const AES_KEY *key, const uint8_t *in,
+                                         uint8_t *out, size_t len);
 
 // CRYPTO_gcm128_encrypt_ctr32 encrypts |len| bytes from |in| to |out| using
 // a CTR function that only handles the bottom 32 bits of the nonce, like
@@ -224,7 +234,7 @@ OPENSSL_EXPORT int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const void *key,
 // passed to |CRYPTO_gcm128_init|. It returns one on success and zero
 // otherwise.
 OPENSSL_EXPORT int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
-                                               const void *key,
+                                               const AES_KEY *key,
                                                const uint8_t *in, uint8_t *out,
                                                size_t len, ctr128_f stream);
 
@@ -234,7 +244,7 @@ OPENSSL_EXPORT int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
 // passed to |CRYPTO_gcm128_init|. It returns one on success and zero
 // otherwise.
 OPENSSL_EXPORT int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
-                                               const void *key,
+                                               const AES_KEY *key,
                                                const uint8_t *in, uint8_t *out,
                                                size_t len, ctr128_f stream);
 
@@ -249,61 +259,113 @@ OPENSSL_EXPORT void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, uint8_t *tag,
                                       size_t len);
 
 
-// CCM.
+// GCM assembly.
 
-typedef struct ccm128_context {
-  block128_f block;
-  ctr128_f ctr;
-  unsigned M, L;
-} CCM128_CONTEXT;
-
-// CRYPTO_ccm128_init initialises |ctx| to use |block| (typically AES) with the
-// specified |M| and |L| parameters. It returns one on success and zero if |M|
-// or |L| is invalid.
-int CRYPTO_ccm128_init(CCM128_CONTEXT *ctx, const void *key, block128_f block,
-                       ctr128_f ctr, unsigned M, unsigned L);
-
-// CRYPTO_ccm128_max_input returns the maximum input length accepted by |ctx|.
-size_t CRYPTO_ccm128_max_input(const CCM128_CONTEXT *ctx);
-
-// CRYPTO_ccm128_encrypt encrypts |len| bytes from |in| to |out| writing the tag
-// to |out_tag|. |key| must be the same key that was passed to
-// |CRYPTO_ccm128_init|. It returns one on success and zero otherwise.
-int CRYPTO_ccm128_encrypt(const CCM128_CONTEXT *ctx, const void *key,
-                          uint8_t *out, uint8_t *out_tag, size_t tag_len,
-                          const uint8_t *nonce, size_t nonce_len,
-                          const uint8_t *in, size_t len, const uint8_t *aad,
-                          size_t aad_len);
-
-// CRYPTO_ccm128_decrypt decrypts |len| bytes from |in| to |out|, writing the
-// expected tag to |out_tag|. |key| must be the same key that was passed to
-// |CRYPTO_ccm128_init|. It returns one on success and zero otherwise.
-int CRYPTO_ccm128_decrypt(const CCM128_CONTEXT *ctx, const void *key,
-                          uint8_t *out, uint8_t *out_tag, size_t tag_len,
-                          const uint8_t *nonce, size_t nonce_len,
-                          const uint8_t *in, size_t len, const uint8_t *aad,
-                          size_t aad_len);
+#if !defined(OPENSSL_NO_ASM) &&                         \
+    (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \
+     defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) || \
+     defined(OPENSSL_PPC64LE))
+#define GHASH_ASM
+#endif
+
+void gcm_init_4bit(u128 Htable[16], const uint64_t H[2]);
+void gcm_gmult_4bit(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_4bit(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                    size_t len);
+
+#if defined(GHASH_ASM)
+
+#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
+#define GCM_FUNCREF_4BIT
+void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_clmul(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_clmul(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                     size_t len);
+
+OPENSSL_INLINE char gcm_ssse3_capable(void) {
+  return (OPENSSL_ia32cap_get()[1] & (1 << (41 - 32))) != 0;
+}
+
+// |gcm_gmult_ssse3| and |gcm_ghash_ssse3| require |Htable| to be
+// 16-byte-aligned, but |gcm_init_ssse3| does not.
+void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in,
+                     size_t len);
+
+#if defined(OPENSSL_X86_64)
+#define GHASH_ASM_X86_64
+void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_avx(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_avx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in,
+                   size_t len);
+
+#define AESNI_GCM
+size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                         const AES_KEY *key, uint8_t ivec[16], uint64_t *Xi);
+size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
+                         const AES_KEY *key, uint8_t ivec[16], uint64_t *Xi);
+#endif  // OPENSSL_X86_64
+
+#if defined(OPENSSL_X86)
+#define GHASH_ASM_X86
+void gcm_gmult_4bit_mmx(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_4bit_mmx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                        size_t len);
+#endif  // OPENSSL_X86
+
+#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
+#define GHASH_ASM_ARM
+#define GCM_FUNCREF_4BIT
+
+OPENSSL_INLINE int gcm_pmull_capable(void) {
+  return CRYPTO_is_ARMv8_PMULL_capable();
+}
+
+void gcm_init_v8(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_v8(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_v8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                  size_t len);
+
+OPENSSL_INLINE int gcm_neon_capable(void) { return CRYPTO_is_NEON_capable(); }
+
+void gcm_init_neon(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_neon(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_neon(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                    size_t len);
+
+#elif defined(OPENSSL_PPC64LE)
+#define GHASH_ASM_PPC64LE
+#define GCM_FUNCREF_4BIT
+void gcm_init_p8(u128 Htable[16], const uint64_t Xi[2]);
+void gcm_gmult_p8(uint64_t Xi[2], const u128 Htable[16]);
+void gcm_ghash_p8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
+                  size_t len);
+#endif
+#endif  // GHASH_ASM
 
 
 // CBC.
 
 // cbc128_f is the type of a function that performs CBC-mode encryption.
 typedef void (*cbc128_f)(const uint8_t *in, uint8_t *out, size_t len,
-                         const void *key, uint8_t ivec[16], int enc);
+                         const AES_KEY *key, uint8_t ivec[16], int enc);
 
 // CRYPTO_cbc128_encrypt encrypts |len| bytes from |in| to |out| using the
 // given IV and block cipher in CBC mode. The input need not be a multiple of
 // 128 bits long, but the output will round up to the nearest 128 bit multiple,
 // zero padding the input if needed. The IV will be updated on return.
 void CRYPTO_cbc128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
-                           const void *key, uint8_t ivec[16], block128_f block);
+                           const AES_KEY *key, uint8_t ivec[16],
+                           block128_f block);
 
 // CRYPTO_cbc128_decrypt decrypts |len| bytes from |in| to |out| using the
 // given IV and block cipher in CBC mode. If |len| is not a multiple of 128
 // bits then only that many bytes will be written, but a multiple of 128 bits
 // is always read from |in|. The IV will be updated on return.
 void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len,
-                           const void *key, uint8_t ivec[16], block128_f block);
+                           const AES_KEY *key, uint8_t ivec[16],
+                           block128_f block);
 
 
 // OFB.
@@ -313,9 +375,9 @@ void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len,
 // requirement that |len| be a multiple of any value and any partial blocks are
 // stored in |ivec| and |*num|, the latter must be zero before the initial
 // call.
-void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out,
-                           size_t len, const void *key, uint8_t ivec[16],
-                           unsigned *num, block128_f block);
+void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                           const AES_KEY *key, uint8_t ivec[16], unsigned *num,
+                           block128_f block);
 
 
 // CFB.
@@ -325,25 +387,25 @@ void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out,
 // |len| be a multiple of any value and any partial blocks are stored in |ivec|
 // and |*num|, the latter must be zero before the initial call.
 void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
-                           const void *key, uint8_t ivec[16], unsigned *num,
+                           const AES_KEY *key, uint8_t ivec[16], unsigned *num,
                            int enc, block128_f block);
 
 // CRYPTO_cfb128_8_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
 // from |in| to |out| using |block| in CFB-8 mode. Prior to the first call
 // |num| should be set to zero.
 void CRYPTO_cfb128_8_encrypt(const uint8_t *in, uint8_t *out, size_t len,
-                             const void *key, uint8_t ivec[16], unsigned *num,
-                             int enc, block128_f block);
+                             const AES_KEY *key, uint8_t ivec[16],
+                             unsigned *num, int enc, block128_f block);
 
 // CRYPTO_cfb128_1_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
 // from |in| to |out| using |block| in CFB-1 mode. Prior to the first call
 // |num| should be set to zero.
 void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits,
-                             const void *key, uint8_t ivec[16], unsigned *num,
-                             int enc, block128_f block);
+                             const AES_KEY *key, uint8_t ivec[16],
+                             unsigned *num, int enc, block128_f block);
 
 size_t CRYPTO_cts128_encrypt_block(const uint8_t *in, uint8_t *out, size_t len,
-                                   const void *key, uint8_t ivec[16],
+                                   const AES_KEY *key, uint8_t ivec[16],
                                    block128_f block);
 
 
@@ -360,10 +422,11 @@ typedef union {
 
 struct polyval_ctx {
   // Note that the order of |S|, |H| and |Htable| is fixed by the MOVBE-based,
-  // x86-64, GHASH assembly.
+  // x86-64, GHASH assembly. Additionally, some assembly routines require
+  // |Htable| to be 16-byte aligned.
   polyval_block S;
   u128 H;
-  u128 Htable[16];
+  alignas(16) u128 Htable[16];
   gmult_func gmult;
   ghash_func ghash;
 };