grpc 1.61.3 → 1.62.0.pre1

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/internal.h

@@ -17,25 +17,172 @@
 
 #include <openssl/base.h>
 
+#include "../../internal.h"
+
 #if defined(__cplusplus)
 extern "C" {
 #endif
 
+// Define SHA{n}[_{variant}]_ASM if sha{n}_block_data_order[_{variant}] is
+// defined in assembly.
+
+#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86)
 
-#if !defined(OPENSSL_NO_ASM) &&                         \
-    (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \
-     defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64))
 #define SHA1_ASM
 #define SHA256_ASM
 #define SHA512_ASM
-void sha1_block_data_order(uint32_t *state, const uint8_t *in,
+
+void sha1_block_data_order(uint32_t *state, const uint8_t *data,
                            size_t num_blocks);
-void sha256_block_data_order(uint32_t *state, const uint8_t *in,
+void sha256_block_data_order(uint32_t *state, const uint8_t *data,
                              size_t num_blocks);
-void sha512_block_data_order(uint64_t *state, const uint8_t *in,
+void sha512_block_data_order(uint64_t *state, const uint8_t *data,
                              size_t num_blocks);
+
+#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_ARM)
+
+#define SHA1_ASM_NOHW
+#define SHA256_ASM_NOHW
+#define SHA512_ASM_NOHW
+
+#define SHA1_ASM_HW
+OPENSSL_INLINE int sha1_hw_capable(void) {
+  return CRYPTO_is_ARMv8_SHA1_capable();
+}
+
+#define SHA1_ASM_NEON
+void sha1_block_data_order_neon(uint32_t *state, const uint8_t *data,
+                                size_t num);
+
+#define SHA256_ASM_HW
+OPENSSL_INLINE int sha256_hw_capable(void) {
+  return CRYPTO_is_ARMv8_SHA256_capable();
+}
+
+#define SHA256_ASM_NEON
+void sha256_block_data_order_neon(uint32_t *state, const uint8_t *data,
+                                  size_t num);
+
+// Armv8.2 SHA-512 instructions are not available in 32-bit.
+#define SHA512_ASM_NEON
+void sha512_block_data_order_neon(uint64_t *state, const uint8_t *data,
+                                  size_t num);
+
+#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64)
+
+#define SHA1_ASM_NOHW
+#define SHA256_ASM_NOHW
+#define SHA512_ASM_NOHW
+
+#define SHA1_ASM_HW
+OPENSSL_INLINE int sha1_hw_capable(void) {
+  return CRYPTO_is_ARMv8_SHA1_capable();
+}
+
+#define SHA256_ASM_HW
+OPENSSL_INLINE int sha256_hw_capable(void) {
+  return CRYPTO_is_ARMv8_SHA256_capable();
+}
+
+#define SHA512_ASM_HW
+OPENSSL_INLINE int sha512_hw_capable(void) {
+  return CRYPTO_is_ARMv8_SHA512_capable();
+}
+
+#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64)
+
+#define SHA1_ASM_NOHW
+#define SHA256_ASM_NOHW
+#define SHA512_ASM_NOHW
+
+#define SHA1_ASM_HW
+OPENSSL_INLINE int sha1_hw_capable(void) {
+  return CRYPTO_is_x86_SHA_capable() && CRYPTO_is_SSSE3_capable();
+}
+
+#define SHA1_ASM_AVX2
+OPENSSL_INLINE int sha1_avx2_capable(void) {
+  return CRYPTO_is_AVX2_capable() && CRYPTO_is_BMI2_capable() &&
+         CRYPTO_is_BMI1_capable();
+}
+void sha1_block_data_order_avx2(uint32_t *state, const uint8_t *data,
+                                size_t num);
+
+#define SHA1_ASM_AVX
+OPENSSL_INLINE int sha1_avx_capable(void) {
+  // Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the
+  // discussion in sha1-586.pl.
+  return CRYPTO_is_AVX_capable() && CRYPTO_is_intel_cpu();
+}
+void sha1_block_data_order_avx(uint32_t *state, const uint8_t *data,
+                               size_t num);
+
+#define SHA1_ASM_SSSE3
+OPENSSL_INLINE int sha1_ssse3_capable(void) {
+  return CRYPTO_is_SSSE3_capable();
+}
+void sha1_block_data_order_ssse3(uint32_t *state, const uint8_t *data,
+                                 size_t num);
+
+#define SHA256_ASM_HW
+OPENSSL_INLINE int sha256_hw_capable(void) {
+  // Note that the original assembly did not check SSSE3.
+  return CRYPTO_is_x86_SHA_capable() && CRYPTO_is_SSSE3_capable();
+}
+
+#define SHA256_ASM_AVX
+OPENSSL_INLINE int sha256_avx_capable(void) {
+  // Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the
+  // discussion in sha1-586.pl.
+  return CRYPTO_is_AVX_capable() && CRYPTO_is_intel_cpu();
+}
+void sha256_block_data_order_avx(uint32_t *state, const uint8_t *data,
+                                 size_t num);
+
+#define SHA256_ASM_SSSE3
+OPENSSL_INLINE int sha256_ssse3_capable(void) {
+  return CRYPTO_is_SSSE3_capable();
+}
+void sha256_block_data_order_ssse3(uint32_t *state, const uint8_t *data,
+                                   size_t num);
+
+#define SHA512_ASM_AVX
+OPENSSL_INLINE int sha512_avx_capable(void) {
+  // Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the
+  // discussion in sha1-586.pl.
+  return CRYPTO_is_AVX_capable() && CRYPTO_is_intel_cpu();
+}
+void sha512_block_data_order_avx(uint64_t *state, const uint8_t *data,
+                                 size_t num);
+
 #endif
 
+#if defined(SHA1_ASM_HW)
+void sha1_block_data_order_hw(uint32_t *state, const uint8_t *data, size_t num);
+#endif
+#if defined(SHA1_ASM_NOHW)
+void sha1_block_data_order_nohw(uint32_t *state, const uint8_t *data,
+                                size_t num);
+#endif
+
+#if defined(SHA256_ASM_HW)
+void sha256_block_data_order_hw(uint32_t *state, const uint8_t *data,
+                                size_t num);
+#endif
+#if defined(SHA256_ASM_NOHW)
+void sha256_block_data_order_nohw(uint32_t *state, const uint8_t *data,
+                                  size_t num);
+#endif
+
+#if defined(SHA512_ASM_HW)
+void sha512_block_data_order_hw(uint64_t *state, const uint8_t *data,
+                                size_t num);
+#endif
+
+#if defined(SHA512_ASM_NOHW)
+void sha512_block_data_order_nohw(uint64_t *state, const uint8_t *data,
+                                  size_t num);
+#endif
 
 #if defined(__cplusplus)
 }  // extern "C"

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha1.c

@@ -100,19 +100,60 @@ int SHA1_Update(SHA_CTX *c, const void *data, size_t len) {
   return 1;
 }
 
+static void sha1_output_state(uint8_t out[SHA_DIGEST_LENGTH],
+                              const SHA_CTX *ctx) {
+  CRYPTO_store_u32_be(out, ctx->h[0]);
+  CRYPTO_store_u32_be(out + 4, ctx->h[1]);
+  CRYPTO_store_u32_be(out + 8, ctx->h[2]);
+  CRYPTO_store_u32_be(out + 12, ctx->h[3]);
+  CRYPTO_store_u32_be(out + 16, ctx->h[4]);
+}
+
 int SHA1_Final(uint8_t out[SHA_DIGEST_LENGTH], SHA_CTX *c) {
   crypto_md32_final(&sha1_block_data_order, c->h, c->data, SHA_CBLOCK, &c->num,
                     c->Nh, c->Nl, /*is_big_endian=*/1);
 
-  CRYPTO_store_u32_be(out, c->h[0]);
-  CRYPTO_store_u32_be(out + 4, c->h[1]);
-  CRYPTO_store_u32_be(out + 8, c->h[2]);
-  CRYPTO_store_u32_be(out + 12, c->h[3]);
-  CRYPTO_store_u32_be(out + 16, c->h[4]);
+  sha1_output_state(out, c);
   FIPS_service_indicator_update_state();
   return 1;
 }
 
+void CRYPTO_fips_186_2_prf(uint8_t *out, size_t out_len,
+                           const uint8_t xkey[SHA_DIGEST_LENGTH]) {
+  // XKEY and XVAL are 160-bit values, but are internally right-padded up to
+  // block size. See FIPS 186-2, Appendix 3.3. This buffer maintains both the
+  // current value of XKEY and the padding.
+  uint8_t block[SHA_CBLOCK] = {0};
+  OPENSSL_memcpy(block, xkey, SHA_DIGEST_LENGTH);
+
+  while (out_len != 0) {
+    // We always use a zero XSEED, so we can merge the inner and outer loops.
+    // XVAL is also always equal to XKEY.
+    SHA_CTX ctx;
+    SHA1_Init(&ctx);
+    SHA1_Transform(&ctx, block);
+
+    // XKEY = (1 + XKEY + w_i) mod 2^b
+    uint32_t carry = 1;
+    for (int i = 4; i >= 0; i--) {
+      uint32_t tmp = CRYPTO_load_u32_be(block + i * 4);
+      tmp = CRYPTO_addc_u32(tmp, ctx.h[i], carry, &carry);
+      CRYPTO_store_u32_be(block + i * 4, tmp);
+    }
+
+    // Output w_i.
+    if (out_len < SHA_DIGEST_LENGTH) {
+      uint8_t buf[SHA_DIGEST_LENGTH];
+      sha1_output_state(buf, &ctx);
+      OPENSSL_memcpy(out, buf, out_len);
+      break;
+    }
+    sha1_output_state(out, &ctx);
+    out += SHA_DIGEST_LENGTH;
+    out_len -= SHA_DIGEST_LENGTH;
+  }
+}
+
 #define Xupdate(a, ix, ia, ib, ic, id)    \
   do {                                    \
     (a) = ((ia) ^ (ib) ^ (ic) ^ (id));    \
@@ -191,8 +232,10 @@ int SHA1_Final(uint8_t out[SHA_DIGEST_LENGTH], SHA_CTX *c) {
 #define X(i)  XX##i
 
 #if !defined(SHA1_ASM)
-static void sha1_block_data_order(uint32_t *state, const uint8_t *data,
-                                  size_t num) {
+
+#if !defined(SHA1_ASM_NOHW)
+static void sha1_block_data_order_nohw(uint32_t *state, const uint8_t *data,
+                                       size_t num) {
   register uint32_t A, B, C, D, E, T;
   uint32_t XX0, XX1, XX2, XX3, XX4, XX5, XX6, XX7, XX8, XX9, XX10,
       XX11, XX12, XX13, XX14, XX15;
@@ -339,7 +382,44 @@ static void sha1_block_data_order(uint32_t *state, const uint8_t *data,
     E = state[4];
   }
 }
+#endif  // !SHA1_ASM_NOHW
+
+static void sha1_block_data_order(uint32_t *state, const uint8_t *data,
+                                  size_t num) {
+#if defined(SHA1_ASM_HW)
+  if (sha1_hw_capable()) {
+    sha1_block_data_order_hw(state, data, num);
+    return;
+  }
+#endif
+#if defined(SHA1_ASM_AVX2)
+  if (sha1_avx2_capable()) {
+    sha1_block_data_order_avx(state, data, num);
+    return;
+  }
+#endif
+#if defined(SHA1_ASM_AVX)
+  if (sha1_avx_capable()) {
+    sha1_block_data_order_avx(state, data, num);
+    return;
+  }
 #endif
+#if defined(SHA1_ASM_SSSE3)
+  if (sha1_ssse3_capable()) {
+    sha1_block_data_order_ssse3(state, data, num);
+    return;
+  }
+#endif
+#if defined(SHA1_ASM_NEON)
+  if (CRYPTO_is_NEON_capable()) {
+    sha1_block_data_order_neon(state, data, num);
+    return;
+  }
+#endif
+  sha1_block_data_order_nohw(state, data, num);
+}
+
+#endif  // !SHA1_ASM
 
 #undef Xupdate
 #undef K_00_19

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha256.c

@@ -114,7 +114,7 @@ uint8_t *SHA256(const uint8_t *data, size_t len,
   return out;
 }
 
-#ifndef SHA256_ASM
+#if !defined(SHA256_ASM)
 static void sha256_block_data_order(uint32_t *state, const uint8_t *in,
                                     size_t num);
 #endif
@@ -172,7 +172,9 @@ int SHA224_Final(uint8_t out[SHA224_DIGEST_LENGTH], SHA256_CTX *ctx) {
   return sha256_final_impl(out, SHA224_DIGEST_LENGTH, ctx);
 }
 
-#ifndef SHA256_ASM
+#if !defined(SHA256_ASM)
+
+#if !defined(SHA256_ASM_NOHW)
 static const uint32_t K256[64] = {
     0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL,
     0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL,
@@ -221,8 +223,8 @@ static const uint32_t K256[64] = {
     ROUND_00_15(i, a, b, c, d, e, f, g, h);            \
   } while (0)
 
-static void sha256_block_data_order(uint32_t *state, const uint8_t *data,
-                                    size_t num) {
+static void sha256_block_data_order_nohw(uint32_t *state, const uint8_t *data,
+                                         size_t num) {
   uint32_t a, b, c, d, e, f, g, h, s0, s1, T1;
   uint32_t X[16];
   int i;
@@ -308,7 +310,39 @@ static void sha256_block_data_order(uint32_t *state, const uint8_t *data,
   }
 }
 
-#endif  // !SHA256_ASM
+#endif  // !defined(SHA256_ASM_NOHW)
+
+static void sha256_block_data_order(uint32_t *state, const uint8_t *data,
+                                    size_t num) {
+#if defined(SHA256_ASM_HW)
+  if (sha256_hw_capable()) {
+    sha256_block_data_order_hw(state, data, num);
+    return;
+  }
+#endif
+#if defined(SHA256_ASM_AVX)
+  if (sha256_avx_capable()) {
+    sha256_block_data_order_avx(state, data, num);
+    return;
+  }
+#endif
+#if defined(SHA256_ASM_SSSE3)
+  if (sha256_ssse3_capable()) {
+    sha256_block_data_order_ssse3(state, data, num);
+    return;
+  }
+#endif
+#if defined(SHA256_ASM_NEON)
+  if (CRYPTO_is_NEON_capable()) {
+    sha256_block_data_order_neon(state, data, num);
+    return;
+  }
+#endif
+  sha256_block_data_order_nohw(state, data, num);
+}
+
+#endif  // !defined(SHA256_ASM)
+
 
 void SHA256_TransformBlocks(uint32_t state[8], const uint8_t *data,
                             size_t num_blocks) {

data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha512.c

@@ -279,7 +279,9 @@ static int sha512_final_impl(uint8_t *out, size_t md_len, SHA512_CTX *sha) {
   return 1;
 }
 
-#ifndef SHA512_ASM
+#if !defined(SHA512_ASM)
+
+#if !defined(SHA512_ASM_NOHW)
 static const uint64_t K512[80] = {
     UINT64_C(0x428a2f98d728ae22), UINT64_C(0x7137449123ef65cd),
     UINT64_C(0xb5c0fbcfec4d3b2f), UINT64_C(0xe9b5dba58189dbbc),
@@ -341,8 +343,8 @@ static const uint64_t K512[80] = {
 #if defined(__i386) || defined(__i386__) || defined(_M_IX86)
 // This code should give better results on 32-bit CPU with less than
 // ~24 registers, both size and performance wise...
-static void sha512_block_data_order(uint64_t *state, const uint8_t *in,
-                                    size_t num) {
+static void sha512_block_data_order_nohw(uint64_t *state, const uint8_t *in,
+                                         size_t num) {
   uint64_t A, E, T;
   uint64_t X[9 + 80], *F;
   int i;
@@ -414,8 +416,8 @@ static void sha512_block_data_order(uint64_t *state, const uint8_t *in,
     ROUND_00_15(i + j, a, b, c, d, e, f, g, h);        \
   } while (0)
 
-static void sha512_block_data_order(uint64_t *state, const uint8_t *in,
-                                    size_t num) {
+static void sha512_block_data_order_nohw(uint64_t *state, const uint8_t *in,
+                                         size_t num) {
   uint64_t a, b, c, d, e, f, g, h, s0, s1, T1;
   uint64_t X[16];
   int i;
@@ -498,6 +500,31 @@ static void sha512_block_data_order(uint64_t *state, const uint8_t *in,
 
 #endif
 
+#endif  // !SHA512_ASM_NOHW
+
+static void sha512_block_data_order(uint64_t *state, const uint8_t *data,
+                                    size_t num) {
+#if defined(SHA512_ASM_HW)
+  if (sha512_hw_capable()) {
+    sha512_block_data_order_hw(state, data, num);
+    return;
+  }
+#endif
+#if defined(SHA512_ASM_AVX)
+  if (sha512_avx_capable()) {
+    sha512_block_data_order_avx(state, data, num);
+    return;
+  }
+#endif
+#if defined(SHA512_ASM_NEON)
+  if (CRYPTO_is_NEON_capable()) {
+    sha512_block_data_order_neon(state, data, num);
+    return;
+  }
+#endif
+  sha512_block_data_order_nohw(state, data, num);
+}
+
 #endif  // !SHA512_ASM
 
 #undef Sigma0