grpc 1.84.1 → 1.85.0.pre1

data/third_party/boringssl-with-bazel/crypto/fipsmodule/aes/cfb.cc.inc

@@ -18,11 +18,13 @@
 #include "internal.h"
 
 
+using namespace bssl;
+
 static_assert(16 % sizeof(size_t) == 0, "block cannot be divided into size_t");
 
-void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
-                           const AES_KEY *key, uint8_t ivec[16], unsigned *num,
-                           int enc, block128_f block) {
+void bssl::CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
+                                 const AES_KEY *key, uint8_t ivec[16],
+                                 unsigned *num, int enc, block128_f block) {
   assert(in && out && key && ivec && num);
 
   unsigned n = *num;
@@ -134,9 +136,9 @@ static void cfbr_encrypt_block(const uint8_t *in, uint8_t *out, unsigned nbits,
 }
 
 // N.B. This expects the input to be packed, MS bit first
-void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits,
-                             const AES_KEY *key, uint8_t ivec[16],
-                             unsigned *num, int enc, block128_f block) {
+void bssl::CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits,
+                                   const AES_KEY *key, uint8_t ivec[16],
+                                   unsigned *num, int enc, block128_f block) {
   size_t n;
   uint8_t c[1], d[1];
 
@@ -151,10 +153,10 @@ void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits,
   }
 }
 
-void CRYPTO_cfb128_8_encrypt(const unsigned char *in, unsigned char *out,
-                             size_t length, const AES_KEY *key,
-                             unsigned char ivec[16], unsigned *num, int enc,
-                             block128_f block) {
+void bssl::CRYPTO_cfb128_8_encrypt(const unsigned char *in, unsigned char *out,
+                                   size_t length, const AES_KEY *key,
+                                   unsigned char ivec[16], unsigned *num,
+                                   int enc, block128_f block) {
   size_t n;
 
   assert(in && out && key && ivec && num);

data/third_party/boringssl-with-bazel/crypto/fipsmodule/aes/ctr.cc.inc

@@ -19,6 +19,8 @@
 #include "../../internal.h"
 
 
+using namespace bssl;
+
 static_assert(16 % sizeof(crypto_word_t) == 0,
               "block cannot be divided into crypto_word_t");
 
@@ -34,10 +36,10 @@ static void ctr96_inc(uint8_t *counter) {
   } while (n);
 }
 
-void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len,
-                                 const AES_KEY *key, uint8_t ivec[16],
-                                 uint8_t ecount_buf[16], unsigned int *num,
-                                 ctr128_f func) {
+void bssl::CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out,
+                                       size_t len, const AES_KEY *key,
+                                       uint8_t ivec[16], uint8_t ecount_buf[16],
+                                       unsigned int *num, ctr128_f func) {
   unsigned int n, ctr32;
 
   assert(key && ecount_buf && num);
@@ -73,7 +75,7 @@ void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len,
     (*func)(in, out, blocks, key, ivec);
     // (*func) does not update ivec, caller does:
     CRYPTO_store_u32_be(ivec + 12, ctr32);
-    // ... overflow was detected, propogate carry.
+    // ... overflow was detected, propagate carry.
     if (ctr32 == 0) {
       ctr96_inc(ivec);
     }

data/third_party/boringssl-with-bazel/crypto/fipsmodule/aes/gcm.cc.inc

@@ -23,12 +23,14 @@
 #include "internal.h"
 
 
+using namespace bssl;
+
 // kSizeTWithoutLower4Bits is a mask that can be used to zero the lower four
-// bits of a |size_t|.
+// bits of a `size_t`.
 static const size_t kSizeTWithoutLower4Bits = (size_t) -16;
 
 
-#define GCM_MUL(key, ctx, Xi) gcm_gmult_nohw((ctx)->Xi, (key)->Htable)
+#define GCM_MUL(key, ctx, Xi) bssl::gcm_gmult_nohw((ctx)->Xi, (key)->Htable)
 #define GHASH(key, ctx, in, len) \
   gcm_ghash_nohw((ctx)->Xi, (key)->Htable, in, len)
 // GHASH_CHUNK is "stride parameter" missioned to mitigate cache
@@ -37,7 +39,7 @@ static const size_t kSizeTWithoutLower4Bits = (size_t) -16;
 #define GHASH_CHUNK (3 * 1024)
 
 #if defined(GHASH_ASM_X86_64) || defined(GHASH_ASM_X86)
-static inline void gcm_reduce_1bit(u128 *V) {
+static void gcm_reduce_1bit(u128 *V) {
   if (sizeof(crypto_word_t) == 8) {
     uint64_t T = UINT64_C(0xe100000000000000) & (0 - (V->hi & 1));
     V->hi = (V->lo << 63) | (V->hi >> 1);
@@ -49,7 +51,7 @@ static inline void gcm_reduce_1bit(u128 *V) {
   }
 }
 
-void gcm_init_ssse3(u128 Htable[16], const uint64_t H[2]) {
+void bssl::gcm_init_ssse3(u128 Htable[16], const uint64_t H[2]) {
   Htable[0].hi = 0;
   Htable[0].lo = 0;
   u128 V;
@@ -77,7 +79,7 @@ void gcm_init_ssse3(u128 Htable[16], const uint64_t H[2]) {
   Htable[14].hi = V.hi ^ Htable[6].hi, Htable[14].lo = V.lo ^ Htable[6].lo;
   Htable[15].hi = V.hi ^ Htable[7].hi, Htable[15].lo = V.lo ^ Htable[7].lo;
 
-  // Treat |Htable| as a 16x16 byte table and transpose it. Thus, Htable[i]
+  // Treat `Htable` as a 16x16 byte table and transpose it. Thus, Htable[i]
   // contains the i'th byte of j*H for all j.
   uint8_t *Hbytes = (uint8_t *)Htable;
   for (int i = 0; i < 16; i++) {
@@ -150,6 +152,10 @@ static size_t hw_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
   if (!len_blocks) {
     return 0;
   }
+  if (impl == gcm_arm64_aes_eor3) {
+    aes_gcm_enc_kernel_eor3(in, len_blocks * 8, out, Xi, ivec, key, Htable);
+    return len_blocks;
+  }
   aes_gcm_enc_kernel(in, len_blocks * 8, out, Xi, ivec, key, Htable);
   return len_blocks;
 }
@@ -162,15 +168,19 @@ static size_t hw_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
   if (!len_blocks) {
     return 0;
   }
+  if (impl == gcm_arm64_aes_eor3) {
+    aes_gcm_dec_kernel_eor3(in, len_blocks * 8, out, Xi, ivec, key, Htable);
+    return len_blocks;
+  }
   aes_gcm_dec_kernel(in, len_blocks * 8, out, Xi, ivec, key, Htable);
   return len_blocks;
 }
 
 #endif  // HW_GCM && AARCH64
 
-void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
-                       u128 out_table[16], const uint8_t gcm_key[16]) {
-  // H is passed to |gcm_init_*| as a pair of byte-swapped, 64-bit values.
+void bssl::CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
+                             u128 out_table[16], const uint8_t gcm_key[16]) {
+  // H is passed to `gcm_init_*` as a pair of byte-swapped, 64-bit values.
   uint64_t H[2] = {CRYPTO_load_u64_be(gcm_key),
                    CRYPTO_load_u64_be(gcm_key + 8)};
 
@@ -240,8 +250,8 @@ void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
   *out_hash = gcm_ghash_nohw;
 }
 
-void CRYPTO_gcm128_init_aes_key(GCM128_KEY *gcm_key, const uint8_t *key,
-                                size_t key_bytes) {
+void bssl::CRYPTO_gcm128_init_aes_key(GCM128_KEY *gcm_key, const uint8_t *key,
+                                      size_t key_bytes) {
   switch (key_bytes) {
     case 16:
       boringssl_fips_inc_counter(fips_counter_evp_aes_128_gcm);
@@ -276,15 +286,17 @@ void CRYPTO_gcm128_init_aes_key(GCM128_KEY *gcm_key, const uint8_t *key,
     gcm_key->impl = gcm_x86_aesni;
   }
 #elif defined(OPENSSL_AARCH64)
-  if (gcm_pmull_capable() && is_hwaes) {
+  if (gcm_eor3_capable() && is_hwaes) {
+    gcm_key->impl = gcm_arm64_aes_eor3;
+  } else if (gcm_pmull_capable() && is_hwaes) {
     gcm_key->impl = gcm_arm64_aes;
   }
 #endif
 #endif
 }
 
-void CRYPTO_gcm128_init_ctx(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
-                            const uint8_t *iv, size_t iv_len) {
+void bssl::CRYPTO_gcm128_init_ctx(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                                  const uint8_t *iv, size_t iv_len) {
 #ifdef GCM_FUNCREF
   void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = key->gmult;
 #endif
@@ -331,8 +343,8 @@ void CRYPTO_gcm128_init_ctx(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
   CRYPTO_store_u32_be(ctx->Yi + 12, ctr);
 }
 
-int CRYPTO_gcm128_aad(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
-                      const uint8_t *aad, size_t aad_len) {
+int bssl::CRYPTO_gcm128_aad(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                            const uint8_t *aad, size_t aad_len) {
 #ifdef GCM_FUNCREF
   void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = key->gmult;
   void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
@@ -385,8 +397,8 @@ int CRYPTO_gcm128_aad(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
   return 1;
 }
 
-int CRYPTO_gcm128_encrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
-                          const uint8_t *in, uint8_t *out, size_t len) {
+int bssl::CRYPTO_gcm128_encrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                                const uint8_t *in, uint8_t *out, size_t len) {
 #ifdef GCM_FUNCREF
   void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = key->gmult;
   void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
@@ -422,9 +434,8 @@ int CRYPTO_gcm128_encrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
   }
 
 #if defined(HW_GCM)
-  // Check |len| to work around a C language bug. See https://crbug.com/1019588.
   if (key->impl != gcm_separate && len > 0) {
-    // |hw_gcm_encrypt| may not process all the input given to it. It may
+    // `hw_gcm_encrypt` may not process all the input given to it. It may
     // not process *any* of its input if it is deemed too small.
     size_t bulk = hw_gcm_encrypt(in, out, len, &key->aes, ctx->Yi, ctx->Xi,
                                  key->Htable, key->impl);
@@ -472,8 +483,8 @@ int CRYPTO_gcm128_encrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
   return 1;
 }
 
-int CRYPTO_gcm128_decrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
-                          const uint8_t *in, uint8_t *out, size_t len) {
+int bssl::CRYPTO_gcm128_decrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                                const uint8_t *in, uint8_t *out, size_t len) {
 #ifdef GCM_FUNCREF
   void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = key->gmult;
   void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
@@ -511,9 +522,8 @@ int CRYPTO_gcm128_decrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
   }
 
 #if defined(HW_GCM)
-  // Check |len| to work around a C language bug. See https://crbug.com/1019588.
   if (key->impl != gcm_separate && len > 0) {
-    // |hw_gcm_decrypt| may not process all the input given to it. It may
+    // `hw_gcm_decrypt` may not process all the input given to it. It may
     // not process *any* of its input if it is deemed too small.
     size_t bulk = hw_gcm_decrypt(in, out, len, &key->aes, ctx->Yi, ctx->Xi,
                                  key->Htable, key->impl);
@@ -563,8 +573,8 @@ int CRYPTO_gcm128_decrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
   return 1;
 }
 
-int CRYPTO_gcm128_finish(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
-                         const uint8_t *tag, size_t len) {
+int bssl::CRYPTO_gcm128_finish(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                               const uint8_t *tag, size_t len) {
 #ifdef GCM_FUNCREF
   void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = key->gmult;
 #endif
@@ -587,14 +597,14 @@ int CRYPTO_gcm128_finish(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
   }
 }
 
-void CRYPTO_gcm128_tag(const GCM128_KEY *key, GCM128_CONTEXT *ctx, uint8_t *tag,
-                       size_t len) {
-  CRYPTO_gcm128_finish(key, ctx, NULL, 0);
+void bssl::CRYPTO_gcm128_tag(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
+                             uint8_t *tag, size_t len) {
+  CRYPTO_gcm128_finish(key, ctx, nullptr, 0);
   OPENSSL_memcpy(tag, ctx->Xi, len <= sizeof(ctx->Xi) ? len : sizeof(ctx->Xi));
 }
 
 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
-int crypto_gcm_clmul_enabled(void) {
+int bssl::crypto_gcm_clmul_enabled() {
 #if defined(GHASH_ASM_X86) || defined(GHASH_ASM_X86_64)
   return CRYPTO_is_PCLMUL_capable() && CRYPTO_is_SSSE3_capable();
 #else

data/third_party/boringssl-with-bazel/crypto/fipsmodule/aes/gcm_nohw.cc.inc

@@ -22,6 +22,8 @@
 #endif
 
 
+using namespace bssl;
+
 // This file contains a constant-time implementation of GHASH based on the notes
 // in https://bearssl.org/constanttime.html#ghash-for-gcm and the reduction
 // algorithm described in
@@ -38,7 +40,7 @@ static void gcm_mul64_nohw(uint64_t *out_lo, uint64_t *out_hi, uint64_t a,
   // One term every four bits means the largest term is 64/4 = 16, which barely
   // overflows into the next term. Using one term every five bits would cost 25
   // multiplications instead of 16. It is faster to mask off the bottom four
-  // bits of |a|, giving a largest term of 60/4 = 15, and apply the bottom bits
+  // bits of `a`, giving a largest term of 60/4 = 15, and apply the bottom bits
   // separately.
   uint64_t a0 = a & UINT64_C(0x1111111111111110);
   uint64_t a1 = a & UINT64_C(0x2222222222222220);
@@ -59,7 +61,7 @@ static void gcm_mul64_nohw(uint64_t *out_lo, uint64_t *out_hi, uint64_t a,
   uint128_t c3 = (a0 * (uint128_t)b3) ^ (a1 * (uint128_t)b2) ^
                  (a2 * (uint128_t)b1) ^ (a3 * (uint128_t)b0);
 
-  // Multiply the bottom four bits of |a| with |b|.
+  // Multiply the bottom four bits of `a` with `b`.
   uint64_t a0_mask = UINT64_C(0) - (a & 1);
   uint64_t a1_mask = UINT64_C(0) - ((a >> 1) & 1);
   uint64_t a2_mask = UINT64_C(0) - ((a >> 2) & 1);
@@ -192,14 +194,14 @@ static void gcm_mul64_nohw(uint64_t *out_lo, uint64_t *out_hi, uint64_t a,
 
 #endif  // BORINGSSL_HAS_UINT128
 
-void gcm_init_nohw(u128 Htable[16], const uint64_t Xi[2]) {
+void bssl::gcm_init_nohw(u128 Htable[16], const uint64_t Xi[2]) {
   // We implement GHASH in terms of POLYVAL, as described in RFC 8452. This
   // avoids a shift by 1 in the multiplication, needed to account for bit
   // reversal losing a bit after multiplication, that is,
   // rev128(X) * rev128(Y) = rev255(X*Y).
   //
   // Per Appendix A, we run mulX_POLYVAL. Note this is the same transformation
-  // applied by |gcm_init_clmul|, etc. Note |Xi| has already been byteswapped.
+  // applied by `gcm_init_clmul`, etc. Note `Xi` has already been byteswapped.
   //
   // See also slide 16 of
   // https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf
@@ -218,12 +220,12 @@ void gcm_init_nohw(u128 Htable[16], const uint64_t Xi[2]) {
   Htable[0].lo ^= carry & 1;
   Htable[0].hi ^= carry & UINT64_C(0xc200000000000000);
 
-  // This implementation does not use the rest of |Htable|.
+  // This implementation does not use the rest of `Htable`.
 }
 
 static void gcm_polyval_nohw(uint64_t Xi[2], const u128 *H) {
-  // Karatsuba multiplication. The product of |Xi| and |H| is stored in |r0|
-  // through |r3|. Note there is no byte or bit reversal because we are
+  // Karatsuba multiplication. The product of `Xi` and `H` is stored in `r0`
+  // through `r3`. Note there is no byte or bit reversal because we are
   // evaluating POLYVAL.
   uint64_t r0, r1;
   gcm_mul64_nohw(&r0, &r1, Xi[0], H->lo);
@@ -236,8 +238,8 @@ static void gcm_polyval_nohw(uint64_t Xi[2], const u128 *H) {
   r2 ^= mid1;
   r1 ^= mid0;
 
-  // Now we multiply our 256-bit result by x^-128 and reduce. |r2| and
-  // |r3| shifts into position and we must multiply |r0| and |r1| by x^-128. We
+  // Now we multiply our 256-bit result by x^-128 and reduce. `r2` and
+  // `r3` shifts into position and we must multiply `r0` and `r1` by x^-128. We
   // have:
   //
   //       1 = x^121 + x^126 + x^127 + x^128
@@ -247,7 +249,7 @@ static void gcm_polyval_nohw(uint64_t Xi[2], const u128 *H) {
 
   // The x^-7, x^-2, and x^-1 terms shift bits past x^0, which would require
   // another reduction steps. Instead, we gather the excess bits, incorporate
-  // them into |r0| and |r1| and reduce once. See slides 17-19
+  // them into `r0` and `r1` and reduce once. See slides 17-19
   // of https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf.
   r1 ^= (r0 << 63) ^ (r0 << 62) ^ (r0 << 57);
 
@@ -274,7 +276,7 @@ static void gcm_polyval_nohw(uint64_t Xi[2], const u128 *H) {
   Xi[1] = r3;
 }
 
-void gcm_gmult_nohw(uint8_t Xi[16], const u128 Htable[16]) {
+void bssl::gcm_gmult_nohw(uint8_t Xi[16], const u128 Htable[16]) {
   uint64_t swapped[2];
   swapped[0] = CRYPTO_load_u64_be(Xi + 8);
   swapped[1] = CRYPTO_load_u64_be(Xi);
@@ -283,8 +285,8 @@ void gcm_gmult_nohw(uint8_t Xi[16], const u128 Htable[16]) {
   CRYPTO_store_u64_be(Xi + 8, swapped[0]);
 }
 
-void gcm_ghash_nohw(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
-                    size_t len) {
+void bssl::gcm_ghash_nohw(uint8_t Xi[16], const u128 Htable[16],
+                          const uint8_t *inp, size_t len) {
   uint64_t swapped[2];
   swapped[0] = CRYPTO_load_u64_be(Xi + 8);
   swapped[1] = CRYPTO_load_u64_be(Xi);