google-protobuf 3.22.2 → 4.30.1

data/ext/google/protobuf_c/shared_convert.c

@@ -0,0 +1,69 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2023 Google Inc.  All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+// -----------------------------------------------------------------------------
+// Ruby <-> upb data conversion functions. Strictly free of dependencies on
+// Ruby interpreter internals.
+
+#include "shared_convert.h"
+
+bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2,
+                           upb_CType type, const upb_MessageDef* msgdef,
+                           upb_Status* status) {
+  switch (type) {
+    case kUpb_CType_Bool:
+      return memcmp(&val1, &val2, 1) == 0;
+    case kUpb_CType_Float:
+    case kUpb_CType_Int32:
+    case kUpb_CType_UInt32:
+    case kUpb_CType_Enum:
+      return memcmp(&val1, &val2, 4) == 0;
+    case kUpb_CType_Double:
+    case kUpb_CType_Int64:
+    case kUpb_CType_UInt64:
+      return memcmp(&val1, &val2, 8) == 0;
+    case kUpb_CType_String:
+    case kUpb_CType_Bytes:
+      return val1.str_val.size == val2.str_val.size &&
+             memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) ==
+                 0;
+    case kUpb_CType_Message: {
+      const upb_MiniTable* m = upb_MessageDef_MiniTable(msgdef);
+      const int options = 0;
+      return upb_Message_IsEqual(val1.msg_val, val2.msg_val, m, options);
+    }
+    default:
+      upb_Status_SetErrorMessage(status, "Internal error, unexpected type");
+      return false;
+  }
+}
+
+uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type,
+                               const upb_MessageDef* msgdef, uint64_t seed,
+                               upb_Status* status) {
+  switch (type) {
+    case kUpb_CType_Bool:
+      return _upb_Hash(&val, 1, seed);
+    case kUpb_CType_Float:
+    case kUpb_CType_Int32:
+    case kUpb_CType_UInt32:
+    case kUpb_CType_Enum:
+      return _upb_Hash(&val, 4, seed);
+    case kUpb_CType_Double:
+    case kUpb_CType_Int64:
+    case kUpb_CType_UInt64:
+      return _upb_Hash(&val, 8, seed);
+    case kUpb_CType_String:
+    case kUpb_CType_Bytes:
+      return _upb_Hash(val.str_val.data, val.str_val.size, seed);
+    case kUpb_CType_Message:
+      return shared_Message_Hash(val.msg_val, msgdef, seed, status);
+    default:
+      upb_Status_SetErrorMessage(status, "Internal error, unexpected type");
+      return 0;
+  }
+}

data/ext/google/protobuf_c/shared_convert.h

@@ -0,0 +1,26 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2023 Google Inc.  All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+// -----------------------------------------------------------------------------
+// Ruby <-> upb data conversion functions. Strictly free of dependencies on
+// Ruby interpreter internals.
+
+#ifndef RUBY_PROTOBUF_SHARED_CONVERT_H_
+#define RUBY_PROTOBUF_SHARED_CONVERT_H_
+
+#include "ruby-upb.h"
+#include "shared_message.h"
+
+bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2,
+                           upb_CType type, const upb_MessageDef* msgdef,
+                           upb_Status* status);
+
+uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type,
+                               const upb_MessageDef* msgdef, uint64_t seed,
+                               upb_Status* status);
+
+#endif  // RUBY_PROTOBUF_SHARED_CONVERT_H_

data/ext/google/protobuf_c/shared_message.c

@@ -0,0 +1,37 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2023 Google Inc.  All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+// -----------------------------------------------------------------------------
+// Ruby Message functions. Strictly free of dependencies on
+// Ruby interpreter internals.
+
+#include "shared_message.h"
+
+// Support function for Message_Hash. Returns a hash value for the given
+// message.
+uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m,
+                             uint64_t seed, upb_Status* status) {
+  upb_Arena* arena = upb_Arena_New();
+  char* data;
+  size_t size;
+
+  // Hash a deterministically serialized payloads with no unknown fields.
+  upb_EncodeStatus encode_status = upb_Encode(
+      msg, upb_MessageDef_MiniTable(m),
+      kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena,
+      &data, &size);
+
+  if (encode_status == kUpb_EncodeStatus_Ok) {
+    uint64_t ret = _upb_Hash(data, size, seed);
+    upb_Arena_Free(arena);
+    return ret;
+  }
+
+  upb_Arena_Free(arena);
+  upb_Status_SetErrorMessage(status, "Error calculating hash");
+  return 0;
+}

data/ext/google/protobuf_c/shared_message.h

@@ -0,0 +1,21 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2023 Google Inc.  All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+// -----------------------------------------------------------------------------
+// Ruby Message functions. Strictly free of dependencies on
+// Ruby interpreter internals.
+
+#ifndef RUBY_PROTOBUF_SHARED_MESSAGE_H_
+#define RUBY_PROTOBUF_SHARED_MESSAGE_H_
+
+#include "ruby-upb.h"
+
+// Returns a hash value for the given message.
+uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m,
+                             uint64_t seed, upb_Status* status);
+
+#endif  // RUBY_PROTOBUF_SHARED_MESSAGE_H_

data/ext/google/protobuf_c/third_party/utf8_range/utf8_range.c

@@ -0,0 +1,207 @@
+// Copyright 2023 Google LLC
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+/* This is a wrapper for the Google range-sse.cc algorithm which checks whether
+ * a sequence of bytes is a valid UTF-8 sequence and finds the longest valid
+ * prefix of the UTF-8 sequence.
+ *
+ * The key difference is that it checks for as much ASCII symbols as possible
+ * and then falls back to the range-sse.cc algorithm. The changes to the
+ * algorithm are cosmetic, mostly to trick the clang compiler to produce optimal
+ * code.
+ *
+ * For API see the utf8_validity.h header.
+ */
+#include "utf8_range.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#if defined(__GNUC__)
+#define FORCE_INLINE_ATTR __attribute__((always_inline))
+#elif defined(_MSC_VER)
+#define FORCE_INLINE_ATTR __forceinline
+#else
+#define FORCE_INLINE_ATTR
+#endif
+
+static FORCE_INLINE_ATTR inline uint64_t utf8_range_UnalignedLoad64(
+    const void* p) {
+  uint64_t t;
+  memcpy(&t, p, sizeof t);
+  return t;
+}
+
+static FORCE_INLINE_ATTR inline int utf8_range_AsciiIsAscii(unsigned char c) {
+  return c < 128;
+}
+
+static FORCE_INLINE_ATTR inline int utf8_range_IsTrailByteOk(const char c) {
+  return (int8_t)(c) <= (int8_t)(0xBF);
+}
+
+/* If return_position is false then it returns 1 if |data| is a valid utf8
+ * sequence, otherwise returns 0.
+ * If return_position is set to true, returns the length in bytes of the prefix
+   of |data| that is all structurally valid UTF-8.
+ */
+static size_t utf8_range_ValidateUTF8Naive(const char* data, const char* end,
+                                           int return_position) {
+  /* We return err_pos in the loop which is always 0 if !return_position */
+  size_t err_pos = 0;
+  size_t codepoint_bytes = 0;
+  /* The early check is done because of early continue's on codepoints of all
+   * sizes, i.e. we first check for ascii and if it is, we call continue, then
+   * for 2 byte codepoints, etc. This is done in order to reduce indentation and
+   * improve readability of the codepoint validity check.
+   */
+  while (data + codepoint_bytes < end) {
+    if (return_position) {
+      err_pos += codepoint_bytes;
+    }
+    data += codepoint_bytes;
+    const size_t len = end - data;
+    const unsigned char byte1 = data[0];
+
+    /* We do not skip many ascii bytes at the same time as this function is
+       used for tail checking (< 16 bytes) and for non x86 platforms. We also
+       don't think that cases where non-ASCII codepoints are followed by ascii
+       happen often. For small strings it also introduces some penalty. For
+       purely ascii UTF8 strings (which is the overwhelming case) we call
+       SkipAscii function which is multiplatform and extremely fast.
+     */
+    /* [00..7F] ASCII -> 1 byte */
+    if (utf8_range_AsciiIsAscii(byte1)) {
+      codepoint_bytes = 1;
+      continue;
+    }
+    /* [C2..DF], [80..BF] -> 2 bytes */
+    if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF &&
+        utf8_range_IsTrailByteOk(data[1])) {
+      codepoint_bytes = 2;
+      continue;
+    }
+    if (len >= 3) {
+      const unsigned char byte2 = data[1];
+      const unsigned char byte3 = data[2];
+
+      /* Is byte2, byte3 between [0x80, 0xBF]
+       * Check for 0x80 was done above.
+       */
+      if (!utf8_range_IsTrailByteOk(byte2) ||
+          !utf8_range_IsTrailByteOk(byte3)) {
+        return err_pos;
+      }
+
+      if (/* E0, A0..BF, 80..BF */
+          ((byte1 == 0xE0 && byte2 >= 0xA0) ||
+           /* E1..EC, 80..BF, 80..BF */
+           (byte1 >= 0xE1 && byte1 <= 0xEC) ||
+           /* ED, 80..9F, 80..BF */
+           (byte1 == 0xED && byte2 <= 0x9F) ||
+           /* EE..EF, 80..BF, 80..BF */
+           (byte1 >= 0xEE && byte1 <= 0xEF))) {
+        codepoint_bytes = 3;
+        continue;
+      }
+      if (len >= 4) {
+        const unsigned char byte4 = data[3];
+        /* Is byte4 between 0x80 ~ 0xBF */
+        if (!utf8_range_IsTrailByteOk(byte4)) {
+          return err_pos;
+        }
+
+        if (/* F0, 90..BF, 80..BF, 80..BF */
+            ((byte1 == 0xF0 && byte2 >= 0x90) ||
+             /* F1..F3, 80..BF, 80..BF, 80..BF */
+             (byte1 >= 0xF1 && byte1 <= 0xF3) ||
+             /* F4, 80..8F, 80..BF, 80..BF */
+             (byte1 == 0xF4 && byte2 <= 0x8F))) {
+          codepoint_bytes = 4;
+          continue;
+        }
+      }
+    }
+    return err_pos;
+  }
+  if (return_position) {
+    err_pos += codepoint_bytes;
+  }
+  /* if return_position is false, this returns 1.
+   * if return_position is true, this returns err_pos.
+   */
+  return err_pos + (1 - return_position);
+}
+
+#if defined(__SSE4_1__) || (defined(__ARM_NEON) && defined(__ARM_64BIT_STATE))
+/* Returns the number of bytes needed to skip backwards to get to the first
+   byte of codepoint.
+ */
+static inline int utf8_range_CodepointSkipBackwards(int32_t codepoint_word) {
+  const int8_t* const codepoint = (const int8_t*)(&codepoint_word);
+  if (!utf8_range_IsTrailByteOk(codepoint[3])) {
+    return 1;
+  } else if (!utf8_range_IsTrailByteOk(codepoint[2])) {
+    return 2;
+  } else if (!utf8_range_IsTrailByteOk(codepoint[1])) {
+    return 3;
+  }
+  return 0;
+}
+#endif  // __SSE4_1__
+
+/* Skipping over ASCII as much as possible, per 8 bytes. It is intentional
+   as most strings to check for validity consist only of 1 byte codepoints.
+ */
+static inline const char* utf8_range_SkipAscii(const char* data,
+                                               const char* end) {
+  while (8 <= end - data &&
+         (utf8_range_UnalignedLoad64(data) & 0x8080808080808080) == 0) {
+    data += 8;
+  }
+  while (data < end && utf8_range_AsciiIsAscii(*data)) {
+    ++data;
+  }
+  return data;
+}
+
+#if defined(__SSE4_1__)
+#include "utf8_range_sse.inc"
+#elif defined(__ARM_NEON) && defined(__ARM_64BIT_STATE)
+#include "utf8_range_neon.inc"
+#endif
+
+static FORCE_INLINE_ATTR inline size_t utf8_range_Validate(
+    const char* data, size_t len, int return_position) {
+  if (len == 0) return 1 - return_position;
+  // Save buffer start address for later use
+  const char* const data_original = data;
+  const char* const end = data + len;
+  data = utf8_range_SkipAscii(data, end);
+  /* SIMD algorithm always outperforms the naive version for any data of
+     length >=16.
+   */
+  if (end - data < 16) {
+    return (return_position ? (data - data_original) : 0) +
+           utf8_range_ValidateUTF8Naive(data, end, return_position);
+  }
+#if defined(__SSE4_1__) || (defined(__ARM_NEON) && defined(__ARM_64BIT_STATE))
+  return utf8_range_ValidateUTF8Simd(
+      data_original, data, end, return_position);
+#else
+  return (return_position ? (data - data_original) : 0) +
+         utf8_range_ValidateUTF8Naive(data, end, return_position);
+#endif
+}
+
+int utf8_range_IsValid(const char* data, size_t len) {
+  return utf8_range_Validate(data, len, /*return_position=*/0) != 0;
+}
+
+size_t utf8_range_ValidPrefix(const char* data, size_t len) {
+  return utf8_range_Validate(data, len, /*return_position=*/1);
+}

data/ext/google/protobuf_c/third_party/utf8_range/utf8_range.h

@@ -1,18 +1,19 @@
 #ifndef THIRD_PARTY_UTF8_RANGE_UTF8_RANGE_H_
 #define THIRD_PARTY_UTF8_RANGE_UTF8_RANGE_H_
 
+#include <stddef.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#if (defined(__ARM_NEON) && defined(__aarch64__)) || defined(__SSE4_1__)
-int utf8_range2(const unsigned char* data, int len);
-#else
-int utf8_naive(const unsigned char* data, int len);
-static inline int utf8_range2(const unsigned char* data, int len) {
-  return utf8_naive(data, len);
-}
-#endif
+// Returns 1 if the sequence of characters is a valid UTF-8 sequence, otherwise
+// 0.
+int utf8_range_IsValid(const char* data, size_t len);
+
+// Returns the length in bytes of the prefix of str that is all
+// structurally valid UTF-8.
+size_t utf8_range_ValidPrefix(const char* data, size_t len);
 
 #ifdef __cplusplus
 }  // extern "C"

data/ext/google/protobuf_c/third_party/utf8_range/utf8_range_neon.inc

@@ -0,0 +1,117 @@
+#include <arm_neon.h>
+
+/* This code is almost the same as SSE implementation, please reference
+ * utf8-range-sse.inc for detailed explanation.
+ * The only difference is the range adjustment step. NEON code is more
+ * straightforward.
+ */
+
+static FORCE_INLINE_ATTR inline size_t utf8_range_ValidateUTF8Simd(
+    const char* data_original, const char* data, const char* end,
+    int return_position) {
+  const uint8x16_t first_len_tbl = {
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
+  };
+  const uint8x16_t first_range_tbl = {
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
+  };
+  const uint8x16_t range_min_tbl = {
+      0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
+      0xC2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+  };
+  const uint8x16_t range_max_tbl = {
+      0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
+      0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  };
+  /* Range adjustment in NEON uint8x16x2 table. Note that lanes are interleaved
+   * in register. The table below is plotted vertically to ease understanding.
+   * The 1st column is for E0~EF, 2nd column for F0~FF.
+   */
+  // clang-format off
+  const uint8_t range_adjust_tbl_data[] = {
+    /* index -> 0~15  16~31 <- index */
+    /*  E0 -> */ 2,     3, /* <- F0  */
+                 0,     0,
+                 0,     0,
+                 0,     0,
+                 0,     4, /* <- F4  */
+                 0,     0,
+                 0,     0,
+                 0,     0,
+                 0,     0,
+                 0,     0,
+                 0,     0,
+                 0,     0,
+                 0,     0,
+    /*  ED -> */ 3,     0,
+                 0,     0,
+                 0,     0,
+  };
+  // clang-format on
+  const uint8x16x2_t range_adjust_tbl = vld2q_u8(range_adjust_tbl_data);
+
+  const uint8x16_t const_1 = vdupq_n_u8(1);
+  const uint8x16_t const_2 = vdupq_n_u8(2);
+  const uint8x16_t const_e0 = vdupq_n_u8(0xE0);
+
+  uint8x16_t prev_input = vdupq_n_u8(0);
+  uint8x16_t prev_first_len = vdupq_n_u8(0);
+  uint8x16_t error = vdupq_n_u8(0);
+
+  while (end - data >= 16) {
+    const uint8x16_t input = vld1q_u8((const uint8_t*)data);
+
+    const uint8x16_t high_nibbles = vshrq_n_u8(input, 4);
+
+    const uint8x16_t first_len = vqtbl1q_u8(first_len_tbl, high_nibbles);
+
+    uint8x16_t range = vqtbl1q_u8(first_range_tbl, high_nibbles);
+
+    range = vorrq_u8(range, vextq_u8(prev_first_len, first_len, 15));
+
+    uint8x16_t shift2 = vextq_u8(prev_first_len, first_len, 14);
+    shift2 = vqsubq_u8(shift2, const_1);
+    range = vorrq_u8(range, shift2);
+
+    uint8x16_t shift3 = vextq_u8(prev_first_len, first_len, 13);
+    shift3 = vqsubq_u8(shift3, const_2);
+    range = vorrq_u8(range, shift3);
+
+    uint8x16_t shift1 = vextq_u8(prev_input, input, 15);
+    shift1 = vsubq_u8(shift1, const_e0);
+    range = vaddq_u8(range, vqtbl2q_u8(range_adjust_tbl, shift1));
+
+    const uint8x16_t min_range = vqtbl1q_u8(range_min_tbl, range);
+    const uint8x16_t max_range = vqtbl1q_u8(range_max_tbl, range);
+
+    if (return_position) {
+      error = vcltq_u8(input, min_range);
+      error = vorrq_u8(error, vcgtq_u8(input, max_range));
+      if (vmaxvq_u32(vreinterpretq_u32_u8(error))) {
+        break;
+      }
+    } else {
+      error = vorrq_u8(error, vcltq_u8(input, min_range));
+      error = vorrq_u8(error, vcgtq_u8(input, max_range));
+    }
+
+    prev_input = input;
+    prev_first_len = first_len;
+
+    data += 16;
+  }
+
+  if (return_position && data == data_original) {
+    return utf8_range_ValidateUTF8Naive(data, end, return_position);
+  }
+  const int32_t prev = vgetq_lane_s32(vreinterpretq_s32_u8(prev_input), 3);
+  data -= utf8_range_CodepointSkipBackwards(prev);
+  if (return_position) {
+    return (data - data_original) +
+           utf8_range_ValidateUTF8Naive(data, end, return_position);
+  }
+  if (vmaxvq_u32(vreinterpretq_u32_u8(error))) {
+    return 0;
+  }
+  return utf8_range_ValidateUTF8Naive(data, end, return_position);
+}