grpc 1.34.0 → 1.35.0.pre1

data/third_party/upb/upb/decode.h

@@ -7,15 +7,34 @@
 
 #include "upb/msg.h"
 
+/* Must be last. */
+#include "upb/port_def.inc"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+enum {
+  /* If set, strings will alias the input buffer instead of copying into the
+   * arena. */
+  UPB_DECODE_ALIAS = 1,
+};
+
+#define UPB_DECODE_MAXDEPTH(depth) ((depth) << 16)
+
+bool _upb_decode(const char *buf, size_t size, upb_msg *msg,
+                 const upb_msglayout *l, upb_arena *arena, int options);
+
+UPB_INLINE
 bool upb_decode(const char *buf, size_t size, upb_msg *msg,
-                const upb_msglayout *l, upb_arena *arena);
+                const upb_msglayout *l, upb_arena *arena) {
+  return _upb_decode(buf, size, msg, l, arena, 0);
+}
 
 #ifdef __cplusplus
 }  /* extern "C" */
 #endif
 
+#include "upb/port_undef.inc"
+
 #endif  /* UPB_DECODE_H_ */

data/third_party/upb/upb/decode.int.h

@@ -0,0 +1,163 @@
+/*
+** Internal implementation details of the decoder that are shared between
+** decode.c and decode_fast.c.
+*/
+
+#ifndef UPB_DECODE_INT_H_
+#define UPB_DECODE_INT_H_
+
+#include <setjmp.h>
+
+#include "upb/msg.h"
+#include "upb/upb.int.h"
+
+/* Must be last. */
+#include "upb/port_def.inc"
+
+#define DECODE_NOGROUP -1
+
+typedef struct upb_decstate {
+  const char *end;         /* Can read up to 16 bytes slop beyond this. */
+  const char *limit_ptr;   /* = end + UPB_MIN(limit, 0) */
+  upb_msg *unknown_msg;    /* If non-NULL, add unknown data at buffer flip. */
+  const char *unknown;     /* Start of unknown data. */
+  int limit;               /* Submessage limit relative to end. */
+  int depth;
+  uint32_t end_group;   /* field number of END_GROUP tag, else DECODE_NOGROUP */
+  bool alias;
+  char patch[32];
+  upb_arena arena;
+  jmp_buf err;
+} upb_decstate;
+
+/* Error function that will abort decoding with longjmp(). We can't declare this
+ * UPB_NORETURN, even though it is appropriate, because if we do then compilers
+ * will "helpfully" refuse to tailcall to it
+ * (see: https://stackoverflow.com/a/55657013), which will defeat a major goal
+ * of our optimizations. That is also why we must declare it in a separate file,
+ * otherwise the compiler will see that it calls longjmp() and deduce that it is
+ * noreturn. */
+const char *fastdecode_err(upb_decstate *d);
+
+extern const uint8_t upb_utf8_offsets[];
+
+UPB_INLINE
+bool decode_verifyutf8_inl(const char *buf, int len) {
+  int i, j;
+  uint8_t offset;
+
+  i = 0;
+  while (i < len) {
+    offset = upb_utf8_offsets[(uint8_t)buf[i]];
+    if (offset == 0 || i + offset > len) {
+      return false;
+    }
+    for (j = i + 1; j < i + offset; j++) {
+      if ((buf[j] & 0xc0) != 0x80) {
+        return false;
+      }
+    }
+    i += offset;
+  }
+  return i == len;
+}
+
+/* x86-64 pointers always have the high 16 bits matching. So we can shift
+ * left 8 and right 8 without loss of information. */
+UPB_INLINE intptr_t decode_totable(const upb_msglayout *tablep) {
+  return ((intptr_t)tablep << 8) | tablep->table_mask;
+}
+
+UPB_INLINE const upb_msglayout *decode_totablep(intptr_t table) {
+  return (const upb_msglayout*)(table >> 8);
+}
+
+UPB_INLINE
+const char *decode_isdonefallback_inl(upb_decstate *d, const char *ptr,
+                                      int overrun) {
+  if (overrun < d->limit) {
+    /* Need to copy remaining data into patch buffer. */
+    UPB_ASSERT(overrun < 16);
+    if (d->unknown_msg) {
+      if (!_upb_msg_addunknown(d->unknown_msg, d->unknown, ptr - d->unknown,
+                               &d->arena)) {
+        return NULL;
+      }
+      d->unknown = &d->patch[0] + overrun;
+    }
+    memset(d->patch + 16, 0, 16);
+    memcpy(d->patch, d->end, 16);
+    ptr = &d->patch[0] + overrun;
+    d->end = &d->patch[16];
+    d->limit -= 16;
+    d->limit_ptr = d->end + d->limit;
+    d->alias = false;
+    UPB_ASSERT(ptr < d->limit_ptr);
+    return ptr;
+  } else {
+    return NULL;
+  }
+}
+
+const char *decode_isdonefallback(upb_decstate *d, const char *ptr,
+                                  int overrun);
+
+UPB_INLINE
+bool decode_isdone(upb_decstate *d, const char **ptr) {
+  int overrun = *ptr - d->end;
+  if (UPB_LIKELY(*ptr < d->limit_ptr)) {
+    return false;
+  } else if (UPB_LIKELY(overrun == d->limit)) {
+    return true;
+  } else {
+    *ptr = decode_isdonefallback(d, *ptr, overrun);
+    return false;
+  }
+}
+
+UPB_INLINE
+const char *fastdecode_tagdispatch(upb_decstate *d, const char *ptr,
+                                    upb_msg *msg, intptr_t table,
+                                    uint64_t hasbits, uint32_t tag) {
+  const upb_msglayout *table_p = decode_totablep(table);
+  uint8_t mask = table;
+  uint64_t data;
+  size_t idx = tag & mask;
+  UPB_ASSUME((idx & 7) == 0);
+  idx >>= 3;
+  data = table_p->fasttable[idx].field_data ^ tag;
+  return table_p->fasttable[idx].field_parser(d, ptr, msg, table, hasbits, data);
+}
+
+UPB_INLINE uint32_t fastdecode_loadtag(const char* ptr) {
+  uint16_t tag;
+  memcpy(&tag, ptr, 2);
+  return tag;
+}
+
+UPB_INLINE void decode_checklimit(upb_decstate *d) {
+  UPB_ASSERT(d->limit_ptr == d->end + UPB_MIN(0, d->limit));
+}
+
+UPB_INLINE int decode_pushlimit(upb_decstate *d, const char *ptr, int size) {
+  int limit = size + (int)(ptr - d->end);
+  int delta = d->limit - limit;
+  decode_checklimit(d);
+  d->limit = limit;
+  d->limit_ptr = d->end + UPB_MIN(0, limit);
+  decode_checklimit(d);
+  return delta;
+}
+
+UPB_INLINE void decode_poplimit(upb_decstate *d, const char *ptr,
+                                int saved_delta) {
+  UPB_ASSERT(ptr - d->end == d->limit);
+  decode_checklimit(d);
+  d->limit += saved_delta;
+  d->limit_ptr = d->end + UPB_MIN(0, d->limit);
+  decode_checklimit(d);
+}
+
+#include "upb/port_undef.inc"
+
+#endif  /* UPB_DECODE_INT_H_ */

data/third_party/upb/upb/decode_fast.c

@@ -0,0 +1,1040 @@
+// Fast decoder: ~3x the speed of decode.c, but x86-64 specific.
+// Also the table size grows by 2x.
+//
+// Could potentially be ported to ARM64 or other 64-bit archs that pass at
+// least six arguments in registers.
+//
+// The overall design is to create specialized functions for every possible
+// field type (eg. oneof boolean field with a 1 byte tag) and then dispatch
+// to the specialized function as quickly as possible.
+
+#include "upb/decode_fast.h"
+
+#include "upb/decode.int.h"
+
+/* Must be last. */
+#include "upb/port_def.inc"
+
+#if UPB_FASTTABLE
+
+// The standard set of arguments passed to each parsing function.
+// Thanks to x86-64 calling conventions, these will stay in registers.
+#define UPB_PARSE_PARAMS                                          \
+  upb_decstate *d, const char *ptr, upb_msg *msg, intptr_t table, \
+      uint64_t hasbits, uint64_t data
+
+#define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data
+
+#define RETURN_GENERIC(m)  \
+  /* fprintf(stderr, m); */ \
+  return fastdecode_generic(d, ptr, msg, table, hasbits, 0);
+
+typedef enum {
+  CARD_s = 0,  /* Singular (optional, non-repeated) */
+  CARD_o = 1,  /* Oneof */
+  CARD_r = 2,  /* Repeated */
+  CARD_p = 3   /* Packed Repeated */
+} upb_card;
+
+UPB_NOINLINE
+static const char *fastdecode_isdonefallback(upb_decstate *d, const char *ptr,
+                                             upb_msg *msg, intptr_t table,
+                                             uint64_t hasbits, int overrun) {
+  ptr = decode_isdonefallback_inl(d, ptr, overrun);
+  if (ptr == NULL) {
+    return fastdecode_err(d);
+  }
+  uint16_t tag = fastdecode_loadtag(ptr);
+  return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, tag);
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_dispatch(upb_decstate *d, const char *ptr,
+                                       upb_msg *msg, intptr_t table,
+                                       uint64_t hasbits) {
+  if (UPB_UNLIKELY(ptr >= d->limit_ptr)) {
+    int overrun = ptr - d->end;
+    if (UPB_LIKELY(overrun == d->limit)) {
+      // Parse is finished.
+      *(uint32_t*)msg |= hasbits;  // Sync hasbits.
+      return ptr;
+    } else {
+      return fastdecode_isdonefallback(d, ptr, msg, table, hasbits, overrun);
+    }
+  }
+
+  // Read two bytes of tag data (for a one-byte tag, the high byte is junk).
+  uint16_t tag = fastdecode_loadtag(ptr);
+  return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, tag);
+}
+
+UPB_FORCEINLINE
+static bool fastdecode_checktag(uint64_t data, int tagbytes) {
+  if (tagbytes == 1) {
+    return (data & 0xff) == 0;
+  } else {
+    return (data & 0xffff) == 0;
+  }
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_longsize(const char *ptr, int *size) {
+  int i;
+  UPB_ASSERT(*size & 0x80);
+  *size &= 0xff;
+  for (i = 0; i < 3; i++) {
+    ptr++;
+    size_t byte = (uint8_t)ptr[-1];
+    *size += (byte - 1) << (7 + 7 * i);
+    if (UPB_LIKELY((byte & 0x80) == 0)) return ptr;
+  }
+  ptr++;
+  size_t byte = (uint8_t)ptr[-1];
+  // len is limited by 2gb not 4gb, hence 8 and not 16 as normally expected
+  // for a 32 bit varint.
+  if (UPB_UNLIKELY(byte >= 8)) return NULL;
+  *size += (byte - 1) << 28;
+  return ptr;
+}
+
+UPB_FORCEINLINE
+static bool fastdecode_boundscheck(const char *ptr, size_t len,
+                                   const char *end) {
+  uintptr_t uptr = (uintptr_t)ptr;
+  uintptr_t uend = (uintptr_t)end + 16;
+  uintptr_t res = uptr + len;
+  return res < uptr || res > uend;
+}
+
+UPB_FORCEINLINE
+static bool fastdecode_boundscheck2(const char *ptr, size_t len,
+                                    const char *end) {
+  // This is one extra branch compared to the more normal:
+  //   return (size_t)(end - ptr) < size;
+  // However it is one less computation if we are just about to use "ptr + len":
+  //   https://godbolt.org/z/35YGPz
+  // In microbenchmarks this shows an overall 4% improvement.
+  uintptr_t uptr = (uintptr_t)ptr;
+  uintptr_t uend = (uintptr_t)end;
+  uintptr_t res = uptr + len;
+  return res < uptr || res > uend;
+}
+
+typedef const char *fastdecode_delimfunc(upb_decstate *d, const char *ptr,
+                                         void *ctx);
+
+UPB_FORCEINLINE
+static const char *fastdecode_delimited(upb_decstate *d, const char *ptr,
+                                        fastdecode_delimfunc *func, void *ctx) {
+  ptr++;
+  int len = (int8_t)ptr[-1];
+  if (fastdecode_boundscheck2(ptr, len, d->limit_ptr)) {
+    // Slow case: Sub-message is >=128 bytes and/or exceeds the current buffer.
+    // If it exceeds the buffer limit, limit/limit_ptr will change during
+    // sub-message parsing, so we need to preserve delta, not limit.
+    if (UPB_UNLIKELY(len & 0x80)) {
+      // Size varint >1 byte (length >= 128).
+      ptr = fastdecode_longsize(ptr, &len);
+      if (!ptr) {
+        // Corrupt wire format: size exceeded INT_MAX.
+        return NULL;
+      }
+    }
+    if (ptr - d->end + (int)len > d->limit) {
+      // Corrupt wire format: invalid limit.
+      return NULL;
+    }
+    int delta = decode_pushlimit(d, ptr, len);
+    ptr = func(d, ptr, ctx);
+    decode_poplimit(d, ptr, delta);
+  } else {
+    // Fast case: Sub-message is <128 bytes and fits in the current buffer.
+    // This means we can preserve limit/limit_ptr verbatim.
+    const char *saved_limit_ptr = d->limit_ptr;
+    int saved_limit = d->limit;
+    d->limit_ptr = ptr + len;
+    d->limit = d->limit_ptr - d->end;
+    UPB_ASSERT(d->limit_ptr == d->end + UPB_MIN(0, d->limit));
+    ptr = func(d, ptr, ctx);
+    d->limit_ptr = saved_limit_ptr;
+    d->limit = saved_limit;
+    UPB_ASSERT(d->limit_ptr == d->end + UPB_MIN(0, d->limit));
+  }
+  return ptr;
+}
+
+/* singular, oneof, repeated field handling ***********************************/
+
+typedef struct {
+  upb_array *arr;
+  void *end;
+} fastdecode_arr;
+
+typedef enum {
+  FD_NEXT_ATLIMIT,
+  FD_NEXT_SAMEFIELD,
+  FD_NEXT_OTHERFIELD
+} fastdecode_next;
+
+typedef struct {
+  void *dst;
+  fastdecode_next next;
+  uint32_t tag;
+} fastdecode_nextret;
+
+UPB_FORCEINLINE
+static void *fastdecode_resizearr(upb_decstate *d, void *dst,
+                                  fastdecode_arr *farr, int valbytes) {
+  if (UPB_UNLIKELY(dst == farr->end)) {
+    size_t old_size = farr->arr->size;
+    size_t old_bytes = old_size * valbytes;
+    size_t new_size = old_size * 2;
+    size_t new_bytes = new_size * valbytes;
+    char *old_ptr = _upb_array_ptr(farr->arr);
+    char *new_ptr = upb_arena_realloc(&d->arena, old_ptr, old_bytes, new_bytes);
+    uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
+    farr->arr->size = new_size;
+    farr->arr->data = _upb_array_tagptr(new_ptr, elem_size_lg2);
+    dst = (void*)(new_ptr + (old_size * valbytes));
+    farr->end = (void*)(new_ptr + (new_size * valbytes));
+  }
+  return dst;
+}
+
+UPB_FORCEINLINE
+static bool fastdecode_tagmatch(uint32_t tag, uint64_t data, int tagbytes) {
+  if (tagbytes == 1) {
+    return (uint8_t)tag == (uint8_t)data;
+  } else {
+    return (uint16_t)tag == (uint16_t)data;
+  }
+}
+
+UPB_FORCEINLINE
+static void fastdecode_commitarr(void *dst, fastdecode_arr *farr,
+                                 int valbytes) {
+  farr->arr->len =
+      (size_t)((char *)dst - (char *)_upb_array_ptr(farr->arr)) / valbytes;
+}
+
+UPB_FORCEINLINE
+static fastdecode_nextret fastdecode_nextrepeated(upb_decstate *d, void *dst,
+                                                  const char **ptr,
+                                                  fastdecode_arr *farr,
+                                                  uint64_t data, int tagbytes,
+                                                  int valbytes) {
+  fastdecode_nextret ret;
+  dst = (char *)dst + valbytes;
+
+  if (UPB_LIKELY(!decode_isdone(d, ptr))) {
+    ret.tag = fastdecode_loadtag(*ptr);
+    if (fastdecode_tagmatch(ret.tag, data, tagbytes)) {
+      ret.next = FD_NEXT_SAMEFIELD;
+    } else {
+      fastdecode_commitarr(dst, farr, valbytes);
+      ret.next = FD_NEXT_OTHERFIELD;
+    }
+  } else {
+    fastdecode_commitarr(dst, farr, valbytes);
+    ret.next = FD_NEXT_ATLIMIT;
+  }
+
+  ret.dst = dst;
+  return ret;
+}
+
+UPB_FORCEINLINE
+static void *fastdecode_fieldmem(upb_msg *msg, uint64_t data) {
+  size_t ofs = data >> 48;
+  return (char *)msg + ofs;
+}
+
+UPB_FORCEINLINE
+static void *fastdecode_getfield(upb_decstate *d, const char *ptr, upb_msg *msg,
+                                 uint64_t *data, uint64_t *hasbits,
+                                 fastdecode_arr *farr, int valbytes,
+                                 upb_card card) {
+  switch (card) {
+    case CARD_s: {
+      uint8_t hasbit_index = *data >> 24;
+      // Set hasbit and return pointer to scalar field.
+      *hasbits |= 1ull << hasbit_index;
+      return fastdecode_fieldmem(msg, *data);
+    }
+    case CARD_o: {
+      uint16_t case_ofs = *data >> 32;
+      uint32_t *oneof_case = UPB_PTR_AT(msg, case_ofs, uint32_t);
+      uint8_t field_number = *data >> 24;
+      *oneof_case = field_number;
+      return fastdecode_fieldmem(msg, *data);
+    }
+    case CARD_r: {
+      // Get pointer to upb_array and allocate/expand if necessary.
+      uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
+      upb_array **arr_p = fastdecode_fieldmem(msg, *data);
+      char *begin;
+      *(uint32_t*)msg |= *hasbits;
+      *hasbits = 0;
+      if (UPB_LIKELY(!*arr_p)) {
+        farr->arr = _upb_array_new(&d->arena, 8, elem_size_lg2);
+        *arr_p = farr->arr;
+      } else {
+        farr->arr = *arr_p;
+      }
+      begin = _upb_array_ptr(farr->arr);
+      farr->end = begin + (farr->arr->size * valbytes);
+      *data = fastdecode_loadtag(ptr);
+      return begin + (farr->arr->len * valbytes);
+    }
+    default:
+      UPB_UNREACHABLE();
+  }
+}
+
+UPB_FORCEINLINE
+static bool fastdecode_flippacked(uint64_t *data, int tagbytes) {
+  *data ^= (0x2 ^ 0x0);  // Patch data to match packed wiretype.
+  return fastdecode_checktag(*data, tagbytes);
+}
+
+/* varint fields **************************************************************/
+
+UPB_FORCEINLINE
+static uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) {
+  if (valbytes == 1) {
+    return val != 0;
+  } else if (zigzag) {
+    if (valbytes == 4) {
+      uint32_t n = val;
+      return (n >> 1) ^ -(int32_t)(n & 1);
+    } else if (valbytes == 8) {
+      return (val >> 1) ^ -(int64_t)(val & 1);
+    }
+    UPB_UNREACHABLE();
+  }
+  return val;
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_varint64(const char *ptr, uint64_t *val) {
+  ptr++;
+  *val = (uint8_t)ptr[-1];
+  if (UPB_UNLIKELY(*val & 0x80)) {
+    int i;
+    for (i = 0; i < 8; i++) {
+      ptr++;
+      uint64_t byte = (uint8_t)ptr[-1];
+      *val += (byte - 1) << (7 + 7 * i);
+      if (UPB_LIKELY((byte & 0x80) == 0)) goto done;
+    }
+    ptr++;
+    uint64_t byte = (uint8_t)ptr[-1];
+    if (byte > 1) {
+      return NULL;
+    }
+    *val += (byte - 1) << 63;
+  }
+done:
+  UPB_ASSUME(ptr != NULL);
+  return ptr;
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_unpackedvarint(UPB_PARSE_PARAMS, int tagbytes,
+                                             int valbytes, upb_card card,
+                                             bool zigzag,
+                                             _upb_field_parser *packed) {
+  uint64_t val;
+  void *dst;
+  fastdecode_arr farr;
+
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) {
+      return packed(UPB_PARSE_ARGS);
+    }
+    RETURN_GENERIC("varint field tag mismatch\n");
+  }
+
+  dst =
+      fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, card);
+  if (card == CARD_r) {
+    if (UPB_UNLIKELY(!dst)) {
+      RETURN_GENERIC("need array resize\n");
+    }
+  }
+
+again:
+  if (card == CARD_r) {
+    dst = fastdecode_resizearr(d, dst, &farr, valbytes);
+  }
+
+  ptr += tagbytes;
+  ptr = fastdecode_varint64(ptr, &val);
+  if (ptr == NULL) return fastdecode_err(d);
+  val = fastdecode_munge(val, valbytes, zigzag);
+  memcpy(dst, &val, valbytes);
+
+  if (card == CARD_r) {
+    fastdecode_nextret ret =
+        fastdecode_nextrepeated(d, dst, &ptr, &farr, data, tagbytes, valbytes);
+    switch (ret.next) {
+      case FD_NEXT_SAMEFIELD:
+        dst = ret.dst;
+        goto again;
+      case FD_NEXT_OTHERFIELD:
+        return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag);
+      case FD_NEXT_ATLIMIT:
+        return ptr;
+    }
+  }
+
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
+}
+
+typedef struct {
+  uint8_t valbytes;
+  bool zigzag;
+  void *dst;
+  fastdecode_arr farr;
+} fastdecode_varintdata;
+
+UPB_FORCEINLINE
+static const char *fastdecode_topackedvarint(upb_decstate *d, const char *ptr,
+                                             void *ctx) {
+  fastdecode_varintdata *data = ctx;
+  void *dst = data->dst;
+  uint64_t val;
+
+  while (!decode_isdone(d, &ptr)) {
+    dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes);
+    ptr = fastdecode_varint64(ptr, &val);
+    if (ptr == NULL) return NULL;
+    val = fastdecode_munge(val, data->valbytes, data->zigzag);
+    memcpy(dst, &val, data->valbytes);
+    dst = (char *)dst + data->valbytes;
+  }
+
+  fastdecode_commitarr(dst, &data->farr, data->valbytes);
+  return ptr;
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_packedvarint(UPB_PARSE_PARAMS, int tagbytes,
+                                           int valbytes, bool zigzag,
+                                           _upb_field_parser *unpacked) {
+  fastdecode_varintdata ctx = {valbytes, zigzag};
+
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    if (fastdecode_flippacked(&data, tagbytes)) {
+      return unpacked(UPB_PARSE_ARGS);
+    } else {
+      RETURN_GENERIC("varint field tag mismatch\n");
+    }
+  }
+
+  ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr,
+                                valbytes, CARD_r);
+  if (UPB_UNLIKELY(!ctx.dst)) {
+    RETURN_GENERIC("need array resize\n");
+  }
+
+  ptr += tagbytes;
+  ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx);
+
+  if (UPB_UNLIKELY(ptr == NULL)) {
+    return fastdecode_err(d);
+  }
+
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_varint(UPB_PARSE_PARAMS, int tagbytes,
+                                     int valbytes, upb_card card, bool zigzag,
+                                     _upb_field_parser *unpacked,
+                                     _upb_field_parser *packed) {
+  if (card == CARD_p) {
+    return fastdecode_packedvarint(UPB_PARSE_ARGS, tagbytes, valbytes, zigzag,
+                                   unpacked);
+  } else {
+    return fastdecode_unpackedvarint(UPB_PARSE_ARGS, tagbytes, valbytes, card,
+                                     zigzag, packed);
+  }
+}
+
+#define z_ZZ true
+#define b_ZZ false
+#define v_ZZ false
+
+/* Generate all combinations:
+ * {s,o,r,p} x {b1,v4,z4,v8,z8} x {1bt,2bt} */
+
+#define F(card, type, valbytes, tagbytes)                                      \
+  UPB_NOINLINE                                                                 \
+  const char *upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
+    return fastdecode_varint(UPB_PARSE_ARGS, tagbytes, valbytes, CARD_##card,  \
+                             type##_ZZ,                                        \
+                             &upb_pr##type##valbytes##_##tagbytes##bt,         \
+                             &upb_pp##type##valbytes##_##tagbytes##bt);        \
+  }
+
+#define TYPES(card, tagbytes) \
+  F(card, b, 1, tagbytes)     \
+  F(card, v, 4, tagbytes)     \
+  F(card, v, 8, tagbytes)     \
+  F(card, z, 4, tagbytes)     \
+  F(card, z, 8, tagbytes)
+
+#define TAGBYTES(card) \
+  TYPES(card, 1)       \
+  TYPES(card, 2)
+
+TAGBYTES(s)
+TAGBYTES(o)
+TAGBYTES(r)
+TAGBYTES(p)
+
+#undef z_ZZ
+#undef b_ZZ
+#undef v_ZZ
+#undef o_ONEOF
+#undef s_ONEOF
+#undef r_ONEOF
+#undef F
+#undef TYPES
+#undef TAGBYTES
+
+
+/* fixed fields ***************************************************************/
+
+UPB_FORCEINLINE
+static const char *fastdecode_unpackedfixed(UPB_PARSE_PARAMS, int tagbytes,
+                                            int valbytes, upb_card card,
+                                            _upb_field_parser *packed) {
+  void *dst;
+  fastdecode_arr farr;
+
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) {
+      return packed(UPB_PARSE_ARGS);
+    }
+    RETURN_GENERIC("fixed field tag mismatch\n");
+  }
+
+  dst =
+      fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, card);
+  if (card == CARD_r) {
+    if (UPB_UNLIKELY(!dst)) {
+      RETURN_GENERIC("couldn't allocate array in arena\n");
+    }
+  }
+
+
+again:
+  if (card == CARD_r) {
+    dst = fastdecode_resizearr(d, dst, &farr, valbytes);
+  }
+
+  ptr += tagbytes;
+  memcpy(dst, ptr, valbytes);
+  ptr += valbytes;
+
+  if (card == CARD_r) {
+    fastdecode_nextret ret =
+        fastdecode_nextrepeated(d, dst, &ptr, &farr, data, tagbytes, valbytes);
+    switch (ret.next) {
+      case FD_NEXT_SAMEFIELD:
+        dst = ret.dst;
+        goto again;
+      case FD_NEXT_OTHERFIELD:
+        return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag);
+      case FD_NEXT_ATLIMIT:
+        return ptr;
+    }
+  }
+
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_packedfixed(UPB_PARSE_PARAMS, int tagbytes,
+                                          int valbytes,
+                                          _upb_field_parser *unpacked) {
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    if (fastdecode_flippacked(&data, tagbytes)) {
+      return unpacked(UPB_PARSE_ARGS);
+    } else {
+      RETURN_GENERIC("varint field tag mismatch\n");
+    }
+  }
+
+  ptr += tagbytes;
+  int size = (uint8_t)ptr[0];
+  ptr++;
+  if (size & 0x80) {
+    ptr = fastdecode_longsize(ptr, &size);
+  }
+
+  if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr)) ||
+      (size % valbytes) != 0) {
+    return fastdecode_err(d);
+  }
+
+  upb_array **arr_p = fastdecode_fieldmem(msg, data);
+  upb_array *arr = *arr_p;
+  uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
+  int elems = size / valbytes;
+
+  if (UPB_LIKELY(!arr)) {
+    *arr_p = arr = _upb_array_new(&d->arena, elems, elem_size_lg2);
+    if (!arr) {
+      return fastdecode_err(d);
+    }
+  } else {
+    _upb_array_resize(arr, elems, &d->arena);
+  }
+
+  char *dst = _upb_array_ptr(arr);
+  memcpy(dst, ptr, size);
+  arr->len = elems;
+
+  return fastdecode_dispatch(d, ptr + size, msg, table, hasbits);
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_fixed(UPB_PARSE_PARAMS, int tagbytes,
+                                    int valbytes, upb_card card,
+                                    _upb_field_parser *unpacked,
+                                    _upb_field_parser *packed) {
+  if (card == CARD_p) {
+    return fastdecode_packedfixed(UPB_PARSE_ARGS, tagbytes, valbytes, unpacked);
+  } else {
+    return fastdecode_unpackedfixed(UPB_PARSE_ARGS, tagbytes, valbytes, card,
+                                    packed);
+  }
+}
+
+/* Generate all combinations:
+ * {s,o,r,p} x {f4,f8} x {1bt,2bt} */
+
+#define F(card, valbytes, tagbytes)                                          \
+  UPB_NOINLINE                                                               \
+  const char *upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) {  \
+    return fastdecode_fixed(UPB_PARSE_ARGS, tagbytes, valbytes, CARD_##card, \
+                            &upb_ppf##valbytes##_##tagbytes##bt,             \
+                            &upb_prf##valbytes##_##tagbytes##bt);            \
+  }
+
+#define TYPES(card, tagbytes) \
+  F(card, 4, tagbytes)        \
+  F(card, 8, tagbytes)
+
+#define TAGBYTES(card) \
+  TYPES(card, 1)       \
+  TYPES(card, 2)
+
+TAGBYTES(s)
+TAGBYTES(o)
+TAGBYTES(r)
+TAGBYTES(p)
+
+#undef F
+#undef TYPES
+#undef TAGBYTES
+
+/* string fields **************************************************************/
+
+typedef const char *fastdecode_copystr_func(struct upb_decstate *d,
+                                            const char *ptr, upb_msg *msg,
+                                            const upb_msglayout *table,
+                                            uint64_t hasbits, upb_strview *dst);
+
+UPB_NOINLINE
+static const char *fastdecode_verifyutf8(upb_decstate *d, const char *ptr,
+                                         upb_msg *msg, intptr_t table,
+                                         uint64_t hasbits, upb_strview *dst) {
+  if (!decode_verifyutf8_inl(dst->data, dst->size)) {
+    return fastdecode_err(d);
+  }
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_longstring(struct upb_decstate *d,
+                                         const char *ptr, upb_msg *msg,
+                                         intptr_t table, uint64_t hasbits,
+                                         upb_strview *dst,
+                                         bool validate_utf8) {
+  int size = (uint8_t)ptr[0];  // Could plumb through hasbits.
+  ptr++;
+  if (size & 0x80) {
+    ptr = fastdecode_longsize(ptr, &size);
+  }
+
+  if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr))) {
+    dst->size = 0;
+    return fastdecode_err(d);
+  }
+
+  if (d->alias) {
+    dst->data = ptr;
+    dst->size = size;
+  } else {
+    char *data = upb_arena_malloc(&d->arena, size);
+    if (!data) {
+      return fastdecode_err(d);
+    }
+    memcpy(data, ptr, size);
+    dst->data = data;
+    dst->size = size;
+  }
+
+  if (validate_utf8) {
+    return fastdecode_verifyutf8(d, ptr + size, msg, table, hasbits, dst);
+  } else {
+    return fastdecode_dispatch(d, ptr + size, msg, table, hasbits);
+  }
+}
+
+UPB_NOINLINE
+static const char *fastdecode_longstring_utf8(struct upb_decstate *d,
+                                         const char *ptr, upb_msg *msg,
+                                         intptr_t table, uint64_t hasbits,
+                                         upb_strview *dst) {
+  return fastdecode_longstring(d, ptr, msg, table, hasbits, dst, true);
+}
+
+UPB_NOINLINE
+static const char *fastdecode_longstring_noutf8(struct upb_decstate *d,
+                                                const char *ptr, upb_msg *msg,
+                                                intptr_t table,
+                                                uint64_t hasbits,
+                                                upb_strview *dst) {
+  return fastdecode_longstring(d, ptr, msg, table, hasbits, dst, false);
+}
+
+UPB_FORCEINLINE
+static void fastdecode_docopy(upb_decstate *d, const char *ptr, uint32_t size,
+                              int copy, char *data, upb_strview *dst) {
+  d->arena.head.ptr += copy;
+  dst->data = data;
+  UPB_UNPOISON_MEMORY_REGION(data, copy);
+  memcpy(data, ptr, copy);
+  UPB_POISON_MEMORY_REGION(data + size, copy - size);
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_copystring(UPB_PARSE_PARAMS, int tagbytes,
+                                         upb_card card, bool validate_utf8) {
+  upb_strview *dst;
+  fastdecode_arr farr;
+  int64_t size;
+  size_t arena_has;
+  size_t common_has;
+  char *buf;
+
+  UPB_ASSERT(!d->alias);
+  UPB_ASSERT(fastdecode_checktag(data, tagbytes));
+
+  dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr,
+                            sizeof(upb_strview), card);
+
+again:
+  if (card == CARD_r) {
+    dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview));
+  }
+
+  size = (uint8_t)ptr[tagbytes];
+  ptr += tagbytes + 1;
+  dst->size = size;
+
+  buf = d->arena.head.ptr;
+  arena_has = _upb_arenahas(&d->arena);
+  common_has = UPB_MIN(arena_has, (d->end - ptr) + 16);
+
+  if (UPB_LIKELY(size <= 15 - tagbytes)) {
+    if (arena_has < 16) goto longstr;
+    d->arena.head.ptr += 16;
+    memcpy(buf, ptr - tagbytes - 1, 16);
+    dst->data = buf + tagbytes + 1;
+  } else if (UPB_LIKELY(size <= 32)) {
+    if (UPB_UNLIKELY(common_has < 32)) goto longstr;
+    fastdecode_docopy(d, ptr, size, 32, buf, dst);
+  } else if (UPB_LIKELY(size <= 64)) {
+    if (UPB_UNLIKELY(common_has < 64)) goto longstr;
+    fastdecode_docopy(d, ptr, size, 64, buf, dst);
+  } else if (UPB_LIKELY(size < 128)) {
+    if (UPB_UNLIKELY(common_has < 128)) goto longstr;
+    fastdecode_docopy(d, ptr, size, 128, buf, dst);
+  } else {
+    goto longstr;
+  }
+
+  ptr += size;
+
+  if (card == CARD_r) {
+    if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) {
+      return fastdecode_err(d);
+    }
+    fastdecode_nextret ret = fastdecode_nextrepeated(
+        d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview));
+    switch (ret.next) {
+      case FD_NEXT_SAMEFIELD:
+        dst = ret.dst;
+        goto again;
+      case FD_NEXT_OTHERFIELD:
+        return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag);
+      case FD_NEXT_ATLIMIT:
+        return ptr;
+    }
+  }
+
+  if (card != CARD_r && validate_utf8) {
+    return fastdecode_verifyutf8(d, ptr, msg, table, hasbits, dst);
+  }
+
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
+
+longstr:
+  ptr--;
+  if (validate_utf8) {
+    return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, dst);
+  } else {
+    return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, dst);
+  }
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_string(UPB_PARSE_PARAMS, int tagbytes,
+                                     upb_card card, _upb_field_parser *copyfunc,
+                                     bool validate_utf8) {
+  upb_strview *dst;
+  fastdecode_arr farr;
+  int64_t size;
+
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    RETURN_GENERIC("string field tag mismatch\n");
+  }
+
+  if (UPB_UNLIKELY(!d->alias)) {
+    return copyfunc(UPB_PARSE_ARGS);
+  }
+
+  dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr,
+                            sizeof(upb_strview), card);
+
+again:
+  if (card == CARD_r) {
+    dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview));
+  }
+
+  size = (int8_t)ptr[tagbytes];
+  ptr += tagbytes + 1;
+  dst->data = ptr;
+  dst->size = size;
+
+  if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->end))) {
+    ptr--;
+    if (validate_utf8) {
+      return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, dst);
+    } else {
+      return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, dst);
+    }
+  }
+
+  ptr += size;
+
+  if (card == CARD_r) {
+    if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) {
+      return fastdecode_err(d);
+    }
+    fastdecode_nextret ret = fastdecode_nextrepeated(
+        d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview));
+    switch (ret.next) {
+      case FD_NEXT_SAMEFIELD:
+        dst = ret.dst;
+        if (UPB_UNLIKELY(!d->alias)) {
+          // Buffer flipped and we can't alias any more. Bounce to copyfunc(),
+          // but via dispatch since we need to reload table data also.
+          fastdecode_commitarr(dst, &farr, sizeof(upb_strview));
+          return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag);
+        }
+        goto again;
+      case FD_NEXT_OTHERFIELD:
+        return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag);
+      case FD_NEXT_ATLIMIT:
+        return ptr;
+    }
+  }
+
+  if (card != CARD_r && validate_utf8) {
+    return fastdecode_verifyutf8(d, ptr, msg, table, hasbits, dst);
+  }
+
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
+}
+
+/* Generate all combinations:
+ * {p,c} x {s,o,r} x {s, b} x {1bt,2bt} */
+
+#define s_VALIDATE true
+#define b_VALIDATE false
+
+#define F(card, tagbytes, type)                                         \
+  UPB_NOINLINE                                                          \
+  const char *upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) {    \
+    return fastdecode_copystring(UPB_PARSE_ARGS, tagbytes, CARD_##card, \
+                                 type##_VALIDATE);                      \
+  }                                                                     \
+  const char *upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) {    \
+    return fastdecode_string(UPB_PARSE_ARGS, tagbytes, CARD_##card,     \
+                             &upb_c##card##type##_##tagbytes##bt,       \
+                             type##_VALIDATE);                          \
+  }
+
+#define UTF8(card, tagbytes) \
+  F(card, tagbytes, s)       \
+  F(card, tagbytes, b)
+
+#define TAGBYTES(card) \
+  UTF8(card, 1)        \
+  UTF8(card, 2)
+
+TAGBYTES(s)
+TAGBYTES(o)
+TAGBYTES(r)
+
+#undef s_VALIDATE
+#undef b_VALIDATE
+#undef F
+#undef TAGBYTES
+
+/* message fields *************************************************************/
+
+UPB_INLINE
+upb_msg *decode_newmsg_ceil(upb_decstate *d, const upb_msglayout *l,
+                            int msg_ceil_bytes) {
+  size_t size = l->size + sizeof(upb_msg_internal);
+  char *msg_data;
+  if (UPB_LIKELY(msg_ceil_bytes > 0 &&
+                 _upb_arenahas(&d->arena) >= msg_ceil_bytes)) {
+    UPB_ASSERT(size <= (size_t)msg_ceil_bytes);
+    msg_data = d->arena.head.ptr;
+    d->arena.head.ptr += size;
+    UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes);
+    memset(msg_data, 0, msg_ceil_bytes);
+    UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size);
+  } else {
+    msg_data = (char*)upb_arena_malloc(&d->arena, size);
+    memset(msg_data, 0, size);
+  }
+  return msg_data + sizeof(upb_msg_internal);
+}
+
+typedef struct {
+  intptr_t table;
+  upb_msg *msg;
+} fastdecode_submsgdata;
+
+UPB_FORCEINLINE
+static const char *fastdecode_tosubmsg(upb_decstate *d, const char *ptr,
+                                       void *ctx) {
+  fastdecode_submsgdata *submsg = ctx;
+  ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0);
+  UPB_ASSUME(ptr != NULL);
+  return ptr;
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_submsg(UPB_PARSE_PARAMS, int tagbytes,
+                                     int msg_ceil_bytes, upb_card card) {
+
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    RETURN_GENERIC("submessage field tag mismatch\n");
+  }
+
+  if (--d->depth == 0) return fastdecode_err(d);
+
+  upb_msg **dst;
+  uint32_t submsg_idx = (data >> 16) & 0xff;
+  const upb_msglayout *tablep = decode_totablep(table);
+  const upb_msglayout *subtablep = tablep->submsgs[submsg_idx];
+  fastdecode_submsgdata submsg = {decode_totable(subtablep)};
+  fastdecode_arr farr;
+
+  if (subtablep->table_mask == (uint8_t)-1) {
+    RETURN_GENERIC("submessage doesn't have fast tables.");
+  }
+
+  dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr,
+                            sizeof(upb_msg *), card);
+
+  if (card == CARD_s) {
+    *(uint32_t*)msg |= hasbits;
+    hasbits = 0;
+  }
+
+again:
+  if (card == CARD_r) {
+    dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_msg*));
+  }
+
+  submsg.msg = *dst;
+
+  if (card == CARD_r || UPB_LIKELY(!submsg.msg)) {
+    *dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes);
+  }
+
+  ptr += tagbytes;
+  ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg);
+
+  if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) {
+    return fastdecode_err(d);
+  }
+
+  if (card == CARD_r) {
+    fastdecode_nextret ret = fastdecode_nextrepeated(
+        d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_msg *));
+    switch (ret.next) {
+      case FD_NEXT_SAMEFIELD:
+        dst = ret.dst;
+        goto again;
+      case FD_NEXT_OTHERFIELD:
+        d->depth++;
+        return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag);
+      case FD_NEXT_ATLIMIT:
+        d->depth++;
+        return ptr;
+    }
+  }
+
+  d->depth++;
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
+}
+
+#define F(card, tagbytes, size_ceil, ceil_arg)                                 \
+  const char *upb_p##card##m_##tagbytes##bt_max##size_ceil##b(                 \
+      UPB_PARSE_PARAMS) {                                                      \
+    return fastdecode_submsg(UPB_PARSE_ARGS, tagbytes, ceil_arg, CARD_##card); \
+  }
+
+#define SIZES(card, tagbytes) \
+  F(card, tagbytes, 64, 64) \
+  F(card, tagbytes, 128, 128) \
+  F(card, tagbytes, 192, 192) \
+  F(card, tagbytes, 256, 256) \
+  F(card, tagbytes, max, -1)
+
+#define TAGBYTES(card) \
+  SIZES(card, 1) \
+  SIZES(card, 2)
+
+TAGBYTES(s)
+TAGBYTES(o)
+TAGBYTES(r)
+
+#undef TAGBYTES
+#undef SIZES
+#undef F
+
+#endif  /* UPB_FASTTABLE */