grpc 1.54.0 → 1.55.0

data/third_party/upb/upb/{decode_fast.c → wire/decode_fast.c}

@@ -35,12 +35,13 @@
 // field type (eg. oneof boolean field with a 1 byte tag) and then dispatch
 // to the specialized function as quickly as possible.
 
-#include "upb/decode_fast.h"
+#include "upb/wire/decode_fast.h"
 
-#include "upb/internal/decode.h"
+#include "upb/collections/array_internal.h"
+#include "upb/wire/decode_internal.h"
 
-/* Must be last. */
-#include "upb/port_def.inc"
+// Must be last.
+#include "upb/port/def.inc"
 
 #if UPB_FASTTABLE
 
@@ -56,7 +57,7 @@
   /* Uncomment either of these for debugging purposes. */ \
   /* fprintf(stderr, m); */                               \
   /*__builtin_trap(); */                                  \
-  return fastdecode_generic(d, ptr, msg, table, hasbits, 0);
+  return _upb_FastDecoder_DecodeGeneric(d, ptr, msg, table, hasbits, 0);
 
 typedef enum {
   CARD_s = 0, /* Singular (optional, non-repeated) */
@@ -68,35 +69,32 @@ typedef enum {
 UPB_NOINLINE
 static const char* fastdecode_isdonefallback(UPB_PARSE_PARAMS) {
   int overrun = data;
-  int status;
-  ptr = decode_isdonefallback_inl(d, ptr, overrun, &status);
-  if (ptr == NULL) {
-    return fastdecode_err(d, status);
-  }
-  data = fastdecode_loadtag(ptr);
-  UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS);
+  ptr = _upb_EpsCopyInputStream_IsDoneFallbackInline(
+      &d->input, ptr, overrun, _upb_Decoder_BufferFlipCallback);
+  data = _upb_FastDecoder_LoadTag(ptr);
+  UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);
 }
 
 UPB_FORCEINLINE
 static const char* fastdecode_dispatch(UPB_PARSE_PARAMS) {
-  if (UPB_UNLIKELY(ptr >= d->limit_ptr)) {
-    int overrun = ptr - d->end;
-    if (UPB_LIKELY(overrun == d->limit)) {
-      // Parse is finished.
+  int overrun;
+  switch (upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun)) {
+    case kUpb_IsDoneStatus_Done:
       *(uint32_t*)msg |= hasbits;  // Sync hasbits.
       const upb_MiniTable* l = decode_totablep(table);
       return UPB_UNLIKELY(l->required_count)
-                 ? decode_checkrequired(d, ptr, msg, l)
+                 ? _upb_Decoder_CheckRequired(d, ptr, msg, l)
                  : ptr;
-    } else {
+    case kUpb_IsDoneStatus_NotDone:
+      break;
+    case kUpb_IsDoneStatus_NeedFallback:
       data = overrun;
       UPB_MUSTTAIL return fastdecode_isdonefallback(UPB_PARSE_ARGS);
-    }
   }
 
   // Read two bytes of tag data (for a one-byte tag, the high byte is junk).
-  data = fastdecode_loadtag(ptr);
-  UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS);
+  data = _upb_FastDecoder_LoadTag(ptr);
+  UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);
 }
 
 UPB_FORCEINLINE
@@ -129,37 +127,17 @@ static const char* fastdecode_longsize(const char* ptr, int* size) {
 }
 
 UPB_FORCEINLINE
-static bool fastdecode_boundscheck(const char* ptr, size_t len,
-                                   const char* end) {
-  uintptr_t uptr = (uintptr_t)ptr;
-  uintptr_t uend = (uintptr_t)end + 16;
-  uintptr_t res = uptr + len;
-  return res < uptr || res > uend;
-}
-
-UPB_FORCEINLINE
-static bool fastdecode_boundscheck2(const char* ptr, size_t len,
-                                    const char* end) {
-  // This is one extra branch compared to the more normal:
-  //   return (size_t)(end - ptr) < size;
-  // However it is one less computation if we are just about to use "ptr + len":
-  //   https://godbolt.org/z/35YGPz
-  // In microbenchmarks this shows an overall 4% improvement.
-  uintptr_t uptr = (uintptr_t)ptr;
-  uintptr_t uend = (uintptr_t)end;
-  uintptr_t res = uptr + len;
-  return res < uptr || res > uend;
-}
-
-typedef const char* fastdecode_delimfunc(upb_Decoder* d, const char* ptr,
-                                         void* ctx);
-
-UPB_FORCEINLINE
-static const char* fastdecode_delimited(upb_Decoder* d, const char* ptr,
-                                        fastdecode_delimfunc* func, void* ctx) {
+static const char* fastdecode_delimited(
+    upb_Decoder* d, const char* ptr,
+    upb_EpsCopyInputStream_ParseDelimitedFunc* func, void* ctx) {
   ptr++;
+
+  // Sign-extend so varint greater than one byte becomes negative, causing
+  // fast delimited parse to fail.
   int len = (int8_t)ptr[-1];
-  if (fastdecode_boundscheck2(ptr, len, d->limit_ptr)) {
+
+  if (!upb_EpsCopyInputStream_TryParseDelimitedFast(&d->input, &ptr, len, func,
+                                                    ctx)) {
     // Slow case: Sub-message is >=128 bytes and/or exceeds the current buffer.
     // If it exceeds the buffer limit, limit/limit_ptr will change during
     // sub-message parsing, so we need to preserve delta, not limit.
@@ -171,25 +149,13 @@ static const char* fastdecode_delimited(upb_Decoder* d, const char* ptr,
         return NULL;
       }
     }
-    if (ptr - d->end + (int)len > d->limit) {
+    if (!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, len)) {
       // Corrupt wire format: invalid limit.
       return NULL;
     }
-    int delta = decode_pushlimit(d, ptr, len);
-    ptr = func(d, ptr, ctx);
-    decode_poplimit(d, ptr, delta);
-  } else {
-    // Fast case: Sub-message is <128 bytes and fits in the current buffer.
-    // This means we can preserve limit/limit_ptr verbatim.
-    const char* saved_limit_ptr = d->limit_ptr;
-    int saved_limit = d->limit;
-    d->limit_ptr = ptr + len;
-    d->limit = d->limit_ptr - d->end;
-    UPB_ASSERT(d->limit_ptr == d->end + UPB_MIN(0, d->limit));
-    ptr = func(d, ptr, ctx);
-    d->limit_ptr = saved_limit_ptr;
-    d->limit = saved_limit;
-    UPB_ASSERT(d->limit_ptr == d->end + UPB_MIN(0, d->limit));
+    int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, len);
+    ptr = func(&d->input, ptr, ctx);
+    upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta);
   }
   return ptr;
 }
@@ -217,14 +183,14 @@ UPB_FORCEINLINE
 static void* fastdecode_resizearr(upb_Decoder* d, void* dst,
                                   fastdecode_arr* farr, int valbytes) {
   if (UPB_UNLIKELY(dst == farr->end)) {
-    size_t old_size = farr->arr->size;
+    size_t old_size = farr->arr->capacity;
     size_t old_bytes = old_size * valbytes;
     size_t new_size = old_size * 2;
     size_t new_bytes = new_size * valbytes;
     char* old_ptr = _upb_array_ptr(farr->arr);
     char* new_ptr = upb_Arena_Realloc(&d->arena, old_ptr, old_bytes, new_bytes);
     uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
-    farr->arr->size = new_size;
+    farr->arr->capacity = new_size;
     farr->arr->data = _upb_array_tagptr(new_ptr, elem_size_lg2);
     dst = (void*)(new_ptr + (old_size * valbytes));
     farr->end = (void*)(new_ptr + (new_size * valbytes));
@@ -244,7 +210,7 @@ static bool fastdecode_tagmatch(uint32_t tag, uint64_t data, int tagbytes) {
 UPB_FORCEINLINE
 static void fastdecode_commitarr(void* dst, fastdecode_arr* farr,
                                  int valbytes) {
-  farr->arr->len =
+  farr->arr->size =
       (size_t)((char*)dst - (char*)_upb_array_ptr(farr->arr)) / valbytes;
 }
 
@@ -257,8 +223,8 @@ static fastdecode_nextret fastdecode_nextrepeated(upb_Decoder* d, void* dst,
   fastdecode_nextret ret;
   dst = (char*)dst + valbytes;
 
-  if (UPB_LIKELY(!decode_isdone(d, ptr))) {
-    ret.tag = fastdecode_loadtag(*ptr);
+  if (UPB_LIKELY(!_upb_Decoder_IsDone(d, ptr))) {
+    ret.tag = _upb_FastDecoder_LoadTag(*ptr);
     if (fastdecode_tagmatch(ret.tag, data, tagbytes)) {
       ret.next = FD_NEXT_SAMEFIELD;
     } else {
@@ -313,9 +279,9 @@ static void* fastdecode_getfield(upb_Decoder* d, const char* ptr,
         farr->arr = *arr_p;
       }
       begin = _upb_array_ptr(farr->arr);
-      farr->end = begin + (farr->arr->size * valbytes);
-      *data = fastdecode_loadtag(ptr);
-      return begin + (farr->arr->len * valbytes);
+      farr->end = begin + (farr->arr->capacity * valbytes);
+      *data = _upb_FastDecoder_LoadTag(ptr);
+      return begin + (farr->arr->size * valbytes);
     }
     default:
       UPB_UNREACHABLE();
@@ -401,7 +367,7 @@ done:
                                                                                \
   ptr += tagbytes;                                                             \
   ptr = fastdecode_varint64(ptr, &val);                                        \
-  if (ptr == NULL) return fastdecode_err(d, kUpb_DecodeStatus_Malformed);      \
+  if (ptr == NULL) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed);  \
   val = fastdecode_munge(val, valbytes, zigzag);                               \
   memcpy(dst, &val, valbytes);                                                 \
                                                                                \
@@ -414,7 +380,7 @@ done:
         goto again;                                                            \
       case FD_NEXT_OTHERFIELD:                                                 \
         data = ret.tag;                                                        \
-        UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS);            \
+        UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);      \
       case FD_NEXT_ATLIMIT:                                                    \
         return ptr;                                                            \
     }                                                                          \
@@ -430,13 +396,14 @@ typedef struct {
 } fastdecode_varintdata;
 
 UPB_FORCEINLINE
-static const char* fastdecode_topackedvarint(upb_Decoder* d, const char* ptr,
-                                             void* ctx) {
+static const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e,
+                                             const char* ptr, void* ctx) {
+  upb_Decoder* d = (upb_Decoder*)e;
   fastdecode_varintdata* data = ctx;
   void* dst = data->dst;
   uint64_t val;
 
-  while (!decode_isdone(d, &ptr)) {
+  while (!_upb_Decoder_IsDone(d, &ptr)) {
     dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes);
     ptr = fastdecode_varint64(ptr, &val);
     if (ptr == NULL) return NULL;
@@ -465,7 +432,7 @@ static const char* fastdecode_topackedvarint(upb_Decoder* d, const char* ptr,
   ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx);      \
                                                                              \
   if (UPB_UNLIKELY(ptr == NULL)) {                                           \
-    return fastdecode_err(d, kUpb_DecodeStatus_Malformed);                   \
+    _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed);               \
   }                                                                          \
                                                                              \
   UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0);
@@ -560,7 +527,7 @@ TAGBYTES(p)
         goto again;                                                           \
       case FD_NEXT_OTHERFIELD:                                                \
         data = ret.tag;                                                       \
-        UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS);           \
+        UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);     \
       case FD_NEXT_ATLIMIT:                                                   \
         return ptr;                                                           \
     }                                                                         \
@@ -579,9 +546,10 @@ TAGBYTES(p)
     ptr = fastdecode_longsize(ptr, &size);                                  \
   }                                                                         \
                                                                             \
-  if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr) ||       \
+  if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckDataSizeAvailable(          \
+                       &d->input, ptr, size) ||                             \
                    (size % valbytes) != 0)) {                               \
-    return fastdecode_err(d, kUpb_DecodeStatus_Malformed);                  \
+    _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed);              \
   }                                                                         \
                                                                             \
   upb_Array** arr_p = fastdecode_fieldmem(msg, data);                       \
@@ -592,15 +560,15 @@ TAGBYTES(p)
   if (UPB_LIKELY(!arr)) {                                                   \
     *arr_p = arr = _upb_Array_New(&d->arena, elems, elem_size_lg2);         \
     if (!arr) {                                                             \
-      return fastdecode_err(d, kUpb_DecodeStatus_Malformed);                \
+      _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed);            \
     }                                                                       \
   } else {                                                                  \
-    _upb_Array_Resize(arr, elems, &d->arena);                               \
+    _upb_Array_ResizeUninitialized(arr, elems, &d->arena);                  \
   }                                                                         \
                                                                             \
   char* dst = _upb_array_ptr(arr);                                          \
   memcpy(dst, ptr, size);                                                   \
-  arr->len = elems;                                                         \
+  arr->size = elems;                                                        \
                                                                             \
   ptr += size;                                                              \
   UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
@@ -658,8 +626,8 @@ static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr,
                                          upb_Message* msg, intptr_t table,
                                          uint64_t hasbits, uint64_t data) {
   upb_StringView* dst = (upb_StringView*)data;
-  if (!decode_verifyutf8_inl(dst->data, dst->size)) {
-    return fastdecode_err(d, kUpb_DecodeStatus_BadUtf8);
+  if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) {
+    _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8);
   }
   UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
 }
@@ -671,25 +639,17 @@ static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr,
     ptr = fastdecode_longsize(ptr, &size);                                     \
   }                                                                            \
                                                                                \
-  if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr))) {         \
+  if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, size))) { \
     dst->size = 0;                                                             \
-    return fastdecode_err(d, kUpb_DecodeStatus_Malformed);                     \
+    _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed);                 \
   }                                                                            \
                                                                                \
-  if (d->options & kUpb_DecodeOption_AliasString) {                            \
-    dst->data = ptr;                                                           \
-    dst->size = size;                                                          \
-  } else {                                                                     \
-    char* data = upb_Arena_Malloc(&d->arena, size);                            \
-    if (!data) {                                                               \
-      return fastdecode_err(d, kUpb_DecodeStatus_OutOfMemory);                 \
-    }                                                                          \
-    memcpy(data, ptr, size);                                                   \
-    dst->data = data;                                                          \
-    dst->size = size;                                                          \
-  }                                                                            \
+  const char* s_ptr = ptr;                                                     \
+  ptr = upb_EpsCopyInputStream_ReadString(&d->input, &s_ptr, size, &d->arena); \
+  if (!ptr) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory);       \
+  dst->data = s_ptr;                                                           \
+  dst->size = size;                                                            \
                                                                                \
-  ptr += size;                                                                 \
   if (validate_utf8) {                                                         \
     data = (uint64_t)dst;                                                      \
     UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS);                 \
@@ -724,105 +684,17 @@ static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size,
   UPB_POISON_MEMORY_REGION(data + size, copy - size);
 }
 
-#define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes,    \
-                              card, validate_utf8)                            \
-  upb_StringView* dst;                                                        \
-  fastdecode_arr farr;                                                        \
-  int64_t size;                                                               \
-  size_t arena_has;                                                           \
-  size_t common_has;                                                          \
-  char* buf;                                                                  \
-                                                                              \
-  UPB_ASSERT((d->options & kUpb_DecodeOption_AliasString) == 0);              \
-  UPB_ASSERT(fastdecode_checktag(data, tagbytes));                            \
-                                                                              \
-  dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr,              \
-                            sizeof(upb_StringView), card);                    \
-                                                                              \
-  again:                                                                      \
-  if (card == CARD_r) {                                                       \
-    dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView));        \
-  }                                                                           \
-                                                                              \
-  size = (uint8_t)ptr[tagbytes];                                              \
-  ptr += tagbytes + 1;                                                        \
-  dst->size = size;                                                           \
-                                                                              \
-  buf = d->arena.head.ptr;                                                    \
-  arena_has = _upb_ArenaHas(&d->arena);                                       \
-  common_has = UPB_MIN(arena_has, (d->end - ptr) + 16);                       \
-                                                                              \
-  if (UPB_LIKELY(size <= 15 - tagbytes)) {                                    \
-    if (arena_has < 16) goto longstr;                                         \
-    d->arena.head.ptr += 16;                                                  \
-    memcpy(buf, ptr - tagbytes - 1, 16);                                      \
-    dst->data = buf + tagbytes + 1;                                           \
-  } else if (UPB_LIKELY(size <= 32)) {                                        \
-    if (UPB_UNLIKELY(common_has < 32)) goto longstr;                          \
-    fastdecode_docopy(d, ptr, size, 32, buf, dst);                            \
-  } else if (UPB_LIKELY(size <= 64)) {                                        \
-    if (UPB_UNLIKELY(common_has < 64)) goto longstr;                          \
-    fastdecode_docopy(d, ptr, size, 64, buf, dst);                            \
-  } else if (UPB_LIKELY(size < 128)) {                                        \
-    if (UPB_UNLIKELY(common_has < 128)) goto longstr;                         \
-    fastdecode_docopy(d, ptr, size, 128, buf, dst);                           \
-  } else {                                                                    \
-    goto longstr;                                                             \
-  }                                                                           \
-                                                                              \
-  ptr += size;                                                                \
-                                                                              \
-  if (card == CARD_r) {                                                       \
-    if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) {      \
-      return fastdecode_err(d, kUpb_DecodeStatus_BadUtf8);                    \
-    }                                                                         \
-    fastdecode_nextret ret = fastdecode_nextrepeated(                         \
-        d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView));         \
-    switch (ret.next) {                                                       \
-      case FD_NEXT_SAMEFIELD:                                                 \
-        dst = ret.dst;                                                        \
-        goto again;                                                           \
-      case FD_NEXT_OTHERFIELD:                                                \
-        data = ret.tag;                                                       \
-        UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS);           \
-      case FD_NEXT_ATLIMIT:                                                   \
-        return ptr;                                                           \
-    }                                                                         \
-  }                                                                           \
-                                                                              \
-  if (card != CARD_r && validate_utf8) {                                      \
-    data = (uint64_t)dst;                                                     \
-    UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS);                \
-  }                                                                           \
-                                                                              \
-  UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);                    \
-                                                                              \
-  longstr:                                                                    \
-  if (card == CARD_r) {                                                       \
-    fastdecode_commitarr(dst + 1, &farr, sizeof(upb_StringView));             \
-  }                                                                           \
-  ptr--;                                                                      \
-  if (validate_utf8) {                                                        \
-    UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table,        \
-                                                   hasbits, (uint64_t)dst);   \
-  } else {                                                                    \
-    UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table,      \
-                                                     hasbits, (uint64_t)dst); \
-  }
-
-#define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card,   \
-                          copyfunc, validate_utf8)                             \
+#define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes,     \
+                              card, validate_utf8)                             \
   upb_StringView* dst;                                                         \
   fastdecode_arr farr;                                                         \
   int64_t size;                                                                \
+  size_t arena_has;                                                            \
+  size_t common_has;                                                           \
+  char* buf;                                                                   \
                                                                                \
-  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {                    \
-    RETURN_GENERIC("string field tag mismatch\n");                             \
-  }                                                                            \
-                                                                               \
-  if (UPB_UNLIKELY((d->options & kUpb_DecodeOption_AliasString) == 0)) {       \
-    UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS);                              \
-  }                                                                            \
+  UPB_ASSERT(!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0));    \
+  UPB_ASSERT(fastdecode_checktag(data, tagbytes));                             \
                                                                                \
   dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr,               \
                             sizeof(upb_StringView), card);                     \
@@ -832,45 +704,49 @@ static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size,
     dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView));         \
   }                                                                            \
                                                                                \
-  size = (int8_t)ptr[tagbytes];                                                \
+  size = (uint8_t)ptr[tagbytes];                                               \
   ptr += tagbytes + 1;                                                         \
-  dst->data = ptr;                                                             \
   dst->size = size;                                                            \
                                                                                \
-  if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->end))) {               \
-    ptr--;                                                                     \
-    if (validate_utf8) {                                                       \
-      return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits,           \
-                                        (uint64_t)dst);                        \
-    } else {                                                                   \
-      return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits,         \
-                                          (uint64_t)dst);                      \
-    }                                                                          \
+  buf = d->arena.head.ptr;                                                     \
+  arena_has = _upb_ArenaHas(&d->arena);                                        \
+  common_has = UPB_MIN(arena_has,                                              \
+                       upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \
+                                                                               \
+  if (UPB_LIKELY(size <= 15 - tagbytes)) {                                     \
+    if (arena_has < 16) goto longstr;                                          \
+    d->arena.head.ptr += 16;                                                   \
+    memcpy(buf, ptr - tagbytes - 1, 16);                                       \
+    dst->data = buf + tagbytes + 1;                                            \
+  } else if (UPB_LIKELY(size <= 32)) {                                         \
+    if (UPB_UNLIKELY(common_has < 32)) goto longstr;                           \
+    fastdecode_docopy(d, ptr, size, 32, buf, dst);                             \
+  } else if (UPB_LIKELY(size <= 64)) {                                         \
+    if (UPB_UNLIKELY(common_has < 64)) goto longstr;                           \
+    fastdecode_docopy(d, ptr, size, 64, buf, dst);                             \
+  } else if (UPB_LIKELY(size < 128)) {                                         \
+    if (UPB_UNLIKELY(common_has < 128)) goto longstr;                          \
+    fastdecode_docopy(d, ptr, size, 128, buf, dst);                            \
+  } else {                                                                     \
+    goto longstr;                                                              \
   }                                                                            \
                                                                                \
   ptr += size;                                                                 \
                                                                                \
   if (card == CARD_r) {                                                        \
-    if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) {       \
-      return fastdecode_err(d, kUpb_DecodeStatus_BadUtf8);                     \
+    if (validate_utf8 &&                                                       \
+        !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) {                \
+      _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8);                 \
     }                                                                          \
     fastdecode_nextret ret = fastdecode_nextrepeated(                          \
         d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView));          \
     switch (ret.next) {                                                        \
       case FD_NEXT_SAMEFIELD:                                                  \
         dst = ret.dst;                                                         \
-        if (UPB_UNLIKELY((d->options & kUpb_DecodeOption_AliasString) == 0)) { \
-          /* Buffer flipped and we can't alias any more. Bounce to */          \
-          /* copyfunc(), but via dispatch since we need to reload table */     \
-          /* data also. */                                                     \
-          fastdecode_commitarr(dst, &farr, sizeof(upb_StringView));            \
-          data = ret.tag;                                                      \
-          UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS);          \
-        }                                                                      \
         goto again;                                                            \
       case FD_NEXT_OTHERFIELD:                                                 \
         data = ret.tag;                                                        \
-        UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS);            \
+        UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);      \
       case FD_NEXT_ATLIMIT:                                                    \
         return ptr;                                                            \
     }                                                                          \
@@ -881,6 +757,88 @@ static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size,
     UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS);                 \
   }                                                                            \
                                                                                \
+  UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);                     \
+                                                                               \
+  longstr:                                                                     \
+  if (card == CARD_r) {                                                        \
+    fastdecode_commitarr(dst + 1, &farr, sizeof(upb_StringView));              \
+  }                                                                            \
+  ptr--;                                                                       \
+  if (validate_utf8) {                                                         \
+    UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table,         \
+                                                   hasbits, (uint64_t)dst);    \
+  } else {                                                                     \
+    UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table,       \
+                                                     hasbits, (uint64_t)dst);  \
+  }
+
+#define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card,  \
+                          copyfunc, validate_utf8)                            \
+  upb_StringView* dst;                                                        \
+  fastdecode_arr farr;                                                        \
+  int64_t size;                                                               \
+                                                                              \
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {                   \
+    RETURN_GENERIC("string field tag mismatch\n");                            \
+  }                                                                           \
+                                                                              \
+  if (UPB_UNLIKELY(                                                           \
+          !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0))) {    \
+    UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS);                             \
+  }                                                                           \
+                                                                              \
+  dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr,              \
+                            sizeof(upb_StringView), card);                    \
+                                                                              \
+  again:                                                                      \
+  if (card == CARD_r) {                                                       \
+    dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView));        \
+  }                                                                           \
+                                                                              \
+  size = (int8_t)ptr[tagbytes];                                               \
+  ptr += tagbytes + 1;                                                        \
+                                                                              \
+  if (UPB_UNLIKELY(                                                           \
+          !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, size))) { \
+    ptr--;                                                                    \
+    if (validate_utf8) {                                                      \
+      return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits,          \
+                                        (uint64_t)dst);                       \
+    } else {                                                                  \
+      return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits,        \
+                                          (uint64_t)dst);                     \
+    }                                                                         \
+  }                                                                           \
+                                                                              \
+  dst->data = ptr;                                                            \
+  dst->size = size;                                                           \
+  ptr = upb_EpsCopyInputStream_ReadStringAliased(&d->input, &dst->data,       \
+                                                 dst->size);                  \
+                                                                              \
+  if (card == CARD_r) {                                                       \
+    if (validate_utf8 &&                                                      \
+        !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) {               \
+      _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8);                \
+    }                                                                         \
+    fastdecode_nextret ret = fastdecode_nextrepeated(                         \
+        d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView));         \
+    switch (ret.next) {                                                       \
+      case FD_NEXT_SAMEFIELD:                                                 \
+        dst = ret.dst;                                                        \
+        goto again;                                                           \
+      case FD_NEXT_OTHERFIELD:                                                \
+        data = ret.tag;                                                       \
+        UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);     \
+      case FD_NEXT_ATLIMIT:                                                   \
+        return ptr;                                                           \
+    }                                                                         \
+  }                                                                           \
+                                                                              \
+  if (card != CARD_r && validate_utf8) {                                      \
+    data = (uint64_t)dst;                                                     \
+    UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS);                \
+  }                                                                           \
+                                                                              \
   UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
 
 /* Generate all combinations:
@@ -949,8 +907,9 @@ typedef struct {
 } fastdecode_submsgdata;
 
 UPB_FORCEINLINE
-static const char* fastdecode_tosubmsg(upb_Decoder* d, const char* ptr,
-                                       void* ctx) {
+static const char* fastdecode_tosubmsg(upb_EpsCopyInputStream* e,
+                                       const char* ptr, void* ctx) {
+  upb_Decoder* d = (upb_Decoder*)e;
   fastdecode_submsgdata* submsg = ctx;
   ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0, 0);
   UPB_ASSUME(ptr != NULL);
@@ -965,7 +924,7 @@ static const char* fastdecode_tosubmsg(upb_Decoder* d, const char* ptr,
   }                                                                       \
                                                                           \
   if (--d->depth == 0) {                                                  \
-    return fastdecode_err(d, kUpb_DecodeStatus_MaxDepthExceeded);         \
+    _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded);     \
   }                                                                       \
                                                                           \
   upb_Message** dst;                                                      \
@@ -1002,7 +961,7 @@ static const char* fastdecode_tosubmsg(upb_Decoder* d, const char* ptr,
   ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg);       \
                                                                           \
   if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) {      \
-    return fastdecode_err(d, kUpb_DecodeStatus_Malformed);                \
+    _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed);            \
   }                                                                       \
                                                                           \
   if (card == CARD_r) {                                                   \
@@ -1015,7 +974,7 @@ static const char* fastdecode_tosubmsg(upb_Decoder* d, const char* ptr,
       case FD_NEXT_OTHERFIELD:                                            \
         d->depth++;                                                       \
         data = ret.tag;                                                   \
-        UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS);       \
+        UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
       case FD_NEXT_ATLIMIT:                                               \
         d->depth++;                                                       \
         return ptr;                                                       \