grpc 1.30.1 → 1.32.0.pre1

data/third_party/upb/upb/table.int.h

@@ -52,19 +52,8 @@ typedef enum {
 
 typedef struct {
   uint64_t val;
-#ifndef NDEBUG
-  /* In debug mode we carry the value type around also so we can check accesses
-   * to be sure the right member is being read. */
-  upb_ctype_t ctype;
-#endif
 } upb_value;
 
-#ifdef NDEBUG
-#define SET_TYPE(dest, val)      UPB_UNUSED(val)
-#else
-#define SET_TYPE(dest, val) dest = val
-#endif
-
 /* Like strdup(), which isn't always available since it's not ANSI C. */
 char *upb_strdup(const char *s, upb_alloc *a);
 /* Variant that works with a length-delimited rather than NULL-delimited string,
@@ -75,15 +64,13 @@ UPB_INLINE char *upb_gstrdup(const char *s) {
   return upb_strdup(s, &upb_alloc_global);
 }
 
-UPB_INLINE void _upb_value_setval(upb_value *v, uint64_t val,
-                                  upb_ctype_t ctype) {
+UPB_INLINE void _upb_value_setval(upb_value *v, uint64_t val) {
   v->val = val;
-  SET_TYPE(v->ctype, ctype);
 }
 
-UPB_INLINE upb_value _upb_value_val(uint64_t val, upb_ctype_t ctype) {
+UPB_INLINE upb_value _upb_value_val(uint64_t val) {
   upb_value ret;
-  _upb_value_setval(&ret, val, ctype);
+  _upb_value_setval(&ret, val);
   return ret;
 }
 
@@ -98,7 +85,6 @@ UPB_INLINE upb_value _upb_value_val(uint64_t val, upb_ctype_t ctype) {
 #define FUNCS(name, membername, type_t, converter, proto_type) \
   UPB_INLINE void upb_value_set ## name(upb_value *val, type_t cval) { \
     val->val = (converter)cval; \
-    SET_TYPE(val->ctype, proto_type); \
   } \
   UPB_INLINE upb_value upb_value_ ## name(type_t val) { \
     upb_value ret; \
@@ -106,7 +92,6 @@ UPB_INLINE upb_value _upb_value_val(uint64_t val, upb_ctype_t ctype) {
     return ret; \
   } \
   UPB_INLINE type_t upb_value_get ## name(upb_value val) { \
-    UPB_ASSERT_DEBUGVAR(val.ctype == proto_type); \
     return (type_t)(converter)val.val; \
   }
 
@@ -124,12 +109,10 @@ FUNCS(fptr,     fptr,         upb_func*,    uintptr_t,  UPB_CTYPE_FPTR)
 
 UPB_INLINE void upb_value_setfloat(upb_value *val, float cval) {
   memcpy(&val->val, &cval, sizeof(cval));
-  SET_TYPE(val->ctype, UPB_CTYPE_FLOAT);
 }
 
 UPB_INLINE void upb_value_setdouble(upb_value *val, double cval) {
   memcpy(&val->val, &cval, sizeof(cval));
-  SET_TYPE(val->ctype, UPB_CTYPE_DOUBLE);
 }
 
 UPB_INLINE upb_value upb_value_float(float cval) {
@@ -173,7 +156,6 @@ typedef struct {
 
 #define UPB_TABVALUE_EMPTY_INIT  {-1}
 
-
 /* upb_table ******************************************************************/
 
 typedef struct _upb_tabent {
@@ -190,7 +172,6 @@ typedef struct _upb_tabent {
 typedef struct {
   size_t count;          /* Number of entries in the hash part. */
   size_t mask;           /* Mask to turn hash value -> bucket. */
-  upb_ctype_t ctype;     /* Type of all values. */
   uint8_t size_lg2;      /* Size of the hashtable part is 2^size_lg2 entries. */
 
   /* Hash table entries.
@@ -200,17 +181,6 @@ typedef struct {
    * initialize const hash tables.  Then we cast away const when we have to.
    */
   const upb_tabent *entries;
-
-#ifndef NDEBUG
-  /* This table's allocator.  We make the user pass it in to every relevant
-   * function and only use this to check it in debug mode.  We do this solely
-   * to keep upb_table as small as possible.  This might seem slightly paranoid
-   * but the plan is to use upb_table for all map fields and extension sets in
-   * a forthcoming message representation, so there could be a lot of these.
-   * If this turns out to be too annoying later, we can change it (since this
-   * is an internal-only header file). */
-  upb_alloc *alloc;
-#endif
 } upb_table;
 
 typedef struct {
@@ -224,12 +194,6 @@ typedef struct {
   size_t array_count;       /* Array part number of elements. */
 } upb_inttable;
 
-#define UPB_INTTABLE_INIT(count, mask, ctype, size_lg2, ent, a, asize, acount) \
-  {UPB_TABLE_INIT(count, mask, ctype, size_lg2, ent), a, asize, acount}
-
-#define UPB_EMPTY_INTTABLE_INIT(ctype) \
-  UPB_INTTABLE_INIT(0, 0, ctype, 0, NULL, NULL, 0, 0)
-
 #define UPB_ARRAY_EMPTYENT -1
 
 UPB_INLINE size_t upb_table_size(const upb_table *t) {
@@ -298,6 +262,7 @@ upb_inttable *upb_inttable_pack(const upb_inttable *t, void *p, size_t *ofs,
                                 size_t size);
 upb_strtable *upb_strtable_pack(const upb_strtable *t, void *p, size_t *ofs,
                                 size_t size);
+void upb_strtable_clear(upb_strtable *t);
 
 /* Inserts the given key into the hashtable with the given value.  The key must
  * not already exist in the hash table.  For string tables, the key must be
@@ -361,15 +326,6 @@ UPB_INLINE bool upb_strtable_remove(upb_strtable *t, const char *key,
  * invalidate iterators. */
 bool upb_inttable_replace(upb_inttable *t, uintptr_t key, upb_value val);
 
-/* Handy routines for treating an inttable like a stack.  May not be mixed with
- * other insert/remove calls. */
-bool upb_inttable_push2(upb_inttable *t, upb_value val, upb_alloc *a);
-upb_value upb_inttable_pop(upb_inttable *t);
-
-UPB_INLINE bool upb_inttable_push(upb_inttable *t, upb_value val) {
-  return upb_inttable_push2(t, val, &upb_alloc_global);
-}
-
 /* Convenience routines for inttables with pointer keys. */
 bool upb_inttable_insertptr2(upb_inttable *t, const void *key, upb_value val,
                              upb_alloc *a);
@@ -399,7 +355,7 @@ UPB_INLINE bool upb_inttable_lookup32(const upb_inttable *t, uint32_t key,
   if (key < t->array_size) {
     upb_tabval arrval = t->array[key];
     if (upb_arrhas(arrval)) {
-      _upb_value_setval(v, arrval.val, t->t.ctype);
+      _upb_value_setval(v, arrval.val);
       return true;
     } else {
       return false;
@@ -409,7 +365,7 @@ UPB_INLINE bool upb_inttable_lookup32(const upb_inttable *t, uint32_t key,
     if (t->t.entries == NULL) return false;
     for (e = upb_getentry(&t->t, upb_inthash(key)); true; e = e->next) {
       if ((uint32_t)e->key == key) {
-        _upb_value_setval(v, e->val.val, t->t.ctype);
+        _upb_value_setval(v, e->val.val);
         return true;
       }
       if (e->next == NULL) return false;
@@ -463,8 +419,7 @@ typedef struct {
 void upb_strtable_begin(upb_strtable_iter *i, const upb_strtable *t);
 void upb_strtable_next(upb_strtable_iter *i);
 bool upb_strtable_done(const upb_strtable_iter *i);
-const char *upb_strtable_iter_key(const upb_strtable_iter *i);
-size_t upb_strtable_iter_keylength(const upb_strtable_iter *i);
+upb_strview upb_strtable_iter_key(const upb_strtable_iter *i);
 upb_value upb_strtable_iter_value(const upb_strtable_iter *i);
 void upb_strtable_iter_setdone(upb_strtable_iter *i);
 bool upb_strtable_iter_isequal(const upb_strtable_iter *i1,
@@ -488,6 +443,10 @@ typedef struct {
   bool array_part;
 } upb_inttable_iter;
 
+UPB_INLINE const upb_tabent *str_tabent(const upb_strtable_iter *i) {
+  return &i->t->t.entries[i->index];
+}
+
 void upb_inttable_begin(upb_inttable_iter *i, const upb_inttable *t);
 void upb_inttable_next(upb_inttable_iter *i);
 bool upb_inttable_done(const upb_inttable_iter *i);

data/third_party/upb/upb/upb.c

@@ -11,17 +11,6 @@
 
 #include "upb/port_def.inc"
 
-/* Guarantee null-termination and provide ellipsis truncation.
- * It may be tempting to "optimize" this by initializing these final
- * four bytes up-front and then being careful never to overwrite them,
- * this is safer and simpler. */
-static void nullz(upb_status *status) {
-  const char *ellipsis = "...";
-  size_t len = strlen(ellipsis);
-  UPB_ASSERT(sizeof(status->msg) > len);
-  memcpy(status->msg + sizeof(status->msg) - len, ellipsis, len);
-}
-
 /* upb_status *****************************************************************/
 
 void upb_status_clear(upb_status *status) {
@@ -37,8 +26,8 @@ const char *upb_status_errmsg(const upb_status *status) { return status->msg; }
 void upb_status_seterrmsg(upb_status *status, const char *msg) {
   if (!status) return;
   status->ok = false;
-  strncpy(status->msg, msg, sizeof(status->msg));
-  nullz(status);
+  strncpy(status->msg, msg, UPB_STATUS_MAX_MESSAGE - 1);
+  status->msg[UPB_STATUS_MAX_MESSAGE - 1] = '\0';
 }
 
 void upb_status_seterrf(upb_status *status, const char *fmt, ...) {
@@ -52,7 +41,16 @@ void upb_status_vseterrf(upb_status *status, const char *fmt, va_list args) {
   if (!status) return;
   status->ok = false;
   _upb_vsnprintf(status->msg, sizeof(status->msg), fmt, args);
-  nullz(status);
+  status->msg[UPB_STATUS_MAX_MESSAGE - 1] = '\0';
+}
+
+void upb_status_vappenderrf(upb_status *status, const char *fmt, va_list args) {
+  size_t len;
+  if (!status) return;
+  status->ok = false;
+  len = strlen(status->msg);
+  _upb_vsnprintf(status->msg + len, sizeof(status->msg) - len, fmt, args);
+  status->msg[UPB_STATUS_MAX_MESSAGE - 1] = '\0';
 }
 
 /* upb_alloc ******************************************************************/
@@ -74,188 +72,216 @@ upb_alloc upb_alloc_global = {&upb_global_allocfunc};
 /* upb_arena ******************************************************************/
 
 /* Be conservative and choose 16 in case anyone is using SSE. */
-static const size_t maxalign = 16;
 
-static size_t align_up_max(size_t size) {
-  return ((size + maxalign - 1) / maxalign) * maxalign;
-}
+typedef struct mem_block {
+  struct mem_block *next;
+  uint32_t size;
+  uint32_t cleanups;
+  /* Data follows. */
+} mem_block;
+
+typedef struct cleanup_ent {
+  upb_cleanup_func *cleanup;
+  void *ud;
+} cleanup_ent;
 
 struct upb_arena {
-  /* We implement the allocator interface.
-   * This must be the first member of upb_arena! */
-  upb_alloc alloc;
+  _upb_arena_head head;
+  uint32_t *cleanups;
 
   /* Allocator to allocate arena blocks.  We are responsible for freeing these
    * when we are destroyed. */
   upb_alloc *block_alloc;
+  uint32_t last_size;
 
-  size_t bytes_allocated;
-  size_t next_block_size;
-  size_t max_block_size;
-
-  /* Linked list of blocks.  Points to an arena_block, defined in env.c */
-  void *block_head;
+  /* When multiple arenas are fused together, each arena points to a parent
+   * arena (root points to itself). The root tracks how many live arenas
+   * reference it. */
+  uint32_t refcount;  /* Only used when a->parent == a */
+  struct upb_arena *parent;
 
-  /* Cleanup entries.  Pointer to a cleanup_ent, defined in env.c */
-  void *cleanup_head;
+  /* Linked list of blocks to free/cleanup. */
+  mem_block *freelist, *freelist_tail;
 };
 
-typedef struct mem_block {
-  struct mem_block *next;
-  size_t size;
-  size_t used;
-  bool owned;
-  /* Data follows. */
-} mem_block;
+static const size_t memblock_reserve = UPB_ALIGN_UP(sizeof(mem_block), 16);
 
-typedef struct cleanup_ent {
-  struct cleanup_ent *next;
-  upb_cleanup_func *cleanup;
-  void *ud;
-} cleanup_ent;
+static upb_arena *arena_findroot(upb_arena *a) {
+  /* Path splitting keeps time complexity down, see:
+   *   https://en.wikipedia.org/wiki/Disjoint-set_data_structure */
+  while (a->parent != a) {
+    upb_arena *next = a->parent;
+    a->parent = next->parent;
+    a = next;
+  }
+  return a;
+}
 
-static void upb_arena_addblock(upb_arena *a, void *ptr, size_t size,
-                               bool owned) {
+static void upb_arena_addblock(upb_arena *a, void *ptr, size_t size) {
   mem_block *block = ptr;
+  upb_arena *root = arena_findroot(a);
 
-  block->next = a->block_head;
-  block->size = size;
-  block->used = align_up_max(sizeof(mem_block));
-  block->owned = owned;
+  /* The block is for arena |a|, but should appear in the freelist of |root|. */
+  block->next = root->freelist;
+  block->size = (uint32_t)size;
+  block->cleanups = 0;
+  root->freelist = block;
+  a->last_size = block->size;
+  if (!root->freelist_tail) root->freelist_tail = block;
 
-  a->block_head = block;
+  a->head.ptr = UPB_PTR_AT(block, memblock_reserve, char);
+  a->head.end = UPB_PTR_AT(block, size, char);
+  a->cleanups = &block->cleanups;
 
   /* TODO(haberman): ASAN poison. */
 }
 
-static mem_block *upb_arena_allocblock(upb_arena *a, size_t size) {
-  size_t block_size = UPB_MAX(size, a->next_block_size) + sizeof(mem_block);
+static bool upb_arena_allocblock(upb_arena *a, size_t size) {
+  size_t block_size = UPB_MAX(size, a->last_size * 2) + memblock_reserve;
   mem_block *block = upb_malloc(a->block_alloc, block_size);
 
-  if (!block) {
-    return NULL;
-  }
+  if (!block) return false;
+  upb_arena_addblock(a, block, block_size);
+  return true;
+}
 
-  upb_arena_addblock(a, block, block_size, true);
-  a->next_block_size = UPB_MIN(block_size * 2, a->max_block_size);
+static bool arena_has(upb_arena *a, size_t size) {
+  _upb_arena_head *h = (_upb_arena_head*)a;
+  return (size_t)(h->end - h->ptr) >= size;
+}
 
-  return block;
+void *_upb_arena_slowmalloc(upb_arena *a, size_t size) {
+  if (!upb_arena_allocblock(a, size)) return NULL;  /* Out of memory. */
+  UPB_ASSERT(arena_has(a, size));
+  return upb_arena_malloc(a, size);
 }
 
 static void *upb_arena_doalloc(upb_alloc *alloc, void *ptr, size_t oldsize,
                                size_t size) {
   upb_arena *a = (upb_arena*)alloc;  /* upb_alloc is initial member. */
-  mem_block *block = a->block_head;
-  void *ret;
-
-  if (size == 0) {
-    return NULL;  /* We are an arena, don't need individual frees. */
-  }
-
-  size = align_up_max(size);
+  return upb_arena_realloc(a, ptr, oldsize, size);
+}
 
-  /* TODO(haberman): special-case if this is a realloc of the last alloc? */
+/* Public Arena API ***********************************************************/
 
-  if (!block || block->size - block->used < size) {
-    /* Slow path: have to allocate a new block. */
-    block = upb_arena_allocblock(a, size);
+upb_arena *arena_initslow(void *mem, size_t n, upb_alloc *alloc) {
+  const size_t first_block_overhead = sizeof(upb_arena) + memblock_reserve;
+  upb_arena *a;
 
-    if (!block) {
-      return NULL;  /* Out of memory. */
-    }
+  /* We need to malloc the initial block. */
+  n = first_block_overhead + 256;
+  if (!alloc || !(mem = upb_malloc(alloc, n))) {
+    return NULL;
   }
 
-  ret = (char*)block + block->used;
-  block->used += size;
+  a = UPB_PTR_AT(mem, n - sizeof(*a), upb_arena);
+  n -= sizeof(*a);
 
-  if (oldsize > 0) {
-    memcpy(ret, ptr, oldsize);  /* Preserve existing data. */
-  }
+  a->head.alloc.func = &upb_arena_doalloc;
+  a->block_alloc = alloc;
+  a->parent = a;
+  a->refcount = 1;
+  a->freelist = NULL;
+  a->freelist_tail = NULL;
 
-  /* TODO(haberman): ASAN unpoison. */
+  upb_arena_addblock(a, mem, n);
 
-  a->bytes_allocated += size;
-  return ret;
+  return a;
 }
 
-/* Public Arena API ***********************************************************/
-
-#define upb_alignof(type) offsetof (struct { char c; type member; }, member)
-
 upb_arena *upb_arena_init(void *mem, size_t n, upb_alloc *alloc) {
-  const size_t first_block_overhead = sizeof(upb_arena) + sizeof(mem_block);
   upb_arena *a;
-  bool owned = false;
 
   /* Round block size down to alignof(*a) since we will allocate the arena
    * itself at the end. */
-  n &= ~(upb_alignof(upb_arena) - 1);
-
-  if (n < first_block_overhead) {
-    /* We need to malloc the initial block. */
-    n = first_block_overhead + 256;
-    owned = true;
-    if (!alloc || !(mem = upb_malloc(alloc, n))) {
-      return NULL;
-    }
+  n = UPB_ALIGN_DOWN(n, UPB_ALIGN_OF(upb_arena));
+
+  if (UPB_UNLIKELY(n < sizeof(upb_arena))) {
+    return arena_initslow(mem, n, alloc);
   }
 
-  a = (void*)((char*)mem + n - sizeof(*a));
+  a = UPB_PTR_AT(mem, n - sizeof(*a), upb_arena);
   n -= sizeof(*a);
 
-  a->alloc.func = &upb_arena_doalloc;
-  a->block_alloc = &upb_alloc_global;
-  a->bytes_allocated = 0;
-  a->next_block_size = 256;
-  a->max_block_size = 16384;
-  a->cleanup_head = NULL;
-  a->block_head = NULL;
+  a->head.alloc.func = &upb_arena_doalloc;
   a->block_alloc = alloc;
-
-  upb_arena_addblock(a, mem, n, owned);
+  a->parent = a;
+  a->refcount = 1;
+  a->last_size = 128;
+  a->head.ptr = mem;
+  a->head.end = UPB_PTR_AT(mem, n, char);
+  a->freelist = NULL;
+  a->cleanups = NULL;
 
   return a;
 }
 
-#undef upb_alignof
-
-void upb_arena_free(upb_arena *a) {
-  cleanup_ent *ent = a->cleanup_head;
-  mem_block *block = a->block_head;
-
-  while (ent) {
-    ent->cleanup(ent->ud);
-    ent = ent->next;
-  }
+static void arena_dofree(upb_arena *a) {
+  mem_block *block = a->freelist;
+  UPB_ASSERT(a->parent == a);
+  UPB_ASSERT(a->refcount == 0);
 
-  /* Must do this after running cleanup functions, because this will delete
-   * the memory we store our cleanup entries in! */
   while (block) {
     /* Load first since we are deleting block. */
     mem_block *next = block->next;
 
-    if (block->owned) {
-      upb_free(a->block_alloc, block);
+    if (block->cleanups > 0) {
+      cleanup_ent *end = UPB_PTR_AT(block, block->size, void);
+      cleanup_ent *ptr = end - block->cleanups;
+
+      for (; ptr < end; ptr++) {
+        ptr->cleanup(ptr->ud);
+      }
     }
 
+    upb_free(a->block_alloc, block);
     block = next;
   }
 }
 
+void upb_arena_free(upb_arena *a) {
+  a = arena_findroot(a);
+  if (--a->refcount == 0) arena_dofree(a);
+}
+
 bool upb_arena_addcleanup(upb_arena *a, void *ud, upb_cleanup_func *func) {
-  cleanup_ent *ent = upb_malloc(&a->alloc, sizeof(cleanup_ent));
-  if (!ent) {
-    return false;  /* Out of memory. */
+  cleanup_ent *ent;
+
+  if (!a->cleanups || !arena_has(a, sizeof(cleanup_ent))) {
+    if (!upb_arena_allocblock(a, 128)) return false;  /* Out of memory. */
+    UPB_ASSERT(arena_has(a, sizeof(cleanup_ent)));
   }
 
+  a->head.end -= sizeof(cleanup_ent);
+  ent = (cleanup_ent*)a->head.end;
+  (*a->cleanups)++;
+
   ent->cleanup = func;
   ent->ud = ud;
-  ent->next = a->cleanup_head;
-  a->cleanup_head = ent;
 
   return true;
 }
 
-size_t upb_arena_bytesallocated(const upb_arena *a) {
-  return a->bytes_allocated;
+void upb_arena_fuse(upb_arena *a1, upb_arena *a2) {
+  upb_arena *r1 = arena_findroot(a1);
+  upb_arena *r2 = arena_findroot(a2);
+
+  if (r1 == r2) return;  /* Already fused. */
+
+  /* We want to join the smaller tree to the larger tree.
+   * So swap first if they are backwards. */
+  if (r1->refcount < r2->refcount) {
+    upb_arena *tmp = r1;
+    r1 = r2;
+    r2 = tmp;
+  }
+
+  /* r1 takes over r2's freelist and refcount. */
+  r1->refcount += r2->refcount;
+  if (r2->freelist_tail) {
+    UPB_ASSERT(r2->freelist_tail->next == NULL);
+    r2->freelist_tail->next = r1->freelist;
+    r1->freelist = r2->freelist;
+  }
+  r2->parent = r1;
 }