grpc 1.30.2 → 1.35.0

data/third_party/upb/upb/upb.c

@@ -1,5 +1,5 @@
 
-#include "upb/upb.h"
+#include "upb/upb.int.h"
 
 #include <errno.h>
 #include <stdarg.h>
@@ -11,17 +11,6 @@
 
 #include "upb/port_def.inc"
 
-/* Guarantee null-termination and provide ellipsis truncation.
- * It may be tempting to "optimize" this by initializing these final
- * four bytes up-front and then being careful never to overwrite them,
- * this is safer and simpler. */
-static void nullz(upb_status *status) {
-  const char *ellipsis = "...";
-  size_t len = strlen(ellipsis);
-  UPB_ASSERT(sizeof(status->msg) > len);
-  memcpy(status->msg + sizeof(status->msg) - len, ellipsis, len);
-}
-
 /* upb_status *****************************************************************/
 
 void upb_status_clear(upb_status *status) {
@@ -37,8 +26,8 @@ const char *upb_status_errmsg(const upb_status *status) { return status->msg; }
 void upb_status_seterrmsg(upb_status *status, const char *msg) {
   if (!status) return;
   status->ok = false;
-  strncpy(status->msg, msg, sizeof(status->msg));
-  nullz(status);
+  strncpy(status->msg, msg, UPB_STATUS_MAX_MESSAGE - 1);
+  status->msg[UPB_STATUS_MAX_MESSAGE - 1] = '\0';
 }
 
 void upb_status_seterrf(upb_status *status, const char *fmt, ...) {
@@ -51,8 +40,17 @@ void upb_status_seterrf(upb_status *status, const char *fmt, ...) {
 void upb_status_vseterrf(upb_status *status, const char *fmt, va_list args) {
   if (!status) return;
   status->ok = false;
-  _upb_vsnprintf(status->msg, sizeof(status->msg), fmt, args);
-  nullz(status);
+  vsnprintf(status->msg, sizeof(status->msg), fmt, args);
+  status->msg[UPB_STATUS_MAX_MESSAGE - 1] = '\0';
+}
+
+void upb_status_vappenderrf(upb_status *status, const char *fmt, va_list args) {
+  size_t len;
+  if (!status) return;
+  status->ok = false;
+  len = strlen(status->msg);
+  vsnprintf(status->msg + len, sizeof(status->msg) - len, fmt, args);
+  status->msg[UPB_STATUS_MAX_MESSAGE - 1] = '\0';
 }
 
 /* upb_alloc ******************************************************************/
@@ -74,188 +72,193 @@ upb_alloc upb_alloc_global = {&upb_global_allocfunc};
 /* upb_arena ******************************************************************/
 
 /* Be conservative and choose 16 in case anyone is using SSE. */
-static const size_t maxalign = 16;
-
-static size_t align_up_max(size_t size) {
-  return ((size + maxalign - 1) / maxalign) * maxalign;
-}
 
-struct upb_arena {
-  /* We implement the allocator interface.
-   * This must be the first member of upb_arena! */
-  upb_alloc alloc;
-
-  /* Allocator to allocate arena blocks.  We are responsible for freeing these
-   * when we are destroyed. */
-  upb_alloc *block_alloc;
-
-  size_t bytes_allocated;
-  size_t next_block_size;
-  size_t max_block_size;
-
-  /* Linked list of blocks.  Points to an arena_block, defined in env.c */
-  void *block_head;
-
-  /* Cleanup entries.  Pointer to a cleanup_ent, defined in env.c */
-  void *cleanup_head;
-};
-
-typedef struct mem_block {
+struct mem_block {
   struct mem_block *next;
-  size_t size;
-  size_t used;
-  bool owned;
+  uint32_t size;
+  uint32_t cleanups;
   /* Data follows. */
-} mem_block;
+};
 
 typedef struct cleanup_ent {
-  struct cleanup_ent *next;
   upb_cleanup_func *cleanup;
   void *ud;
 } cleanup_ent;
 
-static void upb_arena_addblock(upb_arena *a, void *ptr, size_t size,
-                               bool owned) {
+static const size_t memblock_reserve = UPB_ALIGN_UP(sizeof(mem_block), 16);
+
+static upb_arena *arena_findroot(upb_arena *a) {
+  /* Path splitting keeps time complexity down, see:
+   *   https://en.wikipedia.org/wiki/Disjoint-set_data_structure */
+  while (a->parent != a) {
+    upb_arena *next = a->parent;
+    a->parent = next->parent;
+    a = next;
+  }
+  return a;
+}
+
+static void upb_arena_addblock(upb_arena *a, upb_arena *root, void *ptr,
+                               size_t size) {
   mem_block *block = ptr;
 
-  block->next = a->block_head;
-  block->size = size;
-  block->used = align_up_max(sizeof(mem_block));
-  block->owned = owned;
+  /* The block is for arena |a|, but should appear in the freelist of |root|. */
+  block->next = root->freelist;
+  block->size = (uint32_t)size;
+  block->cleanups = 0;
+  root->freelist = block;
+  a->last_size = block->size;
+  if (!root->freelist_tail) root->freelist_tail = block;
 
-  a->block_head = block;
+  a->head.ptr = UPB_PTR_AT(block, memblock_reserve, char);
+  a->head.end = UPB_PTR_AT(block, size, char);
+  a->cleanups = &block->cleanups;
 
-  /* TODO(haberman): ASAN poison. */
+  UPB_POISON_MEMORY_REGION(a->head.ptr, a->head.end - a->head.ptr);
 }
 
-static mem_block *upb_arena_allocblock(upb_arena *a, size_t size) {
-  size_t block_size = UPB_MAX(size, a->next_block_size) + sizeof(mem_block);
-  mem_block *block = upb_malloc(a->block_alloc, block_size);
-
-  if (!block) {
-    return NULL;
-  }
+static bool upb_arena_allocblock(upb_arena *a, size_t size) {
+  upb_arena *root = arena_findroot(a);
+  size_t block_size = UPB_MAX(size, a->last_size * 2) + memblock_reserve;
+  mem_block *block = upb_malloc(root->block_alloc, block_size);
 
-  upb_arena_addblock(a, block, block_size, true);
-  a->next_block_size = UPB_MIN(block_size * 2, a->max_block_size);
+  if (!block) return false;
+  upb_arena_addblock(a, root, block, block_size);
+  return true;
+}
 
-  return block;
+void *_upb_arena_slowmalloc(upb_arena *a, size_t size) {
+  if (!upb_arena_allocblock(a, size)) return NULL;  /* Out of memory. */
+  UPB_ASSERT(_upb_arenahas(a) >= size);
+  return upb_arena_malloc(a, size);
 }
 
 static void *upb_arena_doalloc(upb_alloc *alloc, void *ptr, size_t oldsize,
                                size_t size) {
   upb_arena *a = (upb_arena*)alloc;  /* upb_alloc is initial member. */
-  mem_block *block = a->block_head;
-  void *ret;
-
-  if (size == 0) {
-    return NULL;  /* We are an arena, don't need individual frees. */
-  }
-
-  size = align_up_max(size);
+  return upb_arena_realloc(a, ptr, oldsize, size);
+}
 
-  /* TODO(haberman): special-case if this is a realloc of the last alloc? */
+/* Public Arena API ***********************************************************/
 
-  if (!block || block->size - block->used < size) {
-    /* Slow path: have to allocate a new block. */
-    block = upb_arena_allocblock(a, size);
+upb_arena *arena_initslow(void *mem, size_t n, upb_alloc *alloc) {
+  const size_t first_block_overhead = sizeof(upb_arena) + memblock_reserve;
+  upb_arena *a;
 
-    if (!block) {
-      return NULL;  /* Out of memory. */
-    }
+  /* We need to malloc the initial block. */
+  n = first_block_overhead + 256;
+  if (!alloc || !(mem = upb_malloc(alloc, n))) {
+    return NULL;
   }
 
-  ret = (char*)block + block->used;
-  block->used += size;
+  a = UPB_PTR_AT(mem, n - sizeof(*a), upb_arena);
+  n -= sizeof(*a);
 
-  if (oldsize > 0) {
-    memcpy(ret, ptr, oldsize);  /* Preserve existing data. */
-  }
+  a->head.alloc.func = &upb_arena_doalloc;
+  a->block_alloc = alloc;
+  a->parent = a;
+  a->refcount = 1;
+  a->freelist = NULL;
+  a->freelist_tail = NULL;
 
-  /* TODO(haberman): ASAN unpoison. */
+  upb_arena_addblock(a, a, mem, n);
 
-  a->bytes_allocated += size;
-  return ret;
+  return a;
 }
 
-/* Public Arena API ***********************************************************/
-
-#define upb_alignof(type) offsetof (struct { char c; type member; }, member)
-
 upb_arena *upb_arena_init(void *mem, size_t n, upb_alloc *alloc) {
-  const size_t first_block_overhead = sizeof(upb_arena) + sizeof(mem_block);
   upb_arena *a;
-  bool owned = false;
 
   /* Round block size down to alignof(*a) since we will allocate the arena
    * itself at the end. */
-  n &= ~(upb_alignof(upb_arena) - 1);
-
-  if (n < first_block_overhead) {
-    /* We need to malloc the initial block. */
-    n = first_block_overhead + 256;
-    owned = true;
-    if (!alloc || !(mem = upb_malloc(alloc, n))) {
-      return NULL;
-    }
+  n = UPB_ALIGN_DOWN(n, UPB_ALIGN_OF(upb_arena));
+
+  if (UPB_UNLIKELY(n < sizeof(upb_arena))) {
+    return arena_initslow(mem, n, alloc);
   }
 
-  a = (void*)((char*)mem + n - sizeof(*a));
-  n -= sizeof(*a);
+  a = UPB_PTR_AT(mem, n - sizeof(*a), upb_arena);
 
-  a->alloc.func = &upb_arena_doalloc;
-  a->block_alloc = &upb_alloc_global;
-  a->bytes_allocated = 0;
-  a->next_block_size = 256;
-  a->max_block_size = 16384;
-  a->cleanup_head = NULL;
-  a->block_head = NULL;
+  a->head.alloc.func = &upb_arena_doalloc;
   a->block_alloc = alloc;
-
-  upb_arena_addblock(a, mem, n, owned);
+  a->parent = a;
+  a->refcount = 1;
+  a->last_size = UPB_MAX(128, n);
+  a->head.ptr = mem;
+  a->head.end = UPB_PTR_AT(mem, n - sizeof(*a), char);
+  a->freelist = NULL;
+  a->cleanups = NULL;
 
   return a;
 }
 
-#undef upb_alignof
+static void arena_dofree(upb_arena *a) {
+  mem_block *block = a->freelist;
+  UPB_ASSERT(a->parent == a);
+  UPB_ASSERT(a->refcount == 0);
 
-void upb_arena_free(upb_arena *a) {
-  cleanup_ent *ent = a->cleanup_head;
-  mem_block *block = a->block_head;
-
-  while (ent) {
-    ent->cleanup(ent->ud);
-    ent = ent->next;
-  }
-
-  /* Must do this after running cleanup functions, because this will delete
-   * the memory we store our cleanup entries in! */
   while (block) {
     /* Load first since we are deleting block. */
     mem_block *next = block->next;
 
-    if (block->owned) {
-      upb_free(a->block_alloc, block);
+    if (block->cleanups > 0) {
+      cleanup_ent *end = UPB_PTR_AT(block, block->size, void);
+      cleanup_ent *ptr = end - block->cleanups;
+
+      for (; ptr < end; ptr++) {
+        ptr->cleanup(ptr->ud);
+      }
     }
 
+    upb_free(a->block_alloc, block);
     block = next;
   }
 }
 
+void upb_arena_free(upb_arena *a) {
+  a = arena_findroot(a);
+  if (--a->refcount == 0) arena_dofree(a);
+}
+
 bool upb_arena_addcleanup(upb_arena *a, void *ud, upb_cleanup_func *func) {
-  cleanup_ent *ent = upb_malloc(&a->alloc, sizeof(cleanup_ent));
-  if (!ent) {
-    return false;  /* Out of memory. */
+  cleanup_ent *ent;
+
+  if (!a->cleanups || _upb_arenahas(a) < sizeof(cleanup_ent)) {
+    if (!upb_arena_allocblock(a, 128)) return false;  /* Out of memory. */
+    UPB_ASSERT(_upb_arenahas(a) >= sizeof(cleanup_ent));
   }
 
+  a->head.end -= sizeof(cleanup_ent);
+  ent = (cleanup_ent*)a->head.end;
+  (*a->cleanups)++;
+  UPB_UNPOISON_MEMORY_REGION(ent, sizeof(cleanup_ent));
+
   ent->cleanup = func;
   ent->ud = ud;
-  ent->next = a->cleanup_head;
-  a->cleanup_head = ent;
 
   return true;
 }
 
-size_t upb_arena_bytesallocated(const upb_arena *a) {
-  return a->bytes_allocated;
+void upb_arena_fuse(upb_arena *a1, upb_arena *a2) {
+  upb_arena *r1 = arena_findroot(a1);
+  upb_arena *r2 = arena_findroot(a2);
+
+  if (r1 == r2) return;  /* Already fused. */
+
+  /* We want to join the smaller tree to the larger tree.
+   * So swap first if they are backwards. */
+  if (r1->refcount < r2->refcount) {
+    upb_arena *tmp = r1;
+    r1 = r2;
+    r2 = tmp;
+  }
+
+  /* r1 takes over r2's freelist and refcount. */
+  r1->refcount += r2->refcount;
+  if (r2->freelist_tail) {
+    UPB_ASSERT(r2->freelist_tail->next == NULL);
+    r2->freelist_tail->next = r1->freelist;
+    r1->freelist = r2->freelist;
+  }
+  r2->parent = r1;
 }