grpc 1.72.0 → 1.73.0

data/third_party/upb/upb/hash/common.c

@@ -11,11 +11,16 @@
  * Implementation is heavily inspired by Lua's ltable.c.
  */
 
+#include "upb/hash/common.h"
+
+#include <stdint.h>
 #include <string.h>
 
 #include "upb/base/internal/log2.h"
+#include "upb/base/string_view.h"
 #include "upb/hash/int_table.h"
 #include "upb/hash/str_table.h"
+#include "upb/mem/arena.h"
 
 // Must be last.
 #include "upb/port/def.inc"
@@ -26,21 +31,42 @@
 #define ARRAY_SIZE(x) \
   ((sizeof(x) / sizeof(0 [x])) / ((size_t)(!(sizeof(x) % sizeof(0 [x])))))
 
-static const double MAX_LOAD = 0.85;
-
 /* The minimum utilization of the array part of a mixed hash/array table.  This
  * is a speed/memory-usage tradeoff (though it's not straightforward because of
  * cache effects).  The lower this is, the more memory we'll use. */
 static const double MIN_DENSITY = 0.1;
 
-static bool is_pow2(uint64_t v) { return v == 0 || (v & (v - 1)) == 0; }
+#if defined(__has_builtin)
+#if __has_builtin(__builtin_popcount)
+#define UPB_FAST_POPCOUNT32(i) __builtin_popcount(i)
+#endif
+#elif defined(__GNUC__)
+#define UPB_FAST_POPCOUNT32(i) __builtin_popcount(i)
+#elif defined(_MSC_VER)
+#define UPB_FAST_POPCOUNT32(i) __popcnt(i)
+#endif
 
-static upb_value _upb_value_val(uint64_t val) {
-  upb_value ret;
-  _upb_value_setval(&ret, val);
-  return ret;
+UPB_INLINE int _upb_popcnt32(uint32_t i) {
+#ifdef UPB_FAST_POPCOUNT32
+  return UPB_FAST_POPCOUNT32(i);
+#else
+  int count = 0;
+  while (i != 0) {
+    count += i & 1;
+    i >>= 1;
+  }
+  return count;
+#endif
 }
 
+#undef UPB_FAST_POPCOUNT32
+
+UPB_INLINE uint8_t _upb_log2_table_size(upb_table* t) {
+  return _upb_popcnt32(t->mask);
+}
+
+static bool is_pow2(uint64_t v) { return v == 0 || (v & (v - 1)) == 0; }
+
 static int log2ceil(uint64_t v) {
   int ret = 0;
   bool pow2 = is_pow2(v);
@@ -52,48 +78,50 @@ static int log2ceil(uint64_t v) {
 /* A type to represent the lookup key of either a strtable or an inttable. */
 typedef union {
   uintptr_t num;
-  struct {
-    const char* str;
-    size_t len;
-  } str;
+  upb_StringView str;
 } lookupkey_t;
 
 static lookupkey_t strkey2(const char* str, size_t len) {
-  lookupkey_t k;
-  k.str.str = str;
-  k.str.len = len;
-  return k;
+  return (lookupkey_t){.str = upb_StringView_FromDataAndSize(str, len)};
 }
 
-static lookupkey_t intkey(uintptr_t key) {
-  lookupkey_t k;
-  k.num = key;
-  return k;
-}
+static lookupkey_t intkey(uintptr_t key) { return (lookupkey_t){.num = key}; }
 
-typedef uint32_t hashfunc_t(upb_tabkey key);
-typedef bool eqlfunc_t(upb_tabkey k1, lookupkey_t k2);
+typedef uint32_t hashfunc_t(upb_key key);
+typedef bool eqlfunc_t(upb_key k1, lookupkey_t k2);
 
 /* Base table (shared code) ***************************************************/
 
-static uint32_t upb_inthash(uintptr_t key) { return (uint32_t)key; }
+static uint32_t upb_inthash(uintptr_t key) {
+  if (sizeof(uintptr_t) == 8) {
+    return (uint32_t)key ^ (uint32_t)(key >> 32);
+  } else {
+    UPB_ASSERT(sizeof(uintptr_t) == 4);
+    return (uint32_t)key;
+  }
+}
 
 static const upb_tabent* upb_getentry(const upb_table* t, uint32_t hash) {
   return t->entries + (hash & t->mask);
 }
 
-static bool upb_arrhas(upb_tabval key) { return key.val != (uint64_t)-1; }
-
-static bool isfull(upb_table* t) { return t->count == t->max_count; }
+static bool isfull(upb_table* t) {
+  uint32_t size = upb_table_size(t);
+  // 0.875 load factor
+  return t->count == (size - (size >> 3));
+}
 
 static bool init(upb_table* t, uint8_t size_lg2, upb_Arena* a) {
-  size_t bytes;
-
+  if (size_lg2 >= 32) {
+    return false;
+  }
   t->count = 0;
-  t->size_lg2 = size_lg2;
-  t->mask = upb_table_size(t) ? upb_table_size(t) - 1 : 0;
-  t->max_count = upb_table_size(t) * MAX_LOAD;
-  bytes = upb_table_size(t) * sizeof(upb_tabent);
+  uint32_t size = 1 << size_lg2;
+  t->mask = size - 1;  // 0 mask if size_lg2 is 0
+  if (upb_table_size(t) > (SIZE_MAX / sizeof(upb_tabent))) {
+    return false;
+  }
+  size_t bytes = upb_table_size(t) * sizeof(upb_tabent);
   if (bytes > 0) {
     t->entries = upb_Arena_Malloc(a, bytes);
     if (!t->entries) return false;
@@ -125,7 +153,7 @@ static const upb_tabent* findentry(const upb_table* t, lookupkey_t key,
                                    uint32_t hash, eqlfunc_t* eql) {
   const upb_tabent* e;
 
-  if (t->size_lg2 == 0) return NULL;
+  if (t->count == 0) return NULL;
   e = upb_getentry(t, hash);
   if (upb_tabent_isempty(e)) return NULL;
   while (1) {
@@ -143,9 +171,7 @@ static bool lookup(const upb_table* t, lookupkey_t key, upb_value* v,
                    uint32_t hash, eqlfunc_t* eql) {
   const upb_tabent* e = findentry(t, key, hash, eql);
   if (e) {
-    if (v) {
-      _upb_value_setval(v, e->val.val);
-    }
+    if (v) *v = e->val;
     return true;
   } else {
     return false;
@@ -153,9 +179,8 @@ static bool lookup(const upb_table* t, lookupkey_t key, upb_value* v,
 }
 
 /* The given key must not already exist in the table. */
-static void insert(upb_table* t, lookupkey_t key, upb_tabkey tabkey,
-                   upb_value val, uint32_t hash, hashfunc_t* hashfunc,
-                   eqlfunc_t* eql) {
+static void insert(upb_table* t, lookupkey_t key, upb_key tabkey, upb_value val,
+                   uint32_t hash, hashfunc_t* hashfunc, eqlfunc_t* eql) {
   upb_tabent* mainpos_e;
   upb_tabent* our_e;
 
@@ -195,25 +220,24 @@ static void insert(upb_table* t, lookupkey_t key, upb_tabkey tabkey,
     }
   }
   our_e->key = tabkey;
-  our_e->val.val = val.val;
+  our_e->val = val;
   UPB_ASSERT(findentry(t, key, hash, eql) == our_e);
 }
 
-static bool rm(upb_table* t, lookupkey_t key, upb_value* val,
-               upb_tabkey* removed, uint32_t hash, eqlfunc_t* eql) {
+static bool rm(upb_table* t, lookupkey_t key, upb_value* val, uint32_t hash,
+               eqlfunc_t* eql) {
   upb_tabent* chain = getentry_mutable(t, hash);
   if (upb_tabent_isempty(chain)) return false;
   if (eql(chain->key, key)) {
     /* Element to remove is at the head of its chain. */
     t->count--;
-    if (val) _upb_value_setval(val, chain->val.val);
-    if (removed) *removed = chain->key;
+    if (val) *val = chain->val;
     if (chain->next) {
       upb_tabent* move = (upb_tabent*)chain->next;
       *chain = *move;
-      move->key = 0; /* Make the slot empty. */
+      move->key = upb_key_empty();
     } else {
-      chain->key = 0; /* Make the slot empty. */
+      chain->key = upb_key_empty();
     }
     return true;
   } else {
@@ -226,9 +250,8 @@ static bool rm(upb_table* t, lookupkey_t key, upb_value* val,
       /* Found element to remove. */
       upb_tabent* rm = (upb_tabent*)chain->next;
       t->count--;
-      if (val) _upb_value_setval(val, chain->next->val.val);
-      if (removed) *removed = rm->key;
-      rm->key = 0; /* Make the slot empty. */
+      if (val) *val = chain->next->val;
+      rm->key = upb_key_empty();
       chain->next = rm->next;
       return true;
     } else {
@@ -250,17 +273,21 @@ static size_t begin(const upb_table* t) { return next(t, -1); }
 
 /* upb_strtable ***************************************************************/
 
-/* A simple "subclass" of upb_table that only adds a hash function for strings.
- */
+// A simple "subclass" of upb_table that only adds a hash function for strings.
 
-static upb_tabkey strcopy(lookupkey_t k2, upb_Arena* a) {
-  uint32_t len = (uint32_t)k2.str.len;
-  char* str = upb_Arena_Malloc(a, k2.str.len + sizeof(uint32_t) + 1);
-  if (str == NULL) return 0;
-  memcpy(str, &len, sizeof(uint32_t));
-  if (k2.str.len) memcpy(str + sizeof(uint32_t), k2.str.str, k2.str.len);
-  str[sizeof(uint32_t) + k2.str.len] = '\0';
-  return (uintptr_t)str;
+static upb_SizePrefixString* upb_SizePrefixString_Copy(upb_StringView s,
+                                                       upb_Arena* a) {
+  // A 2GB string will fail at serialization time, but we accept up to 4GB in
+  // memory here.
+  if (s.size > UINT32_MAX) return NULL;
+  upb_SizePrefixString* str =
+      upb_Arena_Malloc(a, sizeof(uint32_t) + s.size + 1);
+  if (str == NULL) return NULL;
+  str->size = s.size;
+  char* data = (char*)str->data;
+  if (s.size) memcpy(data, s.data, s.size);
+  data[s.size] = '\0';
+  return str;
 }
 
 /* Adapted from ABSL's wyhash. */
@@ -402,32 +429,40 @@ uint32_t _upb_Hash(const void* p, size_t n, uint64_t seed) {
   return Wyhash(p, n, seed, kWyhashSalt);
 }
 
-// Returns a seed for upb's hash function. For now this is just a hard-coded
-// constant, but we are going to randomize it soon.
-static uint64_t _upb_Seed(void) { return 0x69835f69597ec1cc; }
+static const void* const _upb_seed;
+
+// Returns a random seed for upb's hash function. This does not provide
+// high-quality randomness, but it should be enough to prevent unit tests from
+// relying on a deterministic map ordering. By returning the address of a
+// variable, we are able to get some randomness for free provided that ASLR is
+// enabled.
+static uint64_t _upb_Seed(void) { return (uint64_t)&_upb_seed; }
 
 static uint32_t _upb_Hash_NoSeed(const char* p, size_t n) {
   return _upb_Hash(p, n, _upb_Seed());
 }
 
-static uint32_t strhash(upb_tabkey key) {
-  uint32_t len;
-  char* str = upb_tabstr(key, &len);
-  return _upb_Hash_NoSeed(str, len);
+static uint32_t strhash(upb_key key) {
+  return _upb_Hash_NoSeed(key.str->data, key.str->size);
+}
+
+static bool streql(upb_key k1, lookupkey_t k2) {
+  const upb_SizePrefixString* k1s = k1.str;
+  const upb_StringView k2s = k2.str;
+  return k1s->size == k2s.size &&
+         (k1s->size == 0 || memcmp(k1s->data, k2s.data, k1s->size) == 0);
 }
 
-static bool streql(upb_tabkey k1, lookupkey_t k2) {
-  uint32_t len;
-  char* str = upb_tabstr(k1, &len);
-  return len == k2.str.len && (len == 0 || memcmp(str, k2.str.str, len) == 0);
+/** Calculates the number of entries required to hold an expected number of
+ * values, within the table's load factor. */
+static size_t _upb_entries_needed_for(size_t expected_size) {
+  size_t need_entries = expected_size + 1 + expected_size / 7;
+  UPB_ASSERT(need_entries - (need_entries >> 3) >= expected_size);
+  return need_entries;
 }
 
 bool upb_strtable_init(upb_strtable* t, size_t expected_size, upb_Arena* a) {
-  // Multiply by approximate reciprocal of MAX_LOAD (0.85), with pow2
-  // denominator.
-  size_t need_entries = (expected_size + 1) * 1204 / 1024;
-  UPB_ASSERT(need_entries >= expected_size * 0.85);
-  int size_lg2 = upb_Log2Ceiling(need_entries);
+  int size_lg2 = upb_Log2Ceiling(_upb_entries_needed_for(expected_size));
   return init(&t->t, size_lg2, a);
 }
 
@@ -442,10 +477,23 @@ bool upb_strtable_resize(upb_strtable* t, size_t size_lg2, upb_Arena* a) {
   if (!init(&new_table.t, size_lg2, a)) return false;
 
   intptr_t iter = UPB_STRTABLE_BEGIN;
-  upb_StringView key;
+  upb_StringView sv;
   upb_value val;
-  while (upb_strtable_next2(t, &key, &val, &iter)) {
-    upb_strtable_insert(&new_table, key.data, key.size, val, a);
+  while (upb_strtable_next2(t, &sv, &val, &iter)) {
+    // Unlike normal insert, does not copy string data or possibly reallocate
+    // the table
+    // The data pointer used in the table is guaranteed to point at a
+    // upb_SizePrefixString, we just need to back up by the size of the uint32_t
+    // length prefix.
+    const upb_SizePrefixString* keystr =
+        (const upb_SizePrefixString*)(sv.data - sizeof(uint32_t));
+    UPB_ASSERT(keystr->data == sv.data);
+    UPB_ASSERT(keystr->size == sv.size);
+
+    lookupkey_t lookupkey = {.str = sv};
+    upb_key tabkey = {.str = keystr};
+    uint32_t hash = _upb_Hash_NoSeed(sv.data, sv.size);
+    insert(&new_table.t, lookupkey, tabkey, val, hash, &strhash, &streql);
   }
   *t = new_table;
   return true;
@@ -453,23 +501,21 @@ bool upb_strtable_resize(upb_strtable* t, size_t size_lg2, upb_Arena* a) {
 
 bool upb_strtable_insert(upb_strtable* t, const char* k, size_t len,
                          upb_value v, upb_Arena* a) {
-  lookupkey_t key;
-  upb_tabkey tabkey;
-  uint32_t hash;
-
   if (isfull(&t->t)) {
     /* Need to resize.  New table of double the size, add old elements to it. */
-    if (!upb_strtable_resize(t, t->t.size_lg2 + 1, a)) {
+    if (!upb_strtable_resize(t, _upb_log2_table_size(&t->t) + 1, a)) {
       return false;
     }
   }
 
-  key = strkey2(k, len);
-  tabkey = strcopy(key, a);
-  if (tabkey == 0) return false;
+  upb_StringView sv = upb_StringView_FromDataAndSize(k, len);
+  upb_SizePrefixString* size_prefix_string = upb_SizePrefixString_Copy(sv, a);
+  if (!size_prefix_string) return false;
 
-  hash = _upb_Hash_NoSeed(key.str.str, key.str.len);
-  insert(&t->t, key, tabkey, v, hash, &strhash, &streql);
+  lookupkey_t lookupkey = {.str = sv};
+  upb_key key = {.str = size_prefix_string};
+  uint32_t hash = _upb_Hash_NoSeed(k, len);
+  insert(&t->t, lookupkey, key, v, hash, &strhash, &streql);
   return true;
 }
 
@@ -482,8 +528,7 @@ bool upb_strtable_lookup2(const upb_strtable* t, const char* key, size_t len,
 bool upb_strtable_remove2(upb_strtable* t, const char* key, size_t len,
                           upb_value* val) {
   uint32_t hash = _upb_Hash_NoSeed(key, len);
-  upb_tabkey tabkey;
-  return rm(&t->t, strkey2(key, len), val, &tabkey, hash, &streql);
+  return rm(&t->t, strkey2(key, len), val, hash, &streql);
 }
 
 /* Iteration */
@@ -504,17 +549,13 @@ bool upb_strtable_done(const upb_strtable_iter* i) {
 }
 
 upb_StringView upb_strtable_iter_key(const upb_strtable_iter* i) {
-  upb_StringView key;
-  uint32_t len;
   UPB_ASSERT(!upb_strtable_done(i));
-  key.data = upb_tabstr(str_tabent(i)->key, &len);
-  key.size = len;
-  return key;
+  return upb_key_strview(str_tabent(i)->key);
 }
 
 upb_value upb_strtable_iter_value(const upb_strtable_iter* i) {
   UPB_ASSERT(!upb_strtable_done(i));
-  return _upb_value_val(str_tabent(i)->val.val);
+  return str_tabent(i)->val;
 }
 
 void upb_strtable_iter_setdone(upb_strtable_iter* i) {
@@ -528,22 +569,79 @@ bool upb_strtable_iter_isequal(const upb_strtable_iter* i1,
   return i1->t == i2->t && i1->index == i2->index;
 }
 
+bool upb_strtable_next2(const upb_strtable* t, upb_StringView* key,
+                        upb_value* val, intptr_t* iter) {
+  size_t tab_idx = next(&t->t, *iter);
+  if (tab_idx < upb_table_size(&t->t)) {
+    upb_tabent* ent = &t->t.entries[tab_idx];
+    *key = upb_key_strview(ent->key);
+    *val = ent->val;
+    *iter = tab_idx;
+    return true;
+  }
+
+  return false;
+}
+
+void upb_strtable_removeiter(upb_strtable* t, intptr_t* iter) {
+  intptr_t i = *iter;
+  upb_tabent* ent = &t->t.entries[i];
+  upb_tabent* prev = NULL;
+
+  // Linear search, not great.
+  upb_tabent* end = &t->t.entries[upb_table_size(&t->t)];
+  for (upb_tabent* e = t->t.entries; e != end; e++) {
+    if (e->next == ent) {
+      prev = e;
+      break;
+    }
+  }
+
+  if (prev) {
+    prev->next = ent->next;
+  }
+
+  t->t.count--;
+  ent->key = upb_key_empty();
+  ent->next = NULL;
+}
+
+void upb_strtable_setentryvalue(upb_strtable* t, intptr_t iter, upb_value v) {
+  t->t.entries[iter].val = v;
+}
+
 /* upb_inttable ***************************************************************/
 
 /* For inttables we use a hybrid structure where small keys are kept in an
  * array and large keys are put in the hash table. */
 
-static uint32_t inthash(upb_tabkey key) { return upb_inthash(key); }
+// The sentinel value used in the dense array part. Note that callers must
+// ensure that inttable is never used with a value of this sentinel type
+// (pointers and u32 values will never be; i32 needs to be handled carefully
+// to avoid sign-extending into this value).
+static const upb_value kInttableSentinel = {.val = UINT64_MAX};
+static uint32_t presence_mask_arr_size(uint32_t array_size) {
+  return (array_size + 7) / 8;  // sizeof(uint8_t) is always 1.
+}
+
+static uint32_t inthash(upb_key key) { return upb_inthash(key.num); }
 
-static bool inteql(upb_tabkey k1, lookupkey_t k2) { return k1 == k2.num; }
+static bool inteql(upb_key k1, lookupkey_t k2) { return k1.num == k2.num; }
+
+static upb_value* mutable_array(upb_inttable* t) {
+  return (upb_value*)t->array;
+}
 
-static upb_tabval* mutable_array(upb_inttable* t) {
-  return (upb_tabval*)t->array;
+static const upb_value* inttable_array_get(const upb_inttable* t,
+                                           uintptr_t key) {
+  UPB_ASSERT(key < t->array_size);
+  const upb_value* val = &t->array[key];
+  return upb_inttable_arrhas(t, key) ? val : NULL;
 }
 
-static upb_tabval* inttable_val(upb_inttable* t, uintptr_t key) {
+static upb_value* inttable_val(upb_inttable* t, uintptr_t key) {
   if (key < t->array_size) {
-    return upb_arrhas(t->array[key]) ? &(mutable_array(t)[key]) : NULL;
+    return (upb_value*)inttable_array_get(t, key);
   } else {
     upb_tabent* e =
         findentry_mutable(&t->t, intkey(key), upb_inthash(key), &inteql);
@@ -551,8 +649,8 @@ static upb_tabval* inttable_val(upb_inttable* t, uintptr_t key) {
   }
 }
 
-static const upb_tabval* inttable_val_const(const upb_inttable* t,
-                                            uintptr_t key) {
+static const upb_value* inttable_val_const(const upb_inttable* t,
+                                           uintptr_t key) {
   return inttable_val((upb_inttable*)t, key);
 }
 
@@ -577,108 +675,117 @@ static void check(upb_inttable* t) {
 #endif
 }
 
-bool upb_inttable_sizedinit(upb_inttable* t, size_t asize, int hsize_lg2,
+bool upb_inttable_sizedinit(upb_inttable* t, uint32_t asize, int hsize_lg2,
                             upb_Arena* a) {
-  size_t array_bytes;
-
   if (!init(&t->t, hsize_lg2, a)) return false;
   /* Always make the array part at least 1 long, so that we know key 0
    * won't be in the hash part, which simplifies things. */
   t->array_size = UPB_MAX(1, asize);
   t->array_count = 0;
-  array_bytes = t->array_size * sizeof(upb_value);
-  t->array = upb_Arena_Malloc(a, array_bytes);
-  if (!t->array) {
+#if UINT32_MAX >= SIZE_MAX
+  if (UPB_UNLIKELY(SIZE_MAX / sizeof(upb_value) < t->array_size)) {
     return false;
   }
+#endif
+
+  // Allocate the array part and the presence mask array in one allocation.
+  size_t array_bytes = t->array_size * sizeof(upb_value);
+  uint32_t presence_bytes = presence_mask_arr_size(t->array_size);
+  uintptr_t total_bytes = array_bytes + presence_bytes;
+  if (UPB_UNLIKELY(total_bytes > SIZE_MAX)) {
+    return false;
+  }
+  void* alloc = upb_Arena_Malloc(a, total_bytes);
+  if (!alloc) {
+    return false;
+  }
+  t->array = alloc;
   memset(mutable_array(t), 0xff, array_bytes);
+  t->presence_mask = (uint8_t*)alloc + array_bytes;
+  memset((uint8_t*)t->presence_mask, 0, presence_bytes);
+
   check(t);
   return true;
 }
 
 bool upb_inttable_init(upb_inttable* t, upb_Arena* a) {
-  return upb_inttable_sizedinit(t, 0, 4, a);
+  // The init size of the table part to match that of strtable.
+  return upb_inttable_sizedinit(t, 0, 3, a);
 }
 
 bool upb_inttable_insert(upb_inttable* t, uintptr_t key, upb_value val,
                          upb_Arena* a) {
-  upb_tabval tabval;
-  tabval.val = val.val;
-  UPB_ASSERT(
-      upb_arrhas(tabval)); /* This will reject (uint64_t)-1.  Fix this. */
-
   if (key < t->array_size) {
-    UPB_ASSERT(!upb_arrhas(t->array[key]));
+    UPB_ASSERT(!upb_inttable_arrhas(t, key));
     t->array_count++;
-    mutable_array(t)[key].val = val.val;
+    mutable_array(t)[key] = val;
+    ((uint8_t*)t->presence_mask)[key / 8] |= (1 << (key % 8));
   } else {
     if (isfull(&t->t)) {
       /* Need to resize the hash part, but we re-use the array part. */
       size_t i;
       upb_table new_table;
 
-      if (!init(&new_table, t->t.size_lg2 + 1, a)) {
+      if (!init(&new_table, _upb_log2_table_size(&t->t) + 1, a)) {
         return false;
       }
 
       for (i = begin(&t->t); i < upb_table_size(&t->t); i = next(&t->t, i)) {
         const upb_tabent* e = &t->t.entries[i];
-        uint32_t hash;
-        upb_value v;
-
-        _upb_value_setval(&v, e->val.val);
-        hash = upb_inthash(e->key);
-        insert(&new_table, intkey(e->key), e->key, v, hash, &inthash, &inteql);
+        insert(&new_table, intkey(e->key.num), e->key, e->val, inthash(e->key),
+               &inthash, &inteql);
       }
 
       UPB_ASSERT(t->t.count == new_table.count);
 
       t->t = new_table;
     }
-    insert(&t->t, intkey(key), key, val, upb_inthash(key), &inthash, &inteql);
+    upb_key tabkey = {.num = key};
+    insert(&t->t, intkey(key), tabkey, val, upb_inthash(key), &inthash,
+           &inteql);
   }
   check(t);
   return true;
 }
 
 bool upb_inttable_lookup(const upb_inttable* t, uintptr_t key, upb_value* v) {
-  const upb_tabval* table_v = inttable_val_const(t, key);
+  const upb_value* table_v = inttable_val_const(t, key);
   if (!table_v) return false;
-  if (v) _upb_value_setval(v, table_v->val);
+  if (v) *v = *table_v;
   return true;
 }
 
 bool upb_inttable_replace(upb_inttable* t, uintptr_t key, upb_value val) {
-  upb_tabval* table_v = inttable_val(t, key);
+  upb_value* table_v = inttable_val(t, key);
   if (!table_v) return false;
-  table_v->val = val.val;
+  *table_v = val;
   return true;
 }
 
 bool upb_inttable_remove(upb_inttable* t, uintptr_t key, upb_value* val) {
   bool success;
   if (key < t->array_size) {
-    if (upb_arrhas(t->array[key])) {
-      upb_tabval empty = UPB_TABVALUE_EMPTY_INIT;
+    if (upb_inttable_arrhas(t, key)) {
       t->array_count--;
       if (val) {
-        _upb_value_setval(val, t->array[key].val);
+        *val = t->array[key];
       }
-      mutable_array(t)[key] = empty;
+      mutable_array(t)[key] = kInttableSentinel;
+      ((uint8_t*)t->presence_mask)[key / 8] &= ~(1 << (key % 8));
       success = true;
     } else {
       success = false;
     }
   } else {
-    success = rm(&t->t, intkey(key), val, NULL, upb_inthash(key), &inteql);
+    success = rm(&t->t, intkey(key), val, upb_inthash(key), &inteql);
   }
   check(t);
   return success;
 }
 
-void upb_inttable_compact(upb_inttable* t, upb_Arena* a) {
+bool upb_inttable_compact(upb_inttable* t, upb_Arena* a) {
   /* A power-of-two histogram of the table keys. */
-  size_t counts[UPB_MAXARRSIZE + 1] = {0};
+  uint32_t counts[UPB_MAXARRSIZE + 1] = {0};
 
   /* The max key in each bucket. */
   uintptr_t max[UPB_MAXARRSIZE + 1] = {0};
@@ -696,11 +803,11 @@ void upb_inttable_compact(upb_inttable* t, upb_Arena* a) {
 
   /* Find the largest power of two that satisfies the MIN_DENSITY
    * definition (while actually having some keys). */
-  size_t arr_count = upb_inttable_count(t);
-  int size_lg2;
-  upb_inttable new_t;
+  uint32_t arr_count = upb_inttable_count(t);
 
-  for (size_lg2 = ARRAY_SIZE(counts) - 1; size_lg2 > 0; size_lg2--) {
+  // Scan all buckets except capped bucket
+  int size_lg2 = ARRAY_SIZE(counts) - 1;
+  for (; size_lg2 > 0; size_lg2--) {
     if (counts[size_lg2] == 0) {
       /* We can halve again without losing any entries. */
       continue;
@@ -713,14 +820,17 @@ void upb_inttable_compact(upb_inttable* t, upb_Arena* a) {
 
   UPB_ASSERT(arr_count <= upb_inttable_count(t));
 
+  upb_inttable new_t;
   {
     /* Insert all elements into new, perfectly-sized table. */
-    size_t arr_size = max[size_lg2] + 1; /* +1 so arr[max] will fit. */
-    size_t hash_count = upb_inttable_count(t) - arr_count;
-    size_t hash_size = hash_count ? (hash_count / MAX_LOAD) + 1 : 0;
+    uintptr_t arr_size = max[size_lg2] + 1; /* +1 so arr[max] will fit. */
+    uint32_t hash_count = upb_inttable_count(t) - arr_count;
+    size_t hash_size = hash_count ? _upb_entries_needed_for(hash_count) : 0;
     int hashsize_lg2 = log2ceil(hash_size);
 
-    upb_inttable_sizedinit(&new_t, arr_size, hashsize_lg2, a);
+    if (!upb_inttable_sizedinit(&new_t, arr_size, hashsize_lg2, a)) {
+      return false;
+    }
 
     {
       intptr_t iter = UPB_INTTABLE_BEGIN;
@@ -732,9 +842,24 @@ void upb_inttable_compact(upb_inttable* t, upb_Arena* a) {
     }
 
     UPB_ASSERT(new_t.array_size == arr_size);
-    UPB_ASSERT(new_t.t.size_lg2 == hashsize_lg2);
   }
   *t = new_t;
+  return true;
+}
+
+void upb_inttable_clear(upb_inttable* t) {
+  // Clear the array part.
+  size_t array_bytes = t->array_size * sizeof(upb_value);
+  t->array_count = 0;
+  // Clear the array by setting all bits to 1, as UINT64_MAX is the sentinel
+  // value for an empty array.
+  memset(mutable_array(t), 0xff, array_bytes);
+  // Clear the presence mask array.
+  memset((uint8_t*)t->presence_mask, 0, presence_mask_arr_size(t->array_size));
+  // Clear the table part.
+  size_t bytes = upb_table_size(&t->t) * sizeof(upb_tabent);
+  t->t.count = 0;
+  memset((char*)t->t.entries, 0, bytes);
 }
 
 // Iteration.
@@ -744,10 +869,10 @@ bool upb_inttable_next(const upb_inttable* t, uintptr_t* key, upb_value* val,
   intptr_t i = *iter;
   if ((size_t)(i + 1) <= t->array_size) {
     while ((size_t)++i < t->array_size) {
-      upb_tabval ent = t->array[i];
-      if (upb_arrhas(ent)) {
+      const upb_value* ent = inttable_array_get(t, i);
+      if (ent) {
         *key = i;
-        *val = _upb_value_val(ent.val);
+        *val = *ent;
         *iter = i;
         return true;
       }
@@ -758,13 +883,18 @@ bool upb_inttable_next(const upb_inttable* t, uintptr_t* key, upb_value* val,
   size_t tab_idx = next(&t->t, i - t->array_size);
   if (tab_idx < upb_table_size(&t->t)) {
     upb_tabent* ent = &t->t.entries[tab_idx];
-    *key = ent->key;
-    *val = _upb_value_val(ent->val.val);
+    *key = ent->key.num;
+    *val = ent->val;
     *iter = tab_idx + t->array_size;
     return true;
+  } else {
+    // We should set the iterator any way. When we are done, the iterator value
+    // is invalidated. `upb_inttable_done` will check on the iterator value to
+    // determine if the iteration is done.
+    *iter = INTPTR_MAX - 1;  // To disambiguate from UPB_INTTABLE_BEGIN, to
+                             // match the behavior of `upb_strtable_iter`.
+    return false;
   }
-
-  return false;
 }
 
 void upb_inttable_removeiter(upb_inttable* t, intptr_t* iter) {
@@ -790,51 +920,42 @@ void upb_inttable_removeiter(upb_inttable* t, intptr_t* iter) {
     }
 
     t->t.count--;
-    ent->key = 0;
+    ent->key = upb_key_empty();
     ent->next = NULL;
   }
 }
 
-bool upb_strtable_next2(const upb_strtable* t, upb_StringView* key,
-                        upb_value* val, intptr_t* iter) {
-  size_t tab_idx = next(&t->t, *iter);
-  if (tab_idx < upb_table_size(&t->t)) {
-    upb_tabent* ent = &t->t.entries[tab_idx];
-    uint32_t len;
-    key->data = upb_tabstr(ent->key, &len);
-    key->size = len;
-    *val = _upb_value_val(ent->val.val);
-    *iter = tab_idx;
-    return true;
+void upb_inttable_setentryvalue(upb_inttable* t, intptr_t iter, upb_value v) {
+  if ((size_t)iter < t->array_size) {
+    mutable_array(t)[iter] = v;
+  } else {
+    upb_tabent* ent = &t->t.entries[iter - t->array_size];
+    ent->val = v;
   }
-
-  return false;
 }
 
-void upb_strtable_removeiter(upb_strtable* t, intptr_t* iter) {
-  intptr_t i = *iter;
-  upb_tabent* ent = &t->t.entries[i];
-  upb_tabent* prev = NULL;
-
-  // Linear search, not great.
-  upb_tabent* end = &t->t.entries[upb_table_size(&t->t)];
-  for (upb_tabent* e = t->t.entries; e != end; e++) {
-    if (e->next == ent) {
-      prev = e;
-      break;
-    }
-  }
-
-  if (prev) {
-    prev->next = ent->next;
+bool upb_inttable_done(const upb_inttable* t, intptr_t iter) {
+  if ((uintptr_t)iter >= t->array_size + upb_table_size(&t->t)) {
+    return true;
+  } else if ((size_t)iter < t->array_size) {
+    return !upb_inttable_arrhas(t, iter);
+  } else {
+    return upb_tabent_isempty(&t->t.entries[iter - t->array_size]);
   }
+}
 
-  t->t.count--;
-  ent->key = 0;
-  ent->next = NULL;
+uintptr_t upb_inttable_iter_key(const upb_inttable* t, intptr_t iter) {
+  UPB_ASSERT(!upb_inttable_done(t, iter));
+  return (size_t)iter < t->array_size
+             ? iter
+             : t->t.entries[iter - t->array_size].key.num;
 }
 
-void upb_strtable_setentryvalue(upb_strtable* t, intptr_t iter, upb_value v) {
-  upb_tabent* ent = &t->t.entries[iter];
-  ent->val.val = v.val;
+upb_value upb_inttable_iter_value(const upb_inttable* t, intptr_t iter) {
+  UPB_ASSERT(!upb_inttable_done(t, iter));
+  if ((size_t)iter < t->array_size) {
+    return t->array[iter];
+  } else {
+    return t->t.entries[iter - t->array_size].val;
+  }
 }