wearefair-grpc 1.3.1.pre.c → 1.4.0.fair

data/src/core/lib/slice/slice_buffer.c

@@ -253,16 +253,18 @@ void grpc_slice_buffer_move_into(grpc_slice_buffer *src,
   src->length = 0;
 }
 
-void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
-                                  grpc_slice_buffer *dst) {
-  size_t output_len = dst->length + n;
-  size_t new_input_len = src->length - n;
+static void slice_buffer_move_first_maybe_ref(grpc_slice_buffer *src, size_t n,
+                                              grpc_slice_buffer *dst,
+                                              bool incref) {
   GPR_ASSERT(src->length >= n);
   if (src->length == n) {
     grpc_slice_buffer_move_into(src, dst);
     return;
   }
 
+  size_t output_len = dst->length + n;
+  size_t new_input_len = src->length - n;
+
   while (src->count > 0) {
     grpc_slice slice = grpc_slice_buffer_take_first(src);
     size_t slice_len = GRPC_SLICE_LENGTH(slice);
@@ -272,11 +274,18 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
     } else if (n == slice_len) {
       grpc_slice_buffer_add(dst, slice);
       break;
-    } else { /* n < slice_len */
-      grpc_slice_buffer_undo_take_first(src, grpc_slice_split_tail(&slice, n));
+    } else if (incref) { /* n < slice_len */
+      grpc_slice_buffer_undo_take_first(
+          src, grpc_slice_split_tail_maybe_ref(&slice, n, GRPC_SLICE_REF_BOTH));
       GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n);
       grpc_slice_buffer_add(dst, slice);
       break;
+    } else { /* n < slice_len */
+      grpc_slice_buffer_undo_take_first(
+          src, grpc_slice_split_tail_maybe_ref(&slice, n, GRPC_SLICE_REF_TAIL));
+      GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n);
+      grpc_slice_buffer_add_indexed(dst, slice);
+      break;
     }
   }
   GPR_ASSERT(dst->length == output_len);
@@ -284,6 +293,16 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
   GPR_ASSERT(src->count > 0);
 }
 
+void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n,
+                                  grpc_slice_buffer *dst) {
+  slice_buffer_move_first_maybe_ref(src, n, dst, true);
+}
+
+void grpc_slice_buffer_move_first_no_ref(grpc_slice_buffer *src, size_t n,
+                                         grpc_slice_buffer *dst) {
+  slice_buffer_move_first_maybe_ref(src, n, dst, false);
+}
+
 void grpc_slice_buffer_move_first_into_buffer(grpc_exec_ctx *exec_ctx,
                                               grpc_slice_buffer *src, size_t n,
                                               void *dst) {

data/src/core/lib/slice/slice_hash_table.c

@@ -42,56 +42,47 @@
 
 struct grpc_slice_hash_table {
   gpr_refcount refs;
+  void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value);
   size_t size;
+  size_t max_num_probes;
   grpc_slice_hash_table_entry* entries;
 };
 
 static bool is_empty(grpc_slice_hash_table_entry* entry) {
-  return entry->vtable == NULL;
+  return entry->value == NULL;
 }
 
-// Helper function for insert and get operations that performs quadratic
-// probing (https://en.wikipedia.org/wiki/Quadratic_probing).
-static size_t grpc_slice_hash_table_find_index(
-    const grpc_slice_hash_table* table, const grpc_slice key, bool find_empty) {
-  size_t hash = grpc_slice_hash(key);
-  for (size_t i = 0; i < table->size; ++i) {
-    const size_t idx = (hash + i * i) % table->size;
+static void grpc_slice_hash_table_add(grpc_slice_hash_table* table,
+                                      grpc_slice key, void* value) {
+  GPR_ASSERT(value != NULL);
+  const size_t hash = grpc_slice_hash(key);
+  for (size_t offset = 0; offset < table->size; ++offset) {
+    const size_t idx = (hash + offset) % table->size;
     if (is_empty(&table->entries[idx])) {
-      return find_empty ? idx : table->size;
-    }
-    if (grpc_slice_eq(table->entries[idx].key, key)) {
-      return idx;
+      table->entries[idx].key = key;
+      table->entries[idx].value = value;
+      // Keep track of the maximum number of probes needed, since this
+      // provides an upper bound for lookups.
+      if (offset > table->max_num_probes) table->max_num_probes = offset;
+      return;
     }
   }
-  return table->size;  // Not found.
-}
-
-static void grpc_slice_hash_table_add(
-    grpc_slice_hash_table* table, grpc_slice key, void* value,
-    const grpc_slice_hash_table_vtable* vtable) {
-  GPR_ASSERT(value != NULL);
-  const size_t idx =
-      grpc_slice_hash_table_find_index(table, key, true /* find_empty */);
-  GPR_ASSERT(idx != table->size);  // Table should never be full.
-  grpc_slice_hash_table_entry* entry = &table->entries[idx];
-  entry->key = grpc_slice_ref_internal(key);
-  entry->value = vtable->copy_value(value);
-  entry->vtable = vtable;
+  GPR_ASSERT(false);  // Table should never be full.
 }
 
 grpc_slice_hash_table* grpc_slice_hash_table_create(
-    size_t num_entries, grpc_slice_hash_table_entry* entries) {
+    size_t num_entries, grpc_slice_hash_table_entry* entries,
+    void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value)) {
   grpc_slice_hash_table* table = gpr_zalloc(sizeof(*table));
   gpr_ref_init(&table->refs, 1);
-  // Quadratic probing gets best performance when the table is no more
-  // than half full.
+  table->destroy_value = destroy_value;
+  // Keep load factor low to improve performance of lookups.
   table->size = num_entries * 2;
   const size_t entry_size = sizeof(grpc_slice_hash_table_entry) * table->size;
   table->entries = gpr_zalloc(entry_size);
   for (size_t i = 0; i < num_entries; ++i) {
     grpc_slice_hash_table_entry* entry = &entries[i];
-    grpc_slice_hash_table_add(table, entry->key, entry->value, entry->vtable);
+    grpc_slice_hash_table_add(table, entry->key, entry->value);
   }
   return table;
 }
@@ -108,7 +99,7 @@ void grpc_slice_hash_table_unref(grpc_exec_ctx* exec_ctx,
       grpc_slice_hash_table_entry* entry = &table->entries[i];
       if (!is_empty(entry)) {
         grpc_slice_unref_internal(exec_ctx, entry->key);
-        entry->vtable->destroy_value(exec_ctx, entry->value);
+        table->destroy_value(exec_ctx, entry->value);
       }
     }
     gpr_free(table->entries);
@@ -118,8 +109,15 @@ void grpc_slice_hash_table_unref(grpc_exec_ctx* exec_ctx,
 
 void* grpc_slice_hash_table_get(const grpc_slice_hash_table* table,
                                 const grpc_slice key) {
-  const size_t idx =
-      grpc_slice_hash_table_find_index(table, key, false /* find_empty */);
-  if (idx == table->size) return NULL;  // Not found.
-  return table->entries[idx].value;
+  const size_t hash = grpc_slice_hash(key);
+  // We cap the number of probes at the max number recorded when
+  // populating the table.
+  for (size_t offset = 0; offset <= table->max_num_probes; ++offset) {
+    const size_t idx = (hash + offset) % table->size;
+    if (is_empty(&table->entries[idx])) break;
+    if (grpc_slice_eq(table->entries[idx].key, key)) {
+      return table->entries[idx].value;
+    }
+  }
+  return NULL;  // Not found.
 }

data/src/core/lib/slice/slice_hash_table.h

@@ -37,33 +37,28 @@
 /** Hash table implementation.
  *
  * This implementation uses open addressing
- * (https://en.wikipedia.org/wiki/Open_addressing) with quadratic
- * probing (https://en.wikipedia.org/wiki/Quadratic_probing).
+ * (https://en.wikipedia.org/wiki/Open_addressing) with linear
+ * probing (https://en.wikipedia.org/wiki/Linear_probing).
  *
  * The keys are \a grpc_slice objects.  The values are arbitrary pointers
- * with a common vtable.
+ * with a common destroy function.
  *
  * Hash tables are intentionally immutable, to avoid the need for locking.
  */
 
 typedef struct grpc_slice_hash_table grpc_slice_hash_table;
 
-typedef struct grpc_slice_hash_table_vtable {
-  void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value);
-  void *(*copy_value)(void *value);
-} grpc_slice_hash_table_vtable;
-
 typedef struct grpc_slice_hash_table_entry {
   grpc_slice key;
   void *value; /* Must not be NULL. */
-  const grpc_slice_hash_table_vtable *vtable;
 } grpc_slice_hash_table_entry;
 
 /** Creates a new hash table of containing \a entries, which is an array
-    of length \a num_entries.
-    Creates its own copy of all keys and values from \a entries. */
+    of length \a num_entries.  Takes ownership of all keys and values in
+    \a entries.  Values will be cleaned up via \a destroy_value(). */
 grpc_slice_hash_table *grpc_slice_hash_table_create(
-    size_t num_entries, grpc_slice_hash_table_entry *entries);
+    size_t num_entries, grpc_slice_hash_table_entry *entries,
+    void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value));
 
 grpc_slice_hash_table *grpc_slice_hash_table_ref(grpc_slice_hash_table *table);
 void grpc_slice_hash_table_unref(grpc_exec_ctx *exec_ctx,

data/src/core/lib/support/atomic.h

@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_H
+#define GRPC_CORE_LIB_SUPPORT_ATOMIC_H
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_HAS_CXX11_ATOMIC
+#include "src/core/lib/support/atomic_with_std.h"
+#else
+#include "src/core/lib/support/atomic_with_atm.h"
+#endif
+
+#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_H */

data/src/core/lib/support/atomic_with_atm.h

@@ -0,0 +1,70 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H
+#define GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H
+
+#include <grpc/support/atm.h>
+
+namespace grpc_core {
+
+enum MemoryOrderRelaxed { memory_order_relaxed };
+
+template <class T>
+class atomic;
+
+template <>
+class atomic<bool> {
+ public:
+  atomic() { gpr_atm_no_barrier_store(&x_, static_cast<gpr_atm>(false)); }
+  explicit atomic(bool x) {
+    gpr_atm_no_barrier_store(&x_, static_cast<gpr_atm>(x));
+  }
+
+  bool compare_exchange_strong(bool& expected, bool update, MemoryOrderRelaxed,
+                               MemoryOrderRelaxed) {
+    if (!gpr_atm_no_barrier_cas(&x_, static_cast<gpr_atm>(expected),
+                                static_cast<gpr_atm>(update))) {
+      expected = gpr_atm_no_barrier_load(&x_) != 0;
+      return false;
+    }
+    return true;
+  }
+
+ private:
+  gpr_atm x_;
+};
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H */

data/src/core/lib/support/atomic_with_std.h

@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H
+#define GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H
+
+#include <atomic>
+
+namespace grpc_core {
+
+template <class T>
+using atomic = std::atomic<T>;
+
+typedef std::memory_order memory_order;
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H */

data/src/core/lib/support/avl.c

@@ -205,8 +205,8 @@ static gpr_avl_node *rebalance(const gpr_avl_vtable *vtable, void *key,
   }
 }
 
-static gpr_avl_node *add(const gpr_avl_vtable *vtable, gpr_avl_node *node,
-                         void *key, void *value) {
+static gpr_avl_node *add_key(const gpr_avl_vtable *vtable, gpr_avl_node *node,
+                             void *key, void *value) {
   long cmp;
   if (node == NULL) {
     return new_node(key, value, NULL, NULL);
@@ -217,17 +217,17 @@ static gpr_avl_node *add(const gpr_avl_vtable *vtable, gpr_avl_node *node,
   } else if (cmp > 0) {
     return rebalance(
         vtable, vtable->copy_key(node->key), vtable->copy_value(node->value),
-        add(vtable, node->left, key, value), ref_node(node->right));
+        add_key(vtable, node->left, key, value), ref_node(node->right));
   } else {
     return rebalance(vtable, vtable->copy_key(node->key),
                      vtable->copy_value(node->value), ref_node(node->left),
-                     add(vtable, node->right, key, value));
+                     add_key(vtable, node->right, key, value));
   }
 }
 
 gpr_avl gpr_avl_add(gpr_avl avl, void *key, void *value) {
   gpr_avl_node *old_root = avl.root;
-  avl.root = add(avl.vtable, avl.root, key, value);
+  avl.root = add_key(avl.vtable, avl.root, key, value);
   assert_invariants(avl.root);
   unref_node(avl.vtable, old_root);
   return avl;
@@ -247,8 +247,8 @@ static gpr_avl_node *in_order_tail(gpr_avl_node *node) {
   return node;
 }
 
-static gpr_avl_node *remove(const gpr_avl_vtable *vtable, gpr_avl_node *node,
-                            void *key) {
+static gpr_avl_node *remove_key(const gpr_avl_vtable *vtable,
+                                gpr_avl_node *node, void *key) {
   long cmp;
   if (node == NULL) {
     return NULL;
@@ -263,27 +263,27 @@ static gpr_avl_node *remove(const gpr_avl_vtable *vtable, gpr_avl_node *node,
       gpr_avl_node *h = in_order_head(node->right);
       return rebalance(vtable, vtable->copy_key(h->key),
                        vtable->copy_value(h->value), ref_node(node->left),
-                       remove(vtable, node->right, h->key));
+                       remove_key(vtable, node->right, h->key));
     } else {
       gpr_avl_node *h = in_order_tail(node->left);
       return rebalance(
           vtable, vtable->copy_key(h->key), vtable->copy_value(h->value),
-          remove(vtable, node->left, h->key), ref_node(node->right));
+          remove_key(vtable, node->left, h->key), ref_node(node->right));
     }
   } else if (cmp > 0) {
-    return rebalance(vtable, vtable->copy_key(node->key),
-                     vtable->copy_value(node->value),
-                     remove(vtable, node->left, key), ref_node(node->right));
+    return rebalance(
+        vtable, vtable->copy_key(node->key), vtable->copy_value(node->value),
+        remove_key(vtable, node->left, key), ref_node(node->right));
   } else {
     return rebalance(vtable, vtable->copy_key(node->key),
                      vtable->copy_value(node->value), ref_node(node->left),
-                     remove(vtable, node->right, key));
+                     remove_key(vtable, node->right, key));
   }
 }
 
 gpr_avl gpr_avl_remove(gpr_avl avl, void *key) {
   gpr_avl_node *old_root = avl.root;
-  avl.root = remove(avl.vtable, avl.root, key);
+  avl.root = remove_key(avl.vtable, avl.root, key);
   assert_invariants(avl.root);
   unref_node(avl.vtable, old_root);
   return avl;