grpc 0.13.0 → 0.13.1.pre1

data/src/core/client_config/lb_policy.h

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -48,7 +48,8 @@ typedef void (*grpc_lb_completion)(void *cb_arg, grpc_subchannel *subchannel,
 struct grpc_lb_policy {
   const grpc_lb_policy_vtable *vtable;
   gpr_atm ref_pair;
-  grpc_pollset_set interested_parties;
+  /* owned pointer to interested parties in load balancing decisions */
+  grpc_pollset_set *interested_parties;
 };
 
 struct grpc_lb_policy_vtable {

data/src/core/client_config/subchannel.c

@@ -36,16 +36,17 @@
 #include <string.h>
 
 #include <grpc/support/alloc.h>
+#include <grpc/support/avl.h>
 
 #include "src/core/channel/channel_args.h"
 #include "src/core/channel/client_channel.h"
 #include "src/core/channel/connected_channel.h"
 #include "src/core/client_config/initial_connect_string.h"
+#include "src/core/client_config/subchannel_index.h"
 #include "src/core/iomgr/timer.h"
 #include "src/core/profiling/timers.h"
 #include "src/core/surface/channel.h"
 #include "src/core/transport/connectivity_state.h"
-#include "src/core/transport/connectivity_state.h"
 
 #define INTERNAL_REF_BITS 16
 #define STRONG_REF_MASK (~(gpr_atm)((1 << INTERNAL_REF_BITS) - 1))
@@ -94,6 +95,8 @@ struct grpc_subchannel {
   struct sockaddr *addr;
   size_t addr_len;
 
+  grpc_subchannel_key *key;
+
   /** initial string to send to peer */
   gpr_slice initial_connect_string;
 
@@ -105,7 +108,7 @@ struct grpc_subchannel {
 
   /** pollset_set tracking who's interested in a connection
       being setup */
-  grpc_pollset_set pollset_set;
+  grpc_pollset_set *pollset_set;
 
   /** active connection, or null; of type grpc_connected_subchannel */
   gpr_atm connected_subchannel;
@@ -206,7 +209,8 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, void *arg,
   gpr_slice_unref(c->initial_connect_string);
   grpc_connectivity_state_destroy(exec_ctx, &c->state_tracker);
   grpc_connector_unref(exec_ctx, c->connector);
-  grpc_pollset_set_destroy(&c->pollset_set);
+  grpc_pollset_set_destroy(c->pollset_set);
+  grpc_subchannel_key_destroy(exec_ctx, c->key);
   gpr_free(c);
 }
 
@@ -222,22 +226,42 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta,
   return old_val;
 }
 
-void grpc_subchannel_ref(grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c
+                                         GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
   gpr_atm old_refs;
   old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS),
                         0 REF_MUTATE_PURPOSE("STRONG_REF"));
   GPR_ASSERT((old_refs & STRONG_REF_MASK) != 0);
+  return c;
 }
 
-void grpc_subchannel_weak_ref(grpc_subchannel *c
-                                  GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c
+                                              GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
   gpr_atm old_refs;
   old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF"));
   GPR_ASSERT(old_refs != 0);
+  return c;
+}
+
+grpc_subchannel *grpc_subchannel_ref_from_weak_ref(
+    grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+  if (!c) return NULL;
+  for (;;) {
+    gpr_atm old_refs = gpr_atm_acq_load(&c->ref_pair);
+    if (old_refs >= (1 << INTERNAL_REF_BITS)) {
+      gpr_atm new_refs = old_refs + (1 << INTERNAL_REF_BITS);
+      if (gpr_atm_rel_cas(&c->ref_pair, old_refs, new_refs)) {
+        return c;
+      }
+    } else {
+      return NULL;
+    }
+  }
 }
 
 static void disconnect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   grpc_connected_subchannel *con;
+  grpc_subchannel_index_unregister(exec_ctx, c->key, c);
   gpr_mu_lock(&c->mu);
   GPR_ASSERT(!c->disconnected);
   c->disconnected = 1;
@@ -276,10 +300,19 @@ static uint32_t random_seed() {
   return (uint32_t)(gpr_time_to_millis(gpr_now(GPR_CLOCK_MONOTONIC)));
 }
 
-grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
+grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
+                                        grpc_connector *connector,
                                         grpc_subchannel_args *args) {
-  grpc_subchannel *c = gpr_malloc(sizeof(*c));
+  grpc_subchannel_key *key = grpc_subchannel_key_create(connector, args);
+  grpc_subchannel *c = grpc_subchannel_index_find(exec_ctx, key);
+  if (c) {
+    grpc_subchannel_key_destroy(exec_ctx, key);
+    return c;
+  }
+
+  c = gpr_malloc(sizeof(*c));
   memset(c, 0, sizeof(*c));
+  c->key = key;
   gpr_atm_no_barrier_store(&c->ref_pair, 1 << INTERNAL_REF_BITS);
   c->connector = connector;
   grpc_connector_ref(c->connector);
@@ -293,7 +326,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
   }
   c->addr = gpr_malloc(args->addr_len);
   memcpy(c->addr, args->addr, args->addr_len);
-  grpc_pollset_set_init(&c->pollset_set);
+  c->pollset_set = grpc_pollset_set_create();
   c->addr_len = args->addr_len;
   grpc_set_initial_connect_string(&c->addr, &c->addr_len,
                                   &c->initial_connect_string);
@@ -305,13 +338,14 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
   grpc_connectivity_state_init(&c->state_tracker, GRPC_CHANNEL_IDLE,
                                "subchannel");
   gpr_mu_init(&c->mu);
-  return c;
+
+  return grpc_subchannel_index_register(exec_ctx, key, c);
 }
 
 static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   grpc_connect_in_args args;
 
-  args.interested_parties = &c->pollset_set;
+  args.interested_parties = c->pollset_set;
   args.addr = c->addr;
   args.addr_len = c->addr_len;
   args.deadline = compute_connect_deadline(c);
@@ -345,7 +379,7 @@ static void on_external_state_watcher_done(grpc_exec_ctx *exec_ctx, void *arg,
   external_state_watcher *w = arg;
   grpc_closure *follow_up = w->notify;
   if (w->pollset_set != NULL) {
-    grpc_pollset_set_del_pollset_set(exec_ctx, &w->subchannel->pollset_set,
+    grpc_pollset_set_del_pollset_set(exec_ctx, w->subchannel->pollset_set,
                                      w->pollset_set);
   }
   gpr_mu_lock(&w->subchannel->mu);
@@ -361,7 +395,6 @@ void grpc_subchannel_notify_on_state_change(
     grpc_exec_ctx *exec_ctx, grpc_subchannel *c,
     grpc_pollset_set *interested_parties, grpc_connectivity_state *state,
     grpc_closure *notify) {
-  int do_connect = 0;
   external_state_watcher *w;
 
   if (state == NULL) {
@@ -381,7 +414,7 @@ void grpc_subchannel_notify_on_state_change(
     w->notify = notify;
     grpc_closure_init(&w->closure, on_external_state_watcher_done, w);
     if (interested_parties != NULL) {
-      grpc_pollset_set_add_pollset_set(exec_ctx, &c->pollset_set,
+      grpc_pollset_set_add_pollset_set(exec_ctx, c->pollset_set,
                                        interested_parties);
     }
     GRPC_SUBCHANNEL_WEAK_REF(c, "external_state_watcher");
@@ -391,17 +424,13 @@ void grpc_subchannel_notify_on_state_change(
     w->next->prev = w->prev->next = w;
     if (grpc_connectivity_state_notify_on_state_change(
             exec_ctx, &c->state_tracker, state, &w->closure)) {
-      do_connect = 1;
       c->connecting = 1;
       /* released by connection */
       GRPC_SUBCHANNEL_WEAK_REF(c, "connecting");
+      start_connect(exec_ctx, c);
     }
     gpr_mu_unlock(&c->mu);
   }
-
-  if (do_connect) {
-    start_connect(exec_ctx, c);
-  }
 }
 
 void grpc_connected_subchannel_process_transport_op(
@@ -539,7 +568,7 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
   GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
   grpc_connected_subchannel_notify_on_state_change(
-      exec_ctx, con, &c->pollset_set, &sw_subchannel->connectivity_state,
+      exec_ctx, con, c->pollset_set, &sw_subchannel->connectivity_state,
       &sw_subchannel->closure);
 
   /* signal completion */
@@ -601,11 +630,12 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) {
   if (c->disconnected) {
     iomgr_success = 0;
   }
-  gpr_mu_unlock(&c->mu);
   if (iomgr_success) {
     update_reconnect_parameters(c);
     continue_connect(exec_ctx, c);
+    gpr_mu_unlock(&c->mu);
   } else {
+    gpr_mu_unlock(&c->mu);
     GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
   }
 }

data/src/core/client_config/subchannel.h

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -48,6 +48,8 @@ typedef struct grpc_subchannel_args grpc_subchannel_args;
 #ifdef GRPC_STREAM_REFCOUNT_DEBUG
 #define GRPC_SUBCHANNEL_REF(p, r) \
   grpc_subchannel_ref((p), __FILE__, __LINE__, (r))
+#define GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(p, r) \
+  grpc_subchannel_ref_from_weak_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_UNREF(cl, p, r) \
   grpc_subchannel_unref((cl), (p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_WEAK_REF(p, r) \
@@ -66,6 +68,8 @@ typedef struct grpc_subchannel_args grpc_subchannel_args;
   , const char *file, int line, const char *reason
 #else
 #define GRPC_SUBCHANNEL_REF(p, r) grpc_subchannel_ref((p))
+#define GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(p, r) \
+  grpc_subchannel_ref_from_weak_ref((p))
 #define GRPC_SUBCHANNEL_UNREF(cl, p, r) grpc_subchannel_unref((cl), (p))
 #define GRPC_SUBCHANNEL_WEAK_REF(p, r) grpc_subchannel_weak_ref((p))
 #define GRPC_SUBCHANNEL_WEAK_UNREF(cl, p, r) \
@@ -79,13 +83,15 @@ typedef struct grpc_subchannel_args grpc_subchannel_args;
 #define GRPC_SUBCHANNEL_REF_EXTRA_ARGS
 #endif
 
-void grpc_subchannel_ref(grpc_subchannel *channel
-                             GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *channel
+                                         GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+grpc_subchannel *grpc_subchannel_ref_from_weak_ref(
+    grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_unref(grpc_exec_ctx *exec_ctx,
                            grpc_subchannel *channel
                                GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-void grpc_subchannel_weak_ref(grpc_subchannel *channel
-                                  GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *channel
+                                              GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx,
                                 grpc_subchannel *channel
                                     GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
@@ -146,6 +152,8 @@ grpc_call_stack *grpc_subchannel_call_get_call_stack(
     grpc_subchannel_call *subchannel_call);
 
 struct grpc_subchannel_args {
+  /* When updating this struct, also update subchannel_index.c */
+
   /** Channel filters for this channel - wrapped factories will likely
       want to mutate this */
   const grpc_channel_filter **filters;
@@ -159,7 +167,8 @@ struct grpc_subchannel_args {
 };
 
 /** create a subchannel given a connector */
-grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
+grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
+                                        grpc_connector *connector,
                                         grpc_subchannel_args *args);
 
 #endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_H */

data/src/core/client_config/subchannel_index.c

@@ -0,0 +1,261 @@
+//
+//
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+//
+
+#include "src/core/client_config/subchannel_index.h"
+
+#include <stdbool.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/avl.h>
+#include <grpc/support/tls.h>
+
+#include "src/core/channel/channel_args.h"
+
+// a map of subchannel_key --> subchannel, used for detecting connections
+// to the same destination in order to share them
+static gpr_avl g_subchannel_index;
+
+static gpr_mu g_mu;
+
+struct grpc_subchannel_key {
+  grpc_connector *connector;
+  grpc_subchannel_args args;
+};
+
+GPR_TLS_DECL(subchannel_index_exec_ctx);
+
+static void enter_ctx(grpc_exec_ctx *exec_ctx) {
+  GPR_ASSERT(gpr_tls_get(&subchannel_index_exec_ctx) == 0);
+  gpr_tls_set(&subchannel_index_exec_ctx, (intptr_t)exec_ctx);
+}
+
+static void leave_ctx(grpc_exec_ctx *exec_ctx) {
+  GPR_ASSERT(gpr_tls_get(&subchannel_index_exec_ctx) == (intptr_t)exec_ctx);
+  gpr_tls_set(&subchannel_index_exec_ctx, 0);
+}
+
+static grpc_exec_ctx *current_ctx() {
+  grpc_exec_ctx *c = (grpc_exec_ctx *)gpr_tls_get(&subchannel_index_exec_ctx);
+  GPR_ASSERT(c != NULL);
+  return c;
+}
+
+static grpc_subchannel_key *create_key(
+    grpc_connector *connector, grpc_subchannel_args *args,
+    grpc_channel_args *(*copy_channel_args)(const grpc_channel_args *args)) {
+  grpc_subchannel_key *k = gpr_malloc(sizeof(*k));
+  k->connector = grpc_connector_ref(connector);
+  k->args.filter_count = args->filter_count;
+  k->args.filters = gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count);
+  memcpy((grpc_channel_filter *)k->args.filters, args->filters,
+         sizeof(*k->args.filters) * k->args.filter_count);
+  k->args.addr_len = args->addr_len;
+  k->args.addr = gpr_malloc(args->addr_len);
+  memcpy(k->args.addr, args->addr, k->args.addr_len);
+  k->args.args = copy_channel_args(args->args);
+  return k;
+}
+
+grpc_subchannel_key *grpc_subchannel_key_create(grpc_connector *connector,
+                                                grpc_subchannel_args *args) {
+  return create_key(connector, args, grpc_channel_args_normalize);
+}
+
+static grpc_subchannel_key *subchannel_key_copy(grpc_subchannel_key *k) {
+  return create_key(k->connector, &k->args, grpc_channel_args_copy);
+}
+
+static int subchannel_key_compare(grpc_subchannel_key *a,
+                                  grpc_subchannel_key *b) {
+  int c = GPR_ICMP(a->connector, b->connector);
+  if (c != 0) return c;
+  c = GPR_ICMP(a->args.addr_len, b->args.addr_len);
+  if (c != 0) return c;
+  c = GPR_ICMP(a->args.filter_count, b->args.filter_count);
+  if (c != 0) return c;
+  c = memcmp(a->args.addr, b->args.addr, a->args.addr_len);
+  if (c != 0) return c;
+  c = memcmp(a->args.filters, b->args.filters,
+             a->args.filter_count * sizeof(*a->args.filters));
+  return grpc_channel_args_compare(a->args.args, b->args.args);
+}
+
+void grpc_subchannel_key_destroy(grpc_exec_ctx *exec_ctx,
+                                 grpc_subchannel_key *k) {
+  grpc_connector_unref(exec_ctx, k->connector);
+  gpr_free(k->args.addr);
+  gpr_free((grpc_channel_args *)k->args.filters);
+  grpc_channel_args_destroy((grpc_channel_args *)k->args.args);
+  gpr_free(k);
+}
+
+static void sck_avl_destroy(void *p) {
+  grpc_subchannel_key_destroy(current_ctx(), p);
+}
+
+static void *sck_avl_copy(void *p) { return subchannel_key_copy(p); }
+
+static long sck_avl_compare(void *a, void *b) {
+  return subchannel_key_compare(a, b);
+}
+
+static void scv_avl_destroy(void *p) {
+  GRPC_SUBCHANNEL_WEAK_UNREF(current_ctx(), p, "subchannel_index");
+}
+
+static void *scv_avl_copy(void *p) {
+  GRPC_SUBCHANNEL_WEAK_REF(p, "subchannel_index");
+  return p;
+}
+
+static const gpr_avl_vtable subchannel_avl_vtable = {
+    .destroy_key = sck_avl_destroy,
+    .copy_key = sck_avl_copy,
+    .compare_keys = sck_avl_compare,
+    .destroy_value = scv_avl_destroy,
+    .copy_value = scv_avl_copy};
+
+void grpc_subchannel_index_init(void) {
+  g_subchannel_index = gpr_avl_create(&subchannel_avl_vtable);
+  gpr_mu_init(&g_mu);
+  gpr_tls_init(&subchannel_index_exec_ctx);
+}
+
+void grpc_subchannel_index_shutdown(void) {
+  gpr_mu_destroy(&g_mu);
+  gpr_avl_unref(g_subchannel_index);
+  gpr_tls_destroy(&subchannel_index_exec_ctx);
+}
+
+grpc_subchannel *grpc_subchannel_index_find(grpc_exec_ctx *exec_ctx,
+                                            grpc_subchannel_key *key) {
+  enter_ctx(exec_ctx);
+
+  // Lock, and take a reference to the subchannel index.
+  // We don't need to do the search under a lock as avl's are immutable.
+  gpr_mu_lock(&g_mu);
+  gpr_avl index = gpr_avl_ref(g_subchannel_index);
+  gpr_mu_unlock(&g_mu);
+
+  grpc_subchannel *c =
+      GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(gpr_avl_get(index, key), "index_find");
+  gpr_avl_unref(index);
+
+  leave_ctx(exec_ctx);
+  return c;
+}
+
+grpc_subchannel *grpc_subchannel_index_register(grpc_exec_ctx *exec_ctx,
+                                                grpc_subchannel_key *key,
+                                                grpc_subchannel *constructed) {
+  enter_ctx(exec_ctx);
+
+  grpc_subchannel *c = NULL;
+
+  while (c == NULL) {
+    // Compare and swap loop:
+    // - take a reference to the current index
+    gpr_mu_lock(&g_mu);
+    gpr_avl index = gpr_avl_ref(g_subchannel_index);
+    gpr_mu_unlock(&g_mu);
+
+    // - Check to see if a subchannel already exists
+    c = gpr_avl_get(index, key);
+    if (c != NULL) {
+      // yes -> we're done
+      GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, constructed, "index_register");
+    } else {
+      // no -> update the avl and compare/swap
+      gpr_avl updated =
+          gpr_avl_add(gpr_avl_ref(index), subchannel_key_copy(key),
+                      GRPC_SUBCHANNEL_WEAK_REF(constructed, "index_register"));
+
+      // it may happen (but it's expected to be unlikely)
+      // that some other thread has changed the index:
+      // compare/swap here to check that, and retry as necessary
+      gpr_mu_lock(&g_mu);
+      if (index.root == g_subchannel_index.root) {
+        GPR_SWAP(gpr_avl, updated, g_subchannel_index);
+        c = constructed;
+      }
+      gpr_mu_unlock(&g_mu);
+
+      gpr_avl_unref(updated);
+    }
+    gpr_avl_unref(index);
+  }
+
+  leave_ctx(exec_ctx);
+
+  return c;
+}
+
+void grpc_subchannel_index_unregister(grpc_exec_ctx *exec_ctx,
+                                      grpc_subchannel_key *key,
+                                      grpc_subchannel *constructed) {
+  enter_ctx(exec_ctx);
+
+  bool done = false;
+  while (!done) {
+    // Compare and swap loop:
+    // - take a reference to the current index
+    gpr_mu_lock(&g_mu);
+    gpr_avl index = gpr_avl_ref(g_subchannel_index);
+    gpr_mu_unlock(&g_mu);
+
+    // Check to see if this key still refers to the previously
+    // registered subchannel
+    grpc_subchannel *c = gpr_avl_get(index, key);
+    if (c != constructed) {
+      gpr_avl_unref(index);
+      break;
+    }
+
+    // compare and swap the update (some other thread may have
+    // mutated the index behind us)
+    gpr_avl updated = gpr_avl_remove(gpr_avl_ref(index), key);
+
+    gpr_mu_lock(&g_mu);
+    if (index.root == g_subchannel_index.root) {
+      GPR_SWAP(gpr_avl, updated, g_subchannel_index);
+      done = true;
+    }
+    gpr_mu_unlock(&g_mu);
+
+    gpr_avl_unref(updated);
+    gpr_avl_unref(index);
+  }
+
+  leave_ctx(exec_ctx);
+}