grpc 0.12.0 → 0.13.0.pre1.1

data/src/core/iomgr/pollset_posix.h

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -122,7 +122,7 @@ int grpc_poll_deadline_to_millis_timeout(gpr_timespec deadline,
    -- mostly for fd_posix's use. */
 void grpc_pollset_kick_ext(grpc_pollset *p,
                            grpc_pollset_worker *specific_worker,
-                           gpr_uint32 flags);
+                           uint32_t flags);
 
 /* turn a pollset into a multipoller: platform specific */
 typedef void (*grpc_platform_become_multipoller_type)(grpc_exec_ctx *exec_ctx,
@@ -139,6 +139,8 @@ void grpc_poll_become_multipoller(grpc_exec_ctx *exec_ctx,
  * be locked) */
 int grpc_pollset_has_workers(grpc_pollset *pollset);
 
+void grpc_remove_fd_from_all_epoll_sets(int fd);
+
 /* override to allow tests to hook poll() usage */
 typedef int (*grpc_poll_function_type)(struct pollfd *, nfds_t, int);
 extern grpc_poll_function_type grpc_poll_function;

data/src/core/iomgr/pollset_windows.c

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -81,15 +81,6 @@ static grpc_pollset_worker *pop_front_worker(
   }
 }
 
-static void push_back_worker(grpc_pollset_worker *root,
-                             grpc_pollset_worker_link_type type,
-                             grpc_pollset_worker *worker) {
-  worker->links[type].next = root;
-  worker->links[type].prev = worker->links[type].next->links[type].prev;
-  worker->links[type].prev->links[type].next =
-      worker->links[type].next->links[type].prev = worker;
-}
-
 static void push_front_worker(grpc_pollset_worker *root,
                               grpc_pollset_worker_link_type type,
                               grpc_pollset_worker *worker) {
@@ -116,7 +107,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   pollset->shutting_down = 1;
   grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
   if (!pollset->is_iocp_worker) {
-    grpc_exec_ctx_enqueue(exec_ctx, closure, 1);
+    grpc_exec_ctx_enqueue(exec_ctx, closure, true, NULL);
   } else {
     pollset->on_shutdown = closure;
   }
@@ -174,7 +165,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       }
 
       if (pollset->shutting_down && pollset->on_shutdown != NULL) {
-        grpc_exec_ctx_enqueue(exec_ctx, pollset->on_shutdown, 1);
+        grpc_exec_ctx_enqueue(exec_ctx, pollset->on_shutdown, true, NULL);
         pollset->on_shutdown = NULL;
       }
       goto done;

data/src/core/iomgr/resolve_address_posix.c

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -152,7 +152,7 @@ done:
 
 /* Callback to be passed to grpc_executor to asynch-ify
  * grpc_blocking_resolve_address */
-static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, int success) {
+static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) {
   request *r = rp;
   grpc_resolved_addresses *resolved =
       grpc_blocking_resolve_address(r->name, r->default_port);

data/src/core/iomgr/resolve_address_windows.c

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -135,7 +135,7 @@ done:
 
 /* Callback to be passed to grpc_executor to asynch-ify
  * grpc_blocking_resolve_address */
-static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, int success) {
+static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) {
   request *r = rp;
   grpc_resolved_addresses *resolved =
       grpc_blocking_resolve_address(r->name, r->default_port);

data/src/core/iomgr/sockaddr_utils.c

@@ -48,8 +48,8 @@
 
 #include "src/core/support/string.h"
 
-static const gpr_uint8 kV4MappedPrefix[] = {0, 0, 0, 0, 0,    0,
-                                            0, 0, 0, 0, 0xff, 0xff};
+static const uint8_t kV4MappedPrefix[] = {0, 0, 0, 0, 0,    0,
+                                          0, 0, 0, 0, 0xff, 0xff};
 
 int grpc_sockaddr_is_v4mapped(const struct sockaddr *addr,
                               struct sockaddr_in *addr4_out) {
@@ -126,14 +126,14 @@ void grpc_sockaddr_make_wildcard4(int port, struct sockaddr_in *wild_out) {
   GPR_ASSERT(port >= 0 && port < 65536);
   memset(wild_out, 0, sizeof(*wild_out));
   wild_out->sin_family = AF_INET;
-  wild_out->sin_port = htons((gpr_uint16)port);
+  wild_out->sin_port = htons((uint16_t)port);
 }
 
 void grpc_sockaddr_make_wildcard6(int port, struct sockaddr_in6 *wild_out) {
   GPR_ASSERT(port >= 0 && port < 65536);
   memset(wild_out, 0, sizeof(*wild_out));
   wild_out->sin6_family = AF_INET6;
-  wild_out->sin6_port = htons((gpr_uint16)port);
+  wild_out->sin6_port = htons((uint16_t)port);
 }
 
 int grpc_sockaddr_to_string(char **out, const struct sockaddr *addr,
@@ -220,11 +220,11 @@ int grpc_sockaddr_set_port(const struct sockaddr *addr, int port) {
   switch (addr->sa_family) {
     case AF_INET:
       GPR_ASSERT(port >= 0 && port < 65536);
-      ((struct sockaddr_in *)addr)->sin_port = htons((gpr_uint16)port);
+      ((struct sockaddr_in *)addr)->sin_port = htons((uint16_t)port);
       return 1;
     case AF_INET6:
       GPR_ASSERT(port >= 0 && port < 65536);
-      ((struct sockaddr_in6 *)addr)->sin6_port = htons((gpr_uint16)port);
+      ((struct sockaddr_in6 *)addr)->sin6_port = htons((uint16_t)port);
       return 1;
     default:
       gpr_log(GPR_ERROR, "Unknown socket family %d in grpc_sockaddr_set_port",

data/src/core/iomgr/sockaddr_win32.h

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -38,9 +38,4 @@
 #include <ws2tcpip.h>
 #include <mswsock.h>
 
-#ifdef __MINGW32__
-/* mingw seems to be missing that definition. */
-const char *inet_ntop(int af, const void *src, char *dst, socklen_t size);
-#endif
-
 #endif /* GRPC_INTERNAL_CORE_IOMGR_SOCKADDR_WIN32_H */

data/src/core/iomgr/tcp_client_posix.c

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -91,7 +91,7 @@ error:
   return 0;
 }
 
-static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, int success) {
+static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, bool success) {
   int done;
   async_connect *ac = acp;
   if (grpc_tcp_trace) {
@@ -111,7 +111,7 @@ static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, int success) {
   }
 }
 
-static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, int success) {
+static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, bool success) {
   async_connect *ac = acp;
   int so_error = 0;
   socklen_t so_error_size;
@@ -206,7 +206,7 @@ finish:
     gpr_free(ac->addr_str);
     gpr_free(ac);
   }
-  grpc_exec_ctx_enqueue(exec_ctx, closure, *ep != NULL);
+  grpc_exec_ctx_enqueue(exec_ctx, closure, *ep != NULL, NULL);
 }
 
 void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
@@ -243,7 +243,7 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
     addr_len = sizeof(addr4_copy);
   }
   if (!prepare_socket(addr, fd)) {
-    grpc_exec_ctx_enqueue(exec_ctx, closure, 0);
+    grpc_exec_ctx_enqueue(exec_ctx, closure, false, NULL);
     return;
   }
 
@@ -259,14 +259,14 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
 
   if (err >= 0) {
     *ep = grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str);
-    grpc_exec_ctx_enqueue(exec_ctx, closure, 1);
+    grpc_exec_ctx_enqueue(exec_ctx, closure, true, NULL);
     goto done;
   }
 
   if (errno != EWOULDBLOCK && errno != EINPROGRESS) {
     gpr_log(GPR_ERROR, "connect error to '%s': %s", addr_str, strerror(errno));
     grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, "tcp_client_connect_error");
-    grpc_exec_ctx_enqueue(exec_ctx, closure, 0);
+    grpc_exec_ctx_enqueue(exec_ctx, closure, false, NULL);
     goto done;
   }
 

data/src/core/iomgr/tcp_client_windows.c

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -74,7 +74,7 @@ static void async_connect_unlock_and_cleanup(async_connect *ac) {
   }
 }
 
-static void on_alarm(grpc_exec_ctx *exec_ctx, void *acp, int occured) {
+static void on_alarm(grpc_exec_ctx *exec_ctx, void *acp, bool occured) {
   async_connect *ac = acp;
   gpr_mu_lock(&ac->mu);
   /* If the alarm didn't occur, it got cancelled. */
@@ -84,7 +84,7 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *acp, int occured) {
   async_connect_unlock_and_cleanup(ac);
 }
 
-static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, int from_iocp) {
+static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, bool from_iocp) {
   async_connect *ac = acp;
   SOCKET sock = ac->socket->socket;
   grpc_endpoint **ep = ac->endpoint;
@@ -215,7 +215,7 @@ failure:
   } else if (sock != INVALID_SOCKET) {
     closesocket(sock);
   }
-  grpc_exec_ctx_enqueue(exec_ctx, on_done, 0);
+  grpc_exec_ctx_enqueue(exec_ctx, on_done, false, NULL);
 }
 
 #endif /* GPR_WINSOCK_SOCKET */

data/src/core/iomgr/tcp_posix.c

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -100,9 +100,9 @@ typedef struct {
 } grpc_tcp;
 
 static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
-                            int success);
+                            bool success);
 static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
-                             int success);
+                             bool success);
 
 static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
   grpc_tcp *tcp = (grpc_tcp *)ep;
@@ -247,7 +247,7 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
 }
 
 static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
-                            int success) {
+                            bool success) {
   grpc_tcp *tcp = (grpc_tcp *)arg;
   GPR_ASSERT(!tcp->finished_edge);
 
@@ -273,7 +273,7 @@ static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     tcp->finished_edge = 0;
     grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure);
   } else {
-    grpc_exec_ctx_enqueue(exec_ctx, &tcp->read_closure, 1);
+    grpc_exec_ctx_enqueue(exec_ctx, &tcp->read_closure, true, NULL);
   }
 }
 
@@ -360,7 +360,7 @@ static flush_result tcp_flush(grpc_tcp *tcp) {
 }
 
 static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
-                             int success) {
+                             bool success) {
   grpc_tcp *tcp = (grpc_tcp *)arg;
   flush_result status;
   grpc_closure *cb;
@@ -407,7 +407,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
   if (buf->length == 0) {
     GPR_TIMER_END("tcp_write", 0);
-    grpc_exec_ctx_enqueue(exec_ctx, cb, 1);
+    grpc_exec_ctx_enqueue(exec_ctx, cb, true, NULL);
     return;
   }
   tcp->outgoing_buffer = buf;
@@ -420,7 +420,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
     tcp->write_cb = cb;
     grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure);
   } else {
-    grpc_exec_ctx_enqueue(exec_ctx, cb, status == FLUSH_DONE);
+    grpc_exec_ctx_enqueue(exec_ctx, cb, status == FLUSH_DONE, NULL);
   }
 
   GPR_TIMER_END("tcp_write", 0);
@@ -473,6 +473,12 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size,
   return &tcp->base;
 }
 
+int grpc_tcp_fd(grpc_endpoint *ep) {
+  grpc_tcp *tcp = (grpc_tcp *)ep;
+  GPR_ASSERT(ep->vtable == &vtable);
+  return grpc_fd_wrapped_fd(tcp->em_fd);
+}
+
 void grpc_tcp_destroy_and_release_fd(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                                      int *fd, grpc_closure *done) {
   grpc_tcp *tcp = (grpc_tcp *)ep;

data/src/core/iomgr/tcp_posix.h

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -56,6 +56,12 @@ extern int grpc_tcp_trace;
 grpc_endpoint *grpc_tcp_create(grpc_fd *fd, size_t read_slice_size,
                                const char *peer_string);
 
+/* Return the tcp endpoint's fd, or -1 if this is not available. Does not
+   release the fd.
+   Requires: ep must be a tcp endpoint.
+ */
+int grpc_tcp_fd(grpc_endpoint *ep);
+
 /* Destroy the tcp endpoint without closing its fd. *fd will be set and done
  * will be called when the endpoint is destroyed.
  * Requires: ep must be a tcp endpoint and fd must not be NULL. */

data/src/core/iomgr/tcp_server.h

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,28 +34,38 @@
 #ifndef GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H
 #define GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H
 
+#include "src/core/iomgr/closure.h"
 #include "src/core/iomgr/endpoint.h"
 
 /* Forward decl of grpc_tcp_server */
 typedef struct grpc_tcp_server grpc_tcp_server;
 
-/* Forward decl of grpc_tcp_listener */
-typedef struct grpc_tcp_listener grpc_tcp_listener;
+typedef struct grpc_tcp_server_acceptor {
+  /* grpc_tcp_server_cb functions share a ref on from_server that is valid
+     until the function returns. */
+  grpc_tcp_server *from_server;
+  /* Indices that may be passed to grpc_tcp_server_port_fd(). */
+  unsigned port_index;
+  unsigned fd_index;
+} grpc_tcp_server_acceptor;
 
 /* Called for newly connected TCP connections. */
 typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg,
-                                   grpc_endpoint *ep);
+                                   grpc_endpoint *ep,
+                                   grpc_tcp_server_acceptor *acceptor);
 
-/* Create a server, initially not bound to any ports */
-grpc_tcp_server *grpc_tcp_server_create(void);
+/* Create a server, initially not bound to any ports. The caller owns one ref.
+   If shutdown_complete is not NULL, it will be used by
+   grpc_tcp_server_unref() when the ref count reaches zero. */
+grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete);
 
 /* Start listening to bound ports */
 void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server,
                            grpc_pollset **pollsets, size_t pollset_count,
                            grpc_tcp_server_cb on_accept_cb, void *cb_arg);
 
-/* Add a port to the server, returning the newly created listener on success,
-   or a null pointer on failure.
+/* Add a port to the server, returning the newly allocated port on success, or
+   -1 on failure.
 
    The :: and 0.0.0.0 wildcard addresses are treated identically, accepting
    both IPv4 and IPv6 connections, but :: is the preferred style.  This usually
@@ -63,21 +73,31 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server,
    but not dualstack sockets. */
 /* TODO(ctiller): deprecate this, and make grpc_tcp_server_add_ports to handle
                   all of the multiple socket port matching logic in one place */
-grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s,
-                                            const void *addr, size_t addr_len);
+int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
+                             size_t addr_len);
 
-/* Returns the file descriptor of the Nth listening socket on this server,
-   or -1 if the index is out of bounds.
+/* Number of fds at the given port_index, or 0 if port_index is out of
+   bounds. */
+unsigned grpc_tcp_server_port_fd_count(grpc_tcp_server *s, unsigned port_index);
 
-   The file descriptor remains owned by the server, and will be cleaned
-   up when grpc_tcp_server_destroy is called. */
-int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned index);
+/* Returns the file descriptor of the Mth (fd_index) listening socket of the Nth
+   (port_index) call to add_port() on this server, or -1 if the indices are out
+   of bounds. The file descriptor remains owned by the server, and will be
+   cleaned up when the ref count reaches zero. */
+int grpc_tcp_server_port_fd(grpc_tcp_server *s, unsigned port_index,
+                            unsigned fd_index);
 
-void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server,
-                             grpc_closure *closure);
+/* Ref s and return s. */
+grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s);
 
-int grpc_tcp_listener_get_port(grpc_tcp_listener *listener);
-void grpc_tcp_listener_ref(grpc_tcp_listener *listener);
-void grpc_tcp_listener_unref(grpc_tcp_listener *listener);
+/* shutdown_starting is called when ref count has reached zero and the server is
+   about to be destroyed. The server will be deleted after it returns. Calling
+   grpc_tcp_server_ref() from it has no effect. */
+void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s,
+                                           grpc_closure *shutdown_starting);
+
+/* If the refcount drops to zero, delete s, and call (exec_ctx==NULL) or enqueue
+   a call (exec_ctx!=NULL) to shutdown_complete. */
+void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s);
 
 #endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H */

data/src/core/iomgr/tcp_server_posix.c

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -73,20 +73,22 @@ static gpr_once s_init_max_accept_queue_size;
 static int s_max_accept_queue_size;
 
 /* one listening port */
+typedef struct grpc_tcp_listener grpc_tcp_listener;
 struct grpc_tcp_listener {
   int fd;
   grpc_fd *emfd;
   grpc_tcp_server *server;
   union {
-    gpr_uint8 untyped[GRPC_MAX_SOCKADDR_SIZE];
+    uint8_t untyped[GRPC_MAX_SOCKADDR_SIZE];
     struct sockaddr sockaddr;
     struct sockaddr_un un;
   } addr;
   size_t addr_len;
   int port;
+  unsigned port_index;
+  unsigned fd_index;
   grpc_closure read_closure;
   grpc_closure destroyed_closure;
-  gpr_refcount refs;
   struct grpc_tcp_listener *next;
   /* When we add a listener, more than one can be created, mainly because of
      IPv6. A sibling will still be in the normal list, but will be flagged
@@ -106,6 +108,7 @@ static void unlink_if_unix_domain_socket(const struct sockaddr_un *un) {
 
 /* the overall server */
 struct grpc_tcp_server {
+  gpr_refcount refs;
   /* Called whenever accept() succeeds on a server port. */
   grpc_tcp_server_cb on_accept_cb;
   void *on_accept_cb_arg;
@@ -122,8 +125,12 @@ struct grpc_tcp_server {
 
   /* linked list of server ports */
   grpc_tcp_listener *head;
+  grpc_tcp_listener *tail;
   unsigned nports;
 
+  /* List of closures passed to shutdown_starting_add(). */
+  grpc_closure_list shutdown_starting;
+
   /* shutdown callback */
   grpc_closure *shutdown_complete;
 
@@ -133,34 +140,42 @@ struct grpc_tcp_server {
   size_t pollset_count;
 };
 
-grpc_tcp_server *grpc_tcp_server_create(void) {
+grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
   grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server));
+  gpr_ref_init(&s->refs, 1);
   gpr_mu_init(&s->mu);
   s->active_ports = 0;
   s->destroyed_ports = 0;
   s->shutdown = 0;
+  s->shutdown_starting.head = NULL;
+  s->shutdown_starting.tail = NULL;
+  s->shutdown_complete = shutdown_complete;
   s->on_accept_cb = NULL;
   s->on_accept_cb_arg = NULL;
   s->head = NULL;
+  s->tail = NULL;
   s->nports = 0;
   return s;
 }
 
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
-  grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1);
+  if (s->shutdown_complete != NULL) {
+    grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, true, NULL);
+  }
 
   gpr_mu_destroy(&s->mu);
 
   while (s->head) {
     grpc_tcp_listener *sp = s->head;
     s->head = sp->next;
-    grpc_tcp_listener_unref(sp);
+    gpr_free(sp);
   }
 
   gpr_free(s);
 }
 
-static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server, int success) {
+static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server,
+                           bool success) {
   grpc_tcp_server *s = server;
   gpr_mu_lock(&s->mu);
   s->destroyed_ports++;
@@ -203,15 +218,12 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   }
 }
 
-void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
-                             grpc_closure *closure) {
+static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   gpr_mu_lock(&s->mu);
 
   GPR_ASSERT(!s->shutdown);
   s->shutdown = 1;
 
-  s->shutdown_complete = closure;
-
   /* shutdown all fd's */
   if (s->active_ports) {
     grpc_tcp_listener *sp;
@@ -306,8 +318,10 @@ error:
 }
 
 /* event manager callback when reads are ready */
-static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   grpc_tcp_listener *sp = arg;
+  grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index,
+                                       sp->fd_index};
   grpc_fd *fdobj;
   size_t i;
 
@@ -355,7 +369,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) {
     }
     sp->server->on_accept_cb(
         exec_ctx, sp->server->on_accept_cb_arg,
-        grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str));
+        grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str),
+        &acceptor);
 
     gpr_free(name);
     gpr_free(addr_str);
@@ -375,7 +390,9 @@ error:
 
 static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, int fd,
                                                const struct sockaddr *addr,
-                                               size_t addr_len) {
+                                               size_t addr_len,
+                                               unsigned port_index,
+                                               unsigned fd_index) {
   grpc_tcp_listener *sp = NULL;
   int port;
   char *addr_str;
@@ -389,17 +406,23 @@ static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, int fd,
     s->nports++;
     GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server");
     sp = gpr_malloc(sizeof(grpc_tcp_listener));
-    sp->next = s->head;
-    s->head = sp;
+    sp->next = NULL;
+    if (s->head == NULL) {
+      s->head = sp;
+    } else {
+      s->tail->next = sp;
+    }
+    s->tail = sp;
     sp->server = s;
     sp->fd = fd;
     sp->emfd = grpc_fd_create(fd, name);
     memcpy(sp->addr.untyped, addr, addr_len);
     sp->addr_len = addr_len;
     sp->port = port;
+    sp->port_index = port_index;
+    sp->fd_index = fd_index;
     sp->is_sibling = 0;
     sp->sibling = NULL;
-    gpr_ref_init(&sp->refs, 1);
     GPR_ASSERT(sp->emfd);
     gpr_mu_unlock(&s->mu);
     gpr_free(addr_str);
@@ -409,8 +432,8 @@ static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, int fd,
   return sp;
 }
 
-grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s,
-                                            const void *addr, size_t addr_len) {
+int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
+                             size_t addr_len) {
   grpc_tcp_listener *sp;
   grpc_tcp_listener *sp2 = NULL;
   int fd;
@@ -423,7 +446,11 @@ grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s,
   struct sockaddr_storage sockname_temp;
   socklen_t sockname_len;
   int port;
-
+  unsigned port_index = 0;
+  unsigned fd_index = 0;
+  if (s->tail != NULL) {
+    port_index = s->tail->port_index + 1;
+  }
   if (((struct sockaddr *)addr)->sa_family == AF_UNIX) {
     unlink_if_unix_domain_socket(addr);
   }
@@ -462,11 +489,13 @@ grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s,
     addr = (struct sockaddr *)&wild6;
     addr_len = sizeof(wild6);
     fd = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode);
-    sp = add_socket_to_server(s, fd, addr, addr_len);
+    sp = add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index);
     if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) {
       goto done;
     }
-
+    if (sp != NULL) {
+      ++fd_index;
+    }
     /* If we didn't get a dualstack socket, also listen on 0.0.0.0. */
     if (port == 0 && sp != NULL) {
       grpc_sockaddr_set_port((struct sockaddr *)&wild4, sp->port);
@@ -485,20 +514,46 @@ grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s,
     addr = (struct sockaddr *)&addr4_copy;
     addr_len = sizeof(addr4_copy);
   }
-  sp = add_socket_to_server(s, fd, addr, addr_len);
-  if (sp != NULL) sp->sibling = sp2;
-  if (sp2 != NULL) sp2->is_sibling = 1;
+  sp = add_socket_to_server(s, fd, addr, addr_len, port_index, fd_index);
+  if (sp2 != NULL && sp != NULL) {
+    sp2->sibling = sp;
+    sp->is_sibling = 1;
+  }
 
 done:
   gpr_free(allocated_addr);
-  return sp;
+  if (sp != NULL) {
+    return sp->port;
+  } else {
+    return -1;
+  }
 }
 
-int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index) {
+unsigned grpc_tcp_server_port_fd_count(grpc_tcp_server *s,
+                                       unsigned port_index) {
+  unsigned num_fds = 0;
   grpc_tcp_listener *sp;
-  for (sp = s->head; sp && port_index != 0; sp = sp->next, port_index--)
+  for (sp = s->head; sp && port_index != 0; sp = sp->next) {
+    if (!sp->is_sibling) {
+      --port_index;
+    }
+  }
+  for (; sp; sp = sp->sibling, ++num_fds)
+    ;
+  return num_fds;
+}
+
+int grpc_tcp_server_port_fd(grpc_tcp_server *s, unsigned port_index,
+                            unsigned fd_index) {
+  grpc_tcp_listener *sp;
+  for (sp = s->head; sp && port_index != 0; sp = sp->next) {
+    if (!sp->is_sibling) {
+      --port_index;
+    }
+  }
+  for (; sp && fd_index != 0; sp = sp->sibling, --fd_index)
     ;
-  if (port_index == 0 && sp) {
+  if (sp) {
     return sp->fd;
   } else {
     return -1;
@@ -531,31 +586,33 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
   gpr_mu_unlock(&s->mu);
 }
 
-int grpc_tcp_listener_get_port(grpc_tcp_listener *listener) {
-  if (listener != NULL) {
-    grpc_tcp_listener *sp = listener;
-    return sp->port;
-  } else {
-    return 0;
-  }
+grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) {
+  gpr_ref(&s->refs);
+  return s;
 }
 
-void grpc_tcp_listener_ref(grpc_tcp_listener *listener) {
-  grpc_tcp_listener *sp = listener;
-  gpr_ref(&sp->refs);
+void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s,
+                                           grpc_closure *shutdown_starting) {
+  gpr_mu_lock(&s->mu);
+  grpc_closure_list_add(&s->shutdown_starting, shutdown_starting, 1);
+  gpr_mu_unlock(&s->mu);
 }
 
-void grpc_tcp_listener_unref(grpc_tcp_listener *listener) {
-  grpc_tcp_listener *sp = listener;
-  if (sp->is_sibling) return;
-  if (gpr_unref(&sp->refs)) {
-    grpc_tcp_listener *sibling = sp->sibling;
-    while (sibling) {
-      sp = sibling;
-      sibling = sp->sibling;
-      gpr_free(sp);
+void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
+  if (gpr_unref(&s->refs)) {
+    /* Complete shutdown_starting work before destroying. */
+    grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT;
+    gpr_mu_lock(&s->mu);
+    grpc_exec_ctx_enqueue_list(&local_exec_ctx, &s->shutdown_starting, NULL);
+    gpr_mu_unlock(&s->mu);
+    if (exec_ctx == NULL) {
+      grpc_exec_ctx_flush(&local_exec_ctx);
+      tcp_server_destroy(&local_exec_ctx, s);
+      grpc_exec_ctx_finish(&local_exec_ctx);
+    } else {
+      grpc_exec_ctx_finish(&local_exec_ctx);
+      tcp_server_destroy(exec_ctx, s);
     }
-    gpr_free(listener);
   }
 }