grpc 1.22.0 → 1.23.0

data/src/core/lib/slice/slice_internal.h

@@ -30,6 +30,7 @@
 #include "src/core/lib/gpr/murmur_hash.h"
 #include "src/core/lib/gprpp/memory.h"
 #include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/slice/slice_utils.h"
 #include "src/core/lib/transport/static_metadata.h"
 
 // Interned slices have specific fast-path operations for hashing. To inline
@@ -95,6 +96,8 @@ extern uint32_t g_hash_seed;
 // In total, this saves us roughly 1-2% latency for unary calls, with smaller
 // calls benefitting. The effect is present, but not as useful, for larger calls
 // where the cost of sending the data dominates.
+// TODO(arjunroy): Investigate if this can be removed with strongly typed
+// grpc_slices.
 struct grpc_slice_refcount {
  public:
   enum class Type {
@@ -171,6 +174,8 @@ struct grpc_slice_refcount {
 
 namespace grpc_core {
 
+extern grpc_slice_refcount kNoopRefcount;
+
 struct InternedSliceRefcount {
   static void Destroy(void* arg) {
     auto* rc = static_cast<InternedSliceRefcount*>(arg);
@@ -312,4 +317,7 @@ grpc_slice grpc_slice_from_moved_string(grpc_core::UniquePtr<char> p);
 // 0. All other slices will return the size of the allocated chars.
 size_t grpc_slice_memory_usage(grpc_slice s);
 
+grpc_core::UnmanagedMemorySlice grpc_slice_sub_no_ref(
+    const grpc_core::UnmanagedMemorySlice& source, size_t begin, size_t end);
+
 #endif /* GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H */

data/src/core/lib/slice/slice_utils.h

@@ -21,6 +21,8 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <cstring>
+
 #include <grpc/slice.h>
 
 // When we compare two slices, and we know the latter is not inlined, we can
@@ -36,6 +38,7 @@
 // x86-64/clang with differs().
 int grpc_slice_differs_refcounted(const grpc_slice& a,
                                   const grpc_slice& b_not_inline);
+
 // When we compare two slices, and we *know* that one of them is static or
 // interned, we can short circuit our slice equality function. The second slice
 // here must be static or interned; slice a can be any slice, inlined or not.
@@ -47,4 +50,121 @@ inline bool grpc_slice_eq_static_interned(const grpc_slice& a,
   return !grpc_slice_differs_refcounted(a, b_static_interned);
 }
 
+// TODO(arjunroy): These type declarations ought to be in
+// src/core/lib/slice/slice_internal.h instead; they are here due to a circular
+// header depedency between slice_internal.h and
+// src/core/lib/transport/metadata.h. We need to fix this circular reference and
+// when we do, move these type declarations.
+//
+// Internal slice type declarations.
+// Externally, a grpc_slice is a grpc_slice is a grpc_slice.
+// Internally, we may have heap allocated slices, static slices, interned
+// slices, and inlined slices. If we know the specific type of slice
+// we're dealing with, we can save cycles (e.g. fast-paths when we know we don't
+// need to take a reference on a slice). Rather than introducing new methods
+// ad-hoc in these cases, we rely on type-system backed overloads to keep
+// internal APIs clean.
+//
+// For each overload, the definition and layout of the underlying slice does not
+// change; this is purely type-system information.
+namespace grpc_core {
+
+// There are two main types of slices: those that have their memory
+// managed by the slice library and those that do not.
+//
+// The following types of slices are not managed:
+// - inlined slices (i.e., refcount is null)
+// - slices that have a custom refcount type (i.e., not STATIC or INTERNED)
+// - slices where the memory is managed by some external agent. The slice is not
+//   ref-counted by grpc, and the programmer is responsible for ensuring the
+//   data is valid for the duration of the period that grpc may access it.
+//
+// The following types of slices are managed:
+// - static metadata slices (i.e., refcount type is STATIC)
+// - interned slices (i.e., refcount type is INTERNED)
+//
+// This categorization is reflected in the following hierarchy:
+//
+// - grpc_slice
+// > - UnmanagedMemorySlice
+//   > - ExternallyManagedSlice
+//   - ManagedMemorySlice
+//   > - InternedSlice
+//     - StaticMetadataSlice
+//
+struct ManagedMemorySlice : public grpc_slice {
+  ManagedMemorySlice() {
+    refcount = nullptr;
+    data.refcounted.bytes = nullptr;
+    data.refcounted.length = 0;
+  }
+  explicit ManagedMemorySlice(const char* string);
+  ManagedMemorySlice(const char* buf, size_t len);
+  explicit ManagedMemorySlice(const grpc_slice* slice);
+  bool Equals(const grpc_slice& other) const {
+    if (refcount == other.refcount) {
+      return true;
+    }
+    return !grpc_slice_differs_refcounted(other, *this);
+  }
+  bool Equals(const char* buf, const size_t len) const {
+    return data.refcounted.length == len &&
+           memcmp(buf, data.refcounted.bytes, len) == 0;
+  }
+};
+struct UnmanagedMemorySlice : public grpc_slice {
+  // TODO(arjunroy): Can we use a default=false param instead of this enum?
+  enum class ForceHeapAllocation {};
+  UnmanagedMemorySlice() {
+    refcount = nullptr;
+    data.inlined.length = 0;
+  }
+  explicit UnmanagedMemorySlice(const char* source);
+  UnmanagedMemorySlice(const char* source, size_t length);
+  // The first constructor creates a slice that may be heap allocated, or
+  // inlined in the slice structure if length is small enough
+  // (< GRPC_SLICE_INLINED_SIZE). The second constructor forces heap alloc.
+  explicit UnmanagedMemorySlice(size_t length);
+  explicit UnmanagedMemorySlice(size_t length, const ForceHeapAllocation&) {
+    HeapInit(length);
+  }
+
+ private:
+  void HeapInit(size_t length);
+};
+
+extern grpc_slice_refcount kNoopRefcount;
+
+struct ExternallyManagedSlice : public UnmanagedMemorySlice {
+  ExternallyManagedSlice()
+      : ExternallyManagedSlice(&kNoopRefcount, 0, nullptr) {}
+  explicit ExternallyManagedSlice(const char* s)
+      : ExternallyManagedSlice(s, strlen(s)) {}
+  ExternallyManagedSlice(const void* s, size_t len)
+      : ExternallyManagedSlice(
+            &kNoopRefcount, len,
+            reinterpret_cast<uint8_t*>(const_cast<void*>(s))) {}
+  ExternallyManagedSlice(grpc_slice_refcount* ref, size_t length,
+                         uint8_t* bytes) {
+    refcount = ref;
+    data.refcounted.length = length;
+    data.refcounted.bytes = bytes;
+  }
+};
+
+struct StaticMetadataSlice : public ManagedMemorySlice {
+  StaticMetadataSlice(grpc_slice_refcount* ref, size_t length, uint8_t* bytes) {
+    refcount = ref;
+    data.refcounted.length = length;
+    data.refcounted.bytes = bytes;
+  }
+};
+
+struct InternedSliceRefcount;
+struct InternedSlice : public ManagedMemorySlice {
+  explicit InternedSlice(InternedSliceRefcount* s);
+};
+
+}  // namespace grpc_core
+
 #endif /* GRPC_CORE_LIB_SLICE_SLICE_UTILS_H */

data/src/core/lib/slice/slice_weak_hash_table.h

@@ -61,13 +61,8 @@ class SliceWeakHashTable : public RefCounted<SliceWeakHashTable<T, Size>> {
   }
 
  private:
-  // So New() can call our private ctor.
-  template <typename T2, typename... Args>
-  friend T2* New(Args&&... args);
-
-  // So Delete() can call our private dtor.
-  template <typename T2>
-  friend void Delete(T2*);
+  GRPC_ALLOW_CLASS_TO_USE_NON_PUBLIC_NEW
+  GRPC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE
 
   SliceWeakHashTable() = default;
   ~SliceWeakHashTable() = default;

data/src/core/lib/surface/call.cc

@@ -1568,6 +1568,10 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
           error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
           goto done_with_error;
         }
+        // TODO(juanlishen): If the user has already specified a compression
+        // algorithm by setting the initial metadata with key of
+        // GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, we shouldn't override that
+        // with the compression algorithm mapped from compression level.
         /* process compression level */
         grpc_metadata& compression_md = call->compression_md;
         compression_md.key = grpc_empty_slice();
@@ -1589,17 +1593,18 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
             effective_compression_level = copts.default_level.level;
           }
         }
+        // Currently, only server side supports compression level setting.
         if (level_set && !call->is_client) {
           const grpc_compression_algorithm calgo =
               compression_algorithm_for_level_locked(
                   call, effective_compression_level);
-          /* the following will be picked up by the compress filter and used
-           * as the call's compression algorithm. */
+          // The following metadata will be checked and removed by the message
+          // compression filter. It will be used as the call's compression
+          // algorithm.
           compression_md.key = GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST;
           compression_md.value = grpc_compression_algorithm_slice(calgo);
           additional_metadata_count++;
         }
-
         if (op->data.send_initial_metadata.count + additional_metadata_count >
             INT_MAX) {
           error = GRPC_CALL_ERROR_INVALID_METADATA;

data/src/core/lib/surface/channel.cc

@@ -235,6 +235,23 @@ grpc_channel* grpc_channel_create(const char* target,
                                   grpc_channel_stack_type channel_stack_type,
                                   grpc_transport* optional_transport,
                                   grpc_resource_user* resource_user) {
+  // We need to make sure that grpc_shutdown() does not shut things down
+  // until after the channel is destroyed.  However, the channel may not
+  // actually be destroyed by the time grpc_channel_destroy() returns,
+  // since there may be other existing refs to the channel.  If those
+  // refs are held by things that are visible to the wrapped language
+  // (such as outstanding calls on the channel), then the wrapped
+  // language can be responsible for making sure that grpc_shutdown()
+  // does not run until after those refs are released.  However, the
+  // channel may also have refs to itself held internally for various
+  // things that need to be cleaned up at channel destruction (e.g.,
+  // LB policies, subchannels, etc), and because these refs are not
+  // visible to the wrapped language, it cannot be responsible for
+  // deferring grpc_shutdown() until after they are released.  To
+  // accommodate that, we call grpc_init() here and then call
+  // grpc_shutdown() when the channel is actually destroyed, thus
+  // ensuring that shutdown is deferred until that point.
+  grpc_init();
   grpc_channel_stack_builder* builder = grpc_channel_stack_builder_create();
   const grpc_core::UniquePtr<char> default_authority =
       get_default_authority(input_args);
@@ -250,6 +267,7 @@ grpc_channel* grpc_channel_create(const char* target,
     if (resource_user != nullptr) {
       grpc_resource_user_free(resource_user, GRPC_RESOURCE_QUOTA_CHANNEL_SIZE);
     }
+    grpc_shutdown();  // Since we won't call destroy_channel().
     return nullptr;
   }
   // We only need to do this for clients here. For servers, this will be
@@ -257,7 +275,12 @@ grpc_channel* grpc_channel_create(const char* target,
   if (grpc_channel_stack_type_is_client(channel_stack_type)) {
     CreateChannelzNode(builder);
   }
-  return grpc_channel_create_with_builder(builder, channel_stack_type);
+  grpc_channel* channel =
+      grpc_channel_create_with_builder(builder, channel_stack_type);
+  if (channel == nullptr) {
+    grpc_shutdown();  // Since we won't call destroy_channel().
+  }
+  return channel;
 }
 
 size_t grpc_channel_get_call_size_estimate(grpc_channel* channel) {
@@ -393,13 +416,11 @@ void* grpc_channel_register_call(grpc_channel* channel, const char* method,
   GPR_ASSERT(!reserved);
   grpc_core::ExecCtx exec_ctx;
 
-  rc->path = grpc_mdelem_from_slices(
-      GRPC_MDSTR_PATH,
-      grpc_slice_intern(grpc_slice_from_static_string(method)));
+  rc->path = grpc_mdelem_from_slices(GRPC_MDSTR_PATH,
+                                     grpc_core::ManagedMemorySlice(method));
   rc->authority =
-      host ? grpc_mdelem_from_slices(
-                 GRPC_MDSTR_AUTHORITY,
-                 grpc_slice_intern(grpc_slice_from_static_string(host)))
+      host ? grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY,
+                                     grpc_core::ManagedMemorySlice(host))
            : GRPC_MDNULL;
   gpr_mu_lock(&channel->registered_call_mu);
   rc->next = channel->registered_calls;
@@ -468,6 +489,8 @@ static void destroy_channel(void* arg, grpc_error* error) {
   gpr_mu_destroy(&channel->registered_call_mu);
   gpr_free(channel->target);
   gpr_free(channel);
+  // See comment in grpc_channel_create() for why we do this.
+  grpc_shutdown();
 }
 
 void grpc_channel_destroy(grpc_channel* channel) {
@@ -488,5 +511,5 @@ grpc_mdelem grpc_channel_get_reffed_status_elem_slowpath(grpc_channel* channel,
   char tmp[GPR_LTOA_MIN_BUFSIZE];
   gpr_ltoa(i, tmp);
   return grpc_mdelem_from_slices(GRPC_MDSTR_GRPC_STATUS,
-                                 grpc_slice_from_copied_string(tmp));
+                                 grpc_core::UnmanagedMemorySlice(tmp));
 }

data/src/core/lib/surface/completion_queue.cc

@@ -857,17 +857,20 @@ static void cq_end_op_for_callback(
   }
 
   auto* functor = static_cast<grpc_experimental_completion_queue_functor*>(tag);
-  if (internal) {
+  if (internal || grpc_iomgr_is_any_background_poller_thread()) {
     grpc_core::ApplicationCallbackExecCtx::Enqueue(functor,
                                                    (error == GRPC_ERROR_NONE));
     GRPC_ERROR_UNREF(error);
-  } else {
-    GRPC_CLOSURE_SCHED(
-        GRPC_CLOSURE_CREATE(
-            functor_callback, functor,
-            grpc_core::Executor::Scheduler(grpc_core::ExecutorJobType::SHORT)),
-        error);
+    return;
   }
+
+  // Schedule the callback on a closure if not internal or triggered
+  // from a background poller thread.
+  GRPC_CLOSURE_SCHED(
+      GRPC_CLOSURE_CREATE(
+          functor_callback, functor,
+          grpc_core::Executor::Scheduler(grpc_core::ExecutorJobType::SHORT)),
+      error);
 }
 
 void grpc_cq_end_op(grpc_completion_queue* cq, void* tag, grpc_error* error,
@@ -1352,6 +1355,13 @@ static void cq_finish_shutdown_callback(grpc_completion_queue* cq) {
   GPR_ASSERT(cqd->shutdown_called);
 
   cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done);
+  if (grpc_iomgr_is_any_background_poller_thread()) {
+    grpc_core::ApplicationCallbackExecCtx::Enqueue(callback, true);
+    return;
+  }
+
+  // Schedule the callback on a closure if not internal or triggered
+  // from a background poller thread.
   GRPC_CLOSURE_SCHED(
       GRPC_CLOSURE_CREATE(
           functor_callback, callback,

data/src/core/lib/surface/init_secure.cc

@@ -78,4 +78,7 @@ void grpc_register_security_filters(void) {
                                    maybe_prepend_server_auth_filter, nullptr);
 }
 
-void grpc_security_init() { grpc_core::SecurityRegisterHandshakerFactories(); }
+void grpc_security_init() {
+  grpc_core::SecurityRegisterHandshakerFactories();
+  grpc_control_plane_credentials_init();
+}

data/src/core/lib/surface/lame_client.cc

@@ -61,10 +61,10 @@ static void fill_metadata(grpc_call_element* elem, grpc_metadata_batch* mdb) {
   char tmp[GPR_LTOA_MIN_BUFSIZE];
   gpr_ltoa(chand->error_code, tmp);
   calld->status.md = grpc_mdelem_from_slices(
-      GRPC_MDSTR_GRPC_STATUS, grpc_slice_from_copied_string(tmp));
+      GRPC_MDSTR_GRPC_STATUS, grpc_core::UnmanagedMemorySlice(tmp));
   calld->details.md = grpc_mdelem_from_slices(
       GRPC_MDSTR_GRPC_MESSAGE,
-      grpc_slice_from_copied_string(chand->error_message));
+      grpc_core::UnmanagedMemorySlice(chand->error_message));
   calld->status.prev = calld->details.next = nullptr;
   calld->status.next = &calld->details;
   calld->details.prev = &calld->status;

data/src/core/lib/surface/server.cc

@@ -31,6 +31,7 @@
 #include <utility>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/channel/connected_channel.h"
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/gpr/mpscq.h"
@@ -111,7 +112,7 @@ struct channel_data {
   uint32_t registered_method_max_probes;
   grpc_closure finish_destroy_channel_closure;
   grpc_closure channel_connectivity_changed;
-  grpc_core::RefCountedPtr<grpc_core::channelz::SocketNode> socket_node;
+  intptr_t channelz_socket_uuid;
 };
 
 typedef struct shutdown_tag {
@@ -710,8 +711,10 @@ static void maybe_finish_shutdown(grpc_server* server) {
     return;
   }
 
+  gpr_mu_lock(&server->mu_call);
   kill_pending_work_locked(
       server, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Server Shutdown"));
+  gpr_mu_unlock(&server->mu_call);
 
   if (server->root_channel_data.next != &server->root_channel_data ||
       server->listeners_destroyed < num_listeners(server)) {
@@ -941,7 +944,6 @@ static grpc_error* init_channel_elem(grpc_channel_element* elem,
 static void destroy_channel_elem(grpc_channel_element* elem) {
   size_t i;
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
-  chand->socket_node.reset();
   if (chand->registered_methods) {
     for (i = 0; i < chand->registered_method_slots; i++) {
       grpc_slice_unref_internal(chand->registered_methods[i].method);
@@ -952,6 +954,11 @@ static void destroy_channel_elem(grpc_channel_element* elem) {
     gpr_free(chand->registered_methods);
   }
   if (chand->server) {
+    if (chand->server->channelz_server != nullptr &&
+        chand->channelz_socket_uuid != 0) {
+      chand->server->channelz_server->RemoveChildSocket(
+          chand->channelz_socket_uuid);
+    }
     gpr_mu_lock(&chand->server->mu_global);
     chand->next->prev = chand->prev;
     chand->prev->next = chand->next;
@@ -1144,7 +1151,8 @@ void grpc_server_get_pollsets(grpc_server* server, grpc_pollset*** pollsets,
 void grpc_server_setup_transport(
     grpc_server* s, grpc_transport* transport, grpc_pollset* accepting_pollset,
     const grpc_channel_args* args,
-    grpc_core::RefCountedPtr<grpc_core::channelz::SocketNode> socket_node,
+    const grpc_core::RefCountedPtr<grpc_core::channelz::SocketNode>&
+        socket_node,
     grpc_resource_user* resource_user) {
   size_t num_registered_methods;
   size_t alloc;
@@ -1166,7 +1174,12 @@ void grpc_server_setup_transport(
   chand->server = s;
   server_ref(s);
   chand->channel = channel;
-  chand->socket_node = std::move(socket_node);
+  if (socket_node != nullptr) {
+    chand->channelz_socket_uuid = socket_node->uuid();
+    s->channelz_server->AddChildSocket(socket_node);
+  } else {
+    chand->channelz_socket_uuid = 0;
+  }
 
   size_t cq_idx;
   for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) {
@@ -1241,28 +1254,6 @@ void grpc_server_setup_transport(
   grpc_transport_perform_op(transport, op);
 }
 
-void grpc_server_populate_server_sockets(
-    grpc_server* s, grpc_core::channelz::ChildSocketsList* server_sockets,
-    intptr_t start_idx) {
-  gpr_mu_lock(&s->mu_global);
-  channel_data* c = nullptr;
-  for (c = s->root_channel_data.next; c != &s->root_channel_data; c = c->next) {
-    if (c->socket_node != nullptr && c->socket_node->uuid() >= start_idx) {
-      server_sockets->push_back(c->socket_node.get());
-    }
-  }
-  gpr_mu_unlock(&s->mu_global);
-}
-
-void grpc_server_populate_listen_sockets(
-    grpc_server* server, grpc_core::channelz::ChildRefsList* listen_sockets) {
-  gpr_mu_lock(&server->mu_global);
-  for (listener* l = server->listeners; l != nullptr; l = l->next) {
-    listen_sockets->push_back(l->socket_uuid);
-  }
-  gpr_mu_unlock(&server->mu_global);
-}
-
 void done_published_shutdown(void* done_arg, grpc_cq_completion* storage) {
   (void)done_arg;
   gpr_free(storage);
@@ -1350,6 +1341,9 @@ void grpc_server_shutdown_and_notify(grpc_server* server,
     GRPC_CLOSURE_INIT(&l->destroy_done, listener_destroy_done, server,
                       grpc_schedule_on_exec_ctx);
     l->destroy(server, l->arg, &l->destroy_done);
+    if (server->channelz_server != nullptr && l->socket_uuid != 0) {
+      server->channelz_server->RemoveChildListenSocket(l->socket_uuid);
+    }
   }
 
   channel_broadcaster_shutdown(&broadcaster, true /* send_goaway */,
@@ -1401,18 +1395,23 @@ void grpc_server_destroy(grpc_server* server) {
   server_unref(server);
 }
 
-void grpc_server_add_listener(grpc_server* server, void* arg,
-                              void (*start)(grpc_server* server, void* arg,
-                                            grpc_pollset** pollsets,
-                                            size_t pollset_count),
-                              void (*destroy)(grpc_server* server, void* arg,
-                                              grpc_closure* on_done),
-                              intptr_t socket_uuid) {
+void grpc_server_add_listener(
+    grpc_server* server, void* listener_arg,
+    void (*start)(grpc_server* server, void* arg, grpc_pollset** pollsets,
+                  size_t pollset_count),
+    void (*destroy)(grpc_server* server, void* arg, grpc_closure* on_done),
+    grpc_core::RefCountedPtr<grpc_core::channelz::ListenSocketNode> node) {
   listener* l = static_cast<listener*>(gpr_malloc(sizeof(listener)));
-  l->arg = arg;
+  l->arg = listener_arg;
   l->start = start;
   l->destroy = destroy;
-  l->socket_uuid = socket_uuid;
+  l->socket_uuid = 0;
+  if (node != nullptr) {
+    l->socket_uuid = node->uuid();
+    if (server->channelz_server != nullptr) {
+      server->channelz_server->AddChildListenSocket(std::move(node));
+    }
+  }
   l->next = server->listeners;
   server->listeners = l;
 }