grpc 1.32.0 → 1.33.0.pre1

data/src/core/lib/channel/channelz.h

@@ -23,6 +23,7 @@
 
 #include <grpc/grpc.h>
 
+#include <set>
 #include <string>
 
 #include "absl/container/inlined_vector.h"
@@ -42,8 +43,9 @@
 // Channel arg key for channelz node.
 #define GRPC_ARG_CHANNELZ_CHANNEL_NODE "grpc.channelz_channel_node"
 
-// Channel arg key to encode the channelz uuid of the channel's parent.
-#define GRPC_ARG_CHANNELZ_PARENT_UUID "grpc.channelz_parent_uuid"
+// Channel arg key for indicating an internal channel.
+#define GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL \
+  "grpc.channelz_is_internal_channel"
 
 /** This is the default value for whether or not to enable channelz. If
  * GRPC_ARG_ENABLE_CHANNELZ is set, it will override this default value. */
@@ -59,10 +61,6 @@ namespace grpc_core {
 
 namespace channelz {
 
-// Helpers for getting and setting GRPC_ARG_CHANNELZ_PARENT_UUID.
-grpc_arg MakeParentUuidArg(intptr_t parent_uuid);
-intptr_t GetParentUuidFromArgs(const grpc_channel_args& args);
-
 class SocketNode;
 class ListenSocketNode;
 
@@ -176,14 +174,12 @@ class CallCountingHelper {
 class ChannelNode : public BaseNode {
  public:
   ChannelNode(std::string target, size_t channel_tracer_max_nodes,
-              intptr_t parent_uuid);
+              bool is_internal_channel);
 
   // Returns the string description of the given connectivity state.
   static const char* GetChannelConnectivityStateChangeString(
       grpc_connectivity_state state);
 
-  intptr_t parent_uuid() const { return parent_uuid_; }
-
   Json RenderJson() override;
 
   // proxy methods to composed classes.
@@ -213,26 +209,22 @@ class ChannelNode : public BaseNode {
   void RemoveChildSubchannel(intptr_t child_uuid);
 
  private:
-  void PopulateChildRefs(Json::Object* json);
-
-  // to allow the channel trace test to access trace_.
+  // Allows the channel trace test to access trace_.
   friend class testing::ChannelNodePeer;
 
+  void PopulateChildRefs(Json::Object* json);
+
   std::string target_;
   CallCountingHelper call_counter_;
   ChannelTrace trace_;
-  const intptr_t parent_uuid_;
 
   // Least significant bit indicates whether the value is set.  Remaining
   // bits are a grpc_connectivity_state value.
   Atomic<int> connectivity_state_{0};
 
-  Mutex child_mu_;  // Guards child maps below.
-  // TODO(roth): We don't actually use the values here, only the keys, so
-  // these should be sets instead of maps, but we don't currently have a set
-  // implementation.  Change this if/when we have one.
-  std::map<intptr_t, bool> child_channels_;
-  std::map<intptr_t, bool> child_subchannels_;
+  Mutex child_mu_;  // Guards sets below.
+  std::set<intptr_t> child_channels_;
+  std::set<intptr_t> child_subchannels_;
 };
 
 // Handles channelz bookkeeping for servers

data/src/core/lib/channel/channelz_registry.cc

@@ -78,8 +78,7 @@ RefCountedPtr<BaseNode> ChannelzRegistry::InternalGet(intptr_t uuid) {
   // Found node.  Return only if its refcount is not zero (i.e., when we
   // know that there is no other thread about to destroy it).
   BaseNode* node = it->second;
-  if (!node->RefIfNonZero()) return nullptr;
-  return RefCountedPtr<BaseNode>(node);
+  return node->RefIfNonZero();
 }
 
 std::string ChannelzRegistry::InternalGetTopChannels(
@@ -91,8 +90,9 @@ std::string ChannelzRegistry::InternalGetTopChannels(
     for (auto it = node_map_.lower_bound(start_channel_id);
          it != node_map_.end(); ++it) {
       BaseNode* node = it->second;
+      RefCountedPtr<BaseNode> node_ref;
       if (node->type() == BaseNode::EntityType::kTopLevelChannel &&
-          node->RefIfNonZero()) {
+          (node_ref = node->RefIfNonZero()) != nullptr) {
         // Check if we are over pagination limit to determine if we need to set
         // the "end" element. If we don't go through this block, we know that
         // when the loop terminates, we have <= to kPaginationLimit.
@@ -100,10 +100,10 @@ std::string ChannelzRegistry::InternalGetTopChannels(
         // refcount, we need to decrease it, but we can't unref while
         // holding the lock, because this may lead to a deadlock.
         if (top_level_channels.size() == kPaginationLimit) {
-          node_after_pagination_limit.reset(node);
+          node_after_pagination_limit = std::move(node_ref);
           break;
         }
-        top_level_channels.emplace_back(node);
+        top_level_channels.emplace_back(std::move(node_ref));
       }
     }
   }
@@ -129,8 +129,9 @@ std::string ChannelzRegistry::InternalGetServers(intptr_t start_server_id) {
     for (auto it = node_map_.lower_bound(start_server_id);
          it != node_map_.end(); ++it) {
       BaseNode* node = it->second;
+      RefCountedPtr<BaseNode> node_ref;
       if (node->type() == BaseNode::EntityType::kServer &&
-          node->RefIfNonZero()) {
+          (node_ref = node->RefIfNonZero()) != nullptr) {
         // Check if we are over pagination limit to determine if we need to set
         // the "end" element. If we don't go through this block, we know that
         // when the loop terminates, we have <= to kPaginationLimit.
@@ -138,10 +139,10 @@ std::string ChannelzRegistry::InternalGetServers(intptr_t start_server_id) {
         // refcount, we need to decrease it, but we can't unref while
         // holding the lock, because this may lead to a deadlock.
         if (servers.size() == kPaginationLimit) {
-          node_after_pagination_limit.reset(node);
+          node_after_pagination_limit = std::move(node_ref);
           break;
         }
-        servers.emplace_back(node);
+        servers.emplace_back(std::move(node_ref));
       }
     }
   }
@@ -164,9 +165,9 @@ void ChannelzRegistry::InternalLogAllEntities() {
   {
     MutexLock lock(&mu_);
     for (auto& p : node_map_) {
-      BaseNode* node = p.second;
-      if (node->RefIfNonZero()) {
-        nodes.emplace_back(node);
+      RefCountedPtr<BaseNode> node = p.second->RefIfNonZero();
+      if (node != nullptr) {
+        nodes.emplace_back(std::move(node));
       }
     }
   }

data/src/core/lib/channel/channelz_registry.h

@@ -23,6 +23,9 @@
 
 #include <stdint.h>
 
+#include <map>
+#include <string>
+
 #include "src/core/lib/channel/channel_trace.h"
 #include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/gprpp/map.h"

data/src/core/lib/gpr/time_precise.cc

@@ -31,6 +31,7 @@
 
 #include "src/core/lib/gpr/time_precise.h"
 
+#ifndef GPR_CYCLE_COUNTER_CUSTOM
 #if GPR_CYCLE_COUNTER_RDTSC_32 || GPR_CYCLE_COUNTER_RDTSC_64
 #if GPR_LINUX
 static bool read_freq_from_kernel(double* freq) {
@@ -163,3 +164,4 @@ gpr_timespec gpr_cycle_counter_sub(gpr_cycle_counter a, gpr_cycle_counter b) {
                       gpr_cycle_counter_to_time(b));
 }
 #endif /* GPR_CYCLE_COUNTER_FALLBACK */
+#endif /* !GPR_CYCLE_COUNTER_CUSTOM */

data/src/core/lib/gpr/time_precise.h

@@ -31,7 +31,10 @@
 // Using gpr_get_cycle_counter() is preferred to using ExecCtx::Get()->Now()
 // whenever possible.
 
-#if GPR_CYCLE_COUNTER_RDTSC_32
+#if GPR_CYCLE_COUNTER_CUSTOM
+typedef int64_t gpr_cycle_counter;
+gpr_cycle_counter gpr_get_cycle_counter();
+#elif GPR_CYCLE_COUNTER_RDTSC_32
 typedef int64_t gpr_cycle_counter;
 inline gpr_cycle_counter gpr_get_cycle_counter() {
   int64_t ret;
@@ -54,7 +57,8 @@ gpr_cycle_counter gpr_get_cycle_counter();
 #else
 #error Must define exactly one of \
     GPR_CYCLE_COUNTER_RDTSC_32, \
-    GPR_CYCLE_COUNTER_RDTSC_64, or \
+    GPR_CYCLE_COUNTER_RDTSC_64, \
+    GPR_CYCLE_COUNTER_CUSTOM, or \
     GPR_CYCLE_COUNTER_FALLBACK
 #endif
 

data/src/core/lib/gprpp/dual_ref_counted.h

@@ -0,0 +1,336 @@
+//
+// Copyright 2020 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef GRPC_CORE_LIB_GPRPP_DUAL_REF_COUNTED_H
+#define GRPC_CORE_LIB_GPRPP_DUAL_REF_COUNTED_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/support/atm.h>
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+
+#include <atomic>
+#include <cassert>
+#include <cinttypes>
+
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gprpp/atomic.h"
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+
+namespace grpc_core {
+
+// DualRefCounted is an interface for reference-counted objects with two
+// classes of refs: strong refs (usually just called "refs") and weak refs.
+// This supports cases where an object needs to start shutting down when
+// all external callers are done with it (represented by strong refs) but
+// cannot be destroyed until all internal callbacks are complete
+// (represented by weak refs).
+//
+// Each class of refs can be incremented and decremented independently.
+// Objects start with 1 strong ref and 0 weak refs at instantiation.
+// When the strong refcount reaches 0, the object's Orphan() method is called.
+// When the weak refcount reaches 0, the object is destroyed.
+//
+// This will be used by CRTP (curiously-recurring template pattern), e.g.:
+//   class MyClass : public RefCounted<MyClass> { ... };
+template <typename Child>
+class DualRefCounted : public Orphanable {
+ public:
+  virtual ~DualRefCounted() = default;
+
+  RefCountedPtr<Child> Ref() GRPC_MUST_USE_RESULT {
+    IncrementRefCount();
+    return RefCountedPtr<Child>(static_cast<Child*>(this));
+  }
+
+  RefCountedPtr<Child> Ref(const DebugLocation& location,
+                           const char* reason) GRPC_MUST_USE_RESULT {
+    IncrementRefCount(location, reason);
+    return RefCountedPtr<Child>(static_cast<Child*>(this));
+  }
+
+  void Unref() {
+    // Convert strong ref to weak ref.
+    const uint64_t prev_ref_pair =
+        refs_.FetchAdd(MakeRefPair(-1, 1), MemoryOrder::ACQ_REL);
+    const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+#ifndef NDEBUG
+    const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+    if (trace_flag_ != nullptr && trace_flag_->enabled()) {
+      gpr_log(GPR_INFO, "%s:%p unref %d -> %d, weak_ref %d -> %d",
+              trace_flag_->name(), this, strong_refs, strong_refs - 1,
+              weak_refs, weak_refs + 1);
+    }
+    GPR_ASSERT(strong_refs > 0);
+#endif
+    if (GPR_UNLIKELY(strong_refs == 1)) {
+      Orphan();
+    }
+    // Now drop the weak ref.
+    WeakUnref();
+  }
+  void Unref(const DebugLocation& location, const char* reason) {
+    const uint64_t prev_ref_pair =
+        refs_.FetchAdd(MakeRefPair(-1, 1), MemoryOrder::ACQ_REL);
+    const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+#ifndef NDEBUG
+    const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+    if (trace_flag_ != nullptr && trace_flag_->enabled()) {
+      gpr_log(GPR_INFO, "%s:%p %s:%d unref %d -> %d, weak_ref %d -> %d) %s",
+              trace_flag_->name(), this, location.file(), location.line(),
+              strong_refs, strong_refs - 1, weak_refs, weak_refs + 1, reason);
+    }
+    GPR_ASSERT(strong_refs > 0);
+#else
+    // Avoid unused-parameter warnings for debug-only parameters
+    (void)location;
+    (void)reason;
+#endif
+    if (GPR_UNLIKELY(strong_refs == 1)) {
+      Orphan();
+    }
+    // Now drop the weak ref.
+    WeakUnref(location, reason);
+  }
+
+  RefCountedPtr<Child> RefIfNonZero() GRPC_MUST_USE_RESULT {
+    uint64_t prev_ref_pair = refs_.Load(MemoryOrder::ACQUIRE);
+    do {
+      const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+#ifndef NDEBUG
+      const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+      if (trace_flag_ != nullptr && trace_flag_->enabled()) {
+        gpr_log(GPR_INFO, "%s:%p ref_if_non_zero %d -> %d (weak_refs=%d)",
+                trace_flag_->name(), this, strong_refs, strong_refs + 1,
+                weak_refs);
+      }
+#endif
+      if (strong_refs == 0) return nullptr;
+    } while (!refs_.CompareExchangeWeak(
+        &prev_ref_pair, prev_ref_pair + MakeRefPair(1, 0), MemoryOrder::ACQ_REL,
+        MemoryOrder::ACQUIRE));
+    return RefCountedPtr<Child>(static_cast<Child*>(this));
+  }
+
+  RefCountedPtr<Child> RefIfNonZero(const DebugLocation& location,
+                                    const char* reason) GRPC_MUST_USE_RESULT {
+    uint64_t prev_ref_pair = refs_.Load(MemoryOrder::ACQUIRE);
+    do {
+      const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+#ifndef NDEBUG
+      const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+      if (trace_flag_ != nullptr && trace_flag_->enabled()) {
+        gpr_log(GPR_INFO,
+                "%s:%p %s:%d ref_if_non_zero %d -> %d (weak_refs=%d) %s",
+                trace_flag_->name(), this, location.file(), location.line(),
+                strong_refs, strong_refs + 1, weak_refs, reason);
+      }
+#else
+      // Avoid unused-parameter warnings for debug-only parameters
+      (void)location;
+      (void)reason;
+#endif
+      if (strong_refs == 0) return nullptr;
+    } while (!refs_.CompareExchangeWeak(
+        &prev_ref_pair, prev_ref_pair + MakeRefPair(1, 0), MemoryOrder::ACQ_REL,
+        MemoryOrder::ACQUIRE));
+    return RefCountedPtr<Child>(static_cast<Child*>(this));
+  }
+
+  WeakRefCountedPtr<Child> WeakRef() GRPC_MUST_USE_RESULT {
+    IncrementWeakRefCount();
+    return WeakRefCountedPtr<Child>(static_cast<Child*>(this));
+  }
+
+  WeakRefCountedPtr<Child> WeakRef(const DebugLocation& location,
+                                   const char* reason) GRPC_MUST_USE_RESULT {
+    IncrementWeakRefCount(location, reason);
+    return WeakRefCountedPtr<Child>(static_cast<Child*>(this));
+  }
+
+  void WeakUnref() {
+#ifndef NDEBUG
+    // Grab a copy of the trace flag before the atomic change, since we
+    // can't safely access it afterwards if we're going to be freed.
+    auto* trace_flag = trace_flag_;
+#endif
+    const uint64_t prev_ref_pair =
+        refs_.FetchSub(MakeRefPair(0, 1), MemoryOrder::ACQ_REL);
+    const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+#ifndef NDEBUG
+    const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+    if (trace_flag != nullptr && trace_flag->enabled()) {
+      gpr_log(GPR_INFO, "%s:%p weak_unref %d -> %d (refs=%d)",
+              trace_flag->name(), this, weak_refs, weak_refs - 1, strong_refs);
+    }
+    GPR_ASSERT(weak_refs > 0);
+#endif
+    if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) {
+      delete static_cast<Child*>(this);
+    }
+  }
+  void WeakUnref(const DebugLocation& location, const char* reason) {
+#ifndef NDEBUG
+    // Grab a copy of the trace flag before the atomic change, since we
+    // can't safely access it afterwards if we're going to be freed.
+    auto* trace_flag = trace_flag_;
+#endif
+    const uint64_t prev_ref_pair =
+        refs_.FetchSub(MakeRefPair(0, 1), MemoryOrder::ACQ_REL);
+    const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+#ifndef NDEBUG
+    const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+    if (trace_flag != nullptr && trace_flag->enabled()) {
+      gpr_log(GPR_INFO, "%s:%p %s:%d weak_unref %d -> %d (refs=%d) %s",
+              trace_flag->name(), this, location.file(), location.line(),
+              weak_refs, weak_refs - 1, strong_refs, reason);
+    }
+    GPR_ASSERT(weak_refs > 0);
+#else
+    // Avoid unused-parameter warnings for debug-only parameters
+    (void)location;
+    (void)reason;
+#endif
+    if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) {
+      delete static_cast<Child*>(this);
+    }
+  }
+
+  // Not copyable nor movable.
+  DualRefCounted(const DualRefCounted&) = delete;
+  DualRefCounted& operator=(const DualRefCounted&) = delete;
+
+ protected:
+  // TraceFlagT is defined to accept both DebugOnlyTraceFlag and TraceFlag.
+  // Note: RefCount tracing is only enabled on debug builds, even when a
+  //       TraceFlag is used.
+  template <typename TraceFlagT = TraceFlag>
+  explicit DualRefCounted(
+      TraceFlagT*
+#ifndef NDEBUG
+          // Leave unnamed if NDEBUG to avoid unused parameter warning
+          trace_flag
+#endif
+      = nullptr,
+      int32_t initial_refcount = 1)
+      :
+#ifndef NDEBUG
+        trace_flag_(trace_flag),
+#endif
+        refs_(MakeRefPair(initial_refcount, 0)) {
+  }
+
+ private:
+  // Allow RefCountedPtr<> to access IncrementRefCount().
+  template <typename T>
+  friend class RefCountedPtr;
+  // Allow WeakRefCountedPtr<> to access IncrementWeakRefCount().
+  template <typename T>
+  friend class WeakRefCountedPtr;
+
+  // First 32 bits are strong refs, next 32 bits are weak refs.
+  static uint64_t MakeRefPair(uint32_t strong, uint32_t weak) {
+    return (static_cast<uint64_t>(strong) << 32) + static_cast<int64_t>(weak);
+  }
+  static uint32_t GetStrongRefs(uint64_t ref_pair) {
+    return static_cast<uint32_t>(ref_pair >> 32);
+  }
+  static uint32_t GetWeakRefs(uint64_t ref_pair) {
+    return static_cast<uint32_t>(ref_pair & 0xffffffffu);
+  }
+
+  void IncrementRefCount() {
+#ifndef NDEBUG
+    const uint64_t prev_ref_pair =
+        refs_.FetchAdd(MakeRefPair(1, 0), MemoryOrder::RELAXED);
+    const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+    const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+    GPR_ASSERT(strong_refs != 0);
+    if (trace_flag_ != nullptr && trace_flag_->enabled()) {
+      gpr_log(GPR_INFO, "%s:%p ref %d -> %d; (weak_refs=%d)",
+              trace_flag_->name(), this, strong_refs, strong_refs + 1,
+              weak_refs);
+    }
+#else
+    refs_.FetchAdd(MakeRefPair(1, 0), MemoryOrder::RELAXED);
+#endif
+  }
+  void IncrementRefCount(const DebugLocation& location, const char* reason) {
+#ifndef NDEBUG
+    const uint64_t prev_ref_pair =
+        refs_.FetchAdd(MakeRefPair(1, 0), MemoryOrder::RELAXED);
+    const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+    const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+    GPR_ASSERT(strong_refs != 0);
+    if (trace_flag_ != nullptr && trace_flag_->enabled()) {
+      gpr_log(GPR_INFO, "%s:%p %s:%d ref %d -> %d (weak_refs=%d) %s",
+              trace_flag_->name(), this, location.file(), location.line(),
+              strong_refs, strong_refs + 1, weak_refs, reason);
+    }
+#else
+    // Use conditionally-important parameters
+    (void)location;
+    (void)reason;
+    refs_.FetchAdd(MakeRefPair(1, 0), MemoryOrder::RELAXED);
+#endif
+  }
+
+  void IncrementWeakRefCount() {
+#ifndef NDEBUG
+    const uint64_t prev_ref_pair =
+        refs_.FetchAdd(MakeRefPair(0, 1), MemoryOrder::RELAXED);
+    const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+    const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+    if (trace_flag_ != nullptr && trace_flag_->enabled()) {
+      gpr_log(GPR_INFO, "%s:%p weak_ref %d -> %d; (refs=%d)",
+              trace_flag_->name(), this, weak_refs, weak_refs + 1, strong_refs);
+    }
+#else
+    refs_.FetchAdd(MakeRefPair(0, 1), MemoryOrder::RELAXED);
+#endif
+  }
+  void IncrementWeakRefCount(const DebugLocation& location,
+                             const char* reason) {
+#ifndef NDEBUG
+    const uint64_t prev_ref_pair =
+        refs_.FetchAdd(MakeRefPair(0, 1), MemoryOrder::RELAXED);
+    const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
+    const uint32_t weak_refs = GetWeakRefs(prev_ref_pair);
+    if (trace_flag_ != nullptr && trace_flag_->enabled()) {
+      gpr_log(GPR_INFO, "%s:%p %s:%d weak_ref %d -> %d (refs=%d) %s",
+              trace_flag_->name(), this, location.file(), location.line(),
+              weak_refs, weak_refs + 1, strong_refs, reason);
+    }
+#else
+    // Use conditionally-important parameters
+    (void)location;
+    (void)reason;
+    refs_.FetchAdd(MakeRefPair(0, 1), MemoryOrder::RELAXED);
+#endif
+  }
+
+#ifndef NDEBUG
+  TraceFlag* trace_flag_;
+#endif
+  Atomic<uint64_t> refs_;
+};
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_LIB_GPRPP_DUAL_REF_COUNTED_H */