grpc 1.57.0 → 1.58.0.pre1

data/src/core/lib/experiments/experiments.h

@@ -51,7 +51,6 @@
 #include <grpc/support/port_platform.h>
 
 #include <stddef.h>
-
 #include "src/core/lib/experiments/config.h"
 
 namespace grpc_core {
@@ -79,8 +78,10 @@ inline bool IsWorkStealingEnabled() { return false; }
 inline bool IsClientPrivacyEnabled() { return false; }
 inline bool IsCanaryClientPrivacyEnabled() { return false; }
 inline bool IsServerPrivacyEnabled() { return false; }
-inline bool IsUniqueMetadataStringsEnabled() { return false; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUE_METADATA_STRINGS
+inline bool IsUniqueMetadataStringsEnabled() { return true; }
 inline bool IsKeepaliveFixEnabled() { return false; }
+inline bool IsKeepaliveServerFixEnabled() { return false; }
 
 #elif defined(GPR_WINDOWS)
 inline bool IsTcpFrameSizeTuningEnabled() { return false; }
@@ -103,8 +104,10 @@ inline bool IsWorkStealingEnabled() { return false; }
 inline bool IsClientPrivacyEnabled() { return false; }
 inline bool IsCanaryClientPrivacyEnabled() { return false; }
 inline bool IsServerPrivacyEnabled() { return false; }
-inline bool IsUniqueMetadataStringsEnabled() { return false; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUE_METADATA_STRINGS
+inline bool IsUniqueMetadataStringsEnabled() { return true; }
 inline bool IsKeepaliveFixEnabled() { return false; }
+inline bool IsKeepaliveServerFixEnabled() { return false; }
 
 #else
 inline bool IsTcpFrameSizeTuningEnabled() { return false; }
@@ -127,8 +130,10 @@ inline bool IsWorkStealingEnabled() { return false; }
 inline bool IsClientPrivacyEnabled() { return false; }
 inline bool IsCanaryClientPrivacyEnabled() { return false; }
 inline bool IsServerPrivacyEnabled() { return false; }
-inline bool IsUniqueMetadataStringsEnabled() { return false; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUE_METADATA_STRINGS
+inline bool IsUniqueMetadataStringsEnabled() { return true; }
 inline bool IsKeepaliveFixEnabled() { return false; }
+inline bool IsKeepaliveServerFixEnabled() { return false; }
 #endif
 
 #else
@@ -139,13 +144,9 @@ inline bool IsTcpRcvLowatEnabled() { return IsExperimentEnabled(1); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_PEER_STATE_BASED_FRAMING
 inline bool IsPeerStateBasedFramingEnabled() { return IsExperimentEnabled(2); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_MEMORY_PRESSURE_CONTROLLER
-inline bool IsMemoryPressureControllerEnabled() {
-  return IsExperimentEnabled(3);
-}
+inline bool IsMemoryPressureControllerEnabled() { return IsExperimentEnabled(3); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_UNCONSTRAINED_MAX_QUOTA_BUFFER_SIZE
-inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() {
-  return IsExperimentEnabled(4);
-}
+inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() { return IsExperimentEnabled(4); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_CLIENT
 inline bool IsEventEngineClientEnabled() { return IsExperimentEnabled(5); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_MONITORING_EXPERIMENT
@@ -157,15 +158,11 @@ inline bool IsFreeLargeAllocatorEnabled() { return IsExperimentEnabled(8); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
 inline bool IsPromiseBasedServerCallEnabled() { return IsExperimentEnabled(9); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_TRANSPORT_SUPPLIES_CLIENT_LATENCY
-inline bool IsTransportSuppliesClientLatencyEnabled() {
-  return IsExperimentEnabled(10);
-}
+inline bool IsTransportSuppliesClientLatencyEnabled() { return IsExperimentEnabled(10); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_LISTENER
 inline bool IsEventEngineListenerEnabled() { return IsExperimentEnabled(11); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_SCHEDULE_CANCELLATION_OVER_WRITE
-inline bool IsScheduleCancellationOverWriteEnabled() {
-  return IsExperimentEnabled(12);
-}
+inline bool IsScheduleCancellationOverWriteEnabled() { return IsExperimentEnabled(12); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_TRACE_RECORD_CALLOPS
 inline bool IsTraceRecordCallopsEnabled() { return IsExperimentEnabled(13); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_DNS
@@ -182,8 +179,10 @@ inline bool IsServerPrivacyEnabled() { return IsExperimentEnabled(18); }
 inline bool IsUniqueMetadataStringsEnabled() { return IsExperimentEnabled(19); }
 #define GRPC_EXPERIMENT_IS_INCLUDED_KEEPALIVE_FIX
 inline bool IsKeepaliveFixEnabled() { return IsExperimentEnabled(20); }
+#define GRPC_EXPERIMENT_IS_INCLUDED_KEEPALIVE_SERVER_FIX
+inline bool IsKeepaliveServerFixEnabled() { return IsExperimentEnabled(21); }
 
-constexpr const size_t kNumExperiments = 21;
+constexpr const size_t kNumExperiments = 22;
 extern const ExperimentMetadata g_experiment_metadata[kNumExperiments];
 
 #endif

data/src/core/lib/gprpp/ref_counted.h

@@ -45,12 +45,14 @@ class RefCount {
  public:
   using Value = intptr_t;
 
+  RefCount() : RefCount(1) {}
+
   // `init` is the initial refcount stored in this object.
   //
   // `trace` is a string to be logged with trace events; if null, no
   // trace logging will be done.  Tracing is a no-op in non-debug builds.
   explicit RefCount(
-      Value init = 1,
+      Value init,
       const char*
 #ifndef NDEBUG
           // Leave unnamed if NDEBUG to avoid unused parameter warning

data/src/core/lib/gprpp/ref_counted_string.cc

@@ -0,0 +1,44 @@
+//
+// Copyright 2023 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/gprpp/ref_counted_string.h"
+
+#include <string.h>
+
+#include <new>
+
+#include <grpc/support/alloc.h>
+
+namespace grpc_core {
+
+RefCountedPtr<RefCountedString> RefCountedString::Make(absl::string_view src) {
+  void* p = gpr_malloc(sizeof(Header) + src.length() + 1);
+  return RefCountedPtr<RefCountedString>(new (p) RefCountedString(src));
+}
+
+RefCountedString::RefCountedString(absl::string_view src)
+    : header_{{}, src.length()} {
+  memcpy(payload_, src.data(), header_.length);
+  // Null terminate because we frequently need to convert to char* still to go
+  // back and forth to the old c-style api.
+  payload_[header_.length] = 0;
+}
+
+void RefCountedString::Destroy() { gpr_free(this); }
+
+}  // namespace grpc_core

data/src/core/lib/gprpp/ref_counted_string.h

@@ -0,0 +1,146 @@
+//
+// Copyright 2023 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef GRPC_SRC_CORE_LIB_GPRPP_REF_COUNTED_STRING_H
+#define GRPC_SRC_CORE_LIB_GPRPP_REF_COUNTED_STRING_H
+
+#include <grpc/support/port_platform.h>
+
+#include <stddef.h>
+
+#include "absl/strings/string_view.h"
+
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+
+namespace grpc_core {
+
+// An immutable reference counted string.
+class RefCountedString {
+ public:
+  static RefCountedPtr<RefCountedString> Make(absl::string_view src);
+
+  // Not copyable.
+  RefCountedString(const RefCountedString&) = delete;
+  RefCountedString& operator=(const RefCountedString&) = delete;
+
+  // Provide the same interface as RefCounted<>.
+  // We reimplement this instead of inheritting to make pointer math
+  // easier in Make().
+  RefCountedPtr<RefCountedString> Ref() {
+    IncrementRefCount();
+    return RefCountedPtr<RefCountedString>(this);
+  }
+  void Unref() {
+    if (header_.rc.Unref()) Destroy();
+  }
+
+  absl::string_view as_string_view() const {
+    return absl::string_view(payload_, header_.length);
+  }
+
+  char* c_str() { return payload_; }
+
+ private:
+  // Allow RefCountedPtr<> to access IncrementRefCount().
+  template <typename T>
+  friend class RefCountedPtr;
+
+  explicit RefCountedString(absl::string_view src);
+  void IncrementRefCount() { header_.rc.Ref(); }
+  void Destroy();
+
+  struct Header {
+    RefCount rc;
+    size_t length;
+  };
+  Header header_;
+  char payload_[];
+};
+
+// Wrapper around RefCountedPtr<RefCountedString> to give value semantics,
+// especially to overloaded operators.
+class RefCountedStringValue {
+ public:
+  RefCountedStringValue() : str_{} {}
+  explicit RefCountedStringValue(absl::string_view str)
+      : str_(RefCountedString::Make(str)) {}
+
+  absl::string_view as_string_view() const {
+    return str_ == nullptr ? absl::string_view() : str_->as_string_view();
+  }
+
+  const char* c_str() const { return str_ == nullptr ? "" : str_->c_str(); }
+
+ private:
+  RefCountedPtr<RefCountedString> str_;
+};
+
+inline bool operator==(const RefCountedStringValue& lhs,
+                       absl::string_view rhs) {
+  return lhs.as_string_view() == rhs;
+}
+inline bool operator==(absl::string_view lhs,
+                       const RefCountedStringValue& rhs) {
+  return lhs == rhs.as_string_view();
+}
+inline bool operator==(const RefCountedStringValue& lhs,
+                       const RefCountedStringValue& rhs) {
+  return lhs.as_string_view() == rhs.as_string_view();
+}
+
+inline bool operator<(const RefCountedStringValue& lhs, absl::string_view rhs) {
+  return lhs.as_string_view() < rhs;
+}
+inline bool operator<(absl::string_view lhs, const RefCountedStringValue& rhs) {
+  return lhs < rhs.as_string_view();
+}
+inline bool operator<(const RefCountedStringValue& lhs,
+                      const RefCountedStringValue& rhs) {
+  return lhs.as_string_view() < rhs.as_string_view();
+}
+
+inline bool operator>(const RefCountedStringValue& lhs, absl::string_view rhs) {
+  return lhs.as_string_view() > rhs;
+}
+inline bool operator>(absl::string_view lhs, const RefCountedStringValue& rhs) {
+  return lhs > rhs.as_string_view();
+}
+inline bool operator>(const RefCountedStringValue& lhs,
+                      const RefCountedStringValue& rhs) {
+  return lhs.as_string_view() > rhs.as_string_view();
+}
+
+// A sorting functor to support heterogeneous lookups in sorted containers.
+struct RefCountedStringValueLessThan {
+  using is_transparent = void;
+  bool operator()(const RefCountedStringValue& lhs,
+                  const RefCountedStringValue& rhs) const {
+    return lhs < rhs;
+  }
+  bool operator()(absl::string_view lhs,
+                  const RefCountedStringValue& rhs) const {
+    return lhs < rhs;
+  }
+  bool operator()(const RefCountedStringValue& lhs,
+                  absl::string_view rhs) const {
+    return lhs < rhs;
+  }
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_GPRPP_REF_COUNTED_STRING_H

data/src/core/lib/gprpp/time.h

@@ -37,8 +37,8 @@
     uint64_t now = grpc_core::Timestamp::FromTimespecRoundDown( \
                        gpr_now(GPR_CLOCK_MONOTONIC))            \
                        .milliseconds_after_process_epoch();     \
-    uint64_t prev_tsamp = prev.exchange(now);                   \
-    if (prev_tsamp == 0 || now - prev_tsamp > (n)*1000) {       \
+    if (prev == 0 || now - prev > (n)*1000) {                   \
+      prev = now;                                               \
       gpr_log(severity, format, __VA_ARGS__);                   \
     }                                                           \
   } while (0)

data/src/core/lib/gprpp/work_serializer.cc

@@ -23,6 +23,7 @@
 #include <atomic>
 #include <functional>
 #include <memory>
+#include <thread>
 #include <utility>
 
 #include <grpc/support/log.h>
@@ -47,6 +48,12 @@ class WorkSerializer::WorkSerializerImpl : public Orphanable {
   void DrainQueue();
   void Orphan() override;
 
+#ifndef NDEBUG
+  bool RunningInWorkSerializer() const {
+    return std::this_thread::get_id() == current_thread_;
+  }
+#endif
+
  private:
   struct CallbackWrapper {
     CallbackWrapper(std::function<void()> cb, const DebugLocation& loc)
@@ -86,6 +93,9 @@ class WorkSerializer::WorkSerializerImpl : public Orphanable {
   // orphaned.
   std::atomic<uint64_t> refs_{MakeRefPair(0, 1)};
   MultiProducerSingleConsumerQueue queue_;
+#ifndef NDEBUG
+  std::thread::id current_thread_;
+#endif
 };
 
 void WorkSerializer::WorkSerializerImpl::Run(std::function<void()> callback,
@@ -102,10 +112,17 @@ void WorkSerializer::WorkSerializerImpl::Run(std::function<void()> callback,
   GPR_DEBUG_ASSERT(GetSize(prev_ref_pair) > 0);
   if (GetOwners(prev_ref_pair) == 0) {
     // We took ownership of the WorkSerializer. Invoke callback and drain queue.
+#ifndef NDEBUG
+    current_thread_ = std::this_thread::get_id();
+#endif
     if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
       gpr_log(GPR_INFO, "  Executing immediately");
     }
     callback();
+    // Delete the callback while still holding the WorkSerializer, so
+    // that any refs being held by the callback via lambda captures will
+    // be destroyed inside the WorkSerializer.
+    callback = nullptr;
     DrainQueueOwned();
   } else {
     // Another thread is holding the WorkSerializer, so decrement the ownership
@@ -158,6 +175,9 @@ void WorkSerializer::WorkSerializerImpl::DrainQueue() {
   const uint64_t prev_ref_pair =
       refs_.fetch_add(MakeRefPair(1, 1), std::memory_order_acq_rel);
   if (GetOwners(prev_ref_pair) == 0) {
+#ifndef NDEBUG
+    current_thread_ = std::this_thread::get_id();
+#endif
     // We took ownership of the WorkSerializer. Drain the queue.
     DrainQueueOwned();
   } else {
@@ -186,6 +206,12 @@ void WorkSerializer::WorkSerializerImpl::DrainQueueOwned() {
     }
     if (GetSize(prev_ref_pair) == 2) {
       // Queue drained. Give up ownership but only if queue remains empty.
+#ifndef NDEBUG
+      // Reset current_thread_ before giving up ownership to avoid TSAN
+      // race.  If we don't wind up giving up ownership, we'll set this
+      // again below before we pull the next callback out of the queue.
+      current_thread_ = std::thread::id();
+#endif
       uint64_t expected = MakeRefPair(1, 1);
       if (refs_.compare_exchange_strong(expected, MakeRefPair(0, 1),
                                         std::memory_order_acq_rel)) {
@@ -200,6 +226,10 @@ void WorkSerializer::WorkSerializerImpl::DrainQueueOwned() {
         delete this;
         return;
       }
+#ifndef NDEBUG
+      // Didn't wind up giving up ownership, so set current_thread_ again.
+      current_thread_ = std::this_thread::get_id();
+#endif
     }
     // There is at least one callback on the queue. Pop the callback from the
     // queue and execute it.
@@ -244,4 +274,10 @@ void WorkSerializer::Schedule(std::function<void()> callback,
 
 void WorkSerializer::DrainQueue() { impl_->DrainQueue(); }
 
+#ifndef NDEBUG
+bool WorkSerializer::RunningInWorkSerializer() const {
+  return impl_->RunningInWorkSerializer();
+}
+#endif
+
 }  // namespace grpc_core

data/src/core/lib/gprpp/work_serializer.h

@@ -75,6 +75,11 @@ class ABSL_LOCKABLE WorkSerializer {
   // Drains the queue of callbacks.
   void DrainQueue();
 
+#ifndef NDEBUG
+  // Returns true if the current thread is running in the WorkSerializer.
+  bool RunningInWorkSerializer() const;
+#endif
+
  private:
   class WorkSerializerImpl;
 

data/src/core/lib/http/httpcli_security_connector.cc

@@ -29,6 +29,7 @@
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
+#include <grpc/impl/channel_arg_names.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>

data/src/core/lib/iomgr/buffer_list.cc

@@ -278,6 +278,8 @@ void TracedBufferList::ProcessTimestamp(struct sock_extended_err* serr,
       elem = elem->next_;
       continue;
     }
+    g_timestamps_callback(elem->arg_, &(elem->ts_),
+                          absl::DeadlineExceededError("Ack timed out"));
     if (prev != nullptr) {
       prev->next_ = elem->next_;
       delete elem;

data/src/core/lib/iomgr/event_engine_shims/endpoint.cc

@@ -74,15 +74,9 @@ class EventEngineEndpointWrapper {
     return fd_;
   }
 
-  absl::string_view PeerAddress() {
-    grpc_core::MutexLock lock(&mu_);
-    return peer_address_;
-  }
+  absl::string_view PeerAddress() { return peer_address_; }
 
-  absl::string_view LocalAddress() {
-    grpc_core::MutexLock lock(&mu_);
-    return local_address_;
-  }
+  absl::string_view LocalAddress() { return local_address_; }
 
   void Ref() { refs_.fetch_add(1, std::memory_order_relaxed); }
   void Unref() {
@@ -119,8 +113,7 @@ class EventEngineEndpointWrapper {
     read_buffer->~SliceBuffer();
     if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       size_t i;
-      gpr_log(GPR_INFO, "TCP: %p READ (peer=%s) error=%s", eeep_->wrapper,
-              std::string(eeep_->wrapper->PeerAddress()).c_str(),
+      gpr_log(GPR_INFO, "TCP: %p READ error=%s", eeep_->wrapper,
               status.ToString().c_str());
       if (gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) {
         for (i = 0; i < pending_read_buffer_->count; i++) {
@@ -270,8 +263,6 @@ class EventEngineEndpointWrapper {
     {
       grpc_core::MutexLock lock(&mu_);
       fd_ = -1;
-      local_address_ = "";
-      peer_address_ = "";
     }
     endpoint_.reset();
     // For the Ref taken in TriggerShutdown
@@ -286,8 +277,10 @@ class EventEngineEndpointWrapper {
   grpc_closure* pending_read_cb_;
   grpc_closure* pending_write_cb_;
   grpc_slice_buffer* pending_read_buffer_;
-  std::string peer_address_;
-  std::string local_address_;
+  const std::string peer_address_{
+      ResolvedAddressToURI(endpoint_->GetPeerAddress()).value_or("")};
+  const std::string local_address_{
+      ResolvedAddressToURI(endpoint_->GetLocalAddress()).value_or("")};
   int fd_{-1};
 };
 
@@ -413,14 +406,6 @@ EventEngineEndpointWrapper::EventEngineEndpointWrapper(
       eeep_(std::make_unique<grpc_event_engine_endpoint>()) {
   eeep_->base.vtable = &grpc_event_engine_endpoint_vtable;
   eeep_->wrapper = this;
-  auto local_addr = ResolvedAddressToURI(endpoint_->GetLocalAddress());
-  if (local_addr.ok()) {
-    local_address_ = *local_addr;
-  }
-  auto peer_addr = ResolvedAddressToURI(endpoint_->GetPeerAddress());
-  if (peer_addr.ok()) {
-    peer_address_ = *peer_addr;
-  }
   if (EventEngineSupportsFd()) {
     fd_ = reinterpret_cast<PosixEndpointWithFdSupport*>(endpoint_.get())
               ->GetWrappedFd();

data/src/core/lib/iomgr/tcp_posix.cc

@@ -1057,7 +1057,7 @@ static void maybe_make_read_slices(grpc_tcp* tcp)
   static const int kBigAlloc = 64 * 1024;
   static const int kSmallAlloc = 8 * 1024;
   if (tcp->incoming_buffer->length <
-      static_cast<size_t>(tcp->min_progress_size)) {
+      std::max<size_t>(tcp->min_progress_size, 1)) {
     size_t allocate_length = tcp->min_progress_size;
     const size_t target_length = static_cast<size_t>(tcp->target_length);
     // If memory pressure is low and we think there will be more than
@@ -1067,8 +1067,8 @@ static void maybe_make_read_slices(grpc_tcp* tcp)
     if (low_memory_pressure && target_length > allocate_length) {
       allocate_length = target_length;
     }
-    int extra_wanted =
-        allocate_length - static_cast<int>(tcp->incoming_buffer->length);
+    int extra_wanted = std::max<int>(
+        1, allocate_length - static_cast<int>(tcp->incoming_buffer->length));
     if (extra_wanted >=
         (low_memory_pressure ? kSmallAlloc * 3 / 2 : kBigAlloc)) {
       while (extra_wanted > 0) {