grpc 1.41.0 → 1.43.1

data/src/core/lib/iomgr/buffer_list.h

@@ -96,8 +96,8 @@ struct Timestamps {
   uint32_t byte_offset; /* byte offset relative to the start of the RPC */
 
 #ifdef GRPC_LINUX_ERRQUEUE
-  grpc_core::tcp_info info; /* tcp_info collected on sendmsg */
-#endif                      /* GRPC_LINUX_ERRQUEUE */
+  tcp_info info; /* tcp_info collected on sendmsg */
+#endif           /* GRPC_LINUX_ERRQUEUE */
 };
 
 /** TracedBuffer is a class to keep track of timestamps for a specific buffer in
@@ -105,7 +105,7 @@ struct Timestamps {
  * this class would only be used by Linux platforms. For all other platforms,
  * TracedBuffer would be an empty class.
  *
- * The timestamps collected are according to grpc_core::Timestamps declared
+ * The timestamps collected are according to Timestamps declared
  * above.
  *
  * A TracedBuffer list is kept track of using the head element of the list. If
@@ -120,33 +120,33 @@ class TracedBuffer {
 
   /** Add a new entry in the TracedBuffer list pointed to by head. Also saves
    * sendmsg_time with the current timestamp. */
-  static void AddNewEntry(grpc_core::TracedBuffer** head, uint32_t seq_no,
-                          int fd, void* arg);
+  static void AddNewEntry(TracedBuffer** head, uint32_t seq_no, int fd,
+                          void* arg);
 
   /** Processes a received timestamp based on sock_extended_err and
    * scm_timestamping structures. It will invoke the timestamps callback if the
    * timestamp type is SCM_TSTAMP_ACK. */
-  static void ProcessTimestamp(grpc_core::TracedBuffer** head,
+  static void ProcessTimestamp(TracedBuffer** head,
                                struct sock_extended_err* serr,
                                struct cmsghdr* opt_stats,
                                struct scm_timestamping* tss);
 
   /** Cleans the list by calling the callback for each traced buffer in the list
    * with timestamps that it has. */
-  static void Shutdown(grpc_core::TracedBuffer** head, void* remaining,
+  static void Shutdown(TracedBuffer** head, void* remaining,
                        grpc_error_handle shutdown_err);
 
  private:
-  uint32_t seq_no_; /* The sequence number for the last byte in the buffer */
-  void* arg_;       /* The arg to pass to timestamps_callback */
-  grpc_core::Timestamps ts_; /* The timestamps corresponding to this buffer */
-  grpc_core::TracedBuffer* next_; /* The next TracedBuffer in the list */
+  uint32_t seq_no_;    /* The sequence number for the last byte in the buffer */
+  void* arg_;          /* The arg to pass to timestamps_callback */
+  Timestamps ts_;      /* The timestamps corresponding to this buffer */
+  TracedBuffer* next_; /* The next TracedBuffer in the list */
 };
 #else  /* GRPC_LINUX_ERRQUEUE */
 class TracedBuffer {
  public:
   /* Phony shutdown function */
-  static void Shutdown(grpc_core::TracedBuffer** /*head*/, void* /*remaining*/,
+  static void Shutdown(TracedBuffer** /*head*/, void* /*remaining*/,
                        grpc_error_handle shutdown_err) {
     GRPC_ERROR_UNREF(shutdown_err);
   }
@@ -156,7 +156,7 @@ class TracedBuffer {
 /** Sets the callback function to call when timestamps for a write are
  *  collected. The callback does not own a reference to error. */
 void grpc_tcp_set_write_timestamps_callback(
-    void (*fn)(void*, grpc_core::Timestamps*, grpc_error_handle error));
+    void (*fn)(void*, Timestamps*, grpc_error_handle error));
 
 } /* namespace grpc_core */
 

data/src/core/lib/iomgr/call_combiner.cc

@@ -33,18 +33,20 @@ DebugOnlyTraceFlag grpc_call_combiner_trace(false, "call_combiner");
 
 namespace {
 
+// grpc_error LSB can be used
+constexpr intptr_t kErrorBit = 1;
+
 grpc_error_handle DecodeCancelStateError(gpr_atm cancel_state) {
-  if (cancel_state & 1) {
-    return reinterpret_cast<grpc_error_handle>(cancel_state &
-                                               ~static_cast<gpr_atm>(1));
+  if (cancel_state & kErrorBit) {
+#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
+    return internal::StatusGetFromHeapPtr(cancel_state & ~kErrorBit);
+#else
+    return reinterpret_cast<grpc_error_handle>(cancel_state & ~kErrorBit);
+#endif
   }
   return GRPC_ERROR_NONE;
 }
 
-gpr_atm EncodeCancelStateError(grpc_error_handle error) {
-  return static_cast<gpr_atm>(1) | reinterpret_cast<gpr_atm>(error);
-}
-
 }  // namespace
 
 CallCombiner::CallCombiner() {
@@ -57,7 +59,14 @@ CallCombiner::CallCombiner() {
 }
 
 CallCombiner::~CallCombiner() {
-  GRPC_ERROR_UNREF(DecodeCancelStateError(cancel_state_));
+  if (cancel_state_ & kErrorBit) {
+#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
+    internal::StatusFreeHeapPtr(cancel_state_ & ~kErrorBit);
+#else
+    GRPC_ERROR_UNREF(reinterpret_cast<grpc_error_handle>(
+        cancel_state_ & ~static_cast<gpr_atm>(kErrorBit)));
+#endif
+  }
 }
 
 #ifdef GRPC_TSAN_ENABLED
@@ -82,8 +91,7 @@ void CallCombiner::TsanClosure(void* arg, grpc_error_handle error) {
   } else {
     lock.reset();
   }
-  grpc_core::Closure::Run(DEBUG_LOCATION, self->original_closure_,
-                          GRPC_ERROR_REF(error));
+  Closure::Run(DEBUG_LOCATION, self->original_closure_, GRPC_ERROR_REF(error));
   if (lock != nullptr) {
     TSAN_ANNOTATE_RWLOCK_RELEASED(&lock->taken, true);
     bool prev = true;
@@ -235,15 +243,24 @@ void CallCombiner::SetNotifyOnCancel(grpc_closure* closure) {
 
 void CallCombiner::Cancel(grpc_error_handle error) {
   GRPC_STATS_INC_CALL_COMBINER_CANCELLED();
+#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
+  intptr_t status_ptr = internal::StatusAllocHeapPtr(error);
+  gpr_atm new_state = kErrorBit | status_ptr;
+#else
+  gpr_atm new_state = kErrorBit | reinterpret_cast<gpr_atm>(error);
+#endif
   while (true) {
     gpr_atm original_state = gpr_atm_acq_load(&cancel_state_);
     grpc_error_handle original_error = DecodeCancelStateError(original_state);
     if (original_error != GRPC_ERROR_NONE) {
+#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
+      internal::StatusFreeHeapPtr(status_ptr);
+#else
       GRPC_ERROR_UNREF(error);
+#endif
       break;
     }
-    if (gpr_atm_full_cas(&cancel_state_, original_state,
-                         EncodeCancelStateError(error))) {
+    if (gpr_atm_full_cas(&cancel_state_, original_state, new_state)) {
       if (original_state != 0) {
         grpc_closure* notify_on_cancel =
             reinterpret_cast<grpc_closure*>(original_state);

data/src/core/lib/iomgr/combiner.cc

@@ -27,11 +27,9 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/gprpp/mpscq.h"
 #include "src/core/lib/iomgr/executor.h"
-#include "src/core/lib/iomgr/iomgr.h"
-#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
 
 grpc_core::DebugOnlyTraceFlag grpc_combiner_trace(false, "combiner");
 
@@ -128,15 +126,11 @@ static void push_first_on_exec_ctx(grpc_core::Combiner* lock) {
 
 static void combiner_exec(grpc_core::Combiner* lock, grpc_closure* cl,
                           grpc_error_handle error) {
-  GPR_TIMER_SCOPE("combiner.execute", 0);
-  GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_ITEMS();
   gpr_atm last = gpr_atm_full_fetch_add(&lock->state, STATE_ELEM_COUNT_LOW_BIT);
   GRPC_COMBINER_TRACE(gpr_log(GPR_INFO,
                               "C:%p grpc_combiner_execute c=%p last=%" PRIdPTR,
                               lock, cl, last));
   if (last == 1) {
-    GRPC_STATS_INC_COMBINER_LOCKS_INITIATED();
-    GPR_TIMER_MARK("combiner.initiated", 0);
     gpr_atm_no_barrier_store(
         &lock->initiating_exec_ctx_or_null,
         reinterpret_cast<gpr_atm>(grpc_core::ExecCtx::Get()));
@@ -175,14 +169,12 @@ static void offload(void* arg, grpc_error_handle /*error*/) {
 }
 
 static void queue_offload(grpc_core::Combiner* lock) {
-  GRPC_STATS_INC_COMBINER_LOCKS_OFFLOADED();
   move_next();
   GRPC_COMBINER_TRACE(gpr_log(GPR_INFO, "C:%p queue_offload", lock));
   grpc_core::Executor::Run(&lock->offload, GRPC_ERROR_NONE);
 }
 
 bool grpc_combiner_continue_exec_ctx() {
-  GPR_TIMER_SCOPE("combiner.continue_exec_ctx", 0);
   grpc_core::Combiner* lock =
       grpc_core::ExecCtx::Get()->combiner_data()->active_combiner;
   if (lock == nullptr) {
@@ -207,9 +199,8 @@ bool grpc_combiner_continue_exec_ctx() {
   // 3. the current thread is not a worker for any background poller
   // 4. the DEFAULT executor is threaded
   if (contended && grpc_core::ExecCtx::Get()->IsReadyToFinish() &&
-      !grpc_iomgr_is_any_background_poller_thread() &&
+      !grpc_iomgr_platform_is_any_background_poller_thread() &&
       grpc_core::Executor::IsThreadedDefault()) {
-    GPR_TIMER_MARK("offload_from_finished_exec_ctx", 0);
     // this execution context wants to move on: schedule remaining work to be
     // picked up on the executor
     queue_offload(lock);
@@ -226,11 +217,9 @@ bool grpc_combiner_continue_exec_ctx() {
     if (n == nullptr) {
       // queue is in an inconsistent state: use this as a cue that we should
       // go off and do something else for a while (and come back later)
-      GPR_TIMER_MARK("delay_busy", 0);
       queue_offload(lock);
       return true;
     }
-    GPR_TIMER_SCOPE("combiner.exec1", 0);
     grpc_closure* cl = reinterpret_cast<grpc_closure*>(n);
     grpc_error_handle cl_err = cl->error_data.error;
 #ifndef NDEBUG
@@ -244,7 +233,6 @@ bool grpc_combiner_continue_exec_ctx() {
     grpc_closure_list_init(&lock->final_list);
     int loops = 0;
     while (c != nullptr) {
-      GPR_TIMER_SCOPE("combiner.exec_1final", 0);
       GRPC_COMBINER_TRACE(
           gpr_log(GPR_INFO, "C:%p execute_final[%d] c=%p", lock, loops, c));
       grpc_closure* next = c->next_data.next;
@@ -258,7 +246,6 @@ bool grpc_combiner_continue_exec_ctx() {
     }
   }
 
-  GPR_TIMER_MARK("unref", 0);
   move_next();
   lock->time_to_execute_final_list = false;
   gpr_atm old_state =
@@ -305,13 +292,10 @@ static void combiner_finally_exec(grpc_core::Combiner* lock,
                                   grpc_closure* closure,
                                   grpc_error_handle error) {
   GPR_ASSERT(lock != nullptr);
-  GPR_TIMER_SCOPE("combiner.execute_finally", 0);
-  GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS();
   GRPC_COMBINER_TRACE(gpr_log(
       GPR_INFO, "C:%p grpc_combiner_execute_finally c=%p; ac=%p", lock, closure,
       grpc_core::ExecCtx::Get()->combiner_data()->active_combiner));
   if (grpc_core::ExecCtx::Get()->combiner_data()->active_combiner != lock) {
-    GPR_TIMER_MARK("slowpath", 0);
     // Using error_data.scratch to store the combiner so that it can be accessed
     // in enqueue_finally.
     closure->error_data.scratch = reinterpret_cast<uintptr_t>(lock);
@@ -327,9 +311,10 @@ static void combiner_finally_exec(grpc_core::Combiner* lock,
 
 static void enqueue_finally(void* closure, grpc_error_handle error) {
   grpc_closure* cl = static_cast<grpc_closure*>(closure);
-  combiner_finally_exec(
-      reinterpret_cast<grpc_core::Combiner*>(cl->error_data.scratch), cl,
-      GRPC_ERROR_REF(error));
+  grpc_core::Combiner* lock =
+      reinterpret_cast<grpc_core::Combiner*>(cl->error_data.scratch);
+  cl->error_data.scratch = 0;
+  combiner_finally_exec(lock, cl, GRPC_ERROR_REF(error));
 }
 
 namespace grpc_core {

data/src/core/lib/iomgr/endpoint.h

@@ -29,7 +29,6 @@
 
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_set.h"
-#include "src/core/lib/iomgr/resource_quota.h"
 
 /* An endpoint caps a streaming channel between two communicating processes.
    Examples may be: a tcp socket, <stdin+stdout>, or some shared memory. */

data/src/core/lib/iomgr/endpoint_cfstream.cc

@@ -59,10 +59,8 @@ struct CFStreamEndpoint {
 
   std::string peer_string;
   std::string local_address;
-  grpc_slice_allocator* slice_allocator;
 };
 static void CFStreamFree(CFStreamEndpoint* ep) {
-  grpc_slice_allocator_destroy(ep->slice_allocator);
   CFRelease(ep->read_stream);
   CFRelease(ep->write_stream);
   CFSTREAM_HANDLE_UNREF(ep->stream_sync, "free");
@@ -110,8 +108,7 @@ static grpc_error_handle CFStreamAnnotateError(grpc_error_handle src_error,
   return grpc_error_set_str(
       grpc_error_set_int(src_error, GRPC_ERROR_INT_GRPC_STATUS,
                          GRPC_STATUS_UNAVAILABLE),
-      GRPC_ERROR_STR_TARGET_ADDRESS,
-      grpc_slice_from_copied_string(ep->peer_string.c_str()));
+      GRPC_ERROR_STR_TARGET_ADDRESS, ep->peer_string);
 }
 
 static void CallReadCb(CFStreamEndpoint* ep, grpc_error_handle error) {
@@ -150,7 +147,7 @@ static void CallWriteCb(CFStreamEndpoint* ep, grpc_error_handle error) {
 static void ReadAction(void* arg, grpc_error_handle error) {
   CFStreamEndpoint* ep = static_cast<CFStreamEndpoint*>(arg);
   GPR_ASSERT(ep->read_cb != nullptr);
-  if (error) {
+  if (error != GRPC_ERROR_NONE) {
     grpc_slice_buffer_reset_and_unref_internal(ep->read_slices);
     CallReadCb(ep, GRPC_ERROR_REF(error));
     EP_UNREF(ep, "read");
@@ -192,7 +189,7 @@ static void ReadAction(void* arg, grpc_error_handle error) {
 static void WriteAction(void* arg, grpc_error_handle error) {
   CFStreamEndpoint* ep = static_cast<CFStreamEndpoint*>(arg);
   GPR_ASSERT(ep->write_cb != nullptr);
-  if (error) {
+  if (error != GRPC_ERROR_NONE) {
     grpc_slice_buffer_reset_and_unref_internal(ep->write_slices);
     CallWriteCb(ep, GRPC_ERROR_REF(error));
     EP_UNREF(ep, "write");
@@ -239,17 +236,6 @@ static void WriteAction(void* arg, grpc_error_handle error) {
   grpc_slice_unref_internal(slice);
 }
 
-static void CFStreamReadAllocationDone(void* arg, grpc_error_handle error) {
-  CFStreamEndpoint* ep = static_cast<CFStreamEndpoint*>(arg);
-  if (error == GRPC_ERROR_NONE) {
-    ep->stream_sync->NotifyOnRead(&ep->read_action);
-  } else {
-    grpc_slice_buffer_reset_and_unref_internal(ep->read_slices);
-    CallReadCb(ep, error);
-    EP_UNREF(ep, "read");
-  }
-}
-
 static void CFStreamRead(grpc_endpoint* ep, grpc_slice_buffer* slices,
                          grpc_closure* cb, bool urgent) {
   CFStreamEndpoint* ep_impl = reinterpret_cast<CFStreamEndpoint*>(ep);
@@ -261,13 +247,10 @@ static void CFStreamRead(grpc_endpoint* ep, grpc_slice_buffer* slices,
   ep_impl->read_cb = cb;
   ep_impl->read_slices = slices;
   grpc_slice_buffer_reset_and_unref_internal(slices);
+  grpc_slice_buffer_add_indexed(
+      slices, GRPC_SLICE_MALLOC(GRPC_TCP_DEFAULT_READ_SLICE_SIZE));
   EP_REF(ep_impl, "read");
-  if (grpc_slice_allocator_allocate(
-          ep_impl->slice_allocator, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, 1,
-          grpc_slice_allocator_intent::kReadBuffer, ep_impl->read_slices,
-          CFStreamReadAllocationDone, ep_impl)) {
-    ep_impl->stream_sync->NotifyOnRead(&ep_impl->read_action);
-  }
+  ep_impl->stream_sync->NotifyOnRead(&ep_impl->read_action);
 }
 
 static void CFStreamWrite(grpc_endpoint* ep, grpc_slice_buffer* slices,
@@ -287,13 +270,15 @@ static void CFStreamWrite(grpc_endpoint* ep, grpc_slice_buffer* slices,
 void CFStreamShutdown(grpc_endpoint* ep, grpc_error_handle why) {
   CFStreamEndpoint* ep_impl = reinterpret_cast<CFStreamEndpoint*>(ep);
   if (grpc_tcp_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "CFStream endpoint:%p shutdown (%p)", ep_impl, why);
+    gpr_log(GPR_DEBUG, "CFStream endpoint:%p shutdown (%s)", ep_impl,
+            grpc_error_std_string(why).c_str());
   }
   CFReadStreamClose(ep_impl->read_stream);
   CFWriteStreamClose(ep_impl->write_stream);
   ep_impl->stream_sync->Shutdown(why);
   if (grpc_tcp_trace.enabled()) {
-    gpr_log(GPR_DEBUG, "CFStream endpoint:%p shutdown DONE (%p)", ep_impl, why);
+    gpr_log(GPR_DEBUG, "CFStream endpoint:%p shutdown DONE (%s)", ep_impl,
+            grpc_error_std_string(why).c_str());
   }
 }
 
@@ -336,10 +321,10 @@ static const grpc_endpoint_vtable vtable = {CFStreamRead,
                                             CFStreamGetFD,
                                             CFStreamCanTrackErr};
 
-grpc_endpoint* grpc_cfstream_endpoint_create(
-    CFReadStreamRef read_stream, CFWriteStreamRef write_stream,
-    const char* peer_string, grpc_slice_allocator* slice_allocator,
-    CFStreamHandle* stream_sync) {
+grpc_endpoint* grpc_cfstream_endpoint_create(CFReadStreamRef read_stream,
+                                             CFWriteStreamRef write_stream,
+                                             const char* peer_string,
+                                             CFStreamHandle* stream_sync) {
   CFStreamEndpoint* ep_impl = new CFStreamEndpoint;
   if (grpc_tcp_trace.enabled()) {
     gpr_log(GPR_DEBUG,
@@ -380,7 +365,6 @@ grpc_endpoint* grpc_cfstream_endpoint_create(
                     static_cast<void*>(ep_impl), grpc_schedule_on_exec_ctx);
   GRPC_CLOSURE_INIT(&ep_impl->write_action, WriteAction,
                     static_cast<void*>(ep_impl), grpc_schedule_on_exec_ctx);
-  ep_impl->slice_allocator = slice_allocator;
 
   return &ep_impl->base;
 }

data/src/core/lib/iomgr/endpoint_cfstream.h

@@ -39,10 +39,10 @@
 #include "src/core/lib/iomgr/cfstream_handle.h"
 #include "src/core/lib/iomgr/endpoint.h"
 
-grpc_endpoint* grpc_cfstream_endpoint_create(
-    CFReadStreamRef read_stream, CFWriteStreamRef write_stream,
-    const char* peer_string, grpc_slice_allocator* slice_allocator,
-    CFStreamHandle* stream_sync);
+grpc_endpoint* grpc_cfstream_endpoint_create(CFReadStreamRef read_stream,
+                                             CFWriteStreamRef write_stream,
+                                             const char* peer_string,
+                                             CFStreamHandle* stream_sync);
 
 #endif /* GRPC_CFSTREAM */
 

data/src/core/lib/iomgr/endpoint_pair_posix.cc

@@ -40,6 +40,7 @@
 #include "src/core/lib/iomgr/socket_utils_posix.h"
 #include "src/core/lib/iomgr/tcp_posix.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
+#include "src/core/lib/resource_quota/api.h"
 
 static void create_sockets(int sv[2]) {
   int flags;
@@ -59,18 +60,15 @@ grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char* name,
   create_sockets(sv);
   grpc_core::ExecCtx exec_ctx;
   std::string final_name = absl::StrCat(name, ":client");
-  grpc_resource_quota* resource_quota =
-      grpc_resource_quota_from_channel_args(args, true);
-  p.client = grpc_tcp_create(
-      grpc_fd_create(sv[1], final_name.c_str(), false), args,
-      "socketpair-server",
-      grpc_slice_allocator_create(resource_quota, "server_endpoint", args));
+  const grpc_channel_args* new_args = grpc_core::CoreConfiguration::Get()
+                                          .channel_args_preconditioning()
+                                          .PreconditionChannelArgs(args);
+  p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name.c_str(), false),
+                             new_args, "socketpair-server");
   final_name = absl::StrCat(name, ":server");
-  p.server = grpc_tcp_create(
-      grpc_fd_create(sv[0], final_name.c_str(), false), args,
-      "socketpair-client",
-      grpc_slice_allocator_create(resource_quota, "client_endpoint", args));
-  grpc_resource_quota_unref_internal(resource_quota);
+  p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name.c_str(), false),
+                             new_args, "socketpair-client");
+  grpc_channel_args_destroy(new_args);
   return p;
 }
 

data/src/core/lib/iomgr/endpoint_pair_windows.cc

@@ -71,24 +71,15 @@ static void create_sockets(SOCKET sv[2]) {
 }
 
 grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(
-    const char* name, grpc_channel_args* channel_args) {
+    const char*, grpc_channel_args* channel_args) {
   SOCKET sv[2];
   grpc_endpoint_pair p;
   create_sockets(sv);
   grpc_core::ExecCtx exec_ctx;
-  grpc_resource_quota* resource_quota =
-      grpc_resource_quota_from_channel_args(channel_args, true);
-  p.client =
-      grpc_tcp_create(grpc_winsocket_create(sv[1], "endpoint:client"),
-                      channel_args, "endpoint:server",
-                      grpc_slice_allocator_create(
-                          resource_quota, "endpoint:server", channel_args));
-  p.server =
-      grpc_tcp_create(grpc_winsocket_create(sv[0], "endpoint:server"),
-                      channel_args, "endpoint:client",
-                      grpc_slice_allocator_create(
-                          resource_quota, "endpoint:client", channel_args));
-  grpc_resource_quota_unref_internal(resource_quota);
+  p.client = grpc_tcp_create(grpc_winsocket_create(sv[1], "endpoint:client"),
+                             channel_args, "endpoint:server");
+  p.server = grpc_tcp_create(grpc_winsocket_create(sv[0], "endpoint:server"),
+                             channel_args, "endpoint:client");
   return p;
 }