grpc 1.62.0 → 1.64.0

data/src/core/lib/transport/batch_builder.cc

@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <grpc/support/port_platform.h>
-
 #include "src/core/lib/transport/batch_builder.h"
 
 #include <type_traits>
 
+#include "absl/log/check.h"
+
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/promise/poll.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/surface/call_trace.h"
@@ -66,25 +68,16 @@ BatchBuilder::Batch::Batch(grpc_transport_stream_op_batch_payload* payload,
 }
 
 BatchBuilder::Batch::~Batch() {
-  auto* arena = party->arena();
   if (grpc_call_trace.enabled()) {
     gpr_log(GPR_DEBUG, "%s[connected] [batch %p] Destroy",
             GetContext<Activity>()->DebugTag().c_str(), this);
   }
-  if (pending_receive_message != nullptr) {
-    arena->DeletePooled(pending_receive_message);
-  }
-  if (pending_receive_initial_metadata != nullptr) {
-    arena->DeletePooled(pending_receive_initial_metadata);
-  }
-  if (pending_receive_trailing_metadata != nullptr) {
-    arena->DeletePooled(pending_receive_trailing_metadata);
-  }
-  if (pending_sends != nullptr) {
-    arena->DeletePooled(pending_sends);
-  }
+  delete pending_receive_message;
+  delete pending_receive_initial_metadata;
+  delete pending_receive_trailing_metadata;
+  delete pending_sends;
   if (batch.cancel_stream) {
-    arena->DeletePooled(batch.payload);
+    delete batch.payload;
   }
 #ifndef NDEBUG
   grpc_stream_unref(stream_refcount, "pending-batch");
@@ -105,13 +98,13 @@ BatchBuilder::Batch* BatchBuilder::GetBatch(Target target) {
     batch_ = GetContext<Arena>()->NewPooled<Batch>(payload_,
                                                    target_->stream_refcount);
   }
-  GPR_ASSERT(batch_ != nullptr);
+  CHECK_NE(batch_, nullptr);
   return batch_;
 }
 
 void BatchBuilder::FlushBatch() {
-  GPR_ASSERT(batch_ != nullptr);
-  GPR_ASSERT(target_.has_value());
+  CHECK_NE(batch_, nullptr);
+  CHECK(target_.has_value());
   if (grpc_call_trace.enabled()) {
     gpr_log(
         GPR_DEBUG, "%sPerform transport stream op batch: %p %s",
@@ -171,8 +164,8 @@ BatchBuilder::Batch* BatchBuilder::MakeCancel(
 
 void BatchBuilder::Cancel(Target target, absl::Status status) {
   auto* batch = MakeCancel(target.stream_refcount, std::move(status));
-  batch->batch.on_complete = NewClosure(
-      [batch](absl::Status) { batch->party->arena()->DeletePooled(batch); });
+  batch->batch.on_complete =
+      NewClosure([batch](absl::Status) { delete batch; });
   batch->PerformWith(target);
 }
 

data/src/core/lib/transport/batch_builder.h

@@ -15,8 +15,6 @@
 #ifndef GRPC_SRC_CORE_LIB_TRANSPORT_BATCH_BUILDER_H
 #define GRPC_SRC_CORE_LIB_TRANSPORT_BATCH_BUILDER_H
 
-#include <grpc/support/port_platform.h>
-
 #include <stdint.h>
 
 #include <memory>
@@ -31,6 +29,7 @@
 
 #include <grpc/status.h>
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/debug/trace.h"
@@ -148,8 +147,7 @@ class BatchBuilder {
     absl::string_view name() const override { return "receive_message"; }
 
     MessageHandle IntoMessageHandle() {
-      return GetContext<Arena>()->MakePooled<Message>(std::move(*payload),
-                                                      flags);
+      return Arena::MakePooled<Message>(std::move(*payload), flags);
     }
 
     absl::optional<SliceBuffer> payload;
@@ -162,8 +160,7 @@ class BatchBuilder {
     using PendingCompletion::PendingCompletion;
 
     Arena::PoolPtr<grpc_metadata_batch> metadata =
-        GetContext<Arena>()->MakePooled<grpc_metadata_batch>(
-            GetContext<Arena>());
+        Arena::MakePooled<grpc_metadata_batch>();
 
    protected:
     ~PendingReceiveMetadata() = default;
@@ -209,7 +206,7 @@ class BatchBuilder {
 
     void IncrementRefCount() { ++refs; }
     void Unref() {
-      if (--refs == 0) party->arena()->DeletePooled(this);
+      if (--refs == 0) delete this;
     }
     RefCountedPtr<Batch> Ref() {
       IncrementRefCount();
@@ -225,7 +222,7 @@ class BatchBuilder {
     template <typename T>
     T* GetInitializedCompletion(T*(Batch::*field)) {
       if (this->*field != nullptr) return this->*field;
-      this->*field = party->arena()->NewPooled<T>(Ref());
+      this->*field = new T(Ref());
       if (grpc_call_trace.enabled()) {
         gpr_log(GPR_DEBUG, "%sAdd batch closure for %s @ %s",
                 DebugPrefix().c_str(),
@@ -330,8 +327,7 @@ inline auto BatchBuilder::SendClientTrailingMetadata(Target target) {
   auto* pc = batch->GetInitializedCompletion(&Batch::pending_sends);
   batch->batch.on_complete = &pc->on_done_closure;
   batch->batch.send_trailing_metadata = true;
-  auto metadata =
-      GetContext<Arena>()->MakePooled<grpc_metadata_batch>(GetContext<Arena>());
+  auto metadata = Arena::MakePooled<grpc_metadata_batch>();
   payload_->send_trailing_metadata.send_trailing_metadata = metadata.get();
   payload_->send_trailing_metadata.sent = nullptr;
   pc->send_trailing_metadata = std::move(metadata);

data/src/core/lib/transport/bdp_estimator.cc

@@ -16,8 +16,6 @@
 //
 //
 
-#include <grpc/support/port_platform.h>
-
 #include "src/core/lib/transport/bdp_estimator.h"
 
 #include <inttypes.h>
@@ -25,6 +23,10 @@
 
 #include <algorithm>
 
+#include "absl/log/check.h"
+
+#include <grpc/support/port_platform.h>
+
 grpc_core::TraceFlag grpc_bdp_estimator_trace(false, "bdp_estimator");
 
 namespace grpc_core {
@@ -53,7 +55,7 @@ Timestamp BdpEstimator::CompletePing() {
             std::string(name_).c_str(), accumulator_, estimate_, dt,
             bw / 125000.0, bw_est_ / 125000.0);
   }
-  GPR_ASSERT(ping_state_ == PingState::STARTED);
+  CHECK(ping_state_ == PingState::STARTED);
   if (accumulator_ > 2 * estimate_ / 3 && bw > bw_est_) {
     estimate_ = std::max(accumulator_, estimate_ * 2);
     bw_est_ = bw;

data/src/core/lib/transport/bdp_estimator.h

@@ -19,15 +19,15 @@
 #ifndef GRPC_SRC_CORE_LIB_TRANSPORT_BDP_ESTIMATOR_H
 #define GRPC_SRC_CORE_LIB_TRANSPORT_BDP_ESTIMATOR_H
 
-#include <grpc/support/port_platform.h>
-
 #include <inttypes.h>
 
 #include <string>
 
+#include "absl/log/check.h"
 #include "absl/strings/string_view.h"
 
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 #include <grpc/support/time.h>
 
 #include "src/core/lib/debug/trace.h"
@@ -55,7 +55,7 @@ class BdpEstimator {
       gpr_log(GPR_INFO, "bdp[%s]:sched acc=%" PRId64 " est=%" PRId64,
               std::string(name_).c_str(), accumulator_, estimate_);
     }
-    GPR_ASSERT(ping_state_ == PingState::UNSCHEDULED);
+    CHECK(ping_state_ == PingState::UNSCHEDULED);
     ping_state_ = PingState::SCHEDULED;
     accumulator_ = 0;
   }
@@ -68,7 +68,7 @@ class BdpEstimator {
       gpr_log(GPR_INFO, "bdp[%s]:start acc=%" PRId64 " est=%" PRId64,
               std::string(name_).c_str(), accumulator_, estimate_);
     }
-    GPR_ASSERT(ping_state_ == PingState::SCHEDULED);
+    CHECK(ping_state_ == PingState::SCHEDULED);
     ping_state_ = PingState::STARTED;
     ping_start_time_ = gpr_now(GPR_CLOCK_MONOTONIC);
   }

data/src/core/lib/transport/{call_size_estimator.cc → call_arena_allocator.cc}

@@ -12,12 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <grpc/support/port_platform.h>
-
-#include "src/core/lib/transport/call_size_estimator.h"
+#include "src/core/lib/transport/call_arena_allocator.h"
 
 #include <algorithm>
 
+#include <grpc/support/port_platform.h>
+
 namespace grpc_core {
 
 void CallSizeEstimator::UpdateCallSizeEstimate(size_t size) {

data/src/core/lib/transport/{call_size_estimator.h → call_arena_allocator.h}

@@ -12,14 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_SRC_CORE_LIB_TRANSPORT_CALL_SIZE_ESTIMATOR_H
-#define GRPC_SRC_CORE_LIB_TRANSPORT_CALL_SIZE_ESTIMATOR_H
-
-#include <grpc/support/port_platform.h>
+#ifndef GRPC_SRC_CORE_LIB_TRANSPORT_CALL_ARENA_ALLOCATOR_H
+#define GRPC_SRC_CORE_LIB_TRANSPORT_CALL_ARENA_ALLOCATOR_H
 
 #include <stddef.h>
 
 #include <atomic>
+#include <cstddef>
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/resource_quota/arena.h"
+#include "src/core/lib/resource_quota/memory_quota.h"
 
 namespace grpc_core {
 
@@ -47,6 +52,22 @@ class CallSizeEstimator {
   std::atomic<size_t> call_size_estimate_;
 };
 
+class CallArenaAllocator : public RefCounted<CallArenaAllocator> {
+ public:
+  CallArenaAllocator(MemoryAllocator allocator, size_t initial_size)
+      : allocator_(std::move(allocator)), call_size_estimator_(initial_size) {}
+
+  Arena* MakeArena() {
+    return Arena::Create(call_size_estimator_.CallSizeEstimate(), &allocator_);
+  }
+
+  void Destroy(Arena* arena) { arena->Destroy(); }
+
+ private:
+  MemoryAllocator allocator_;
+  CallSizeEstimator call_size_estimator_;
+};
+
 }  // namespace grpc_core
 
-#endif  // GRPC_SRC_CORE_LIB_TRANSPORT_CALL_SIZE_ESTIMATOR_H
+#endif  // GRPC_SRC_CORE_LIB_TRANSPORT_CALL_ARENA_ALLOCATOR_H

data/src/core/lib/transport/call_filters.cc

@@ -12,11 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <grpc/support/port_platform.h>
-
 #include "src/core/lib/transport/call_filters.h"
 
+#include "absl/log/check.h"
+
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/gprpp/crash.h"
+#include "src/core/lib/transport/metadata.h"
 
 namespace grpc_core {
 
@@ -42,7 +45,7 @@ Poll<ResultOr<T>> OperationExecutor<T>::Start(
   if (layout->promise_size == 0) {
     // No call state ==> instantaneously ready
     auto r = InitStep(std::move(input), call_data);
-    GPR_ASSERT(r.ready());
+    CHECK(r.ready());
     return r;
   }
   promise_data_ =
@@ -52,6 +55,7 @@ Poll<ResultOr<T>> OperationExecutor<T>::Start(
 
 template <typename T>
 Poll<ResultOr<T>> OperationExecutor<T>::InitStep(T input, void* call_data) {
+  CHECK(input != nullptr);
   while (true) {
     if (ops_ == end_ops_) {
       return ResultOr<T>{std::move(input), nullptr};
@@ -71,7 +75,7 @@ Poll<ResultOr<T>> OperationExecutor<T>::InitStep(T input, void* call_data) {
 
 template <typename T>
 Poll<ResultOr<T>> OperationExecutor<T>::Step(void* call_data) {
-  GPR_DEBUG_ASSERT(promise_data_ != nullptr);
+  DCHECK_NE(promise_data_, nullptr);
   auto p = ContinueStep(call_data);
   if (p.ready()) {
     gpr_free_aligned(promise_data_);
@@ -107,7 +111,7 @@ Poll<T> InfallibleOperationExecutor<T>::Start(
   if (layout->promise_size == 0) {
     // No call state ==> instantaneously ready
     auto r = InitStep(std::move(input), call_data);
-    GPR_ASSERT(r.ready());
+    CHECK(r.ready());
     return r;
   }
   promise_data_ =
@@ -135,7 +139,7 @@ Poll<T> InfallibleOperationExecutor<T>::InitStep(T input, void* call_data) {
 
 template <typename T>
 Poll<T> InfallibleOperationExecutor<T>::Step(void* call_data) {
-  GPR_DEBUG_ASSERT(promise_data_ != nullptr);
+  DCHECK_NE(promise_data_, nullptr);
   auto p = ContinueStep(call_data);
   if (p.ready()) {
     gpr_free_aligned(promise_data_);
@@ -165,21 +169,10 @@ template class InfallibleOperationExecutor<ServerMetadataHandle>;
 ///////////////////////////////////////////////////////////////////////////////
 // CallFilters
 
-CallFilters::CallFilters() : stack_(nullptr), call_data_(nullptr) {}
-
-CallFilters::CallFilters(RefCountedPtr<Stack> stack)
-    : stack_(std::move(stack)),
-      call_data_(gpr_malloc_aligned(stack_->data_.call_data_size,
-                                    stack_->data_.call_data_alignment)) {
-  for (const auto& constructor : stack_->data_.filter_constructor) {
-    constructor.call_init(Offset(call_data_, constructor.call_offset),
-                          constructor.channel_data);
-  }
-  client_initial_metadata_state_.Start();
-  client_to_server_message_state_.Start();
-  server_initial_metadata_state_.Start();
-  server_to_client_message_state_.Start();
-}
+CallFilters::CallFilters(ClientMetadataHandle client_initial_metadata)
+    : stack_(nullptr),
+      call_data_(nullptr),
+      client_initial_metadata_(std::move(client_initial_metadata)) {}
 
 CallFilters::~CallFilters() {
   if (call_data_ != nullptr) {
@@ -191,7 +184,7 @@ CallFilters::~CallFilters() {
 }
 
 void CallFilters::SetStack(RefCountedPtr<Stack> stack) {
-  GPR_ASSERT(call_data_ == nullptr);
+  CHECK_EQ(call_data_, nullptr);
   stack_ = std::move(stack);
   call_data_ = gpr_malloc_aligned(stack_->data_.call_data_size,
                                   stack_->data_.call_data_alignment);
@@ -212,15 +205,54 @@ void CallFilters::Finalize(const grpc_call_final_info* final_info) {
   }
 }
 
-void CallFilters::CancelDueToFailedPipeOperation() {
+void CallFilters::CancelDueToFailedPipeOperation(SourceLocation but_where) {
   // We expect something cancelled before now
   if (server_trailing_metadata_ == nullptr) return;
-  gpr_log(GPR_DEBUG, "Cancelling due to failed pipe operation");
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+    gpr_log(but_where.file(), but_where.line(), GPR_LOG_SEVERITY_DEBUG,
+            "Cancelling due to failed pipe operation: %s",
+            DebugString().c_str());
+  }
   server_trailing_metadata_ =
       ServerMetadataFromStatus(absl::CancelledError("Failed pipe operation"));
   server_trailing_metadata_waiter_.Wake();
 }
 
+void CallFilters::PushServerTrailingMetadata(ServerMetadataHandle md) {
+  CHECK(md != nullptr);
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+    gpr_log(GPR_INFO, "%s PushServerTrailingMetadata[%p]: %s into %s",
+            GetContext<Activity>()->DebugTag().c_str(), this,
+            md->DebugString().c_str(), DebugString().c_str());
+  }
+  CHECK(md != nullptr);
+  if (server_trailing_metadata_ != nullptr) return;
+  server_trailing_metadata_ = std::move(md);
+  client_initial_metadata_state_.CloseWithError();
+  server_initial_metadata_state_.CloseSending();
+  client_to_server_message_state_.CloseWithError();
+  server_to_client_message_state_.CloseSending();
+  server_trailing_metadata_waiter_.Wake();
+}
+
+std::string CallFilters::DebugString() const {
+  std::vector<std::string> components = {
+      absl::StrFormat("this:%p", this),
+      absl::StrCat("client_initial_metadata:",
+                   client_initial_metadata_state_.DebugString()),
+      ServerInitialMetadataPromises::DebugString("server_initial_metadata",
+                                                 this),
+      ClientToServerMessagePromises::DebugString("client_to_server_message",
+                                                 this),
+      ServerToClientMessagePromises::DebugString("server_to_client_message",
+                                                 this),
+      absl::StrCat("server_trailing_metadata:",
+                   server_trailing_metadata_ == nullptr
+                       ? "not-set"
+                       : server_trailing_metadata_->DebugString())};
+  return absl::StrCat("CallFilters{", absl::StrJoin(components, ", "), "}");
+};
+
 ///////////////////////////////////////////////////////////////////////////////
 // CallFilters::Stack
 
@@ -256,11 +288,54 @@ RefCountedPtr<CallFilters::Stack> CallFilters::StackBuilder::Build() {
 // CallFilters::PipeState
 
 void filters_detail::PipeState::Start() {
-  GPR_DEBUG_ASSERT(!started_);
+  DCHECK(!started_);
   started_ = true;
   wait_recv_.Wake();
 }
 
+void filters_detail::PipeState::CloseWithError() {
+  if (state_ == ValueState::kClosed) return;
+  state_ = ValueState::kError;
+  wait_recv_.Wake();
+  wait_send_.Wake();
+}
+
+Poll<bool> filters_detail::PipeState::PollClosed() {
+  switch (state_) {
+    case ValueState::kIdle:
+    case ValueState::kWaiting:
+    case ValueState::kQueued:
+    case ValueState::kReady:
+    case ValueState::kProcessing:
+      return wait_recv_.pending();
+    case ValueState::kClosed:
+      return false;
+    case ValueState::kError:
+      return true;
+  }
+  GPR_UNREACHABLE_CODE(return Pending{});
+}
+
+void filters_detail::PipeState::CloseSending() {
+  switch (state_) {
+    case ValueState::kIdle:
+      state_ = ValueState::kClosed;
+      break;
+    case ValueState::kWaiting:
+      state_ = ValueState::kClosed;
+      wait_recv_.Wake();
+      break;
+    case ValueState::kClosed:
+    case ValueState::kError:
+      break;
+    case ValueState::kQueued:
+    case ValueState::kReady:
+    case ValueState::kProcessing:
+      Crash("Only one push allowed to be outstanding");
+      break;
+  }
+}
+
 void filters_detail::PipeState::BeginPush() {
   switch (state_) {
     case ValueState::kIdle:
@@ -287,6 +362,10 @@ void filters_detail::PipeState::DropPush() {
     case ValueState::kReady:
     case ValueState::kProcessing:
     case ValueState::kWaiting:
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        gpr_log(GPR_INFO, "%p drop push in state %s", this,
+                DebugString().c_str());
+      }
       state_ = ValueState::kError;
       wait_recv_.Wake();
       break;
@@ -303,6 +382,10 @@ void filters_detail::PipeState::DropPull() {
     case ValueState::kReady:
     case ValueState::kProcessing:
     case ValueState::kWaiting:
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        gpr_log(GPR_INFO, "%p drop pull in state %s", this,
+                DebugString().c_str());
+      }
       state_ = ValueState::kError;
       wait_send_.Wake();
       break;
@@ -315,9 +398,12 @@ void filters_detail::PipeState::DropPull() {
 
 Poll<StatusFlag> filters_detail::PipeState::PollPush() {
   switch (state_) {
-    case ValueState::kIdle:
     // Read completed and new read started => we see waiting here
     case ValueState::kWaiting:
+      state_ = ValueState::kReady;
+      wait_recv_.Wake();
+      return wait_send_.pending();
+    case ValueState::kIdle:
     case ValueState::kClosed:
       return Success{};
     case ValueState::kQueued:
@@ -330,7 +416,7 @@ Poll<StatusFlag> filters_detail::PipeState::PollPush() {
   GPR_UNREACHABLE_CODE(return Pending{});
 }
 
-Poll<StatusFlag> filters_detail::PipeState::PollPull() {
+Poll<ValueOrFailure<bool>> filters_detail::PipeState::PollPull() {
   switch (state_) {
     case ValueState::kWaiting:
       return wait_recv_.pending();
@@ -341,10 +427,11 @@ Poll<StatusFlag> filters_detail::PipeState::PollPull() {
     case ValueState::kQueued:
       if (!started_) return wait_recv_.pending();
       state_ = ValueState::kProcessing;
-      return Success{};
+      return true;
     case ValueState::kProcessing:
       Crash("Only one pull allowed to be outstanding");
     case ValueState::kClosed:
+      return false;
     case ValueState::kError:
       return Failure{};
   }
@@ -368,4 +455,32 @@ void filters_detail::PipeState::AckPull() {
   }
 }
 
+std::string filters_detail::PipeState::DebugString() const {
+  const char* state_str = "<<invalid-value>>";
+  switch (state_) {
+    case ValueState::kIdle:
+      state_str = "Idle";
+      break;
+    case ValueState::kWaiting:
+      state_str = "Waiting";
+      break;
+    case ValueState::kQueued:
+      state_str = "Queued";
+      break;
+    case ValueState::kReady:
+      state_str = "Ready";
+      break;
+    case ValueState::kProcessing:
+      state_str = "Processing";
+      break;
+    case ValueState::kClosed:
+      state_str = "Closed";
+      break;
+    case ValueState::kError:
+      state_str = "Error";
+      break;
+  }
+  return absl::StrCat(state_str, started_ ? "" : " (not started)");
+}
+
 }  // namespace grpc_core