grpc 1.60.0 → 1.62.0

data/src/core/lib/surface/server.h

@@ -211,26 +211,6 @@ class Server : public InternallyRefCounted<Server>,
  private:
   struct RequestedCall;
 
-  struct ChannelRegisteredMethod {
-    ChannelRegisteredMethod() = default;
-    ChannelRegisteredMethod(RegisteredMethod* server_registered_method_arg,
-                            uint32_t flags_arg, bool has_host_arg,
-                            Slice method_arg, Slice host_arg)
-        : server_registered_method(server_registered_method_arg),
-          flags(flags_arg),
-          has_host(has_host_arg),
-          method(std::move(method_arg)),
-          host(std::move(host_arg)) {}
-
-    ~ChannelRegisteredMethod() = default;
-
-    RegisteredMethod* server_registered_method = nullptr;
-    uint32_t flags;
-    bool has_host;
-    Slice method;
-    Slice host;
-  };
-
   class RequestMatcherInterface;
   class RealRequestMatcherFilterStack;
   class RealRequestMatcherPromises;
@@ -238,7 +218,7 @@ class Server : public InternallyRefCounted<Server>,
   class AllocatingRequestMatcherBatch;
   class AllocatingRequestMatcherRegistered;
 
-  class ChannelData {
+  class ChannelData final : public ServerTransport::Acceptor {
    public:
     ChannelData() = default;
     ~ChannelData();
@@ -251,60 +231,37 @@ class Server : public InternallyRefCounted<Server>,
     Channel* channel() const { return channel_.get(); }
     size_t cq_idx() const { return cq_idx_; }
 
-    ChannelRegisteredMethod* GetRegisteredMethod(const grpc_slice& host,
-                                                 const grpc_slice& path);
-
-    ChannelRegisteredMethod* GetRegisteredMethod(const absl::string_view& host,
-                                                 const absl::string_view& path);
+    RegisteredMethod* GetRegisteredMethod(const absl::string_view& host,
+                                          const absl::string_view& path);
     // Filter vtable functions.
     static grpc_error_handle InitChannelElement(
         grpc_channel_element* elem, grpc_channel_element_args* args);
     static void DestroyChannelElement(grpc_channel_element* elem);
     static ArenaPromise<ServerMetadataHandle> MakeCallPromise(
         grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory);
+    void InitCall(RefCountedPtr<CallSpineInterface> call);
+
+    Arena* CreateArena() override;
+    absl::StatusOr<CallInitiator> CreateCall(
+        ClientMetadata& client_initial_metadata, Arena* arena) override;
 
    private:
     class ConnectivityWatcher;
 
     static void AcceptStream(void* arg, Transport* /*transport*/,
                              const void* transport_server_data);
-    static void SetRegisteredMethodOnMetadata(void* arg,
-                                              ServerMetadata* metadata);
+    void SetRegisteredMethodOnMetadata(ClientMetadata& metadata);
 
     void Destroy() ABSL_EXCLUSIVE_LOCKS_REQUIRED(server_->mu_global_);
 
     static void FinishDestroy(void* arg, grpc_error_handle error);
 
-    struct StringViewStringViewPairHash
-        : absl::flat_hash_set<
-              std::pair<absl::string_view, absl::string_view>>::hasher {
-      using is_transparent = void;
-    };
-
-    struct StringViewStringViewPairEq
-        : std::equal_to<std::pair<absl::string_view, absl::string_view>> {
-      using is_transparent = void;
-    };
-
     RefCountedPtr<Server> server_;
     RefCountedPtr<Channel> channel_;
     // The index into Server::cqs_ of the CQ used as a starting point for
     // where to publish new incoming calls.
     size_t cq_idx_;
     absl::optional<std::list<ChannelData*>::iterator> list_position_;
-    // A hash-table of the methods and hosts of the registered methods.
-    // TODO(vjpai): Convert this to an STL map type as opposed to a direct
-    // bucket implementation. (Consider performance impact, hash function to
-    // use, etc.)
-    std::unique_ptr<std::vector<ChannelRegisteredMethod>>
-        old_registered_methods_;
-    // Map of registered methods.
-    absl::flat_hash_map<std::pair<std::string, std::string> /*host, method*/,
-                        std::unique_ptr<ChannelRegisteredMethod>,
-                        StringViewStringViewPairHash,
-                        StringViewStringViewPairEq>
-        registered_methods_;
-    uint32_t registered_method_max_probes_;
     grpc_closure finish_destroy_channel_closure_;
     intptr_t channelz_socket_uuid_;
   };
@@ -413,6 +370,17 @@ class Server : public InternallyRefCounted<Server>,
     grpc_cq_completion completion;
   };
 
+  struct StringViewStringViewPairHash
+      : absl::flat_hash_set<
+            std::pair<absl::string_view, absl::string_view>>::hasher {
+    using is_transparent = void;
+  };
+
+  struct StringViewStringViewPairEq
+      : std::equal_to<std::pair<absl::string_view, absl::string_view>> {
+    using is_transparent = void;
+  };
+
   static void ListenerDestroyDone(void* arg, grpc_error_handle error);
 
   static void DoneShutdownEvent(void* server,
@@ -498,7 +466,11 @@ class Server : public InternallyRefCounted<Server>,
   bool starting_ ABSL_GUARDED_BY(mu_global_) = false;
   CondVar starting_cv_;
 
-  std::vector<std::unique_ptr<RegisteredMethod>> registered_methods_;
+  // Map of registered methods.
+  absl::flat_hash_map<std::pair<std::string, std::string> /*host, method*/,
+                      std::unique_ptr<RegisteredMethod>,
+                      StringViewStringViewPairHash, StringViewStringViewPairEq>
+      registered_methods_;
 
   // Request matcher for unregistered methods.
   std::unique_ptr<RequestMatcherInterface> unregistered_request_matcher_;
@@ -521,7 +493,7 @@ class Server : public InternallyRefCounted<Server>,
           0,
           channel_args_.GetInt(GRPC_ARG_SERVER_MAX_PENDING_REQUESTS_HARD_LIMIT)
               .value_or(3000)))};
-  Duration max_time_in_pending_queue_{Duration::Seconds(30)};
+  const Duration max_time_in_pending_queue_;
   absl::BitGen bitgen_ ABSL_GUARDED_BY(mu_call_);
 
   std::list<ChannelData*> channels_;

data/src/core/lib/surface/version.cc

@@ -23,6 +23,6 @@
 
 #include <grpc/grpc.h>
 
-const char* grpc_version_string(void) { return "37.0.0"; }
+const char* grpc_version_string(void) { return "39.0.0"; }
 
-const char* grpc_g_stands_for(void) { return "gjallarhorn"; }
+const char* grpc_g_stands_for(void) { return "guardian"; }

data/src/core/lib/surface/wait_for_cq_end_op.cc

@@ -0,0 +1,75 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/surface/wait_for_cq_end_op.h"
+
+#include <atomic>
+
+#include "src/core/lib/gprpp/match.h"
+#include "src/core/lib/promise/trace.h"
+
+namespace grpc_core {
+Poll<Empty> WaitForCqEndOp::operator()() {
+  if (grpc_trace_promise_primitives.enabled()) {
+    gpr_log(GPR_INFO, "%sWaitForCqEndOp[%p] %s",
+            Activity::current()->DebugTag().c_str(), this,
+            StateString(state_).c_str());
+  }
+  if (auto* n = absl::get_if<NotStarted>(&state_)) {
+    if (n->is_closure) {
+      ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(n->tag),
+                   std::move(n->error));
+      return Empty{};
+    } else {
+      auto not_started = std::move(*n);
+      auto& started =
+          state_.emplace<Started>(GetContext<Activity>()->MakeOwningWaker());
+      grpc_cq_end_op(
+          not_started.cq, not_started.tag, std::move(not_started.error),
+          [](void* p, grpc_cq_completion*) {
+            auto started = static_cast<Started*>(p);
+            auto wakeup = std::move(started->waker);
+            started->done.store(true, std::memory_order_release);
+            wakeup.Wakeup();
+          },
+          &started, &started.completion);
+    }
+  }
+  auto& started = absl::get<Started>(state_);
+  if (started.done.load(std::memory_order_acquire)) {
+    return Empty{};
+  } else {
+    return Pending{};
+  }
+}
+
+std::string WaitForCqEndOp::StateString(const State& state) {
+  return Match(
+      state,
+      [](const NotStarted& x) {
+        return absl::StrFormat(
+            "NotStarted{is_closure=%s, tag=%p, error=%s, cq=%p}",
+            x.is_closure ? "true" : "false", x.tag, x.error.ToString(), x.cq);
+      },
+      [](const Started& x) {
+        return absl::StrFormat(
+            "Started{completion=%p, done=%s}", &x.completion,
+            x.done.load(std::memory_order_relaxed) ? "true" : "false");
+      },
+      [](const Invalid&) -> std::string { return "Invalid{}"; });
+}
+
+}  // namespace grpc_core

data/src/core/lib/surface/wait_for_cq_end_op.h

@@ -0,0 +1,72 @@
+// Copyright 2023 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_SURFACE_WAIT_FOR_CQ_END_OP_H
+#define GRPC_SRC_CORE_LIB_SURFACE_WAIT_FOR_CQ_END_OP_H
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/promise/activity.h"
+#include "src/core/lib/surface/completion_queue.h"
+
+namespace grpc_core {
+
+// Defines a promise that calls grpc_cq_end_op() (on first poll) and then waits
+// for the callback supplied to grpc_cq_end_op() to be called, before resolving
+// to Empty{}
+class WaitForCqEndOp {
+ public:
+  WaitForCqEndOp(bool is_closure, void* tag, grpc_error_handle error,
+                 grpc_completion_queue* cq)
+      : state_{NotStarted{is_closure, tag, std::move(error), cq}} {}
+
+  Poll<Empty> operator()();
+
+  WaitForCqEndOp(const WaitForCqEndOp&) = delete;
+  WaitForCqEndOp& operator=(const WaitForCqEndOp&) = delete;
+  WaitForCqEndOp(WaitForCqEndOp&& other) noexcept
+      : state_(std::move(absl::get<NotStarted>(other.state_))) {
+    other.state_.emplace<Invalid>();
+  }
+  WaitForCqEndOp& operator=(WaitForCqEndOp&& other) noexcept {
+    state_ = std::move(absl::get<NotStarted>(other.state_));
+    other.state_.emplace<Invalid>();
+    return *this;
+  }
+
+ private:
+  struct NotStarted {
+    bool is_closure;
+    void* tag;
+    grpc_error_handle error;
+    grpc_completion_queue* cq;
+  };
+  struct Started {
+    explicit Started(Waker waker) : waker(std::move(waker)) {}
+    Waker waker;
+    grpc_cq_completion completion;
+    std::atomic<bool> done{false};
+  };
+  struct Invalid {};
+  using State = absl::variant<NotStarted, Started, Invalid>;
+
+  static std::string StateString(const State& state);
+
+  State state_{Invalid{}};
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_SURFACE_WAIT_FOR_CQ_END_OP_H

data/src/core/lib/transport/batch_builder.cc

@@ -55,8 +55,7 @@ BatchBuilder::PendingCompletion::PendingCompletion(RefCountedPtr<Batch> batch)
 
 BatchBuilder::Batch::Batch(grpc_transport_stream_op_batch_payload* payload,
                            grpc_stream_refcount* stream_refcount)
-    : party(static_cast<Party*>(Activity::current())->Ref()),
-      stream_refcount(stream_refcount) {
+    : party(GetContext<Party>()->Ref()), stream_refcount(stream_refcount) {
   batch.payload = payload;
   batch.is_traced = GetContext<CallContext>()->traced();
 #ifndef NDEBUG
@@ -70,7 +69,7 @@ BatchBuilder::Batch::~Batch() {
   auto* arena = party->arena();
   if (grpc_call_trace.enabled()) {
     gpr_log(GPR_DEBUG, "%s[connected] [batch %p] Destroy",
-            Activity::current()->DebugTag().c_str(), this);
+            GetContext<Activity>()->DebugTag().c_str(), this);
   }
   if (pending_receive_message != nullptr) {
     arena->DeletePooled(pending_receive_message);

data/src/core/lib/transport/batch_builder.h

@@ -202,7 +202,7 @@ class BatchBuilder {
     ~Batch();
     Batch(const Batch&) = delete;
     Batch& operator=(const Batch&) = delete;
-    std::string DebugPrefix(Activity* activity = Activity::current()) const {
+    std::string DebugPrefix(Activity* activity = GetContext<Activity>()) const {
       return absl::StrFormat("%s[connected] [batch %p] ", activity->DebugTag(),
                              this);
     }

data/src/core/lib/transport/call_factory.cc

@@ -0,0 +1,41 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/transport/call_factory.h"
+
+#include "src/core/lib/debug/stats.h"
+#include "src/core/lib/resource_quota/resource_quota.h"
+
+namespace grpc_core {
+
+CallFactory::CallFactory(const ChannelArgs& args)
+    : call_size_estimator_(1024),
+      allocator_(args.GetObject<ResourceQuota>()
+                     ->memory_quota()
+                     ->CreateMemoryOwner()) {}
+
+Arena* CallFactory::CreateArena() {
+  const size_t initial_size = call_size_estimator_.CallSizeEstimate();
+  global_stats().IncrementCallInitialSize(initial_size);
+  return Arena::Create(initial_size, &allocator_);
+}
+
+void CallFactory::DestroyArena(Arena* arena) {
+  call_size_estimator_.UpdateCallSizeEstimate(arena->TotalUsedBytes());
+  arena->Destroy();
+}
+
+}  // namespace grpc_core

data/src/core/lib/transport/call_factory.h

@@ -0,0 +1,56 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_TRANSPORT_CALL_FACTORY_H
+#define GRPC_SRC_CORE_LIB_TRANSPORT_CALL_FACTORY_H
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/resource_quota/arena.h"
+#include "src/core/lib/transport/call_size_estimator.h"
+#include "src/core/lib/transport/call_spine.h"
+
+namespace grpc_core {
+
+// CallFactory creates calls.
+class CallFactory : public RefCounted<CallFactory> {
+ public:
+  explicit CallFactory(const ChannelArgs& args);
+
+  // Create an arena for a call.
+  // We do this as a separate step so that servers can create arenas without
+  // creating the call into it - in the case that we have a HTTP/2 rapid reset
+  // like attack this saves a lot of cpu time.
+  Arena* CreateArena();
+  // Destroy an arena created by CreateArena.
+  // Updates the call size estimator so that we always create arenas of about
+  // the right size.
+  void DestroyArena(Arena* arena);
+
+  // Create a call. The call will be created in the given arena.
+  // It is the CallFactory's responsibility to ensure that the CallHandler
+  // associated with the call is eventually handled by something (typically a
+  // CallDestination, but this is not strictly required).
+  virtual CallInitiator CreateCall(ClientMetadataHandle md, Arena* arena) = 0;
+
+ private:
+  CallSizeEstimator call_size_estimator_;
+  MemoryAllocator allocator_;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_TRANSPORT_CALL_FACTORY_H