grpc 1.60.0 → 1.61.0

data/src/core/lib/surface/server.h

@@ -211,26 +211,6 @@ class Server : public InternallyRefCounted<Server>,
  private:
   struct RequestedCall;
 
-  struct ChannelRegisteredMethod {
-    ChannelRegisteredMethod() = default;
-    ChannelRegisteredMethod(RegisteredMethod* server_registered_method_arg,
-                            uint32_t flags_arg, bool has_host_arg,
-                            Slice method_arg, Slice host_arg)
-        : server_registered_method(server_registered_method_arg),
-          flags(flags_arg),
-          has_host(has_host_arg),
-          method(std::move(method_arg)),
-          host(std::move(host_arg)) {}
-
-    ~ChannelRegisteredMethod() = default;
-
-    RegisteredMethod* server_registered_method = nullptr;
-    uint32_t flags;
-    bool has_host;
-    Slice method;
-    Slice host;
-  };
-
   class RequestMatcherInterface;
   class RealRequestMatcherFilterStack;
   class RealRequestMatcherPromises;
@@ -238,7 +218,7 @@ class Server : public InternallyRefCounted<Server>,
   class AllocatingRequestMatcherBatch;
   class AllocatingRequestMatcherRegistered;
 
-  class ChannelData {
+  class ChannelData final : public ServerTransport::Acceptor {
    public:
     ChannelData() = default;
     ~ChannelData();
@@ -251,60 +231,37 @@ class Server : public InternallyRefCounted<Server>,
     Channel* channel() const { return channel_.get(); }
     size_t cq_idx() const { return cq_idx_; }
 
-    ChannelRegisteredMethod* GetRegisteredMethod(const grpc_slice& host,
-                                                 const grpc_slice& path);
-
-    ChannelRegisteredMethod* GetRegisteredMethod(const absl::string_view& host,
-                                                 const absl::string_view& path);
+    RegisteredMethod* GetRegisteredMethod(const absl::string_view& host,
+                                          const absl::string_view& path);
     // Filter vtable functions.
     static grpc_error_handle InitChannelElement(
         grpc_channel_element* elem, grpc_channel_element_args* args);
     static void DestroyChannelElement(grpc_channel_element* elem);
     static ArenaPromise<ServerMetadataHandle> MakeCallPromise(
         grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory);
+    void InitCall(RefCountedPtr<CallSpineInterface> call);
+
+    Arena* CreateArena() override;
+    absl::StatusOr<CallInitiator> CreateCall(
+        ClientMetadata& client_initial_metadata, Arena* arena) override;
 
    private:
     class ConnectivityWatcher;
 
     static void AcceptStream(void* arg, Transport* /*transport*/,
                              const void* transport_server_data);
-    static void SetRegisteredMethodOnMetadata(void* arg,
-                                              ServerMetadata* metadata);
+    void SetRegisteredMethodOnMetadata(ClientMetadata& metadata);
 
     void Destroy() ABSL_EXCLUSIVE_LOCKS_REQUIRED(server_->mu_global_);
 
     static void FinishDestroy(void* arg, grpc_error_handle error);
 
-    struct StringViewStringViewPairHash
-        : absl::flat_hash_set<
-              std::pair<absl::string_view, absl::string_view>>::hasher {
-      using is_transparent = void;
-    };
-
-    struct StringViewStringViewPairEq
-        : std::equal_to<std::pair<absl::string_view, absl::string_view>> {
-      using is_transparent = void;
-    };
-
     RefCountedPtr<Server> server_;
     RefCountedPtr<Channel> channel_;
     // The index into Server::cqs_ of the CQ used as a starting point for
     // where to publish new incoming calls.
     size_t cq_idx_;
     absl::optional<std::list<ChannelData*>::iterator> list_position_;
-    // A hash-table of the methods and hosts of the registered methods.
-    // TODO(vjpai): Convert this to an STL map type as opposed to a direct
-    // bucket implementation. (Consider performance impact, hash function to
-    // use, etc.)
-    std::unique_ptr<std::vector<ChannelRegisteredMethod>>
-        old_registered_methods_;
-    // Map of registered methods.
-    absl::flat_hash_map<std::pair<std::string, std::string> /*host, method*/,
-                        std::unique_ptr<ChannelRegisteredMethod>,
-                        StringViewStringViewPairHash,
-                        StringViewStringViewPairEq>
-        registered_methods_;
-    uint32_t registered_method_max_probes_;
     grpc_closure finish_destroy_channel_closure_;
     intptr_t channelz_socket_uuid_;
   };
@@ -413,6 +370,17 @@ class Server : public InternallyRefCounted<Server>,
     grpc_cq_completion completion;
   };
 
+  struct StringViewStringViewPairHash
+      : absl::flat_hash_set<
+            std::pair<absl::string_view, absl::string_view>>::hasher {
+    using is_transparent = void;
+  };
+
+  struct StringViewStringViewPairEq
+      : std::equal_to<std::pair<absl::string_view, absl::string_view>> {
+    using is_transparent = void;
+  };
+
   static void ListenerDestroyDone(void* arg, grpc_error_handle error);
 
   static void DoneShutdownEvent(void* server,
@@ -498,7 +466,11 @@ class Server : public InternallyRefCounted<Server>,
   bool starting_ ABSL_GUARDED_BY(mu_global_) = false;
   CondVar starting_cv_;
 
-  std::vector<std::unique_ptr<RegisteredMethod>> registered_methods_;
+  // Map of registered methods.
+  absl::flat_hash_map<std::pair<std::string, std::string> /*host, method*/,
+                      std::unique_ptr<RegisteredMethod>,
+                      StringViewStringViewPairHash, StringViewStringViewPairEq>
+      registered_methods_;
 
   // Request matcher for unregistered methods.
   std::unique_ptr<RequestMatcherInterface> unregistered_request_matcher_;
@@ -521,7 +493,7 @@ class Server : public InternallyRefCounted<Server>,
           0,
           channel_args_.GetInt(GRPC_ARG_SERVER_MAX_PENDING_REQUESTS_HARD_LIMIT)
               .value_or(3000)))};
-  Duration max_time_in_pending_queue_{Duration::Seconds(30)};
+  const Duration max_time_in_pending_queue_;
   absl::BitGen bitgen_ ABSL_GUARDED_BY(mu_call_);
 
   std::list<ChannelData*> channels_;

data/src/core/lib/surface/version.cc

@@ -23,6 +23,6 @@
 
 #include <grpc/grpc.h>
 
-const char* grpc_version_string(void) { return "37.0.0"; }
+const char* grpc_version_string(void) { return "38.0.0"; }
 
-const char* grpc_g_stands_for(void) { return "gjallarhorn"; }
+const char* grpc_g_stands_for(void) { return "grand"; }

data/src/core/lib/surface/wait_for_cq_end_op.h

@@ -0,0 +1,94 @@
+// Copyright 2023 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_SURFACE_WAIT_FOR_CQ_END_OP_H
+#define GRPC_SRC_CORE_LIB_SURFACE_WAIT_FOR_CQ_END_OP_H
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/promise/activity.h"
+#include "src/core/lib/surface/completion_queue.h"
+
+namespace grpc_core {
+
+// Defines a promise that calls grpc_cq_end_op() (on first poll) and then waits
+// for the callback supplied to grpc_cq_end_op() to be called, before resolving
+// to Empty{}
+class WaitForCqEndOp {
+ public:
+  WaitForCqEndOp(bool is_closure, void* tag, grpc_error_handle error,
+                 grpc_completion_queue* cq)
+      : state_{NotStarted{is_closure, tag, std::move(error), cq}} {}
+
+  Poll<Empty> operator()() {
+    if (auto* n = absl::get_if<NotStarted>(&state_)) {
+      if (n->is_closure) {
+        ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(n->tag),
+                     std::move(n->error));
+        return Empty{};
+      } else {
+        auto not_started = std::move(*n);
+        auto& started =
+            state_.emplace<Started>(Activity::current()->MakeOwningWaker());
+        grpc_cq_end_op(
+            not_started.cq, not_started.tag, std::move(not_started.error),
+            [](void* p, grpc_cq_completion*) {
+              auto started = static_cast<Started*>(p);
+              started->done.store(true, std::memory_order_release);
+            },
+            &started, &started.completion);
+      }
+    }
+    auto& started = absl::get<Started>(state_);
+    if (started.done.load(std::memory_order_acquire)) {
+      return Empty{};
+    } else {
+      return Pending{};
+    }
+  }
+
+  WaitForCqEndOp(const WaitForCqEndOp&) = delete;
+  WaitForCqEndOp& operator=(const WaitForCqEndOp&) = delete;
+  WaitForCqEndOp(WaitForCqEndOp&& other) noexcept
+      : state_(std::move(absl::get<NotStarted>(other.state_))) {
+    other.state_.emplace<Invalid>();
+  }
+  WaitForCqEndOp& operator=(WaitForCqEndOp&& other) noexcept {
+    state_ = std::move(absl::get<NotStarted>(other.state_));
+    other.state_.emplace<Invalid>();
+    return *this;
+  }
+
+ private:
+  struct NotStarted {
+    bool is_closure;
+    void* tag;
+    grpc_error_handle error;
+    grpc_completion_queue* cq;
+  };
+  struct Started {
+    explicit Started(Waker waker) : waker(std::move(waker)) {}
+    Waker waker;
+    grpc_cq_completion completion;
+    std::atomic<bool> done{false};
+  };
+  struct Invalid {};
+  using State = absl::variant<NotStarted, Started, Invalid>;
+  State state_{Invalid{}};
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_SURFACE_WAIT_FOR_CQ_END_OP_H

data/src/core/lib/transport/call_final_info.cc

@@ -0,0 +1,38 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/transport/call_final_info.h"
+
+#include <utility>
+
+static void move64bits(uint64_t* from, uint64_t* to) {
+  *to += *from;
+  *from = 0;
+}
+
+void grpc_transport_move_one_way_stats(grpc_transport_one_way_stats* from,
+                                       grpc_transport_one_way_stats* to) {
+  move64bits(&from->framing_bytes, &to->framing_bytes);
+  move64bits(&from->data_bytes, &to->data_bytes);
+  move64bits(&from->header_bytes, &to->header_bytes);
+}
+
+void grpc_transport_move_stats(grpc_transport_stream_stats* from,
+                               grpc_transport_stream_stats* to) {
+  grpc_transport_move_one_way_stats(&from->incoming, &to->incoming);
+  grpc_transport_move_one_way_stats(&from->outgoing, &to->outgoing);
+  to->latency = std::exchange(from->latency, gpr_inf_future(GPR_TIMESPAN));
+}

data/src/core/lib/transport/call_final_info.h

@@ -0,0 +1,54 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_TRANSPORT_CALL_FINAL_INFO_H
+#define GRPC_SRC_CORE_LIB_TRANSPORT_CALL_FINAL_INFO_H
+
+#include <grpc/support/port_platform.h>
+
+#include <cstdint>
+
+#include <grpc/status.h>
+#include <grpc/support/time.h>
+
+struct grpc_transport_one_way_stats {
+  uint64_t framing_bytes = 0;
+  uint64_t data_bytes = 0;
+  uint64_t header_bytes = 0;
+};
+
+struct grpc_transport_stream_stats {
+  grpc_transport_one_way_stats incoming;
+  grpc_transport_one_way_stats outgoing;
+  gpr_timespec latency = gpr_inf_future(GPR_TIMESPAN);
+};
+
+void grpc_transport_move_one_way_stats(grpc_transport_one_way_stats* from,
+                                       grpc_transport_one_way_stats* to);
+
+void grpc_transport_move_stats(grpc_transport_stream_stats* from,
+                               grpc_transport_stream_stats* to);
+
+struct grpc_call_stats {
+  grpc_transport_stream_stats transport_stream_stats;
+  gpr_timespec latency;  // From call creating to enqueing of received status
+};
+/// Information about the call upon completion.
+struct grpc_call_final_info {
+  grpc_call_stats stats;
+  grpc_status_code final_status = GRPC_STATUS_OK;
+  const char* error_string = nullptr;
+};
+
+#endif  // GRPC_SRC_CORE_LIB_TRANSPORT_CALL_FINAL_INFO_H

data/src/core/lib/transport/connectivity_state.cc

@@ -91,8 +91,9 @@ class AsyncConnectivityStateWatcherInterface::Notifier {
 
 void AsyncConnectivityStateWatcherInterface::Notify(
     grpc_connectivity_state state, const absl::Status& status) {
-  new Notifier(Ref(), state, status,
-               work_serializer_);  // Deletes itself when done.
+  // Deletes itself when done.
+  new Notifier(RefAsSubclass<AsyncConnectivityStateWatcherInterface>(), state,
+               status, work_serializer_);
 }
 
 //

data/src/core/lib/transport/connectivity_state.h

@@ -128,6 +128,10 @@ class ConnectivityStateTracker {
   // Not thread safe; access must be serialized with an external lock.
   absl::Status status() const { return status_; }
 
+  // Returns the number of watchers.
+  // Not thread safe; access must be serialized with an external lock.
+  size_t NumWatchers() const { return watchers_.size(); }
+
  private:
   const char* name_;
   std::atomic<grpc_connectivity_state> state_{grpc_connectivity_state()};

data/src/core/lib/transport/metadata_batch.h

@@ -410,7 +410,8 @@ struct GrpcLbClientStatsMetadata {
   static const char* DisplayMemento(MementoType) {
     return "<internal-lb-stats>";
   }
-  static MementoType ParseMemento(Slice, bool, MetadataParseErrorFn) {
+  static MementoType ParseMemento(Slice, bool, MetadataParseErrorFn error) {
+    error("not a valid value for grpclb_client_stats", Slice());
     return nullptr;
   }
 };
@@ -649,9 +650,8 @@ class ParseHelper {
     return ParsedMetadata<Container>(
         typename ParsedMetadata<Container>::FromSlicePair{},
         Slice::FromCopiedString(key),
-        IsUniquelyUnownedEnabled() && will_keep_past_request_lifetime_
-            ? value_.TakeUniquelyOwned()
-            : std::move(value_),
+        will_keep_past_request_lifetime_ ? value_.TakeUniquelyOwned()
+                                         : std::move(value_),
         transport_size_);
   }
 

data/src/core/lib/transport/transport.cc

@@ -35,6 +35,9 @@
 #include "src/core/lib/event_engine/default_event_engine.h"
 #include "src/core/lib/gprpp/time.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/promise/for_each.h"
+#include "src/core/lib/promise/promise.h"
+#include "src/core/lib/promise/try_seq.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/transport/error_utils.h"
 
@@ -83,25 +86,6 @@ void grpc_stream_ref_init(grpc_stream_refcount* refcount, int /*initial_refs*/,
                                                              : nullptr);
 }
 
-static void move64bits(uint64_t* from, uint64_t* to) {
-  *to += *from;
-  *from = 0;
-}
-
-void grpc_transport_move_one_way_stats(grpc_transport_one_way_stats* from,
-                                       grpc_transport_one_way_stats* to) {
-  move64bits(&from->framing_bytes, &to->framing_bytes);
-  move64bits(&from->data_bytes, &to->data_bytes);
-  move64bits(&from->header_bytes, &to->header_bytes);
-}
-
-void grpc_transport_move_stats(grpc_transport_stream_stats* from,
-                               grpc_transport_stream_stats* to) {
-  grpc_transport_move_one_way_stats(&from->incoming, &to->incoming);
-  grpc_transport_move_one_way_stats(&from->outgoing, &to->outgoing);
-  to->latency = std::exchange(from->latency, gpr_inf_future(GPR_TIMESPAN));
-}
-
 namespace grpc_core {
 void Transport::SetPollingEntity(grpc_stream* stream,
                                  grpc_polling_entity* pollset_or_pollset_set) {
@@ -268,4 +252,71 @@ std::string Message::DebugString() const {
   return out;
 }
 
+void ForwardCall(CallHandler call_handler, CallInitiator call_initiator,
+                 ClientMetadataHandle client_initial_metadata) {
+  // Send initial metadata.
+  call_initiator.SpawnGuarded(
+      "send_initial_metadata",
+      [client_initial_metadata = std::move(client_initial_metadata),
+       call_initiator]() mutable {
+        return call_initiator.PushClientInitialMetadata(
+            std::move(client_initial_metadata));
+      });
+  // Read messages from handler into initiator.
+  call_handler.SpawnGuarded(
+      "read_messages", [call_handler, call_initiator]() mutable {
+        return ForEach(OutgoingMessages(call_handler),
+                       [call_initiator](MessageHandle msg) mutable {
+                         // Need to spawn a job into the initiator's activity to
+                         // push the message in.
+                         return call_initiator.SpawnWaitable(
+                             "send_message",
+                             [msg = std::move(msg), call_initiator]() mutable {
+                               return call_initiator.CancelIfFails(
+                                   call_initiator.PushMessage(std::move(msg)));
+                             });
+                       });
+      });
+  call_initiator.SpawnInfallible("read_the_things", [call_initiator,
+                                                     call_handler]() mutable {
+    return Seq(
+        call_initiator.CancelIfFails(TrySeq(
+            call_initiator.PullServerInitialMetadata(),
+            [call_handler](ServerMetadataHandle md) mutable {
+              call_handler.SpawnGuarded(
+                  "recv_initial_metadata",
+                  [md = std::move(md), call_handler]() mutable {
+                    return call_handler.PushServerInitialMetadata(
+                        std::move(md));
+                  });
+              return Success{};
+            },
+            ForEach(OutgoingMessages(call_initiator),
+                    [call_handler](MessageHandle msg) mutable {
+                      return call_handler.SpawnWaitable(
+                          "recv_message",
+                          [msg = std::move(msg), call_handler]() mutable {
+                            return call_handler.CancelIfFails(
+                                call_handler.PushMessage(std::move(msg)));
+                          });
+                    }),
+            ImmediateOkStatus())),
+        call_initiator.PullServerTrailingMetadata(),
+        [call_handler](ServerMetadataHandle md) mutable {
+          call_handler.SpawnGuarded(
+              "recv_trailing_metadata",
+              [md = std::move(md), call_handler]() mutable {
+                return call_handler.PushServerTrailingMetadata(std::move(md));
+              });
+          return Empty{};
+        });
+  });
+}
+
+CallInitiatorAndHandler MakeCall(
+    grpc_event_engine::experimental::EventEngine* event_engine, Arena* arena) {
+  auto spine = CallSpine::Create(event_engine, arena);
+  return {CallInitiator(spine), CallHandler(spine)};
+}
+
 }  // namespace grpc_core