grpc 1.69.0 → 1.70.1

data/include/grpc/support/atm.h

@@ -79,17 +79,4 @@
 #error could not determine platform for atm
 #endif
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/** Adds \a delta to \a *value, clamping the result to the range specified
-    by \a min and \a max.  Returns the new value. */
-gpr_atm gpr_atm_no_barrier_clamped_add(gpr_atm* value, gpr_atm delta,
-                                       gpr_atm min, gpr_atm max);
-
-#ifdef __cplusplus
-}
-#endif
-
 #endif /* GRPC_SUPPORT_ATM_H */

data/src/core/call/request_buffer.cc

@@ -0,0 +1,224 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/core/call/request_buffer.h"
+
+#include <cstdint>
+
+#include "absl/strings/str_cat.h"
+#include "absl/types/optional.h"
+#include "src/core/util/match.h"
+
+namespace grpc_core {
+
+RequestBuffer::Buffering::Buffering() {}
+
+RequestBuffer::RequestBuffer() : state_(absl::in_place_type_t<Buffering>()) {}
+
+ValueOrFailure<size_t> RequestBuffer::PushClientInitialMetadata(
+    ClientMetadataHandle md) {
+  MutexLock lock(&mu_);
+  if (absl::get_if<Cancelled>(&state_)) return Failure{};
+  auto& buffering = absl::get<Buffering>(state_);
+  CHECK_EQ(buffering.initial_metadata.get(), nullptr);
+  buffering.initial_metadata = std::move(md);
+  buffering.buffered += buffering.initial_metadata->TransportSize();
+  WakeupAsyncAllPullers();
+  return buffering.buffered;
+}
+
+Poll<ValueOrFailure<size_t>> RequestBuffer::PollPushMessage(
+    MessageHandle& message) {
+  MutexLock lock(&mu_);
+  if (absl::get_if<Cancelled>(&state_)) return Failure{};
+  size_t buffered = 0;
+  if (auto* buffering = absl::get_if<Buffering>(&state_)) {
+    if (winner_ != nullptr) return PendingPush();
+    buffering->buffered += message->payload()->Length();
+    buffered = buffering->buffered;
+    buffering->messages.push_back(std::move(message));
+  } else {
+    auto& streaming = absl::get<Streaming>(state_);
+    CHECK_EQ(streaming.end_of_stream, false);
+    if (streaming.message != nullptr) {
+      return PendingPush();
+    }
+    streaming.message = std::move(message);
+  }
+  WakeupAsyncAllPullers();
+  return buffered;
+}
+
+StatusFlag RequestBuffer::FinishSends() {
+  MutexLock lock(&mu_);
+  if (absl::get_if<Cancelled>(&state_)) return Failure{};
+  if (auto* buffering = absl::get_if<Buffering>(&state_)) {
+    Buffered buffered(std::move(buffering->initial_metadata),
+                      std::move(buffering->messages));
+    state_.emplace<Buffered>(std::move(buffered));
+  } else {
+    auto& streaming = absl::get<Streaming>(state_);
+    CHECK_EQ(streaming.end_of_stream, false);
+    streaming.end_of_stream = true;
+  }
+  WakeupAsyncAllPullers();
+  return Success{};
+}
+
+void RequestBuffer::Cancel(absl::Status error) {
+  MutexLock lock(&mu_);
+  if (absl::holds_alternative<Cancelled>(state_)) return;
+  state_.emplace<Cancelled>(std::move(error));
+  WakeupAsyncAllPullers();
+}
+
+void RequestBuffer::Commit(Reader* winner) {
+  MutexLock lock(&mu_);
+  CHECK_EQ(winner_, nullptr);
+  winner_ = winner;
+  if (auto* buffering = absl::get_if<Buffering>(&state_)) {
+    if (buffering->initial_metadata != nullptr &&
+        winner->message_index_ == buffering->messages.size() &&
+        winner->pulled_client_initial_metadata_) {
+      state_.emplace<Streaming>();
+    }
+  } else if (auto* buffered = absl::get_if<Buffered>(&state_)) {
+    CHECK_NE(buffered->initial_metadata.get(), nullptr);
+    if (winner->message_index_ == buffered->messages.size()) {
+      state_.emplace<Streaming>().end_of_stream = true;
+    }
+  }
+  WakeupAsyncAllPullersExcept(winner);
+}
+
+void RequestBuffer::WakeupAsyncAllPullersExcept(Reader* except_reader) {
+  for (auto wakeup_reader : readers_) {
+    if (wakeup_reader == except_reader) continue;
+    wakeup_reader->pull_waker_.WakeupAsync();
+  }
+}
+
+Poll<ValueOrFailure<ClientMetadataHandle>>
+RequestBuffer::Reader::PollPullClientInitialMetadata() {
+  MutexLock lock(&buffer_->mu_);
+  if (buffer_->winner_ != nullptr && buffer_->winner_ != this) {
+    error_ = absl::CancelledError("Another call was chosen");
+    return Failure{};
+  }
+  if (auto* buffering = absl::get_if<Buffering>(&buffer_->state_)) {
+    if (buffering->initial_metadata.get() == nullptr) {
+      return buffer_->PendingPull(this);
+    }
+    pulled_client_initial_metadata_ = true;
+    auto result = ClaimObject(buffering->initial_metadata);
+    buffer_->MaybeSwitchToStreaming();
+    return std::move(result);
+  }
+  if (auto* buffered = absl::get_if<Buffered>(&buffer_->state_)) {
+    pulled_client_initial_metadata_ = true;
+    return ClaimObject(buffered->initial_metadata);
+  }
+  error_ = absl::get<Cancelled>(buffer_->state_).error;
+  return Failure{};
+}
+
+Poll<ValueOrFailure<absl::optional<MessageHandle>>>
+RequestBuffer::Reader::PollPullMessage() {
+  ReleasableMutexLock lock(&buffer_->mu_);
+  if (buffer_->winner_ != nullptr && buffer_->winner_ != this) {
+    error_ = absl::CancelledError("Another call was chosen");
+    return Failure{};
+  }
+  if (auto* buffering = absl::get_if<Buffering>(&buffer_->state_)) {
+    if (message_index_ == buffering->messages.size()) {
+      return buffer_->PendingPull(this);
+    }
+    const auto idx = message_index_;
+    auto result = ClaimObject(buffering->messages[idx]);
+    ++message_index_;
+    buffer_->MaybeSwitchToStreaming();
+    return std::move(result);
+  }
+  if (auto* buffered = absl::get_if<Buffered>(&buffer_->state_)) {
+    if (message_index_ == buffered->messages.size()) return absl::nullopt;
+    const auto idx = message_index_;
+    ++message_index_;
+    return ClaimObject(buffered->messages[idx]);
+  }
+  if (auto* streaming = absl::get_if<Streaming>(&buffer_->state_)) {
+    if (streaming->message == nullptr) {
+      if (streaming->end_of_stream) return absl::nullopt;
+      return buffer_->PendingPull(this);
+    }
+    auto msg = std::move(streaming->message);
+    auto waker = std::move(buffer_->push_waker_);
+    lock.Release();
+    waker.Wakeup();
+    return std::move(msg);
+  }
+  error_ = absl::get<Cancelled>(buffer_->state_).error;
+  return Failure{};
+}
+
+std::string RequestBuffer::DebugString(Reader* caller)
+    ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+  return absl::StrCat(
+      "have_winner=",
+      (winner_ == nullptr ? "no" : (winner_ == caller ? "this" : "other")),
+      " num_readers=", readers_.size(),
+      " push_waker=", push_waker_.DebugString(),
+      Match(
+          state_,
+          [](const Buffering& buffering) {
+            return absl::StrCat(
+                " buffering initial_metadata=",
+                (buffering.initial_metadata != nullptr
+                     ? buffering.initial_metadata->DebugString()
+                     : "null"),
+                " messages=[",
+                absl::StrJoin(
+                    buffering.messages, ",",
+                    [](std::string* output, const MessageHandle& hdl) {
+                      absl::StrAppend(output, hdl->DebugString());
+                    }),
+                "] buffered=", buffering.buffered);
+          },
+          [](const Buffered& buffered) {
+            return absl::StrCat(
+                " buffered initial_metadata=",
+                (buffered.initial_metadata != nullptr
+                     ? buffered.initial_metadata->DebugString()
+                     : "null"),
+                " messages=[",
+                absl::StrJoin(
+                    buffered.messages, ",",
+                    [](std::string* output, const MessageHandle& hdl) {
+                      absl::StrAppend(
+                          output, hdl != nullptr ? hdl->DebugString() : "null");
+                    }),
+                "]");
+          },
+          [](const Streaming& streaming) {
+            return absl::StrCat(
+                " streaming message=",
+                (streaming.message != nullptr ? streaming.message->DebugString()
+                                              : "null"),
+                " end_of_stream=", streaming.end_of_stream);
+          },
+          [](const Cancelled& cancelled) {
+            return absl::StrCat(" cancelled error=",
+                                cancelled.error.ToString());
+          }));
+}
+}  // namespace grpc_core

data/src/core/call/request_buffer.h

@@ -0,0 +1,192 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_CALL_REQUEST_BUFFER_H
+#define GRPC_SRC_CORE_CALL_REQUEST_BUFFER_H
+
+#include <utility>
+
+#include "src/core/lib/transport/call_spine.h"
+#include "src/core/lib/transport/message.h"
+#include "src/core/lib/transport/metadata.h"
+
+namespace grpc_core {
+
+// Outbound request buffer.
+// Collects client->server metadata and messages whilst in its initial buffering
+// mode. In buffering mode it can have zero or more Reader objects attached to
+// it.
+// The buffer can later be switched to committed mode, at which point it
+// will have exactly one Reader object attached to it.
+// Callers can choose to switch to committed mode based upon policy of their
+// choice.
+class RequestBuffer {
+ public:
+  // One reader of the request buffer.
+  class Reader {
+   public:
+    explicit Reader(RequestBuffer* buffer) ABSL_LOCKS_EXCLUDED(buffer->mu_)
+        : buffer_(buffer) {
+      buffer->AddReader(this);
+    }
+    ~Reader() ABSL_LOCKS_EXCLUDED(buffer_->mu_) { buffer_->RemoveReader(this); }
+
+    Reader(const Reader&) = delete;
+    Reader& operator=(const Reader&) = delete;
+
+    // Pull client initial metadata. Returns a promise that resolves to
+    // ValueOrFailure<ClientMetadataHandle>.
+    GRPC_MUST_USE_RESULT auto PullClientInitialMetadata() {
+      return [this]() { return PollPullClientInitialMetadata(); };
+    }
+    // Pull a message. Returns a promise that resolves to a
+    // ValueOrFailure<absl::optional<MessageHandle>>.
+    GRPC_MUST_USE_RESULT auto PullMessage() {
+      return [this]() { return PollPullMessage(); };
+    }
+
+    absl::Status TakeError() { return std::move(error_); }
+
+   private:
+    friend class RequestBuffer;
+
+    Poll<ValueOrFailure<ClientMetadataHandle>> PollPullClientInitialMetadata();
+    Poll<ValueOrFailure<absl::optional<MessageHandle>>> PollPullMessage();
+
+    template <typename T>
+    T ClaimObject(T& object) ABSL_EXCLUSIVE_LOCKS_REQUIRED(buffer_->mu_) {
+      if (buffer_->winner_ == this) return std::move(object);
+      return CopyObject(object);
+    }
+
+    ClientMetadataHandle CopyObject(const ClientMetadataHandle& md) {
+      return Arena::MakePooled<ClientMetadata>(md->Copy());
+    }
+
+    MessageHandle CopyObject(const MessageHandle& msg) {
+      return Arena::MakePooled<Message>(msg->payload()->Copy(), msg->flags());
+    }
+
+    RequestBuffer* const buffer_;
+    bool pulled_client_initial_metadata_ = false;
+    size_t message_index_ = 0;
+    absl::Status error_;
+    Waker pull_waker_;
+  };
+
+  RequestBuffer();
+
+  // Push ClientInitialMetadata into the buffer.
+  // This is instantaneous, and returns success with the amount of data
+  // buffered, or failure.
+  ValueOrFailure<size_t> PushClientInitialMetadata(ClientMetadataHandle md);
+  // Resolves to a ValueOrFailure<size_t> where the size_t is the amount of data
+  // buffered (or 0 if we're in committed mode).
+  GRPC_MUST_USE_RESULT auto PushMessage(MessageHandle message) {
+    return [this, message = std::move(message)]() mutable {
+      return PollPushMessage(message);
+    };
+  }
+  // Push end of stream (client half-closure).
+  StatusFlag FinishSends();
+  // Cancel the request, propagate failure to all readers.
+  void Cancel(absl::Status error = absl::CancelledError());
+
+  // Switch to committed mode - needs to be called exactly once with the winning
+  // reader. All other readers will see failure.
+  void Commit(Reader* winner);
+
+  bool committed() const {
+    MutexLock lock(&mu_);
+    return winner_ != nullptr;
+  }
+
+ private:
+  // Buffering state: we're collecting metadata and messages.
+  struct Buffering {
+    Buffering();
+    // Initial metadata, or nullptr if not yet received.
+    ClientMetadataHandle initial_metadata;
+    // Buffered messages.
+    absl::InlinedVector<MessageHandle, 1> messages;
+    // Amount of data buffered.
+    size_t buffered = 0;
+  };
+  // Buffered state: all messages have been collected (the client has finished
+  // sending).
+  struct Buffered {
+    Buffered(ClientMetadataHandle md,
+             absl::InlinedVector<MessageHandle, 1> msgs)
+        : initial_metadata(std::move(md)), messages(std::move(msgs)) {}
+    ClientMetadataHandle initial_metadata;
+    absl::InlinedVector<MessageHandle, 1> messages;
+  };
+  // Streaming state: we're streaming messages to the server.
+  // This implies winner_ is set.
+  struct Streaming {
+    MessageHandle message;
+    bool end_of_stream = false;
+  };
+  // Cancelled state: the request has been cancelled.
+  struct Cancelled {
+    explicit Cancelled(absl::Status error) : error(std::move(error)) {}
+    absl::Status error;
+  };
+  using State = absl::variant<Buffering, Buffered, Streaming, Cancelled>;
+
+  Poll<ValueOrFailure<size_t>> PollPushMessage(MessageHandle& message);
+  Pending PendingPull(Reader* reader) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+    reader->pull_waker_ = Activity::current()->MakeOwningWaker();
+    return Pending{};
+  }
+  Pending PendingPush() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+    push_waker_ = Activity::current()->MakeOwningWaker();
+    return Pending{};
+  }
+  void MaybeSwitchToStreaming() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+    auto& buffering = absl::get<Buffering>(state_);
+    if (winner_ == nullptr) return;
+    if (winner_->message_index_ < buffering.messages.size()) return;
+    state_.emplace<Streaming>();
+    push_waker_.Wakeup();
+  }
+
+  void WakeupAsyncAllPullers() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+    WakeupAsyncAllPullersExcept(nullptr);
+  }
+  void WakeupAsyncAllPullersExcept(Reader* except_reader)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+
+  void AddReader(Reader* reader) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+    readers_.insert(reader);
+  }
+
+  void RemoveReader(Reader* reader) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+    readers_.erase(reader);
+  }
+
+  std::string DebugString(Reader* caller) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+
+  mutable Mutex mu_;
+  Reader* winner_ ABSL_GUARDED_BY(mu_){nullptr};
+  State state_ ABSL_GUARDED_BY(mu_);
+  // TODO(ctiller): change this to an intrusively linked list to avoid
+  // allocations.
+  absl::flat_hash_set<Reader*> readers_ ABSL_GUARDED_BY(mu_);
+  Waker push_waker_ ABSL_GUARDED_BY(mu_);
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_CALL_REQUEST_BUFFER_H

data/src/core/client_channel/client_channel.cc

@@ -51,6 +51,7 @@
 #include "src/core/client_channel/dynamic_filters.h"
 #include "src/core/client_channel/global_subchannel_pool.h"
 #include "src/core/client_channel/local_subchannel_pool.h"
+#include "src/core/client_channel/retry_interceptor.h"
 #include "src/core/client_channel/subchannel.h"
 #include "src/core/client_channel/subchannel_interface_internal.h"
 #include "src/core/config/core_configuration.h"
@@ -1291,13 +1292,11 @@ void ClientChannel::UpdateServiceConfigInDataPlaneLocked(
       GRPC_CLIENT_CHANNEL, builder);
   // Add filters returned by the config selector (e.g., xDS HTTP filters).
   config_selector->AddFilters(builder);
-  // TODO(roth, ctiller): When we implement the retry interceptor, that
-  // needs to be added *after* the filters added by the config selector.
   const bool enable_retries =
       !channel_args_.WantMinimalStack() &&
       channel_args_.GetBool(GRPC_ARG_ENABLE_RETRIES).value_or(true);
   if (enable_retries) {
-    Crash("call v3 stack does not yet support retries");
+    builder.Add<RetryInterceptor>();
   }
   // Create call destination.
   auto top_of_stack_call_destination = builder.Build(call_destination_);

data/src/core/client_channel/client_channel_args.h

@@ -0,0 +1,21 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_CLIENT_CHANNEL_CLIENT_CHANNEL_ARGS_H
+#define GRPC_SRC_CORE_CLIENT_CHANNEL_CLIENT_CHANNEL_ARGS_H
+
+// Channel arg key for server URI string.
+#define GRPC_ARG_SERVER_URI "grpc.server_uri"
+
+#endif  // GRPC_SRC_CORE_CLIENT_CHANNEL_CLIENT_CHANNEL_ARGS_H

data/src/core/client_channel/client_channel_filter.h

@@ -35,6 +35,7 @@
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 #include "src/core/channelz/channelz.h"
+#include "src/core/client_channel/client_channel_args.h"
 #include "src/core/client_channel/client_channel_factory.h"
 #include "src/core/client_channel/config_selector.h"
 #include "src/core/client_channel/dynamic_filters.h"
@@ -78,9 +79,6 @@
 // Calls on a disconnected client channel are queued until a connection is
 // established.
 
-// Channel arg key for server URI string.
-#define GRPC_ARG_SERVER_URI "grpc.server_uri"
-
 // Max number of batches that can be pending on a call at any given
 // time.  This includes one batch for each of the following ops:
 //   recv_initial_metadata