grpc 1.64.0 → 1.65.2

data/src/core/lib/surface/call.h

@@ -36,9 +36,7 @@
 
 #include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/channel/context.h"
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gpr/time_precise.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/time.h"
 #include "src/core/lib/iomgr/closure.h"
@@ -52,6 +50,7 @@
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/transport/transport.h"
 #include "src/core/server/server_interface.h"
+#include "src/core/util/time_precise.h"
 
 typedef void (*grpc_ioreq_completion_func)(grpc_call* call, int success,
                                            void* user_data);
@@ -78,21 +77,23 @@ typedef struct grpc_call_create_args {
 
 namespace grpc_core {
 
+template <>
+struct ArenaContextType<census_context> {
+  static void Destroy(census_context*) {}
+};
+
 class Call : public CppImplOf<Call, grpc_call>,
              public grpc_event_engine::experimental::EventEngine::
                  Closure /* for deadlines */ {
  public:
-  Arena* arena() { return arena_; }
+  Arena* arena() { return arena_.get(); }
   bool is_client() const { return is_client_; }
 
-  virtual void ContextSet(grpc_context_index elem, void* value,
-                          void (*destroy)(void* value)) = 0;
-  virtual void* ContextGet(grpc_context_index elem) const = 0;
   virtual bool Completed() = 0;
   void CancelWithStatus(grpc_status_code status, const char* description);
   virtual void CancelWithError(grpc_error_handle error) = 0;
   virtual void SetCompletionQueue(grpc_completion_queue* cq) = 0;
-  char* GetPeer();
+  virtual char* GetPeer() = 0;
   virtual grpc_call_error StartBatch(const grpc_op* ops, size_t nops,
                                      void* notify_tag,
                                      bool is_notify_tag_closure) = 0;
@@ -111,10 +112,7 @@ class Call : public CppImplOf<Call, grpc_call>,
     return deadline_;
   }
 
-  grpc_compression_algorithm test_only_compression_algorithm() {
-    return incoming_compression_algorithm_;
-  }
-  uint32_t test_only_message_flags() { return test_only_last_message_flags_; }
+  virtual uint32_t test_only_message_flags() = 0;
   CompressionAlgorithmSet encodings_accepted_by_peer() {
     return encodings_accepted_by_peer_;
   }
@@ -124,12 +122,20 @@ class Call : public CppImplOf<Call, grpc_call>,
   virtual grpc_call_stack* call_stack() = 0;
 
   // Return the EventEngine used for this call's async execution.
-  virtual grpc_event_engine::experimental::EventEngine* event_engine()
-      const = 0;
+  grpc_event_engine::experimental::EventEngine* event_engine() const {
+    return event_engine_;
+  }
 
   // Implementation of EventEngine::Closure, called when deadline expires
   void Run() final;
 
+  gpr_cycle_counter start_time() const { return start_time_; }
+
+  void set_traced(bool traced) { traced_ = traced; }
+  bool traced() const { return traced_; }
+
+  virtual grpc_compression_algorithm incoming_compression_algorithm() = 0;
+
  protected:
   // The maximum number of concurrent batches possible.
   // Based upon the maximum number of individually queueable ops in the batch
@@ -157,25 +163,12 @@ class Call : public CppImplOf<Call, grpc_call>,
     Call* sibling_prev = nullptr;
   };
 
-  Call(Arena* arena, bool is_client, Timestamp send_deadline,
-       RefCountedPtr<Channel> channel)
-      : channel_(std::move(channel)),
-        arena_(arena),
-        send_deadline_(send_deadline),
-        is_client_(is_client) {
-    DCHECK_NE(arena_, nullptr);
-    DCHECK(channel_ != nullptr);
-  }
+  Call(bool is_client, Timestamp send_deadline, RefCountedPtr<Arena> arena,
+       grpc_event_engine::experimental::EventEngine* event_engine);
   ~Call() override = default;
 
-  void DeleteThis();
-
   ParentCall* GetOrCreateParentCall();
   ParentCall* parent_call();
-  Channel* channel() const {
-    DCHECK(channel_ != nullptr);
-    return channel_.get();
-  }
 
   absl::Status InitParent(Call* parent, uint32_t propagation_mask);
   void PublishToParent(Call* parent);
@@ -207,36 +200,30 @@ class Call : public CppImplOf<Call, grpc_call>,
   // internal headers against external modification.
   void PrepareOutgoingInitialMetadata(const grpc_op& op,
                                       grpc_metadata_batch& md);
-  void NoteLastMessageFlags(uint32_t flags) {
-    test_only_last_message_flags_ = flags;
-  }
-  grpc_compression_algorithm incoming_compression_algorithm() const {
-    return incoming_compression_algorithm_;
-  }
 
   void HandleCompressionAlgorithmDisabled(
       grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
   void HandleCompressionAlgorithmNotAccepted(
       grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
 
-  gpr_cycle_counter start_time() const { return start_time_; }
+  virtual grpc_compression_options compression_options() = 0;
+
+  virtual void SetIncomingCompressionAlgorithm(
+      grpc_compression_algorithm algorithm) = 0;
 
  private:
-  RefCountedPtr<Channel> channel_;
-  Arena* const arena_;
+  const RefCountedPtr<Arena> arena_;
   std::atomic<ParentCall*> parent_call_{nullptr};
   ChildCall* child_ = nullptr;
   Timestamp send_deadline_;
   const bool is_client_;
   // flag indicating that cancellation is inherited
   bool cancellation_is_inherited_ = false;
-  // Compression algorithm for *incoming* data
-  grpc_compression_algorithm incoming_compression_algorithm_ =
-      GRPC_COMPRESS_NONE;
+  // Is this call traced?
+  bool traced_ = false;
   // Supported encodings (compression algorithms), a bitset.
   // Always support no compression.
   CompressionAlgorithmSet encodings_accepted_by_peer_{GRPC_COMPRESS_NONE};
-  uint32_t test_only_last_message_flags_ = 0;
   // Peer name is protected by a mutex because it can be accessed by the
   // application at the same moment as it is being set by the completion
   // of the recv_initial_metadata op.  The mutex should be mostly uncontended.
@@ -247,91 +234,14 @@ class Call : public CppImplOf<Call, grpc_call>,
   Timestamp deadline_ ABSL_GUARDED_BY(deadline_mu_) = Timestamp::InfFuture();
   grpc_event_engine::experimental::EventEngine::TaskHandle ABSL_GUARDED_BY(
       deadline_mu_) deadline_task_;
+  grpc_event_engine::experimental::EventEngine* const event_engine_;
   gpr_cycle_counter start_time_ = gpr_get_cycle_counter();
 };
 
-class BasicPromiseBasedCall;
-class ServerPromiseBasedCall;
-
-class ServerCallContext {
- public:
-  virtual void PublishInitialMetadata(
-      ClientMetadataHandle metadata,
-      grpc_metadata_array* publish_initial_metadata) = 0;
-
-  // Construct the top of the server call promise for the v2 filter stack.
-  // TODO(ctiller): delete when v3 is available.
-  virtual ArenaPromise<ServerMetadataHandle> MakeTopOfServerCallPromise(
-      CallArgs call_args, grpc_completion_queue* cq,
-      absl::FunctionRef<void(grpc_call* call)> publish) = 0;
-
-  // Server stream data as supplied by the transport (so we can link the
-  // transport stream up with the call again).
-  // TODO(ctiller): legacy API - once we move transports to promises we'll
-  // create the promise directly and not need to pass around this token.
-  virtual const void* server_stream_data() = 0;
-
- protected:
-  ~ServerCallContext() = default;
-};
-
-// TODO(ctiller): move more call things into this type
-class CallContext {
- public:
-  explicit CallContext(BasicPromiseBasedCall* call) : call_(call) {}
-
-  // Run some action in the call activity context. This is needed to adapt some
-  // legacy systems to promises, and will likely disappear once that conversion
-  // is complete.
-  void RunInContext(absl::AnyInvocable<void()> fn);
-
-  // TODO(ctiller): remove this once transport APIs are promise based
-  void IncrementRefCount(const char* reason = "call_context");
-
-  // TODO(ctiller): remove this once transport APIs are promise based
-  void Unref(const char* reason = "call_context");
-
-  RefCountedPtr<CallContext> Ref() {
-    IncrementRefCount();
-    return RefCountedPtr<CallContext>(this);
-  }
-
-  grpc_call_stats* call_stats() { return &call_stats_; }
-  gpr_atm* peer_string_atm_ptr();
-  gpr_cycle_counter call_start_time() { return start_time_; }
-
-  ServerCallContext* server_call_context();
-
-  void set_traced(bool traced) { traced_ = traced; }
-  bool traced() const { return traced_; }
-
-  // TEMPORARY HACK
-  // Create a call spine object for this call.
-  // Said object should only be created once.
-  // Allows interop between the v2 call stack and the v3 (which is required by
-  // transports).
-  RefCountedPtr<CallSpineInterface> MakeCallSpine(CallArgs call_args);
-  grpc_call* c_call();
-
- private:
-  friend class PromiseBasedCall;
-  // Call final info.
-  grpc_call_stats call_stats_;
-  // TODO(ctiller): remove this once transport APIs are promise based and we
-  // don't need refcounting here.
-  BasicPromiseBasedCall* const call_;
-  gpr_cycle_counter start_time_ = gpr_get_cycle_counter();
-  // Is this call traced?
-  bool traced_ = false;
-};
-
 template <>
-struct ContextType<CallContext> {};
-
-// TODO(ctiller): remove once call-v3 finalized
-RefCountedPtr<CallSpineInterface> MakeServerCall(
-    ClientMetadataHandle client_initial_metadata, ServerInterface* server,
-    Channel* channel, Arena* arena);
+struct ArenaContextType<Call> {
+  static void Destroy(Call*) {}
+};
 
 }  // namespace grpc_core
 
@@ -362,28 +272,19 @@ grpc_call* grpc_call_from_top_element(grpc_call_element* surface_element);
 void grpc_call_log_batch(const char* file, int line, gpr_log_severity severity,
                          const grpc_op* ops, size_t nops);
 
-// Set a context pointer.
-// No thread safety guarantees are made wrt this value.
-// TODO(#9731): add exec_ctx to destroy
-void grpc_call_context_set(grpc_call* call, grpc_context_index elem,
-                           void* value, void (*destroy)(void* value));
-// Get a context pointer.
-void* grpc_call_context_get(grpc_call* call, grpc_context_index elem);
-
-#define GRPC_CALL_LOG_BATCH(sev, ops, nops)        \
-  do {                                             \
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace)) { \
-      grpc_call_log_batch(sev, ops, nops);         \
-    }                                              \
+void grpc_call_tracer_set(grpc_call* call, grpc_core::ClientCallTracer* tracer);
+
+void* grpc_call_tracer_get(grpc_call* call);
+
+#define GRPC_CALL_LOG_BATCH(sev, ops, nops) \
+  do {                                      \
+    if (GRPC_TRACE_FLAG_ENABLED(api)) {     \
+      grpc_call_log_batch(sev, ops, nops);  \
+    }                                       \
   } while (0)
 
 uint8_t grpc_call_is_client(grpc_call* call);
 
-// Get the estimated memory size for a call BESIDES the call stack. Combined
-// with the size of the call stack, it helps estimate the arena size for the
-// initial call.
-size_t grpc_call_get_initial_size_estimate();
-
 // Return an appropriate compression algorithm for the requested compression \a
 // level in the context of \a call.
 grpc_compression_algorithm grpc_call_compression_for_level(
@@ -397,7 +298,4 @@ bool grpc_call_is_trailers_only(const grpc_call* call);
 // Returns the authority for the call, as seen on the server side.
 absl::string_view grpc_call_server_authority(const grpc_call* call);
 
-extern grpc_core::TraceFlag grpc_call_error_trace;
-extern grpc_core::TraceFlag grpc_compression_trace;
-
 #endif  // GRPC_SRC_CORE_LIB_SURFACE_CALL_H

data/src/core/lib/surface/call_log_batch.cc

@@ -31,10 +31,10 @@
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
-#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 #include "src/core/lib/surface/call.h"
+#include "src/core/util/string.h"
 
 static void add_metadata(const grpc_metadata* md, size_t count,
                          std::vector<std::string>* b) {

data/src/core/lib/surface/call_utils.cc

@@ -0,0 +1,276 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/core/lib/surface/call_utils.h"
+
+#include <inttypes.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+
+#include "absl/log/check.h"
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "absl/strings/string_view.h"
+
+#include <grpc/byte_buffer.h>
+#include <grpc/compression.h>
+#include <grpc/event_engine/event_engine.h>
+#include <grpc/grpc.h>
+#include <grpc/impl/call.h>
+#include <grpc/impl/propagation_bits.h>
+#include <grpc/slice.h>
+#include <grpc/slice_buffer.h>
+#include <grpc/status.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/atm.h>
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/lib/channel/status_util.h"
+#include "src/core/lib/gprpp/crash.h"
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/match.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/context.h"
+#include "src/core/lib/promise/poll.h"
+#include "src/core/lib/promise/status_flag.h"
+#include "src/core/lib/slice/slice_buffer.h"
+#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/surface/validate_metadata.h"
+#include "src/core/lib/transport/metadata.h"
+#include "src/core/lib/transport/metadata_batch.h"
+
+namespace grpc_core {
+
+void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array,
+                          bool is_client) {
+  const auto md_count = md->count();
+  if (md_count > array->capacity) {
+    array->capacity =
+        std::max(array->capacity + md->count(), array->capacity * 3 / 2);
+    array->metadata = static_cast<grpc_metadata*>(
+        gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity));
+  }
+  PublishToAppEncoder encoder(array, md, is_client);
+  md->Encode(&encoder);
+}
+
+void CToMetadata(grpc_metadata* metadata, size_t count,
+                 grpc_metadata_batch* b) {
+  for (size_t i = 0; i < count; i++) {
+    grpc_metadata* md = &metadata[i];
+    auto key = StringViewFromSlice(md->key);
+    // Filter "content-length metadata"
+    if (key == "content-length") continue;
+    b->Append(key, Slice(CSliceRef(md->value)),
+              [md](absl::string_view error, const Slice& value) {
+                gpr_log(GPR_DEBUG, "Append error: %s",
+                        absl::StrCat("key=", StringViewFromSlice(md->key),
+                                     " error=", error,
+                                     " value=", value.as_string_view())
+                            .c_str());
+              });
+  }
+}
+
+const char* GrpcOpTypeName(grpc_op_type op) {
+  switch (op) {
+    case GRPC_OP_SEND_INITIAL_METADATA:
+      return "SendInitialMetadata";
+    case GRPC_OP_SEND_MESSAGE:
+      return "SendMessage";
+    case GRPC_OP_SEND_STATUS_FROM_SERVER:
+      return "SendStatusFromServer";
+    case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
+      return "SendCloseFromClient";
+    case GRPC_OP_RECV_MESSAGE:
+      return "RecvMessage";
+    case GRPC_OP_RECV_CLOSE_ON_SERVER:
+      return "RecvCloseOnServer";
+    case GRPC_OP_RECV_INITIAL_METADATA:
+      return "RecvInitialMetadata";
+    case GRPC_OP_RECV_STATUS_ON_CLIENT:
+      return "RecvStatusOnClient";
+  }
+  Crash("Unreachable");
+}
+
+////////////////////////////////////////////////////////////////////////
+// WaitForCqEndOp
+
+Poll<Empty> WaitForCqEndOp::operator()() {
+  GRPC_TRACE_LOG(promise_primitives, INFO)
+      << Activity::current()->DebugTag() << "WaitForCqEndOp[" << this << "] "
+      << StateString(state_);
+  if (auto* n = absl::get_if<NotStarted>(&state_)) {
+    if (n->is_closure) {
+      ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(n->tag),
+                   std::move(n->error));
+      return Empty{};
+    } else {
+      auto not_started = std::move(*n);
+      auto& started =
+          state_.emplace<Started>(GetContext<Activity>()->MakeOwningWaker());
+      grpc_cq_end_op(
+          not_started.cq, not_started.tag, std::move(not_started.error),
+          [](void* p, grpc_cq_completion*) {
+            auto started = static_cast<Started*>(p);
+            auto wakeup = std::move(started->waker);
+            started->done.store(true, std::memory_order_release);
+            wakeup.Wakeup();
+          },
+          &started, &started.completion);
+    }
+  }
+  auto& started = absl::get<Started>(state_);
+  if (started.done.load(std::memory_order_acquire)) {
+    return Empty{};
+  } else {
+    return Pending{};
+  }
+}
+
+std::string WaitForCqEndOp::StateString(const State& state) {
+  return Match(
+      state,
+      [](const NotStarted& x) {
+        return absl::StrFormat(
+            "NotStarted{is_closure=%s, tag=%p, error=%s, cq=%p}",
+            x.is_closure ? "true" : "false", x.tag, x.error.ToString(), x.cq);
+      },
+      [](const Started& x) {
+        return absl::StrFormat(
+            "Started{completion=%p, done=%s}", &x.completion,
+            x.done.load(std::memory_order_relaxed) ? "true" : "false");
+      },
+      [](const Invalid&) -> std::string { return "Invalid{}"; });
+}
+
+////////////////////////////////////////////////////////////////////////
+// MessageReceiver
+
+StatusFlag MessageReceiver::FinishRecvMessage(
+    ValueOrFailure<absl::optional<MessageHandle>> result) {
+  if (!result.ok()) {
+    GRPC_TRACE_LOG(call, INFO) << Activity::current()->DebugTag()
+                               << "[call] RecvMessage: outstanding_recv "
+                                  "finishes: received end-of-stream with error";
+    *recv_message_ = nullptr;
+    recv_message_ = nullptr;
+    return Failure{};
+  }
+  if (!result->has_value()) {
+    GRPC_TRACE_LOG(call, INFO) << Activity::current()->DebugTag()
+                               << "[call] RecvMessage: outstanding_recv "
+                                  "finishes: received end-of-stream";
+    *recv_message_ = nullptr;
+    recv_message_ = nullptr;
+    return Success{};
+  }
+  MessageHandle& message = **result;
+  test_only_last_message_flags_ = message->flags();
+  if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
+      (incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) {
+    *recv_message_ = grpc_raw_compressed_byte_buffer_create(
+        nullptr, 0, incoming_compression_algorithm_);
+  } else {
+    *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
+  }
+  grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
+                              &(*recv_message_)->data.raw.slice_buffer);
+  GRPC_TRACE_LOG(call, INFO)
+      << Activity::current()->DebugTag()
+      << "[call] RecvMessage: outstanding_recv "
+         "finishes: received "
+      << (*recv_message_)->data.raw.slice_buffer.length << " byte message";
+  recv_message_ = nullptr;
+  return Success{};
+}
+
+////////////////////////////////////////////////////////////////////////
+// MakeErrorString
+
+std::string MakeErrorString(const ServerMetadata* trailing_metadata) {
+  std::string out = absl::StrCat(
+      trailing_metadata->get(GrpcStatusFromWire()).value_or(false)
+          ? "Error received from peer"
+          : "Error generated by client",
+      "grpc_status: ",
+      grpc_status_code_to_string(trailing_metadata->get(GrpcStatusMetadata())
+                                     .value_or(GRPC_STATUS_UNKNOWN)));
+  if (const Slice* message =
+          trailing_metadata->get_pointer(GrpcMessageMetadata())) {
+    absl::StrAppend(&out, "\ngrpc_message: ", message->as_string_view());
+  }
+  if (auto annotations = trailing_metadata->get_pointer(GrpcStatusContext())) {
+    absl::StrAppend(&out, "\nStatus Context:");
+    for (const std::string& annotation : *annotations) {
+      absl::StrAppend(&out, "\n  ", annotation);
+    }
+  }
+  return out;
+}
+
+bool ValidateMetadata(size_t count, grpc_metadata* metadata) {
+  if (count > INT_MAX) {
+    return false;
+  }
+  for (size_t i = 0; i < count; i++) {
+    grpc_metadata* md = &metadata[i];
+    if (!GRPC_LOG_IF_ERROR("validate_metadata",
+                           grpc_validate_header_key_is_legal(md->key))) {
+      return false;
+    } else if (!grpc_is_binary_header_internal(md->key) &&
+               !GRPC_LOG_IF_ERROR(
+                   "validate_metadata",
+                   grpc_validate_header_nonbin_value_is_legal(md->value))) {
+      return false;
+    } else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
+      // HTTP2 hpack encoding has a maximum limit.
+      return false;
+    }
+  }
+  return true;
+}
+
+void EndOpImmediately(grpc_completion_queue* cq, void* notify_tag,
+                      bool is_notify_tag_closure) {
+  if (!is_notify_tag_closure) {
+    CHECK(grpc_cq_begin_op(cq, notify_tag));
+    grpc_cq_end_op(
+        cq, notify_tag, absl::OkStatus(),
+        [](void*, grpc_cq_completion* completion) { gpr_free(completion); },
+        nullptr,
+        static_cast<grpc_cq_completion*>(
+            gpr_malloc(sizeof(grpc_cq_completion))));
+  } else {
+    Closure::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(notify_tag),
+                 absl::OkStatus());
+  }
+}
+
+}  // namespace grpc_core