grpc 1.39.0 → 1.40.0.pre1

data/src/core/ext/filters/client_channel/client_channel.h

@@ -39,6 +39,7 @@
 #include "src/core/ext/filters/client_channel/service_config.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
 #include "src/core/ext/filters/client_channel/subchannel_pool_interface.h"
+#include "src/core/lib/channel/call_tracer.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/polling_entity.h"
@@ -132,9 +133,11 @@ class ClientChannel {
   void RemoveConnectivityWatcher(
       AsyncConnectivityStateWatcherInterface* watcher);
 
-  RefCountedPtr<LoadBalancedCall> CreateLoadBalancedCall(
+  OrphanablePtr<LoadBalancedCall> CreateLoadBalancedCall(
       const grpc_call_element_args& args, grpc_polling_entity* pollent,
-      grpc_closure* on_call_destruction_complete);
+      grpc_closure* on_call_destruction_complete,
+      ConfigSelector::CallDispatchController* call_dispatch_controller,
+      bool is_transparent_retry);
 
  private:
   class CallData;
@@ -273,7 +276,6 @@ class ClientChannel {
   // Fields set at construction and never modified.
   //
   const bool deadline_checking_enabled_;
-  const bool enable_retries_;
   grpc_channel_stack* owning_stack_;
   ClientChannelFactory* client_channel_factory_;
   const grpc_channel_args* channel_args_;
@@ -369,12 +371,10 @@ class ClientChannel {
 // ClientChannel::LoadBalancedCall
 //
 
-// This object is ref-counted, but it cannot inherit from RefCounted<>,
-// because it is allocated on the arena and can't free its memory when
-// its refcount goes to zero.  So instead, it manually implements the
-// same API as RefCounted<>, so that it can be used with RefCountedPtr<>.
+// TODO(roth): As part of simplifying cancellation in the filter stack,
+// this should no longer need to be ref-counted.
 class ClientChannel::LoadBalancedCall
-    : public RefCounted<LoadBalancedCall, PolymorphicRefCount, kUnrefCallDtor> {
+    : public InternallyRefCounted<LoadBalancedCall, kUnrefCallDtor> {
  public:
   // If on_call_destruction_complete is non-null, then it will be
   // invoked once the LoadBalancedCall is completely destroyed.
@@ -382,11 +382,15 @@ class ClientChannel::LoadBalancedCall
   // the LB call has a subchannel call and ensuring that the
   // on_call_destruction_complete closure passed down from the surface
   // is not invoked until after the subchannel call stack is destroyed.
-  LoadBalancedCall(ClientChannel* chand, const grpc_call_element_args& args,
-                   grpc_polling_entity* pollent,
-                   grpc_closure* on_call_destruction_complete);
+  LoadBalancedCall(
+      ClientChannel* chand, const grpc_call_element_args& args,
+      grpc_polling_entity* pollent, grpc_closure* on_call_destruction_complete,
+      ConfigSelector::CallDispatchController* call_dispatch_controller,
+      bool is_transparent_retry);
   ~LoadBalancedCall() override;
 
+  void Orphan() override;
+
   void StartTransportStreamOpBatch(grpc_transport_stream_op_batch* batch);
 
   // Invoked by channel for queued LB picks when the picker is updated.
@@ -438,10 +442,10 @@ class ClientChannel::LoadBalancedCall
   // Resumes all pending batches on subchannel_call_.
   void PendingBatchesResume();
 
-  static void RecvTrailingMetadataReadyForLoadBalancingPolicy(
-      void* arg, grpc_error_handle error);
-  void InjectRecvTrailingMetadataReadyForLoadBalancingPolicy(
-      grpc_transport_stream_op_batch* batch);
+  static void SendInitialMetadataOnComplete(void* arg, grpc_error_handle error);
+  static void RecvInitialMetadataReady(void* arg, grpc_error_handle error);
+  static void RecvMessageReady(void* arg, grpc_error_handle error);
+  static void RecvTrailingMetadataReady(void* arg, grpc_error_handle error);
 
   void CreateSubchannelCall();
   // Invoked when a pick is completed, on both success or failure.
@@ -459,7 +463,6 @@ class ClientChannel::LoadBalancedCall
   // that uses any one of them, we should store them in the call
   // context.  This will save per-call memory overhead.
   grpc_slice path_;  // Request path.
-  gpr_cycle_counter call_start_time_;
   grpc_millis deadline_;
   Arena* arena_;
   grpc_call_stack* owning_call_;
@@ -467,6 +470,11 @@ class ClientChannel::LoadBalancedCall
   grpc_call_context_element* call_context_;
   grpc_polling_entity* pollent_;
   grpc_closure* on_call_destruction_complete_;
+  ConfigSelector::CallDispatchController* call_dispatch_controller_;
+
+  CallTracer::CallAttemptTracer* call_attempt_tracer_;
+
+  gpr_cycle_counter lb_call_start_time_ = gpr_get_cycle_counter();
 
   // Set when we get a cancel_stream op.
   grpc_error_handle cancel_error_ = GRPC_ERROR_NONE;
@@ -486,14 +494,30 @@ class ClientChannel::LoadBalancedCall
 
   RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
   const LoadBalancingPolicy::BackendMetricData* backend_metric_data_ = nullptr;
-  std::function<void(grpc_error_handle, LoadBalancingPolicy::MetadataInterface*,
+  std::function<void(absl::Status, LoadBalancingPolicy::MetadataInterface*,
                      LoadBalancingPolicy::CallState*)>
       lb_recv_trailing_metadata_ready_;
 
   RefCountedPtr<SubchannelCall> subchannel_call_;
 
-  // For intercepting recv_trailing_metadata_ready for the LB policy.
+  // For intercepting send_initial_metadata on_complete.
+  gpr_atm* peer_string_ = nullptr;
+  grpc_closure send_initial_metadata_on_complete_;
+  grpc_closure* original_send_initial_metadata_on_complete_ = nullptr;
+
+  // For intercepting recv_initial_metadata_ready.
+  grpc_metadata_batch* recv_initial_metadata_ = nullptr;
+  grpc_closure recv_initial_metadata_ready_;
+  grpc_closure* original_recv_initial_metadata_ready_ = nullptr;
+
+  // For intercepting recv_message_ready.
+  OrphanablePtr<ByteStream>* recv_message_ = nullptr;
+  grpc_closure recv_message_ready_;
+  grpc_closure* original_recv_message_ready_ = nullptr;
+
+  // For intercepting recv_trailing_metadata_ready.
   grpc_metadata_batch* recv_trailing_metadata_ = nullptr;
+  grpc_transport_stream_stats* transport_stream_stats_ = nullptr;
   grpc_closure recv_trailing_metadata_ready_;
   grpc_closure* original_recv_trailing_metadata_ready_ = nullptr;
 

data/src/core/ext/filters/client_channel/config_selector.h

@@ -44,6 +44,22 @@ namespace grpc_core {
 // MethodConfig and provide input to LB policies on a per-call basis.
 class ConfigSelector : public RefCounted<ConfigSelector> {
  public:
+  using CallAttributes = std::map<const char*, absl::string_view>;
+
+  // An interface to be used by the channel when dispatching calls.
+  class CallDispatchController {
+   public:
+    virtual ~CallDispatchController() = default;
+
+    // Called by the channel to decide if it should retry the call upon a
+    // failure.
+    virtual bool ShouldRetry() = 0;
+
+    // Called by the channel when no more LB picks will be performed for
+    // the call.
+    virtual void Commit() = 0;
+  };
+
   struct GetCallConfigArgs {
     grpc_slice* path;
     grpc_metadata_batch* initial_metadata;
@@ -60,11 +76,9 @@ class ConfigSelector : public RefCounted<ConfigSelector> {
     // the call to ensure that method_configs lives long enough.
     RefCountedPtr<ServiceConfig> service_config;
     // Call attributes that will be accessible to LB policy implementations.
-    std::map<const char*, absl::string_view> call_attributes;
-    // A callback that, if set, will be invoked when the call is
-    // committed (i.e., when we know that we will never again need to
-    // ask the picker for a subchannel for this call).
-    std::function<void()> on_call_committed;
+    CallAttributes call_attributes;
+    // Call dispatch controller.
+    CallDispatchController* call_dispatch_controller = nullptr;
   };
 
   ~ConfigSelector() override = default;
@@ -85,7 +99,6 @@ class ConfigSelector : public RefCounted<ConfigSelector> {
   // The channel will call this when the resolver returns a new ConfigSelector
   // to determine what set of dynamic filters will be configured.
   virtual std::vector<const grpc_channel_filter*> GetFilters() { return {}; }
-
   // Modifies channel args to be passed to the dynamic filter stack.
   // Takes ownership of argument.  Caller takes ownership of result.
   virtual grpc_channel_args* ModifyChannelArgs(grpc_channel_args* args) {

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc

@@ -606,7 +606,6 @@ const char* GrpcLb::Serverlist::ShouldDrop() {
 //
 
 GrpcLb::PickResult GrpcLb::Picker::Pick(PickArgs args) {
-  PickResult result;
   // Check if we should drop the call.
   const char* drop_token =
       serverlist_ == nullptr ? nullptr : serverlist_->ShouldDrop();
@@ -619,16 +618,16 @@ GrpcLb::PickResult GrpcLb::Picker::Pick(PickArgs args) {
     if (client_stats_ != nullptr) {
       client_stats_->AddCallDropped(drop_token);
     }
-    result.type = PickResult::PICK_COMPLETE;
-    return result;
+    return PickResult::Drop(
+        absl::UnavailableError("drop directed by grpclb balancer"));
   }
   // Forward pick to child policy.
-  result = child_picker_->Pick(args);
+  PickResult result = child_picker_->Pick(args);
   // If pick succeeded, add LB token to initial metadata.
-  if (result.type == PickResult::PICK_COMPLETE &&
-      result.subchannel != nullptr) {
+  auto* complete_pick = absl::get_if<PickResult::Complete>(&result.result);
+  if (complete_pick != nullptr) {
     const SubchannelWrapper* subchannel_wrapper =
-        static_cast<SubchannelWrapper*>(result.subchannel.get());
+        static_cast<SubchannelWrapper*>(complete_pick->subchannel.get());
     // Encode client stats object into metadata for use by
     // client_load_reporting filter.
     GrpcLbClientStats* client_stats = subchannel_wrapper->client_stats();
@@ -654,7 +653,7 @@ GrpcLb::PickResult GrpcLb::Picker::Pick(PickArgs args) {
       args.initial_metadata->Add(kGrpcLbLbTokenMetadataKey, lb_token);
     }
     // Unwrap subchannel to pass up to the channel.
-    result.subchannel = subchannel_wrapper->wrapped_subchannel();
+    complete_pick->subchannel = subchannel_wrapper->wrapped_subchannel();
   }
   return result;
 }

data/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc

@@ -114,10 +114,7 @@ class PickFirst : public LoadBalancingPolicy {
         : subchannel_(std::move(subchannel)) {}
 
     PickResult Pick(PickArgs /*args*/) override {
-      PickResult result;
-      result.type = PickResult::PICK_COMPLETE;
-      result.subchannel = subchannel_;
-      return result;
+      return PickResult::Complete(subchannel_);
     }
 
    private:
@@ -197,12 +194,10 @@ void PickFirst::AttemptToConnectUsingLatestUpdateArgsLocked() {
     // (If we are idle, then this will happen in ExitIdleLocked() if we
     // haven't gotten a non-empty update by the time the application tries
     // to start a new call.)
-    grpc_error_handle error =
-        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
-                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+    absl::Status status = absl::UnavailableError("Empty update");
     channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
-        absl::make_unique<TransientFailurePicker>(error));
+        GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+        absl::make_unique<TransientFailurePicker>(status));
     return;
   }
   // If one of the subchannels in the new list is already in state
@@ -314,13 +309,11 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
       p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
       // Set our state to that of the pending subchannel list.
       if (p->subchannel_list_->in_transient_failure()) {
-        grpc_error_handle error = grpc_error_set_int(
-            GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                "selected subchannel failed; switching to pending update"),
-            GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+        absl::Status status = absl::UnavailableError(
+            "selected subchannel failed; switching to pending update");
         p->channel_control_helper()->UpdateState(
-            GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
-            absl::make_unique<TransientFailurePicker>(error));
+            GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+            absl::make_unique<TransientFailurePicker>(status));
       } else {
         p->channel_control_helper()->UpdateState(
             GRPC_CHANNEL_CONNECTING, absl::Status(),
@@ -393,13 +386,11 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
         subchannel_list()->set_in_transient_failure(true);
         // Only report new state in case 1.
         if (subchannel_list() == p->subchannel_list_.get()) {
-          grpc_error_handle error = grpc_error_set_int(
-              GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                  "failed to connect to all addresses"),
-              GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+          absl::Status status =
+              absl::UnavailableError("failed to connect to all addresses");
           p->channel_control_helper()->UpdateState(
-              GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
-              absl::make_unique<TransientFailurePicker>(error));
+              GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+              absl::make_unique<TransientFailurePicker>(status));
         }
       }
       sd->CheckConnectivityStateAndStartWatchingLocked();

data/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc

@@ -472,12 +472,10 @@ void PriorityLb::TryNextPriorityLocked(bool report_connecting) {
             this);
   }
   current_child_from_before_update_ = nullptr;
-  grpc_error_handle error = grpc_error_set_int(
-      GRPC_ERROR_CREATE_FROM_STATIC_STRING("no ready priority"),
-      GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+  absl::Status status = absl::UnavailableError("no ready priority");
   channel_control_helper()->UpdateState(
-      GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
-      absl::make_unique<TransientFailurePicker>(error));
+      GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+      absl::make_unique<TransientFailurePicker>(status));
 }
 
 void PriorityLb::SelectPriorityLocked(uint32_t priority) {

data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc

@@ -380,18 +380,12 @@ RingHash::Picker::Picker(RefCountedPtr<RingHash> parent,
 }
 
 RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
-  PickResult result;
-  // Initialize to PICK_FAILED.
-  result.type = PickResult::PICK_FAILED;
   auto hash =
       args.call_state->ExperimentalGetCallAttribute(kRequestRingHashAttribute);
   uint64_t h;
   if (!absl::SimpleAtoi(hash, &h)) {
-    result.error = grpc_error_set_int(
-        GRPC_ERROR_CREATE_FROM_COPIED_STRING(
-            absl::StrCat("xds ring hash value is not a number").c_str()),
-        GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_INTERNAL);
-    return result;
+    return PickResult::Fail(
+        absl::InternalError("xds ring hash value is not a number"));
   }
   // Ported from https://github.com/RJ/ketama/blob/master/libketama/ketama.c
   // (ketama_get_server) NOTE: The algorithm depends on using signed integers
@@ -431,15 +425,12 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
       };
   switch (ring_[first_index].connectivity_state) {
     case GRPC_CHANNEL_READY:
-      result.type = PickResult::PICK_COMPLETE;
-      result.subchannel = ring_[first_index].subchannel;
-      return result;
+      return PickResult::Complete(ring_[first_index].subchannel);
     case GRPC_CHANNEL_IDLE:
       ScheduleSubchannelConnectionAttempt(ring_[first_index].subchannel);
-      // fallthrough
+      ABSL_FALLTHROUGH_INTENDED;
     case GRPC_CHANNEL_CONNECTING:
-      result.type = PickResult::PICK_QUEUE;
-      return result;
+      return PickResult::Queue();
     default:  // GRPC_CHANNEL_TRANSIENT_FAILURE
       break;
   }
@@ -455,18 +446,15 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
       continue;
     }
     if (entry.connectivity_state == GRPC_CHANNEL_READY) {
-      result.type = PickResult::PICK_COMPLETE;
-      result.subchannel = entry.subchannel;
-      return result;
+      return PickResult::Complete(entry.subchannel);
     }
     if (!found_second_subchannel) {
       switch (entry.connectivity_state) {
         case GRPC_CHANNEL_IDLE:
           ScheduleSubchannelConnectionAttempt(entry.subchannel);
-          // fallthrough
+          ABSL_FALLTHROUGH_INTENDED;
         case GRPC_CHANNEL_CONNECTING:
-          result.type = PickResult::PICK_QUEUE;
-          return result;
+          return PickResult::Queue();
         default:
           break;
       }
@@ -483,13 +471,8 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
       }
     }
   }
-  result.error =
-      grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(
-                             absl::StrCat("xds ring hash found a subchannel "
-                                          "that is in TRANSIENT_FAILURE state")
-                                 .c_str()),
-                         GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_INTERNAL);
-  return result;
+  return PickResult::Fail(absl::UnavailableError(
+      "xds ring hash found a subchannel that is in TRANSIENT_FAILURE state"));
 }
 
 //
@@ -580,13 +563,11 @@ bool RingHash::RingHashSubchannelList::UpdateRingHashConnectivityStateLocked() {
                                   this));
     return false;
   }
-  grpc_error* error =
-      grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                             "connections to backend failing or idle"),
-                         GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+  absl::Status status =
+      absl::UnavailableError("connections to backend failing or idle");
   p->channel_control_helper()->UpdateState(
-      GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
-      absl::make_unique<TransientFailurePicker>(error));
+      GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+      absl::make_unique<TransientFailurePicker>(status));
   return true;
 }
 
@@ -724,12 +705,10 @@ void RingHash::UpdateLocked(UpdateArgs args) {
       this, &grpc_lb_ring_hash_trace, std::move(addresses), *args.args);
   if (subchannel_list_->num_subchannels() == 0) {
     // If the new list is empty, immediately transition to TRANSIENT_FAILURE.
-    grpc_error* error =
-        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
-                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+    absl::Status status = absl::UnavailableError("Empty update");
     channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
-        absl::make_unique<TransientFailurePicker>(error));
+        GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+        absl::make_unique<TransientFailurePicker>(status));
   } else {
     // Start watching the new list.
     subchannel_list_->StartWatchingLocked();

data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc

@@ -213,10 +213,7 @@ RoundRobin::PickResult RoundRobin::Picker::Pick(PickArgs /*args*/) {
             parent_, this, last_picked_index_,
             subchannels_[last_picked_index_].get());
   }
-  PickResult result;
-  result.type = PickResult::PICK_COMPLETE;
-  result.subchannel = subchannels_[last_picked_index_];
-  return result;
+  return PickResult::Complete(subchannels_[last_picked_index_]);
 }
 
 //
@@ -331,13 +328,11 @@ void RoundRobin::RoundRobinSubchannelList::
         absl::make_unique<QueuePicker>(p->Ref(DEBUG_LOCATION, "QueuePicker")));
   } else if (num_transient_failure_ == num_subchannels()) {
     /* 3) TRANSIENT_FAILURE */
-    grpc_error_handle error =
-        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                               "connections to all backends failing"),
-                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+    absl::Status status =
+        absl::UnavailableError("connections to all backends failing");
     p->channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
-        absl::make_unique<TransientFailurePicker>(error));
+        GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+        absl::make_unique<TransientFailurePicker>(status));
   }
 }
 
@@ -449,12 +444,10 @@ void RoundRobin::UpdateLocked(UpdateArgs args) {
   if (latest_pending_subchannel_list_->num_subchannels() == 0) {
     // If the new list is empty, immediately promote the new list to the
     // current list and transition to TRANSIENT_FAILURE.
-    grpc_error_handle error =
-        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
-                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+    absl::Status status = absl::UnavailableError("Empty update");
     channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
-        absl::make_unique<TransientFailurePicker>(error));
+        GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+        absl::make_unique<TransientFailurePicker>(status));
     subchannel_list_ = std::move(latest_pending_subchannel_list_);
   } else if (subchannel_list_ == nullptr) {
     // If there is no current list, immediately promote the new list to