grpc 1.40.0.pre1 → 1.42.0.pre1

data/src/core/ext/filters/client_channel/retry_filter.cc

@@ -27,8 +27,8 @@
 #include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/filters/client_channel/retry_service_config.h"
 #include "src/core/ext/filters/client_channel/retry_throttle.h"
-#include "src/core/ext/filters/client_channel/service_config.h"
-#include "src/core/ext/filters/client_channel/service_config_call_data.h"
+#include "src/core/ext/service_config/service_config.h"
+#include "src/core/ext/service_config/service_config_call_data.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
@@ -88,9 +88,7 @@
 
 // TODO(roth): In subsequent PRs:
 // - add support for transparent retries (including initial metadata)
-// - figure out how to record stats in census for retries
-//   (census filter is on top of this one)
-// - add census stats for retries
+// - implement hedging
 
 // By default, we buffer 256 KiB per RPC for retries.
 // TODO(roth): Do we have any data to suggest a better value?
@@ -337,9 +335,10 @@ class RetryFilter::CallData {
         call_attempt_->lb_call_committed_ = true;
         auto* calld = call_attempt_->calld_;
         if (calld->retry_committed_) {
-          auto* service_config_call_data = static_cast<ServiceConfigCallData*>(
-              calld->call_context_[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA]
-                  .value);
+          auto* service_config_call_data =
+              static_cast<ClientChannelServiceConfigCallData*>(
+                  calld->call_context_[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA]
+                      .value);
           service_config_call_data->call_dispatch_controller()->Commit();
         }
       }
@@ -421,27 +420,22 @@ class RetryFilter::CallData {
     // BatchData.batch.payload points to this.
     grpc_transport_stream_op_batch_payload batch_payload_;
     // For send_initial_metadata.
-    // Note that we need to make a copy of the initial metadata for each
-    // call attempt instead of just referring to the copy in call_data,
-    // because filters in the subchannel stack may modify the metadata,
-    // so we need to start in a pristine state for each attempt of the call.
-    grpc_linked_mdelem* send_initial_metadata_storage_;
-    grpc_metadata_batch send_initial_metadata_;
+    grpc_linked_mdelem retry_attempts_metadata_;
+    grpc_metadata_batch send_initial_metadata_{calld_->arena_};
     // For send_message.
     // TODO(roth): Restructure this to eliminate use of ManualConstructor.
     ManualConstructor<ByteStreamCache::CachingByteStream> send_message_;
     // For send_trailing_metadata.
-    grpc_linked_mdelem* send_trailing_metadata_storage_;
-    grpc_metadata_batch send_trailing_metadata_;
+    grpc_metadata_batch send_trailing_metadata_{calld_->arena_};
     // For intercepting recv_initial_metadata.
-    grpc_metadata_batch recv_initial_metadata_;
+    grpc_metadata_batch recv_initial_metadata_{calld_->arena_};
     grpc_closure recv_initial_metadata_ready_;
     bool trailing_metadata_available_ = false;
     // For intercepting recv_message.
     grpc_closure recv_message_ready_;
     OrphanablePtr<ByteStream> recv_message_;
     // For intercepting recv_trailing_metadata.
-    grpc_metadata_batch recv_trailing_metadata_;
+    grpc_metadata_batch recv_trailing_metadata_{calld_->arena_};
     grpc_transport_stream_stats collect_stats_;
     grpc_closure recv_trailing_metadata_ready_;
     // These fields indicate which ops have been started and completed on
@@ -539,6 +533,8 @@ class RetryFilter::CallData {
   CallCombiner* call_combiner_;
   grpc_call_context_element* call_context_;
 
+  grpc_error_handle cancelled_from_surface_ = GRPC_ERROR_NONE;
+
   RefCountedPtr<CallStackDestructionBarrier> call_stack_destruction_barrier_;
 
   // TODO(roth): As part of implementing hedging, we will need to maintain a
@@ -572,8 +568,7 @@ class RetryFilter::CallData {
   // Cached data for retrying send ops.
   // send_initial_metadata
   bool seen_send_initial_metadata_ = false;
-  grpc_linked_mdelem* send_initial_metadata_storage_ = nullptr;
-  grpc_metadata_batch send_initial_metadata_;
+  grpc_metadata_batch send_initial_metadata_{arena_};
   uint32_t send_initial_metadata_flags_;
   // TODO(roth): As part of implementing hedging, we'll probably need to
   // have the LB call set a value in CallAttempt and then propagate it
@@ -598,8 +593,7 @@ class RetryFilter::CallData {
   absl::InlinedVector<ByteStreamCache*, 3> send_messages_;
   // send_trailing_metadata
   bool seen_send_trailing_metadata_ = false;
-  grpc_linked_mdelem* send_trailing_metadata_storage_ = nullptr;
-  grpc_metadata_batch send_trailing_metadata_;
+  grpc_metadata_batch send_trailing_metadata_{arena_};
 };
 
 //
@@ -1159,8 +1153,9 @@ bool RetryFilter::CallData::CallAttempt::ShouldRetry(
     }
   }
   // Check with call dispatch controller.
-  auto* service_config_call_data = static_cast<ServiceConfigCallData*>(
-      calld_->call_context_[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value);
+  auto* service_config_call_data =
+      static_cast<ClientChannelServiceConfigCallData*>(
+          calld_->call_context_[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value);
   if (!service_config_call_data->call_dispatch_controller()->ShouldRetry()) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) {
       gpr_log(
@@ -1308,18 +1303,6 @@ RetryFilter::CallData::CallAttempt::BatchData::~BatchData() {
             call_attempt_->calld_->chand_, call_attempt_->calld_,
             call_attempt_.get(), this);
   }
-  if (batch_.send_initial_metadata) {
-    grpc_metadata_batch_destroy(&call_attempt_->send_initial_metadata_);
-  }
-  if (batch_.send_trailing_metadata) {
-    grpc_metadata_batch_destroy(&call_attempt_->send_trailing_metadata_);
-  }
-  if (batch_.recv_initial_metadata) {
-    grpc_metadata_batch_destroy(&call_attempt_->recv_initial_metadata_);
-  }
-  if (batch_.recv_trailing_metadata) {
-    grpc_metadata_batch_destroy(&call_attempt_->recv_trailing_metadata_);
-  }
   GRPC_CALL_STACK_UNREF(call_attempt_->calld_->owning_call_, "Retry BatchData");
   call_attempt_.reset(DEBUG_LOCATION, "~BatchData");
 }
@@ -1363,9 +1346,8 @@ void RetryFilter::CallData::CallAttempt::BatchData::
     return;
   }
   // Return metadata.
-  grpc_metadata_batch_move(
-      &call_attempt_->recv_initial_metadata_,
-      pending->batch->payload->recv_initial_metadata.recv_initial_metadata);
+  *pending->batch->payload->recv_initial_metadata.recv_initial_metadata =
+      std::move(call_attempt_->recv_initial_metadata_);
   // Propagate trailing_metadata_available.
   *pending->batch->payload->recv_initial_metadata.trailing_metadata_available =
       call_attempt_->trailing_metadata_available_;
@@ -1565,11 +1547,12 @@ void GetCallStatus(grpc_millis deadline, grpc_metadata_batch* md_batch,
       *is_lb_drop = true;
     }
   } else {
-    GPR_ASSERT(md_batch->idx.named.grpc_status != nullptr);
-    *status =
-        grpc_get_status_code_from_metadata(md_batch->idx.named.grpc_status->md);
-    if (md_batch->idx.named.grpc_retry_pushback_ms != nullptr) {
-      *server_pushback_md = &md_batch->idx.named.grpc_retry_pushback_ms->md;
+    GPR_ASSERT(md_batch->legacy_index()->named.grpc_status != nullptr);
+    *status = grpc_get_status_code_from_metadata(
+        md_batch->legacy_index()->named.grpc_status->md);
+    if (md_batch->legacy_index()->named.grpc_retry_pushback_ms != nullptr) {
+      *server_pushback_md =
+          &md_batch->legacy_index()->named.grpc_retry_pushback_ms->md;
     }
   }
   GRPC_ERROR_UNREF(error);
@@ -1601,9 +1584,8 @@ void RetryFilter::CallData::CallAttempt::BatchData::
       &call_attempt_->collect_stats_,
       pending->batch->payload->recv_trailing_metadata.collect_stats);
   // Return metadata.
-  grpc_metadata_batch_move(
-      &call_attempt_->recv_trailing_metadata_,
-      pending->batch->payload->recv_trailing_metadata.recv_trailing_metadata);
+  *pending->batch->payload->recv_trailing_metadata.recv_trailing_metadata =
+      std::move(call_attempt_->recv_trailing_metadata_);
   // Add closure.
   closures->Add(pending->batch->payload->recv_trailing_metadata
                     .recv_trailing_metadata_ready,
@@ -1920,18 +1902,12 @@ void RetryFilter::CallData::CallAttempt::BatchData::
   //
   // If we've already completed one or more attempts, add the
   // grpc-retry-attempts header.
-  call_attempt_->send_initial_metadata_storage_ =
-      static_cast<grpc_linked_mdelem*>(
-          calld->arena_->Alloc(sizeof(grpc_linked_mdelem) *
-                               (calld->send_initial_metadata_.list.count +
-                                (calld->num_attempts_completed_ > 0))));
   grpc_metadata_batch_copy(&calld->send_initial_metadata_,
-                           &call_attempt_->send_initial_metadata_,
-                           call_attempt_->send_initial_metadata_storage_);
-  if (GPR_UNLIKELY(call_attempt_->send_initial_metadata_.idx.named
-                       .grpc_previous_rpc_attempts != nullptr)) {
-    grpc_metadata_batch_remove(&call_attempt_->send_initial_metadata_,
-                               GRPC_BATCH_GRPC_PREVIOUS_RPC_ATTEMPTS);
+                           &call_attempt_->send_initial_metadata_);
+  if (GPR_UNLIKELY(call_attempt_->send_initial_metadata_.legacy_index()
+                       ->named.grpc_previous_rpc_attempts != nullptr)) {
+    call_attempt_->send_initial_metadata_.Remove(
+        GRPC_BATCH_GRPC_PREVIOUS_RPC_ATTEMPTS);
   }
   if (GPR_UNLIKELY(calld->num_attempts_completed_ > 0)) {
     grpc_mdelem retry_md = grpc_mdelem_create(
@@ -1939,9 +1915,8 @@ void RetryFilter::CallData::CallAttempt::BatchData::
         *retry_count_strings[calld->num_attempts_completed_ - 1], nullptr);
     grpc_error_handle error = grpc_metadata_batch_add_tail(
         &call_attempt_->send_initial_metadata_,
-        &call_attempt_->send_initial_metadata_storage_
-             [calld->send_initial_metadata_.list.count],
-        retry_md, GRPC_BATCH_GRPC_PREVIOUS_RPC_ATTEMPTS);
+        &call_attempt_->retry_attempts_metadata_, retry_md,
+        GRPC_BATCH_GRPC_PREVIOUS_RPC_ATTEMPTS);
     if (GPR_UNLIKELY(error != GRPC_ERROR_NONE)) {
       gpr_log(GPR_ERROR, "error adding retry metadata: %s",
               grpc_error_std_string(error).c_str());
@@ -1983,13 +1958,8 @@ void RetryFilter::CallData::CallAttempt::BatchData::
   // We need to make a copy of the metadata batch for each attempt, since
   // the filters in the subchannel stack may modify this batch, and we don't
   // want those modifications to be passed forward to subsequent attempts.
-  call_attempt_->send_trailing_metadata_storage_ =
-      static_cast<grpc_linked_mdelem*>(
-          calld->arena_->Alloc(sizeof(grpc_linked_mdelem) *
-                               calld->send_trailing_metadata_.list.count));
   grpc_metadata_batch_copy(&calld->send_trailing_metadata_,
-                           &call_attempt_->send_trailing_metadata_,
-                           call_attempt_->send_trailing_metadata_storage_);
+                           &call_attempt_->send_trailing_metadata_);
   call_attempt_->started_send_trailing_metadata_ = true;
   batch_.send_trailing_metadata = true;
   batch_.payload->send_trailing_metadata.send_trailing_metadata =
@@ -2000,7 +1970,7 @@ void RetryFilter::CallData::CallAttempt::BatchData::
     AddRetriableRecvInitialMetadataOp() {
   call_attempt_->started_recv_initial_metadata_ = true;
   batch_.recv_initial_metadata = true;
-  grpc_metadata_batch_init(&call_attempt_->recv_initial_metadata_);
+  call_attempt_->recv_initial_metadata_.Clear();
   batch_.payload->recv_initial_metadata.recv_initial_metadata =
       &call_attempt_->recv_initial_metadata_;
   batch_.payload->recv_initial_metadata.trailing_metadata_available =
@@ -2027,7 +1997,7 @@ void RetryFilter::CallData::CallAttempt::BatchData::
     AddRetriableRecvTrailingMetadataOp() {
   call_attempt_->started_recv_trailing_metadata_ = true;
   batch_.recv_trailing_metadata = true;
-  grpc_metadata_batch_init(&call_attempt_->recv_trailing_metadata_);
+  call_attempt_->recv_trailing_metadata_.Clear();
   batch_.payload->recv_trailing_metadata.recv_trailing_metadata =
       &call_attempt_->recv_trailing_metadata_;
   batch_.payload->recv_trailing_metadata.collect_stats =
@@ -2141,6 +2111,7 @@ RetryFilter::CallData::~CallData() {
   for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) {
     GPR_ASSERT(pending_batches_[i].batch == nullptr);
   }
+  GRPC_ERROR_UNREF(cancelled_from_surface_);
 }
 
 void RetryFilter::CallData::StartTransportStreamOpBatch(
@@ -2173,6 +2144,9 @@ void RetryFilter::CallData::StartTransportStreamOpBatch(
       call_attempt_->CancelFromSurface(batch);
       return;
     }
+    // Save cancel_error in case subsequent batches are started.
+    GRPC_ERROR_UNREF(cancelled_from_surface_);
+    cancelled_from_surface_ = GRPC_ERROR_REF(cancel_error);
     // Cancel retry timer.
     if (retry_timer_pending_) {
       if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) {
@@ -2201,6 +2175,15 @@ void RetryFilter::CallData::StartTransportStreamOpBatch(
   }
   // If we do not yet have a call attempt, create one.
   if (call_attempt_ == nullptr) {
+    // If we were previously cancelled from the surface, cancel this
+    // batch instead of creating a call attempt.
+    if (cancelled_from_surface_ != GRPC_ERROR_NONE) {
+      PendingBatchClear(pending);
+      // Note: This will release the call combiner.
+      grpc_transport_stream_op_batch_finish_with_failure(
+          batch, GRPC_ERROR_REF(cancelled_from_surface_), call_combiner_);
+      return;
+    }
     // If there is no retry policy, then commit retries immediately.
     // This ensures that the code below will always jump to the fast path.
     // TODO(roth): Remove this special case when we implement
@@ -2231,8 +2214,9 @@ void RetryFilter::CallData::StartTransportStreamOpBatch(
                 chand_, this);
       }
       PendingBatchClear(pending);
-      auto* service_config_call_data = static_cast<ServiceConfigCallData*>(
-          call_context_[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value);
+      auto* service_config_call_data =
+          static_cast<ClientChannelServiceConfigCallData*>(
+              call_context_[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value);
       committed_call_ = CreateLoadBalancedCall(
           service_config_call_data->call_dispatch_controller());
       committed_call_->StartTransportStreamOpBatch(batch);
@@ -2288,14 +2272,9 @@ void RetryFilter::CallData::MaybeCacheSendOpsForBatch(PendingBatch* pending) {
   // Save a copy of metadata for send_initial_metadata ops.
   if (batch->send_initial_metadata) {
     seen_send_initial_metadata_ = true;
-    GPR_ASSERT(send_initial_metadata_storage_ == nullptr);
     grpc_metadata_batch* send_initial_metadata =
         batch->payload->send_initial_metadata.send_initial_metadata;
-    send_initial_metadata_storage_ =
-        static_cast<grpc_linked_mdelem*>(arena_->Alloc(
-            sizeof(grpc_linked_mdelem) * send_initial_metadata->list.count));
-    grpc_metadata_batch_copy(send_initial_metadata, &send_initial_metadata_,
-                             send_initial_metadata_storage_);
+    grpc_metadata_batch_copy(send_initial_metadata, &send_initial_metadata_);
     send_initial_metadata_flags_ =
         batch->payload->send_initial_metadata.send_initial_metadata_flags;
     peer_string_ = batch->payload->send_initial_metadata.peer_string;
@@ -2309,14 +2288,9 @@ void RetryFilter::CallData::MaybeCacheSendOpsForBatch(PendingBatch* pending) {
   // Save metadata batch for send_trailing_metadata ops.
   if (batch->send_trailing_metadata) {
     seen_send_trailing_metadata_ = true;
-    GPR_ASSERT(send_trailing_metadata_storage_ == nullptr);
     grpc_metadata_batch* send_trailing_metadata =
         batch->payload->send_trailing_metadata.send_trailing_metadata;
-    send_trailing_metadata_storage_ =
-        static_cast<grpc_linked_mdelem*>(arena_->Alloc(
-            sizeof(grpc_linked_mdelem) * send_trailing_metadata->list.count));
-    grpc_metadata_batch_copy(send_trailing_metadata, &send_trailing_metadata_,
-                             send_trailing_metadata_storage_);
+    grpc_metadata_batch_copy(send_trailing_metadata, &send_trailing_metadata_);
   }
 }
 
@@ -2325,7 +2299,7 @@ void RetryFilter::CallData::FreeCachedSendInitialMetadata() {
     gpr_log(GPR_INFO, "chand=%p calld=%p: destroying send_initial_metadata",
             chand_, this);
   }
-  grpc_metadata_batch_destroy(&send_initial_metadata_);
+  send_initial_metadata_.Clear();
 }
 
 void RetryFilter::CallData::FreeCachedSendMessage(size_t idx) {
@@ -2342,7 +2316,7 @@ void RetryFilter::CallData::FreeCachedSendTrailingMetadata() {
     gpr_log(GPR_INFO, "chand=%p calld=%p: destroying send_trailing_metadata",
             chand_, this);
   }
-  grpc_metadata_batch_destroy(&send_trailing_metadata_);
+  send_trailing_metadata_.Clear();
 }
 
 void RetryFilter::CallData::FreeAllCachedSendOpData() {
@@ -2391,8 +2365,8 @@ RetryFilter::CallData::PendingBatch* RetryFilter::CallData::PendingBatchesAdd(
   // gRPC clients do not send trailing metadata.
   if (batch->send_initial_metadata) {
     pending_send_initial_metadata_ = true;
-    bytes_buffered_for_retry_ += grpc_metadata_batch_size(
-        batch->payload->send_initial_metadata.send_initial_metadata);
+    bytes_buffered_for_retry_ += batch->payload->send_initial_metadata
+                                     .send_initial_metadata->TransportSize();
   }
   if (batch->send_message) {
     pending_send_message_ = true;
@@ -2528,8 +2502,9 @@ void RetryFilter::CallData::RetryCommit(CallAttempt* call_attempt) {
     // call dispatch controller down into the LB call, and it won't be
     // our problem anymore.
     if (call_attempt->lb_call_committed()) {
-      auto* service_config_call_data = static_cast<ServiceConfigCallData*>(
-          call_context_[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value);
+      auto* service_config_call_data =
+          static_cast<ClientChannelServiceConfigCallData*>(
+              call_context_[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value);
       service_config_call_data->call_dispatch_controller()->Commit();
     }
     // Free cached send ops.

data/src/core/ext/filters/client_channel/retry_service_config.h

@@ -22,7 +22,7 @@
 #include <memory>
 
 #include "src/core/ext/filters/client_channel/retry_throttle.h"
-#include "src/core/ext/filters/client_channel/service_config_parser.h"
+#include "src/core/ext/service_config/service_config_parser.h"
 #include "src/core/lib/channel/status_util.h"
 #include "src/core/lib/iomgr/exec_ctx.h"  // for grpc_millis
 

data/src/core/ext/filters/client_channel/retry_throttle.cc

@@ -23,6 +23,7 @@
 #include <limits.h>
 #include <string.h>
 
+#include <map>
 #include <string>
 
 #include <grpc/support/alloc.h>
@@ -30,7 +31,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 
-#include "src/core/lib/avl/avl.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 
 namespace grpc_core {
 namespace internal {
@@ -114,55 +115,24 @@ void ServerRetryThrottleData::RecordSuccess() {
       static_cast<gpr_atm>(throttle_data->max_milli_tokens_));
 }
 
-//
-// avl vtable for string -> server_retry_throttle_data map
-//
-
-namespace {
-
-void* copy_server_name(void* key, void* /*unused*/) {
-  return gpr_strdup(static_cast<const char*>(key));
-}
-
-long compare_server_name(void* key1, void* key2, void* /*unused*/) {
-  return strcmp(static_cast<const char*>(key1), static_cast<const char*>(key2));
-}
-
-void destroy_server_retry_throttle_data(void* value, void* /*unused*/) {
-  ServerRetryThrottleData* throttle_data =
-      static_cast<ServerRetryThrottleData*>(value);
-  throttle_data->Unref();
-}
-
-void* copy_server_retry_throttle_data(void* value, void* /*unused*/) {
-  ServerRetryThrottleData* throttle_data =
-      static_cast<ServerRetryThrottleData*>(value);
-  return throttle_data->Ref().release();
-}
-
-void destroy_server_name(void* key, void* /*unused*/) { gpr_free(key); }
-
-const grpc_avl_vtable avl_vtable = {
-    destroy_server_name, copy_server_name, compare_server_name,
-    destroy_server_retry_throttle_data, copy_server_retry_throttle_data};
-
-}  // namespace
-
 //
 // ServerRetryThrottleMap
 //
 
+using StringToDataMap =
+    std::map<std::string, RefCountedPtr<ServerRetryThrottleData>>;
 static gpr_mu g_mu;
-static grpc_avl g_avl;
+static StringToDataMap* g_map;
 
 void ServerRetryThrottleMap::Init() {
   gpr_mu_init(&g_mu);
-  g_avl = grpc_avl_create(&avl_vtable);
+  g_map = new StringToDataMap();
 }
 
 void ServerRetryThrottleMap::Shutdown() {
   gpr_mu_destroy(&g_mu);
-  grpc_avl_unref(g_avl, nullptr);
+  delete g_map;
+  g_map = nullptr;
 }
 
 RefCountedPtr<ServerRetryThrottleData> ServerRetryThrottleMap::GetDataForServer(
@@ -170,23 +140,22 @@ RefCountedPtr<ServerRetryThrottleData> ServerRetryThrottleMap::GetDataForServer(
     intptr_t milli_token_ratio) {
   RefCountedPtr<ServerRetryThrottleData> result;
   gpr_mu_lock(&g_mu);
+  auto it = g_map->find(server_name);
   ServerRetryThrottleData* throttle_data =
-      static_cast<ServerRetryThrottleData*>(
-          grpc_avl_get(g_avl, const_cast<char*>(server_name.c_str()), nullptr));
+      it == g_map->end() ? nullptr : it->second.get();
   if (throttle_data == nullptr ||
       throttle_data->max_milli_tokens() != max_milli_tokens ||
       throttle_data->milli_token_ratio() != milli_token_ratio) {
     // Entry not found, or found with old parameters.  Create a new one.
-    result = MakeRefCounted<ServerRetryThrottleData>(
-        max_milli_tokens, milli_token_ratio, throttle_data);
-    g_avl = grpc_avl_add(g_avl, gpr_strdup(server_name.c_str()),
-                         result->Ref().release(), nullptr);
-  } else {
-    // Entry found.  Return a new ref to it.
-    result = throttle_data->Ref();
+    it = g_map
+             ->emplace(server_name,
+                       MakeRefCounted<ServerRetryThrottleData>(
+                           max_milli_tokens, milli_token_ratio, throttle_data))
+             .first;
+    throttle_data = it->second.get();
   }
   gpr_mu_unlock(&g_mu);
-  return result;
+  return throttle_data->Ref();
 }
 
 }  // namespace internal

data/src/core/ext/filters/client_channel/server_address.h

@@ -128,7 +128,7 @@ class ServerAddressWeightAttribute : public ServerAddress::AttributeInterface {
   int Cmp(const AttributeInterface* other) const override {
     const auto* other_locality_attr =
         static_cast<const ServerAddressWeightAttribute*>(other);
-    return GPR_ICMP(weight_, other_locality_attr->weight_);
+    return QsortCompare(weight_, other_locality_attr->weight_);
   }
 
   std::string ToString() const override {

data/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc

@@ -19,11 +19,11 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/ext/filters/client_channel/service_config_call_data.h"
+#include "src/core/ext/service_config/service_config_call_data.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/channel_stack_builder.h"
-#include "src/core/lib/surface/channel_init.h"
+#include "src/core/lib/config/core_configuration.h"
 
 namespace grpc_core {
 
@@ -59,26 +59,43 @@ class ServiceConfigChannelArgChannelData {
 
 class ServiceConfigChannelArgCallData {
  public:
-  ServiceConfigChannelArgCallData(grpc_call_element* elem,
-                                  const grpc_call_element_args* args) {
-    ServiceConfigChannelArgChannelData* chand =
-        static_cast<ServiceConfigChannelArgChannelData*>(elem->channel_data);
-    RefCountedPtr<ServiceConfig> service_config = chand->service_config();
-    if (service_config != nullptr) {
-      GPR_DEBUG_ASSERT(args->context != nullptr);
-      const auto* method_params_vector =
-          service_config->GetMethodParsedConfigVector(args->path);
-      args->arena->New<ServiceConfigCallData>(
-          std::move(service_config), method_params_vector, args->context);
-    }
+  ServiceConfigChannelArgCallData(
+      RefCountedPtr<ServiceConfig> service_config,
+      const ServiceConfigParser::ParsedConfigVector* method_config,
+      const grpc_call_element_args* args)
+      : call_context_(args->context),
+        service_config_call_data_(std::move(service_config), method_config,
+                                  /*call_attributes=*/{}) {
+    GPR_DEBUG_ASSERT(args->context != nullptr);
+    // No need to set the destroy function, since it will be cleaned up
+    // when this filter is destroyed in the filter stack.
+    args->context[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value =
+        &service_config_call_data_;
+  }
+
+  ~ServiceConfigChannelArgCallData() {
+    // Remove the entry from call context, just in case anyone above us
+    // tries to look at it during call stack destruction.
+    call_context_[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value = nullptr;
   }
+
+ private:
+  grpc_call_context_element* call_context_;
+  ServiceConfigCallData service_config_call_data_;
 };
 
 grpc_error_handle ServiceConfigChannelArgInitCallElem(
     grpc_call_element* elem, const grpc_call_element_args* args) {
-  ServiceConfigChannelArgCallData* calld =
-      static_cast<ServiceConfigChannelArgCallData*>(elem->call_data);
-  new (calld) ServiceConfigChannelArgCallData(elem, args);
+  auto* chand =
+      static_cast<ServiceConfigChannelArgChannelData*>(elem->channel_data);
+  auto* calld = static_cast<ServiceConfigChannelArgCallData*>(elem->call_data);
+  RefCountedPtr<ServiceConfig> service_config = chand->service_config();
+  const ServiceConfigParser::ParsedConfigVector* method_config = nullptr;
+  if (service_config != nullptr) {
+    method_config = service_config->GetMethodParsedConfigVector(args->path);
+  }
+  new (calld) ServiceConfigChannelArgCallData(std::move(service_config),
+                                              method_config, args);
   return GRPC_ERROR_NONE;
 }
 
@@ -117,27 +134,23 @@ const grpc_channel_filter ServiceConfigChannelArgFilter = {
     grpc_channel_next_get_info,
     "service_config_channel_arg"};
 
-bool maybe_add_service_config_channel_arg_filter(
-    grpc_channel_stack_builder* builder, void* /* arg */) {
-  const grpc_channel_args* channel_args =
-      grpc_channel_stack_builder_get_channel_arguments(builder);
-  if (grpc_channel_args_want_minimal_stack(channel_args) ||
-      grpc_channel_args_find_string(channel_args, GRPC_ARG_SERVICE_CONFIG) ==
-          nullptr) {
-    return true;
-  }
-  return grpc_channel_stack_builder_prepend_filter(
-      builder, &ServiceConfigChannelArgFilter, nullptr, nullptr);
-}
-
 }  // namespace
 
-}  // namespace grpc_core
-
-void grpc_service_config_channel_arg_filter_init(void) {
-  grpc_channel_init_register_stage(
+void RegisterServiceConfigChannelArgFilter(
+    CoreConfiguration::Builder* builder) {
+  builder->channel_init()->RegisterStage(
       GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY,
-      grpc_core::maybe_add_service_config_channel_arg_filter, nullptr);
+      [](grpc_channel_stack_builder* builder) {
+        const grpc_channel_args* channel_args =
+            grpc_channel_stack_builder_get_channel_arguments(builder);
+        if (grpc_channel_args_want_minimal_stack(channel_args) ||
+            grpc_channel_args_find_string(channel_args,
+                                          GRPC_ARG_SERVICE_CONFIG) == nullptr) {
+          return true;
+        }
+        return grpc_channel_stack_builder_prepend_filter(
+            builder, &ServiceConfigChannelArgFilter, nullptr, nullptr);
+      });
 }
 
-void grpc_service_config_channel_arg_filter_shutdown(void) {}
+}  // namespace grpc_core