grpc 1.78.0 → 1.80.0.pre1

data/src/core/client_channel/client_channel.cc

@@ -275,7 +275,7 @@ class ClientChannel::SubchannelWrapper::WatcherWrapper
         << subchannel_wrapper_->subchannel_.get()
         << " watcher=" << watcher_.get()
         << " state=" << ConnectivityStateName(state) << " status=" << status;
-    if (!IsTransportStateWatcherEnabled()) {
+    if (!IsSubchannelConnectionScalingEnabled()) {
       auto keepalive_throttling = status.GetPayload(kKeepaliveThrottlingKey);
       if (keepalive_throttling.has_value()) {
         int new_keepalive_time_ms = -1;
@@ -679,7 +679,7 @@ ClientChannel::ClientChannel(
       client_channel_factory_(client_channel_factory),
       channelz_node_(channel_args_.GetObject<channelz::ChannelNode>()),
       idle_timeout_(GetClientIdleTimeout(channel_args_)),
-      resolver_data_for_calls_(ResolverDataForCalls{}),
+      resolver_data_for_calls_(nullptr),
       picker_(nullptr),
       call_destination_(
           call_destination_factory->CreateCallDestination(picker_)),
@@ -906,6 +906,72 @@ grpc_call* ClientChannel::CreateCall(
                         compression_options(), std::move(arena), Ref());
 }
 
+namespace {
+
+class FilterChainImpl final : public FilterChain {
+ public:
+  explicit FilterChainImpl(RefCountedPtr<UnstartedCallDestination> destination)
+      : destination_(std::move(destination)) {}
+
+  UnstartedCallDestination* destination() const { return destination_.get(); }
+
+ private:
+  RefCountedPtr<UnstartedCallDestination> destination_;
+};
+
+class FilterChainBuilderImpl final : public FilterChainBuilder {
+ public:
+  FilterChainBuilderImpl(
+      bool enable_retries, const ChannelArgs& channel_args,
+      Blackboard* blackboard,
+      std::function<void(ServerMetadata&)> on_server_trailing_metadata,
+      RefCountedPtr<UnstartedCallDestination> destination)
+      : enable_retries_(enable_retries),
+        channel_args_(channel_args),
+        blackboard_(blackboard),
+        on_server_trailing_metadata_(std::move(on_server_trailing_metadata)),
+        destination_(std::move(destination)) {}
+
+  absl::StatusOr<RefCountedPtr<FilterChain>> Build() override {
+    if (builder_ == nullptr) InitBuilder();
+    if (enable_retries_) builder_->Add<RetryInterceptor>(nullptr);
+    auto top_of_stack_destination = builder_->Build(destination_);
+    if (!top_of_stack_destination.ok()) {
+      return MaybeRewriteIllegalStatusCode(top_of_stack_destination.status(),
+                                           "channel construction");
+    }
+    builder_.reset();
+    return MakeRefCounted<FilterChainImpl>(
+        std::move(*top_of_stack_destination));
+  }
+
+ private:
+  void AddFilter(const FilterHandle& filter_handle,
+                 RefCountedPtr<const FilterConfig> config) override {
+    if (builder_ == nullptr) InitBuilder();
+    filter_handle.AddToBuilder(builder_.get(), std::move(config));
+  }
+
+  void InitBuilder() {
+    builder_ =
+        std::make_unique<InterceptionChainBuilder>(channel_args_, blackboard_);
+    if (on_server_trailing_metadata_ != nullptr) {
+      builder_->AddOnServerTrailingMetadata(on_server_trailing_metadata_);
+    }
+    CoreConfiguration::Get().channel_init().AddToInterceptionChainBuilder(
+        GRPC_CLIENT_CHANNEL, *builder_);
+  }
+
+  const bool enable_retries_;
+  const ChannelArgs channel_args_;
+  const Blackboard* blackboard_;
+  const std::function<void(ServerMetadata&)> on_server_trailing_metadata_;
+  const RefCountedPtr<UnstartedCallDestination> destination_;
+  std::unique_ptr<InterceptionChainBuilder> builder_;
+};
+
+}  // namespace
+
 void ClientChannel::StartCall(UnstartedCallHandler unstarted_handler) {
   // Increment call count.
   if (idle_timeout_ != Duration::Zero()) idle_state_.IncreaseCallCount();
@@ -924,31 +990,32 @@ void ClientChannel::StartCall(UnstartedCallHandler unstarted_handler) {
             // Wait for the resolver result.
             CheckDelayed(self->resolver_data_for_calls_.NextWhen(
                 [wait_for_ready](
-                    const absl::StatusOr<ResolverDataForCalls> result) {
+                    const absl::StatusOr<RefCountedPtr<ConfigSelector>>
+                        config_selector) {
                   bool got_result = false;
                   // If the resolver reports an error but the call is
                   // wait_for_ready, keep waiting for the next result
                   // instead of failing the call.
-                  if (!result.ok()) {
+                  if (!config_selector.ok()) {
                     got_result = !wait_for_ready;
                   } else {
                     // Not an error.  Make sure we actually have a result.
-                    got_result = result->config_selector != nullptr;
+                    got_result = *config_selector != nullptr;
                   }
                   return got_result;
                 })),
             // Handle resolver result.
             [self, unstarted_handler](
-                std::tuple<absl::StatusOr<ResolverDataForCalls>, bool>
+                std::tuple<absl::StatusOr<RefCountedPtr<ConfigSelector>>, bool>
                     result_and_delayed) mutable {
-              auto& resolver_data = std::get<0>(result_and_delayed);
-              const bool was_queued = std::get<1>(result_and_delayed);
-              if (!resolver_data.ok()) return resolver_data.status();
+              auto& [config_selector, was_queued] = result_and_delayed;
+              if (!config_selector.ok()) return config_selector.status();
               // Apply service config to call.
-              absl::Status status = self->ApplyServiceConfigToCall(
-                  *resolver_data->config_selector,
-                  unstarted_handler.UnprocessedClientInitialMetadata());
-              if (!status.ok()) return status;
+              absl::StatusOr<RefCountedPtr<const FilterChain>> filter_chain =
+                  self->ApplyServiceConfigToCall(
+                      **config_selector,
+                      unstarted_handler.UnprocessedClientInitialMetadata());
+              if (!filter_chain.ok()) return filter_chain.status();
               // If the call was queued, add trace annotation.
               if (was_queued) {
                 auto* call_tracer = MaybeGetContext<CallSpan>();
@@ -959,8 +1026,10 @@ void ClientChannel::StartCall(UnstartedCallHandler unstarted_handler) {
               }
               // Start the call on the destination provided by the
               // resolver.
-              resolver_data->call_destination->StartCall(
-                  std::move(unstarted_handler));
+              auto destination =
+                  DownCast<const FilterChainImpl*>(filter_chain->get())
+                      ->destination();
+              destination->StartCall(std::move(unstarted_handler));
               return absl::OkStatus();
             });
       });
@@ -992,7 +1061,7 @@ void ClientChannel::DestroyResolverAndLbPolicyLocked() {
     resolver_.reset();
     saved_service_config_.reset();
     saved_config_selector_.reset();
-    resolver_data_for_calls_.Set(ResolverDataForCalls{nullptr, nullptr});
+    resolver_data_for_calls_.Set(nullptr);
     // Clear LB policy if set.
     if (lb_policy_ != nullptr) {
       GRPC_TRACE_LOG(client_channel, INFO)
@@ -1324,35 +1393,26 @@ void ClientChannel::UpdateServiceConfigInDataPlaneLocked(
   ChannelArgs new_args = args.SetObject(this).SetObject(saved_service_config_);
   // Construct filter stack.
   auto new_blackboard = MakeRefCounted<Blackboard>();
-  InterceptionChainBuilder builder(new_args, new_blackboard.get());
-  if (idle_timeout_ != Duration::Zero()) {
-    builder.AddOnServerTrailingMetadata([this](ServerMetadata&) {
-      if (idle_state_.DecreaseCallCount()) StartIdleTimer();
-    });
-  }
-  CoreConfiguration::Get().channel_init().AddToInterceptionChainBuilder(
-      GRPC_CLIENT_CHANNEL, builder);
-  // Add filters returned by the config selector (e.g., xDS HTTP filters).
-  config_selector->AddFilters(builder, blackboard_.get(), new_blackboard.get());
   const bool enable_retries =
       !channel_args_.WantMinimalStack() &&
       channel_args_.GetBool(GRPC_ARG_ENABLE_RETRIES).value_or(true);
   if (enable_retries) {
     RetryInterceptor::UpdateBlackboard(*saved_service_config_,
                                        blackboard_.get(), new_blackboard.get());
-    builder.Add<RetryInterceptor>(nullptr);
   }
-  blackboard_ = std::move(new_blackboard);
-  // Create call destination.
-  auto top_of_stack_call_destination = builder.Build(call_destination_);
-  // Send result to data plane.
-  if (!top_of_stack_call_destination.ok()) {
-    resolver_data_for_calls_.Set(MaybeRewriteIllegalStatusCode(
-        top_of_stack_call_destination.status(), "channel construction"));
-  } else {
-    resolver_data_for_calls_.Set(ResolverDataForCalls{
-        std::move(config_selector), std::move(*top_of_stack_call_destination)});
+  std::function<void(ServerMetadata&)> on_server_trailing_metadata;
+  if (idle_timeout_ != Duration::Zero()) {
+    on_server_trailing_metadata = [this](ServerMetadata&) {
+      if (idle_state_.DecreaseCallCount()) StartIdleTimer();
+    };
   }
+  FilterChainBuilderImpl filter_chain_builder(
+      enable_retries, new_args, new_blackboard.get(),
+      std::move(on_server_trailing_metadata), call_destination_);
+  config_selector->BuildFilterChains(filter_chain_builder, blackboard_.get(),
+                                     new_blackboard.get());
+  blackboard_ = std::move(new_blackboard);
+  resolver_data_for_calls_.Set(std::move(config_selector));
 }
 
 void ClientChannel::UpdateStateLocked(grpc_connectivity_state state,
@@ -1423,7 +1483,8 @@ void ClientChannel::StartIdleTimer() {
       std::move(arena)));
 }
 
-absl::Status ClientChannel::ApplyServiceConfigToCall(
+absl::StatusOr<RefCountedPtr<const FilterChain>>
+ClientChannel::ApplyServiceConfigToCall(
     ConfigSelector& config_selector,
     ClientMetadata& client_initial_metadata) const {
   GRPC_TRACE_LOG(client_channel_call, INFO)
@@ -1438,11 +1499,12 @@ absl::Status ClientChannel::ApplyServiceConfigToCall(
       GetContext<Arena>()->New<ClientChannelServiceConfigCallData>(
           GetContext<Arena>());
   // Use the ConfigSelector to determine the config for the call.
-  absl::Status call_config_status = config_selector.GetCallConfig(
-      {&client_initial_metadata, GetContext<Arena>(),
-       service_config_call_data});
-  if (!call_config_status.ok()) {
-    return MaybeRewriteIllegalStatusCode(call_config_status, "ConfigSelector");
+  auto filter_chain = config_selector.GetCallConfig({&client_initial_metadata,
+                                                     GetContext<Arena>(),
+                                                     service_config_call_data});
+  if (!filter_chain.ok()) {
+    return MaybeRewriteIllegalStatusCode(filter_chain.status(),
+                                         "ConfigSelector");
   }
   // Apply our own method params to the call.
   auto* method_params = DownCast<ClientChannelMethodParsedConfig*>(
@@ -1467,7 +1529,7 @@ absl::Status ClientChannel::ApplyServiceConfigToCall(
       wait_for_ready->value = method_params->wait_for_ready().value();
     }
   }
-  return absl::OkStatus();
+  return filter_chain;
 }
 
 }  // namespace grpc_core

data/src/core/client_channel/client_channel.h

@@ -166,7 +166,7 @@ class ClientChannel : public Channel {
 
   // Applies service config settings from config_selector to the call.
   // May modify call context and client_initial_metadata.
-  absl::Status ApplyServiceConfigToCall(
+  absl::StatusOr<RefCountedPtr<const FilterChain>> ApplyServiceConfigToCall(
       ConfigSelector& config_selector,
       ClientMetadata& client_initial_metadata) const;
 
@@ -192,11 +192,8 @@ class ClientChannel : public Channel {
   //
   // Fields related to name resolution.
   //
-  struct ResolverDataForCalls {
-    RefCountedPtr<ConfigSelector> config_selector;
-    RefCountedPtr<UnstartedCallDestination> call_destination;
-  };
-  Observable<absl::StatusOr<ResolverDataForCalls>> resolver_data_for_calls_;
+  Observable<absl::StatusOr<RefCountedPtr<ConfigSelector>>>
+      resolver_data_for_calls_;
 
   //
   // Fields related to LB picks.

data/src/core/client_channel/client_channel_filter.cc

@@ -217,7 +217,7 @@ class ClientChannelFilter::CallData {
   grpc_closure* original_recv_trailing_metadata_ready_ = nullptr;
   grpc_closure recv_trailing_metadata_ready_;
 
-  RefCountedPtr<DynamicFilters> dynamic_filters_;
+  RefCountedPtr<const DynamicFilters> dynamic_filters_;
   RefCountedPtr<DynamicFilters::Call> dynamic_call_;
 
   BufferedCall buffered_call_;
@@ -300,7 +300,7 @@ class DynamicTerminationFilter::CallData final {
                       const grpc_call_final_info* /*final_info*/,
                       grpc_closure* then_schedule_closure) {
     auto* calld = static_cast<CallData*>(elem->call_data);
-    RefCountedPtr<SubchannelCall> subchannel_call;
+    RefCountedPtr<Subchannel::Call> subchannel_call;
     if (GPR_LIKELY(calld->lb_call_ != nullptr)) {
       subchannel_call = calld->lb_call_->subchannel_call();
     }
@@ -504,8 +504,13 @@ class ClientChannelFilter::SubchannelWrapper final
     watcher_map_.erase(it);
   }
 
-  RefCountedPtr<ConnectedSubchannel> connected_subchannel() const {
-    return subchannel_->connected_subchannel();
+  absl::Status Ping(grpc_closure* on_initiate, grpc_closure* on_ack) {
+    return subchannel_->Ping(on_initiate, on_ack);
+  }
+
+  RefCountedPtr<Subchannel::Call> CreateCall(Subchannel::CreateCallArgs args,
+                                             grpc_error_handle* error) {
+    return subchannel_->CreateCall(args, error);
   }
 
   void RequestConnection() override { subchannel_->RequestConnection(); }
@@ -599,7 +604,7 @@ class ClientChannelFilter::SubchannelWrapper final
           << parent_.get() << " subchannel " << parent_->subchannel_.get()
           << " watcher=" << watcher_.get()
           << " state=" << ConnectivityStateName(state) << " status=" << status;
-      if (!IsTransportStateWatcherEnabled()) {
+      if (!IsSubchannelConnectionScalingEnabled()) {
         auto keepalive_throttling = status.GetPayload(kKeepaliveThrottlingKey);
         if (keepalive_throttling.has_value()) {
           int new_keepalive_time_ms = -1;
@@ -1412,6 +1417,46 @@ void ClientChannelFilter::UpdateServiceConfigInControlPlaneLocked(
       << saved_config_selector_.get();
 }
 
+namespace {
+
+// Filter chain builder impl to inject into ConfigSelector.
+class LegacyFilterChainBuilder final : public FilterChainBuilder {
+ public:
+  LegacyFilterChainBuilder(bool enable_retries, const ChannelArgs& channel_args,
+                           const Blackboard* blackboard)
+      : enable_retries_(enable_retries),
+        channel_args_(channel_args),
+        blackboard_(blackboard) {}
+
+  absl::StatusOr<RefCountedPtr<FilterChain>> Build() override {
+    if (enable_retries_) {
+      filters_.push_back({&RetryFilter::kFilterVtable, nullptr});
+    } else {
+      filters_.push_back({&DynamicTerminationFilter::kFilterVtable, nullptr});
+    }
+    RefCountedPtr<DynamicFilters> dynamic_filters =
+        DynamicFilters::Create(channel_args_, std::move(filters_), blackboard_);
+    if (dynamic_filters == nullptr) {
+      return absl::InternalError("error constructing dynamic filter stack");
+    }
+    filters_.clear();
+    return dynamic_filters;
+  }
+
+ private:
+  void AddFilter(const FilterHandle& filter_handle,
+                 RefCountedPtr<const FilterConfig> config) override {
+    filter_handle.AddToBuilder(&filters_, std::move(config));
+  }
+
+  const bool enable_retries_;
+  const ChannelArgs channel_args_;
+  const Blackboard* blackboard_;
+  std::vector<FilterAndConfig> filters_;
+};
+
+}  // namespace
+
 void ClientChannelFilter::UpdateServiceConfigInDataPlaneLocked(
     const ChannelArgs& args) {
   // Grab ref to service config.
@@ -1432,19 +1477,15 @@ void ClientChannelFilter::UpdateServiceConfigInDataPlaneLocked(
       new_args.GetBool(GRPC_ARG_ENABLE_RETRIES).value_or(true);
   // Construct dynamic filter stack.
   auto new_blackboard = MakeRefCounted<Blackboard>();
-  std::vector<const grpc_channel_filter*> filters =
-      config_selector->GetFilters(blackboard_.get(), new_blackboard.get());
   if (enable_retries) {
     RetryFilter::UpdateBlackboard(*service_config, blackboard_.get(),
                                   new_blackboard.get());
-    filters.push_back(&RetryFilter::kVtable);
-  } else {
-    filters.push_back(&DynamicTerminationFilter::kFilterVtable);
   }
+  LegacyFilterChainBuilder filter_chain_builder(enable_retries, new_args,
+                                                new_blackboard.get());
+  config_selector->BuildFilterChains(filter_chain_builder, blackboard_.get(),
+                                     new_blackboard.get());
   blackboard_ = std::move(new_blackboard);
-  RefCountedPtr<DynamicFilters> dynamic_filters =
-      DynamicFilters::Create(new_args, std::move(filters), blackboard_.get());
-  GRPC_CHECK(dynamic_filters != nullptr);
   // Grab data plane lock to update service config.
   //
   // We defer unreffing the old values (and deallocating memory) until
@@ -1457,7 +1498,6 @@ void ClientChannelFilter::UpdateServiceConfigInDataPlaneLocked(
     // Old values will be unreffed after lock is released.
     service_config_.swap(service_config);
     config_selector_.swap(config_selector);
-    dynamic_filters_.swap(dynamic_filters);
     // Re-process queued calls asynchronously.
     ReprocessQueuedResolverCalls();
   }
@@ -1495,13 +1535,11 @@ void ClientChannelFilter::DestroyResolverAndLbPolicyLocked() {
     // after we release the lock.
     RefCountedPtr<ServiceConfig> service_config_to_unref;
     RefCountedPtr<ConfigSelector> config_selector_to_unref;
-    RefCountedPtr<DynamicFilters> dynamic_filters_to_unref;
     {
       MutexLock lock(&resolution_mu_);
       received_service_config_data_ = false;
       service_config_to_unref = std::move(service_config_);
       config_selector_to_unref = std::move(config_selector_);
-      dynamic_filters_to_unref = std::move(dynamic_filters_);
     }
     // Clear LB policy if set.
     if (lb_policy_ != nullptr) {
@@ -1604,17 +1642,11 @@ grpc_error_handle ClientChannelFilter::DoPingLocked(grpc_transport_op* op) {
       // Complete pick.
       [op](LoadBalancingPolicy::PickResult::Complete* complete_pick)
           ABSL_EXCLUSIVE_LOCKS_REQUIRED(
-              *ClientChannelFilter::work_serializer_) {
-            SubchannelWrapper* subchannel = static_cast<SubchannelWrapper*>(
-                complete_pick->subchannel.get());
-            RefCountedPtr<ConnectedSubchannel> connected_subchannel =
-                subchannel->connected_subchannel();
-            if (connected_subchannel == nullptr) {
-              return GRPC_ERROR_CREATE("LB pick for ping not connected");
-            }
-            connected_subchannel->Ping(op->send_ping.on_initiate,
-                                       op->send_ping.on_ack);
-            return absl::OkStatus();
+              *ClientChannelFilter::work_serializer_) -> grpc_error_handle {
+            SubchannelWrapper* subchannel =
+                DownCast<SubchannelWrapper*>(complete_pick->subchannel.get());
+            return subchannel->Ping(op->send_ping.on_initiate,
+                                    op->send_ping.on_ack);
           },
       // Queue pick.
       [](LoadBalancingPolicy::PickResult::Queue* /*queue_pick*/) {
@@ -1850,14 +1882,14 @@ grpc_error_handle ClientChannelFilter::CallData::ApplyServiceConfigToCallLocked(
   auto* service_config_call_data =
       arena_->New<ClientChannelServiceConfigCallData>(arena_);
   // Use the ConfigSelector to determine the config for the call.
-  absl::Status call_config_status =
-      (*config_selector)
-          ->GetCallConfig(
-              {send_initial_metadata(), arena_, service_config_call_data});
-  if (!call_config_status.ok()) {
+  auto filter_chain = (*config_selector)
+                          ->GetCallConfig({send_initial_metadata(), arena_,
+                                           service_config_call_data});
+  if (!filter_chain.ok()) {
     return absl_status_to_grpc_error(
-        MaybeRewriteIllegalStatusCode(call_config_status, "ConfigSelector"));
+        MaybeRewriteIllegalStatusCode(filter_chain.status(), "ConfigSelector"));
   }
+  dynamic_filters_ = filter_chain->TakeAsSubclass<const DynamicFilters>();
   // Apply our own method params to the call.
   auto* method_params = static_cast<ClientChannelMethodParsedConfig*>(
       service_config_call_data->GetMethodParsedConfig(
@@ -1938,7 +1970,6 @@ bool ClientChannelFilter::CallData::CheckResolutionLocked(
   }
   // Result found.
   *config_selector = chand()->config_selector_;
-  dynamic_filters_ = chand()->dynamic_filters_;
   return true;
 }
 
@@ -2106,7 +2137,7 @@ void ClientChannelFilter::CallData::CreateDynamicCall() {
                                      call_start_time_, deadline_,
                                      arena_,           call_combiner_};
   grpc_error_handle error;
-  DynamicFilters* channel_stack = args.channel_stack.get();
+  const DynamicFilters* channel_stack = args.channel_stack.get();
   GRPC_TRACE_LOG(client_channel_call, INFO)
       << "chand=" << chand() << " calld=" << this
       << ": creating dynamic call stack on channel_stack=" << channel_stack;
@@ -2461,7 +2492,7 @@ ClientChannelFilter::LoadBalancedCall::PickSubchannel(bool was_queued) {
 
 bool ClientChannelFilter::LoadBalancedCall::PickSubchannelImpl(
     LoadBalancingPolicy::SubchannelPicker* picker, grpc_error_handle* error) {
-  GRPC_CHECK(connected_subchannel_ == nullptr);
+  GRPC_CHECK(subchannel_call_ == nullptr);
   // Perform LB pick.
   LoadBalancingPolicy::PickArgs pick_args;
   Slice* path = send_initial_metadata()->get_pointer(HttpPathMetadata());
@@ -2475,7 +2506,7 @@ bool ClientChannelFilter::LoadBalancedCall::PickSubchannelImpl(
   return HandlePickResult<bool>(
       &result,
       // CompletePick
-      [this](LoadBalancingPolicy::PickResult::Complete* complete_pick) {
+      [this, &error](LoadBalancingPolicy::PickResult::Complete* complete_pick) {
         GRPC_TRACE_LOG(client_channel_lb_call, INFO)
             << "chand=" << chand_ << " lb_call=" << this
             << ": LB pick succeeded: subchannel="
@@ -2485,12 +2516,24 @@ bool ClientChannelFilter::LoadBalancedCall::PickSubchannelImpl(
         // holding the data plane mutex.
         SubchannelWrapper* subchannel =
             static_cast<SubchannelWrapper*>(complete_pick->subchannel.get());
-        connected_subchannel_ = subchannel->connected_subchannel();
+        Subchannel::CreateCallArgs call_args = {
+            pollent_, /*start_time=*/0, arena_->GetContext<Call>()->deadline(),
+            // TODO(roth): When we implement hedging support, we will probably
+            // need to use a separate call arena for each subchannel call.
+            arena_, call_combiner_};
+        subchannel_call_ = subchannel->CreateCall(call_args, error);
+        if (subchannel_call_ != nullptr &&
+            on_call_destruction_complete_ != nullptr) {
+          subchannel_call_->SetAfterCallStackDestroy(
+              on_call_destruction_complete_);
+          on_call_destruction_complete_ = nullptr;
+        }
+        if (!error->ok()) return true;
         // If the subchannel has no connected subchannel (e.g., if the
         // subchannel has moved out of state READY but the LB policy hasn't
         // yet seen that change and given us a new picker), then just
         // queue the pick.  We'll try again as soon as we get a new picker.
-        if (connected_subchannel_ == nullptr) {
+        if (subchannel_call_ == nullptr) {
           GRPC_TRACE_LOG(client_channel_lb_call, INFO)
               << "chand=" << chand_ << " lb_call=" << this
               << ": subchannel returned by LB picker "
@@ -2726,7 +2769,7 @@ void ClientChannelFilter::LoadBalancedCall::TryPick(bool was_queued) {
       buffered_call_.Fail(*result, BufferedCall::YieldCallCombiner);
       return;
     }
-    CreateSubchannelCall();
+    StartSubchannelCall();
   }
 }
 
@@ -2759,32 +2802,15 @@ void ClientChannelFilter::LoadBalancedCall::RetryPickLocked() {
                absl::OkStatus());
 }
 
-void ClientChannelFilter::LoadBalancedCall::CreateSubchannelCall() {
-  SubchannelCall::Args call_args = {
-      connected_subchannel_->Ref(), pollent_, /*start_time=*/0,
-      arena_->GetContext<Call>()->deadline(),
-      // TODO(roth): When we implement hedging support, we will probably
-      // need to use a separate call arena for each subchannel call.
-      arena_, call_combiner_};
-  grpc_error_handle error;
-  subchannel_call_ = SubchannelCall::Create(std::move(call_args), &error);
+void ClientChannelFilter::LoadBalancedCall::StartSubchannelCall() {
   GRPC_TRACE_LOG(client_channel_lb_call, INFO)
       << "chand=" << chand_ << " lb_call=" << this
-      << ": create subchannel_call=" << subchannel_call_.get()
-      << ": error=" << StatusToString(error);
-  if (on_call_destruction_complete_ != nullptr) {
-    subchannel_call_->SetAfterCallStackDestroy(on_call_destruction_complete_);
-    on_call_destruction_complete_ = nullptr;
-  }
-  if (GPR_UNLIKELY(!error.ok())) {
-    buffered_call_.Fail(error, BufferedCall::YieldCallCombiner);
-  } else {
-    buffered_call_.Resume([subchannel_call = subchannel_call_](
-                              grpc_transport_stream_op_batch* batch) {
-      // Note: This will release the call combiner.
-      subchannel_call->StartTransportStreamOpBatch(batch);
-    });
-  }
+      << ": starting subchannel_call=" << subchannel_call_.get();
+  buffered_call_.Resume([subchannel_call = subchannel_call_](
+                            grpc_transport_stream_op_batch* batch) {
+    // Note: This will release the call combiner.
+    subchannel_call->StartTransportStreamOpBatch(batch);
+  });
 }
 
 }  // namespace grpc_core

data/src/core/client_channel/client_channel_filter.h

@@ -276,8 +276,6 @@ class ClientChannelFilter final {
   RefCountedPtr<ServiceConfig> service_config_ ABSL_GUARDED_BY(resolution_mu_);
   RefCountedPtr<ConfigSelector> config_selector_
       ABSL_GUARDED_BY(resolution_mu_);
-  RefCountedPtr<DynamicFilters> dynamic_filters_
-      ABSL_GUARDED_BY(resolution_mu_);
 
   //
   // Fields related to LB picks.  Guarded by lb_mu_.
@@ -369,7 +367,7 @@ class ClientChannelFilter::LoadBalancedCall final
   void RetryPickLocked()
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(&ClientChannelFilter::lb_mu_);
 
-  RefCountedPtr<SubchannelCall> subchannel_call() const {
+  RefCountedPtr<Subchannel::Call> subchannel_call() const {
     return subchannel_call_;
   }
 
@@ -423,7 +421,7 @@ class ClientChannelFilter::LoadBalancedCall final
   // and when it is queued and the channel gets a new picker.
   void TryPick(bool was_queued);
 
-  void CreateSubchannelCall();
+  void StartSubchannelCall();
 
   ClientChannelFilter* chand_;
   // When we start a new attempt for a call, we might not have cleaned up the
@@ -443,7 +441,6 @@ class ClientChannelFilter::LoadBalancedCall final
 
   absl::AnyInvocable<void()> on_commit_;
 
-  RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
   const BackendMetricData* backend_metric_data_ = nullptr;
   std::unique_ptr<LoadBalancingPolicy::SubchannelCallTrackerInterface>
       lb_subchannel_call_tracker_;
@@ -459,7 +456,7 @@ class ClientChannelFilter::LoadBalancedCall final
   LbQueuedCallCanceller* lb_call_canceller_
       ABSL_GUARDED_BY(&ClientChannelFilter::lb_mu_) = nullptr;
 
-  RefCountedPtr<SubchannelCall> subchannel_call_;
+  RefCountedPtr<Subchannel::Call> subchannel_call_;
 
   // For intercepting recv_initial_metadata_ready.
   grpc_metadata_batch* recv_initial_metadata_ = nullptr;

data/src/core/client_channel/client_channel_internal.h

@@ -45,6 +45,11 @@
 #define GRPC_ARG_MAX_CONNECTIONS_PER_SUBCHANNEL \
   "grpc.internal.max_connections_per_subchannel"
 
+// EXPERIMENTAL: Fail requests at the client if the client is over max
+// concurrent streams, so they may be retried elsewhere.
+#define GRPC_ARG_MAX_CONCURRENT_STREAMS_REJECT_ON_CLIENT \
+  "grpc.http.max_concurrent_streams_reject_on_client"
+
 namespace grpc_core {
 
 // Internal type for LB call state interface.  Provides an interface for