grpc 1.12.0 → 1.13.0.pre1

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h

@@ -21,47 +21,52 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <stdbool.h>
+#include <grpc/support/atm.h>
 
-#include <grpc/impl/codegen/grpc_types.h>
+#include "src/core/lib/gprpp/inlined_vector.h"
+#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 
-typedef struct grpc_grpclb_client_stats grpc_grpclb_client_stats;
+namespace grpc_core {
 
-typedef struct {
-  char* token;
-  int64_t count;
-} grpc_grpclb_drop_token_count;
+class GrpcLbClientStats : public RefCounted<GrpcLbClientStats> {
+ public:
+  struct DropTokenCount {
+    UniquePtr<char> token;
+    int64_t count;
 
-typedef struct {
-  grpc_grpclb_drop_token_count* token_counts;
-  size_t num_entries;
-} grpc_grpclb_dropped_call_counts;
+    DropTokenCount(UniquePtr<char> token, int64_t count)
+        : token(std::move(token)), count(count) {}
+  };
 
-grpc_grpclb_client_stats* grpc_grpclb_client_stats_create();
-grpc_grpclb_client_stats* grpc_grpclb_client_stats_ref(
-    grpc_grpclb_client_stats* client_stats);
-void grpc_grpclb_client_stats_unref(grpc_grpclb_client_stats* client_stats);
+  typedef InlinedVector<DropTokenCount, 10> DroppedCallCounts;
 
-void grpc_grpclb_client_stats_add_call_started(
-    grpc_grpclb_client_stats* client_stats);
-void grpc_grpclb_client_stats_add_call_finished(
-    bool finished_with_client_failed_to_send, bool finished_known_received,
-    grpc_grpclb_client_stats* client_stats);
+  GrpcLbClientStats() {}
 
-// This method is not thread-safe; caller must synchronize.
-void grpc_grpclb_client_stats_add_call_dropped_locked(
-    char* token, grpc_grpclb_client_stats* client_stats);
+  void AddCallStarted();
+  void AddCallFinished(bool finished_with_client_failed_to_send,
+                       bool finished_known_received);
 
-// This method is not thread-safe; caller must synchronize.
-void grpc_grpclb_client_stats_get_locked(
-    grpc_grpclb_client_stats* client_stats, int64_t* num_calls_started,
-    int64_t* num_calls_finished,
-    int64_t* num_calls_finished_with_client_failed_to_send,
-    int64_t* num_calls_finished_known_received,
-    grpc_grpclb_dropped_call_counts** drop_token_counts);
+  // This method is not thread-safe; caller must synchronize.
+  void AddCallDroppedLocked(char* token);
 
-void grpc_grpclb_dropped_call_counts_destroy(
-    grpc_grpclb_dropped_call_counts* drop_entries);
+  // This method is not thread-safe; caller must synchronize.
+  void GetLocked(int64_t* num_calls_started, int64_t* num_calls_finished,
+                 int64_t* num_calls_finished_with_client_failed_to_send,
+                 int64_t* num_calls_finished_known_received,
+                 UniquePtr<DroppedCallCounts>* drop_token_counts);
+
+ private:
+  // This field must only be accessed via *_locked() methods.
+  UniquePtr<DroppedCallCounts> drop_token_counts_;
+  // These fields may be accessed from multiple threads at a time.
+  gpr_atm num_calls_started_ = 0;
+  gpr_atm num_calls_finished_ = 0;
+  gpr_atm num_calls_finished_with_client_failed_to_send_ = 0;
+  gpr_atm num_calls_finished_known_received_ = 0;
+};
+
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_GRPCLB_GRPCLB_CLIENT_STATS_H \
         */

data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc

@@ -29,7 +29,7 @@ static bool count_serverlist(pb_istream_t* stream, const pb_field_t* field,
                              void** arg) {
   grpc_grpclb_serverlist* sl = static_cast<grpc_grpclb_serverlist*>(*arg);
   grpc_grpclb_server server;
-  if (!pb_decode(stream, grpc_lb_v1_Server_fields, &server)) {
+  if (GPR_UNLIKELY(!pb_decode(stream, grpc_lb_v1_Server_fields, &server))) {
     gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(stream));
     return false;
   }
@@ -52,7 +52,7 @@ static bool decode_serverlist(pb_istream_t* stream, const pb_field_t* field,
   GPR_ASSERT(dec_arg->serverlist->num_servers >= dec_arg->decoding_idx);
   grpc_grpclb_server* server =
       static_cast<grpc_grpclb_server*>(gpr_zalloc(sizeof(grpc_grpclb_server)));
-  if (!pb_decode(stream, grpc_lb_v1_Server_fields, server)) {
+  if (GPR_UNLIKELY(!pb_decode(stream, grpc_lb_v1_Server_fields, server))) {
     gpr_free(server);
     gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(stream));
     return false;
@@ -89,16 +89,16 @@ static bool encode_string(pb_ostream_t* stream, const pb_field_t* field,
 
 static bool encode_drops(pb_ostream_t* stream, const pb_field_t* field,
                          void* const* arg) {
-  grpc_grpclb_dropped_call_counts* drop_entries =
-      static_cast<grpc_grpclb_dropped_call_counts*>(*arg);
+  grpc_core::GrpcLbClientStats::DroppedCallCounts* drop_entries =
+      static_cast<grpc_core::GrpcLbClientStats::DroppedCallCounts*>(*arg);
   if (drop_entries == nullptr) return true;
-  for (size_t i = 0; i < drop_entries->num_entries; ++i) {
+  for (size_t i = 0; i < drop_entries->size(); ++i) {
     if (!pb_encode_tag_for_field(stream, field)) return false;
     grpc_lb_v1_ClientStatsPerToken drop_message;
     drop_message.load_balance_token.funcs.encode = encode_string;
-    drop_message.load_balance_token.arg = drop_entries->token_counts[i].token;
+    drop_message.load_balance_token.arg = (*drop_entries)[i].token.get();
     drop_message.has_num_calls = true;
-    drop_message.num_calls = drop_entries->token_counts[i].count;
+    drop_message.num_calls = (*drop_entries)[i].count;
     if (!pb_encode_submessage(stream, grpc_lb_v1_ClientStatsPerToken_fields,
                               &drop_message)) {
       return false;
@@ -108,7 +108,7 @@ static bool encode_drops(pb_ostream_t* stream, const pb_field_t* field,
 }
 
 grpc_grpclb_request* grpc_grpclb_load_report_request_create_locked(
-    grpc_grpclb_client_stats* client_stats) {
+    grpc_core::GrpcLbClientStats* client_stats) {
   grpc_grpclb_request* req = static_cast<grpc_grpclb_request*>(
       gpr_zalloc(sizeof(grpc_grpclb_request)));
   req->has_client_stats = true;
@@ -120,13 +120,15 @@ grpc_grpclb_request* grpc_grpclb_load_report_request_create_locked(
   req->client_stats.has_num_calls_finished_with_client_failed_to_send = true;
   req->client_stats.has_num_calls_finished_known_received = true;
   req->client_stats.calls_finished_with_drop.funcs.encode = encode_drops;
-  grpc_grpclb_client_stats_get_locked(
-      client_stats, &req->client_stats.num_calls_started,
+  grpc_core::UniquePtr<grpc_core::GrpcLbClientStats::DroppedCallCounts>
+      drop_counts;
+  client_stats->GetLocked(
+      &req->client_stats.num_calls_started,
       &req->client_stats.num_calls_finished,
       &req->client_stats.num_calls_finished_with_client_failed_to_send,
-      &req->client_stats.num_calls_finished_known_received,
-      reinterpret_cast<grpc_grpclb_dropped_call_counts**>(
-          &req->client_stats.calls_finished_with_drop.arg));
+      &req->client_stats.num_calls_finished_known_received, &drop_counts);
+  // Will be deleted in grpc_grpclb_request_destroy().
+  req->client_stats.calls_finished_with_drop.arg = drop_counts.release();
   return req;
 }
 
@@ -149,10 +151,10 @@ grpc_slice grpc_grpclb_request_encode(const grpc_grpclb_request* request) {
 
 void grpc_grpclb_request_destroy(grpc_grpclb_request* request) {
   if (request->has_client_stats) {
-    grpc_grpclb_dropped_call_counts* drop_entries =
-        static_cast<grpc_grpclb_dropped_call_counts*>(
+    grpc_core::GrpcLbClientStats::DroppedCallCounts* drop_entries =
+        static_cast<grpc_core::GrpcLbClientStats::DroppedCallCounts*>(
             request->client_stats.calls_finished_with_drop.arg);
-    grpc_grpclb_dropped_call_counts_destroy(drop_entries);
+    grpc_core::Delete(drop_entries);
   }
   gpr_free(request);
 }
@@ -165,7 +167,8 @@ grpc_grpclb_initial_response* grpc_grpclb_initial_response_parse(
                              GRPC_SLICE_LENGTH(encoded_grpc_grpclb_response));
   grpc_grpclb_response res;
   memset(&res, 0, sizeof(grpc_grpclb_response));
-  if (!pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res)) {
+  if (GPR_UNLIKELY(
+          !pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res))) {
     gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream));
     return nullptr;
   }
@@ -195,7 +198,7 @@ grpc_grpclb_serverlist* grpc_grpclb_response_parse_serverlist(
   res.server_list.servers.funcs.decode = count_serverlist;
   res.server_list.servers.arg = sl;
   bool status = pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res);
-  if (!status) {
+  if (GPR_UNLIKELY(!status)) {
     gpr_free(sl);
     gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream));
     return nullptr;
@@ -211,7 +214,7 @@ grpc_grpclb_serverlist* grpc_grpclb_response_parse_serverlist(
     res.server_list.servers.arg = &decode_arg;
     status = pb_decode(&stream_at_start, grpc_lb_v1_LoadBalanceResponse_fields,
                        &res);
-    if (!status) {
+    if (GPR_UNLIKELY(!status)) {
       grpc_grpclb_destroy_serverlist(sl);
       gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream));
       return nullptr;

data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h

@@ -42,7 +42,7 @@ typedef struct {
 /** Create a request for a gRPC LB service under \a lb_service_name */
 grpc_grpclb_request* grpc_grpclb_request_create(const char* lb_service_name);
 grpc_grpclb_request* grpc_grpclb_load_report_request_create_locked(
-    grpc_grpclb_client_stats* client_stats);
+    grpc_core::GrpcLbClientStats* client_stats);
 
 /** Protocol Buffers v3-encode \a request */
 grpc_slice grpc_grpclb_request_encode(const grpc_grpclb_request* request);

data/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc

@@ -490,7 +490,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
         if (grpc_lb_pick_first_trace.enabled()) {
           gpr_log(GPR_INFO,
                   "Servicing pending pick with selected subchannel %p",
-                  p->selected_);
+                  p->selected_->subchannel());
         }
         GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
       }

data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc

@@ -608,7 +608,7 @@ void RoundRobin::PingOneLocked(grpc_closure* on_initiate,
 
 void RoundRobin::UpdateLocked(const grpc_channel_args& args) {
   const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES);
-  if (arg == nullptr || arg->type != GRPC_ARG_POINTER) {
+  if (GPR_UNLIKELY(arg == nullptr || arg->type != GRPC_ARG_POINTER)) {
     gpr_log(GPR_ERROR, "[RR %p] update provided no addresses; ignoring", this);
     // If we don't have a current subchannel list, go into TRANSIENT_FAILURE.
     // Otherwise, keep using the current subchannel list (ignore this update).

data/src/core/ext/filters/client_channel/resolver.h

@@ -105,9 +105,7 @@ class Resolver : public InternallyRefCountedWithTracing<Resolver> {
   GRPC_ABSTRACT_BASE_CLASS
 
  protected:
-  // So Delete() can access our protected dtor.
-  template <typename T>
-  friend void Delete(T*);
+  GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE
 
   /// Does NOT take ownership of the reference to \a combiner.
   // TODO(roth): Once we have a C++-like interface for combiners, this

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc

@@ -346,7 +346,7 @@ void AresDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) {
     RefCountedPtr<Resolver> self = r->Ref(DEBUG_LOCATION, "retry-timer");
     self.release();
     if (timeout > 0) {
-      gpr_log(GPR_DEBUG, "retrying in %" PRIdPTR " milliseconds", timeout);
+      gpr_log(GPR_DEBUG, "retrying in %" PRId64 " milliseconds", timeout);
     } else {
       gpr_log(GPR_DEBUG, "retrying immediately");
     }
@@ -381,8 +381,8 @@ void AresDnsResolver::MaybeStartResolvingLocked() {
       const grpc_millis last_resolution_ago =
           grpc_core::ExecCtx::Get()->Now() - last_resolution_timestamp_;
       gpr_log(GPR_DEBUG,
-              "In cooldown from last resolution (from %" PRIdPTR
-              " ms ago). Will resolve again in %" PRIdPTR " ms",
+              "In cooldown from last resolution (from %" PRId64
+              " ms ago). Will resolve again in %" PRId64 " ms",
               last_resolution_ago, ms_until_next_resolution);
       have_next_resolution_timer_ = true;
       // TODO(roth): We currently deal with this ref manually.  Once the

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc

@@ -18,7 +18,7 @@
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/iomgr/port.h"
-#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET)
+#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER)
 
 #include <ares.h>
 #include <sys/ioctl.h>
@@ -348,4 +348,4 @@ void grpc_ares_ev_driver_start(grpc_ares_ev_driver* ev_driver) {
   gpr_mu_unlock(&ev_driver->mu);
 }
 
-#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET) */
+#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) */

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc

@@ -270,7 +270,6 @@ static void on_hostbyname_done_cb(void* arg, int status, int timeouts,
 static void on_srv_query_done_cb(void* arg, int status, int timeouts,
                                  unsigned char* abuf, int alen) {
   grpc_ares_request* r = static_cast<grpc_ares_request*>(arg);
-  grpc_core::ExecCtx exec_ctx;
   gpr_log(GPR_DEBUG, "on_query_srv_done_cb");
   if (status == ARES_SUCCESS) {
     gpr_log(GPR_DEBUG, "on_query_srv_done_cb ARES_SUCCESS");

data/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc

@@ -218,7 +218,7 @@ void NativeDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) {
         r->Ref(DEBUG_LOCATION, "next_resolution_timer");
     self.release();
     if (timeout > 0) {
-      gpr_log(GPR_DEBUG, "retrying in %" PRIdPTR " milliseconds", timeout);
+      gpr_log(GPR_DEBUG, "retrying in %" PRId64 " milliseconds", timeout);
     } else {
       gpr_log(GPR_DEBUG, "retrying immediately");
     }
@@ -254,8 +254,8 @@ void NativeDnsResolver::MaybeStartResolvingLocked() {
       const grpc_millis last_resolution_ago =
           grpc_core::ExecCtx::Get()->Now() - last_resolution_timestamp_;
       gpr_log(GPR_DEBUG,
-              "In cooldown from last resolution (from %" PRIdPTR
-              " ms ago). Will resolve again in %" PRIdPTR " ms",
+              "In cooldown from last resolution (from %" PRId64
+              " ms ago). Will resolve again in %" PRId64 " ms",
               last_resolution_ago, ms_until_next_resolution);
       have_next_resolution_timer_ = true;
       // TODO(roth): We currently deal with this ref manually.  Once the
@@ -310,7 +310,7 @@ class NativeDnsResolverFactory : public ResolverFactory {
  public:
   OrphanablePtr<Resolver> CreateResolver(
       const ResolverArgs& args) const override {
-    if (0 != strcmp(args.uri->authority, "")) {
+    if (GPR_UNLIKELY(0 != strcmp(args.uri->authority, ""))) {
       gpr_log(GPR_ERROR, "authority based dns uri's not supported");
       return OrphanablePtr<Resolver>(nullptr);
     }

data/src/core/ext/filters/client_channel/subchannel.cc

@@ -408,7 +408,7 @@ static void on_external_state_watcher_done(void* arg, grpc_error* error) {
   gpr_mu_unlock(&w->subchannel->mu);
   GRPC_SUBCHANNEL_WEAK_UNREF(w->subchannel, "external_state_watcher");
   gpr_free(w);
-  GRPC_CLOSURE_RUN(follow_up, GRPC_ERROR_REF(error));
+  GRPC_CLOSURE_SCHED(follow_up, GRPC_ERROR_REF(error));
 }
 
 static void on_alarm(void* arg, grpc_error* error) {
@@ -467,7 +467,7 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) {
     if (time_til_next <= 0) {
       gpr_log(GPR_INFO, "Subchannel %p: Retry immediately", c);
     } else {
-      gpr_log(GPR_INFO, "Subchannel %p: Retry in %" PRIdPTR " milliseconds", c,
+      gpr_log(GPR_INFO, "Subchannel %p: Retry in %" PRId64 " milliseconds", c,
               time_til_next);
     }
     GRPC_CLOSURE_INIT(&c->on_alarm, on_alarm, c, grpc_schedule_on_exec_ctx);
@@ -803,7 +803,7 @@ grpc_error* ConnectedSubchannel::CreateCall(const CallArgs& args,
   };
   grpc_error* error = grpc_call_stack_init(
       channel_stack_, 1, subchannel_call_destroy, *call, &call_args);
-  if (error != GRPC_ERROR_NONE) {
+  if (GPR_UNLIKELY(error != GRPC_ERROR_NONE)) {
     const char* error_string = grpc_error_string(error);
     gpr_log(GPR_ERROR, "error: %s", error_string);
     return error;

data/src/core/ext/filters/http/client_authority_filter.cc

@@ -59,8 +59,9 @@ void authority_start_transport_stream_op_batch(
       initial_metadata->idx.named.authority == nullptr) {
     grpc_error* error = grpc_metadata_batch_add_head(
         initial_metadata, &calld->authority_storage,
-        grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY,
-                                grpc_slice_ref(chand->default_authority)));
+        grpc_mdelem_from_slices(
+            GRPC_MDSTR_AUTHORITY,
+            grpc_slice_ref_internal(chand->default_authority)));
     if (error != GRPC_ERROR_NONE) {
       grpc_transport_stream_op_batch_finish_with_failure(batch, error,
                                                          calld->call_combiner);
@@ -102,7 +103,7 @@ grpc_error* init_channel_elem(grpc_channel_element* elem,
         "GRPC_ARG_DEFAULT_AUTHORITY channel arg. must be a string");
   }
   chand->default_authority =
-      grpc_slice_from_copied_string(default_authority_str);
+      grpc_slice_intern(grpc_slice_from_static_string(default_authority_str));
   GPR_ASSERT(!args->is_last);
   return GRPC_ERROR_NONE;
 }
@@ -110,7 +111,7 @@ grpc_error* init_channel_elem(grpc_channel_element* elem,
 /* Destructor for channel data */
 void destroy_channel_elem(grpc_channel_element* elem) {
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
-  grpc_slice_unref(chand->default_authority);
+  grpc_slice_unref_internal(chand->default_authority);
 }
 }  // namespace
 

data/src/core/ext/filters/http/message_compress/message_compress_filter.cc

@@ -116,8 +116,8 @@ static grpc_error* process_send_initial_metadata(
   if (initial_metadata->idx.named.grpc_internal_encoding_request != nullptr) {
     grpc_mdelem md =
         initial_metadata->idx.named.grpc_internal_encoding_request->md;
-    if (!grpc_compression_algorithm_parse(GRPC_MDVALUE(md),
-                                          &compression_algorithm)) {
+    if (GPR_UNLIKELY(!grpc_compression_algorithm_parse(
+            GRPC_MDVALUE(md), &compression_algorithm))) {
       char* val = grpc_slice_to_c_string(GRPC_MDVALUE(md));
       gpr_log(GPR_ERROR,
               "Invalid compression algorithm: '%s' (unknown). Ignoring.", val);
@@ -125,8 +125,8 @@ static grpc_error* process_send_initial_metadata(
       calld->message_compression_algorithm = GRPC_MESSAGE_COMPRESS_NONE;
       stream_compression_algorithm = GRPC_STREAM_COMPRESS_NONE;
     }
-    if (!GPR_BITGET(channeld->enabled_algorithms_bitset,
-                    compression_algorithm)) {
+    if (GPR_UNLIKELY(!GPR_BITGET(channeld->enabled_algorithms_bitset,
+                                 compression_algorithm))) {
       char* val = grpc_slice_to_c_string(GRPC_MDVALUE(md));
       gpr_log(GPR_ERROR,
               "Invalid compression algorithm: '%s' (previously disabled). "

data/src/core/ext/filters/http/server/http_server_filter.cc

@@ -35,43 +35,37 @@
 #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
 namespace {
+
 struct call_data {
   grpc_call_combiner* call_combiner;
 
+  // Outgoing headers to add to send_initial_metadata.
   grpc_linked_mdelem status;
   grpc_linked_mdelem content_type;
 
-  /* did this request come with path query containing request payload */
-  bool seen_path_with_query;
-  /* flag to ensure payload_bin is delivered only once */
-  bool payload_bin_delivered;
+  // If we see the recv_message contents in the GET query string, we
+  // store it here.
+  grpc_core::ManualConstructor<grpc_core::SliceBufferByteStream> read_stream;
+  bool have_read_stream;
 
+  // State for intercepting recv_initial_metadata.
+  grpc_closure recv_initial_metadata_ready;
+  grpc_closure* original_recv_initial_metadata_ready;
   grpc_metadata_batch* recv_initial_metadata;
   uint32_t* recv_initial_metadata_flags;
-  /** Closure to call when finished with the hs_on_recv hook */
-  grpc_closure* on_done_recv;
-  /** Closure to call when we retrieve read message from the path URI
-   */
-  grpc_closure* recv_message_ready;
-  grpc_closure* on_complete;
-  grpc_core::OrphanablePtr<grpc_core::ByteStream>* pp_recv_message;
-  grpc_core::ManualConstructor<grpc_core::SliceBufferByteStream> read_stream;
+  bool seen_recv_initial_metadata_ready;
 
-  /** Receive closures are chained: we inject this closure as the on_done_recv
-      up-call on transport_op, and remember to call our on_done_recv member
-      after handling it. */
-  grpc_closure hs_on_recv;
-  grpc_closure hs_on_complete;
-  grpc_closure hs_recv_message_ready;
+  // State for intercepting recv_message.
+  grpc_closure* original_recv_message_ready;
+  grpc_closure recv_message_ready;
+  grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message;
+  bool seen_recv_message_ready;
 };
 
-struct channel_data {
-  uint8_t unused;
-};
 }  // namespace
 
-static grpc_error* server_filter_outgoing_metadata(grpc_call_element* elem,
-                                                   grpc_metadata_batch* b) {
+static grpc_error* hs_filter_outgoing_metadata(grpc_call_element* elem,
+                                               grpc_metadata_batch* b) {
   if (b->idx.named.grpc_message != nullptr) {
     grpc_slice pct_encoded_msg = grpc_percent_encode_slice(
         GRPC_MDVALUE(b->idx.named.grpc_message->md),
@@ -86,8 +80,8 @@ static grpc_error* server_filter_outgoing_metadata(grpc_call_element* elem,
   return GRPC_ERROR_NONE;
 }
 
-static void add_error(const char* error_name, grpc_error** cumulative,
-                      grpc_error* new_err) {
+static void hs_add_error(const char* error_name, grpc_error** cumulative,
+                         grpc_error* new_err) {
   if (new_err == GRPC_ERROR_NONE) return;
   if (*cumulative == GRPC_ERROR_NONE) {
     *cumulative = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_name);
@@ -95,8 +89,8 @@ static void add_error(const char* error_name, grpc_error** cumulative,
   *cumulative = grpc_error_add_child(*cumulative, new_err);
 }
 
-static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
-                                                   grpc_metadata_batch* b) {
+static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
+                                               grpc_metadata_batch* b) {
   call_data* calld = static_cast<call_data*>(elem->call_data);
   grpc_error* error = GRPC_ERROR_NONE;
   static const char* error_name = "Failed processing incoming headers";
@@ -119,14 +113,14 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
       *calld->recv_initial_metadata_flags &=
           ~GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST;
     } else {
-      add_error(error_name, &error,
-                grpc_attach_md_to_error(
-                    GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
-                    b->idx.named.method->md));
+      hs_add_error(error_name, &error,
+                   grpc_attach_md_to_error(
+                       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
+                       b->idx.named.method->md));
     }
     grpc_metadata_batch_remove(b, b->idx.named.method);
   } else {
-    add_error(
+    hs_add_error(
         error_name, &error,
         grpc_error_set_str(
             GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
@@ -135,31 +129,31 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
 
   if (b->idx.named.te != nullptr) {
     if (!grpc_mdelem_eq(b->idx.named.te->md, GRPC_MDELEM_TE_TRAILERS)) {
-      add_error(error_name, &error,
-                grpc_attach_md_to_error(
-                    GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
-                    b->idx.named.te->md));
+      hs_add_error(error_name, &error,
+                   grpc_attach_md_to_error(
+                       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
+                       b->idx.named.te->md));
     }
     grpc_metadata_batch_remove(b, b->idx.named.te);
   } else {
-    add_error(error_name, &error,
-              grpc_error_set_str(
-                  GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
-                  GRPC_ERROR_STR_KEY, grpc_slice_from_static_string("te")));
+    hs_add_error(error_name, &error,
+                 grpc_error_set_str(
+                     GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
+                     GRPC_ERROR_STR_KEY, grpc_slice_from_static_string("te")));
   }
 
   if (b->idx.named.scheme != nullptr) {
     if (!grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTP) &&
         !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTPS) &&
         !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_GRPC)) {
-      add_error(error_name, &error,
-                grpc_attach_md_to_error(
-                    GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
-                    b->idx.named.scheme->md));
+      hs_add_error(error_name, &error,
+                   grpc_attach_md_to_error(
+                       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
+                       b->idx.named.scheme->md));
     }
     grpc_metadata_batch_remove(b, b->idx.named.scheme);
   } else {
-    add_error(
+    hs_add_error(
         error_name, &error,
         grpc_error_set_str(
             GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
@@ -196,10 +190,11 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.path == nullptr) {
-    add_error(error_name, &error,
-              grpc_error_set_str(
-                  GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
-                  GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":path")));
+    hs_add_error(
+        error_name, &error,
+        grpc_error_set_str(
+            GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
+            GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":path")));
   } else if (*calld->recv_initial_metadata_flags &
              GRPC_INITIAL_METADATA_CACHEABLE_REQUEST) {
     /* We have a cacheable request made with GET verb. The path contains the
@@ -235,7 +230,7 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
               GRPC_SLICE_LENGTH(query_slice), k_url_safe));
       calld->read_stream.Init(&read_slice_buffer, 0);
       grpc_slice_buffer_destroy_internal(&read_slice_buffer);
-      calld->seen_path_with_query = true;
+      calld->have_read_stream = true;
       grpc_slice_unref_internal(query_slice);
     } else {
       gpr_log(GPR_ERROR, "GET request without QUERY");
@@ -246,17 +241,17 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
     grpc_linked_mdelem* el = b->idx.named.host;
     grpc_mdelem md = GRPC_MDELEM_REF(el->md);
     grpc_metadata_batch_remove(b, el);
-    add_error(error_name, &error,
-              grpc_metadata_batch_add_head(
-                  b, el,
-                  grpc_mdelem_from_slices(
-                      GRPC_MDSTR_AUTHORITY,
-                      grpc_slice_ref_internal(GRPC_MDVALUE(md)))));
+    hs_add_error(error_name, &error,
+                 grpc_metadata_batch_add_head(
+                     b, el,
+                     grpc_mdelem_from_slices(
+                         GRPC_MDSTR_AUTHORITY,
+                         grpc_slice_ref_internal(GRPC_MDVALUE(md)))));
     GRPC_MDELEM_UNREF(md);
   }
 
   if (b->idx.named.authority == nullptr) {
-    add_error(
+    hs_add_error(
         error_name, &error,
         grpc_error_set_str(
             GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"),
@@ -266,49 +261,55 @@ static grpc_error* server_filter_incoming_metadata(grpc_call_element* elem,
   return error;
 }
 
-static void hs_on_recv(void* user_data, grpc_error* err) {
+static void hs_recv_initial_metadata_ready(void* user_data, grpc_error* err) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(user_data);
   call_data* calld = static_cast<call_data*>(elem->call_data);
+  calld->seen_recv_initial_metadata_ready = true;
   if (err == GRPC_ERROR_NONE) {
-    err = server_filter_incoming_metadata(elem, calld->recv_initial_metadata);
+    err = hs_filter_incoming_metadata(elem, calld->recv_initial_metadata);
+    if (calld->seen_recv_message_ready) {
+      // We've already seen the recv_message callback, but we previously
+      // deferred it, so we need to return it here.
+      // Replace the recv_message byte stream if needed.
+      if (calld->have_read_stream) {
+        calld->recv_message->reset(calld->read_stream.get());
+        calld->have_read_stream = false;
+      }
+      // Re-enter call combiner for original_recv_message_ready, since the
+      // surface code will release the call combiner for each callback it
+      // receives.
+      GRPC_CALL_COMBINER_START(
+          calld->call_combiner, calld->original_recv_message_ready,
+          GRPC_ERROR_REF(err),
+          "resuming recv_message_ready from recv_initial_metadata_ready");
+    }
   } else {
     GRPC_ERROR_REF(err);
   }
-  GRPC_CLOSURE_RUN(calld->on_done_recv, err);
-}
-
-static void hs_on_complete(void* user_data, grpc_error* err) {
-  grpc_call_element* elem = static_cast<grpc_call_element*>(user_data);
-  call_data* calld = static_cast<call_data*>(elem->call_data);
-  /* Call recv_message_ready if we got the payload via the path field */
-  if (calld->seen_path_with_query && calld->recv_message_ready != nullptr) {
-    calld->pp_recv_message->reset(
-        calld->payload_bin_delivered ? nullptr
-                                     : reinterpret_cast<grpc_core::ByteStream*>(
-                                           calld->read_stream.get()));
-    // Re-enter call combiner for recv_message_ready, since the surface
-    // code will release the call combiner for each callback it receives.
-    GRPC_CALL_COMBINER_START(calld->call_combiner, calld->recv_message_ready,
-                             GRPC_ERROR_REF(err),
-                             "resuming recv_message_ready from on_complete");
-    calld->recv_message_ready = nullptr;
-    calld->payload_bin_delivered = true;
-  }
-  GRPC_CLOSURE_RUN(calld->on_complete, GRPC_ERROR_REF(err));
+  GRPC_CLOSURE_RUN(calld->original_recv_initial_metadata_ready, err);
 }
 
 static void hs_recv_message_ready(void* user_data, grpc_error* err) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(user_data);
   call_data* calld = static_cast<call_data*>(elem->call_data);
-  if (calld->seen_path_with_query) {
-    // Do nothing. This is probably a GET request, and payload will be
-    // returned in hs_on_complete callback.
+  calld->seen_recv_message_ready = true;
+  if (calld->seen_recv_initial_metadata_ready) {
+    // We've already seen the recv_initial_metadata callback, so
+    // replace the recv_message byte stream if needed and invoke the
+    // original recv_message callback immediately.
+    if (calld->have_read_stream) {
+      calld->recv_message->reset(calld->read_stream.get());
+      calld->have_read_stream = false;
+    }
+    GRPC_CLOSURE_RUN(calld->original_recv_message_ready, GRPC_ERROR_REF(err));
+  } else {
+    // We have not yet seen the recv_initial_metadata callback, so we
+    // need to wait to see if this is a GET request.
     // Note that we release the call combiner here, so that other
     // callbacks can run.
-    GRPC_CALL_COMBINER_STOP(calld->call_combiner,
-                            "pausing recv_message_ready until on_complete");
-  } else {
-    GRPC_CLOSURE_RUN(calld->recv_message_ready, GRPC_ERROR_REF(err));
+    GRPC_CALL_COMBINER_STOP(
+        calld->call_combiner,
+        "pausing recv_message_ready until recv_initial_metadata_ready");
   }
 }
 
@@ -320,18 +321,18 @@ static grpc_error* hs_mutate_op(grpc_call_element* elem,
   if (op->send_initial_metadata) {
     grpc_error* error = GRPC_ERROR_NONE;
     static const char* error_name = "Failed sending initial metadata";
-    add_error(error_name, &error,
-              grpc_metadata_batch_add_head(
-                  op->payload->send_initial_metadata.send_initial_metadata,
-                  &calld->status, GRPC_MDELEM_STATUS_200));
-    add_error(error_name, &error,
-              grpc_metadata_batch_add_tail(
-                  op->payload->send_initial_metadata.send_initial_metadata,
-                  &calld->content_type,
-                  GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC));
-    add_error(
+    hs_add_error(error_name, &error,
+                 grpc_metadata_batch_add_head(
+                     op->payload->send_initial_metadata.send_initial_metadata,
+                     &calld->status, GRPC_MDELEM_STATUS_200));
+    hs_add_error(error_name, &error,
+                 grpc_metadata_batch_add_tail(
+                     op->payload->send_initial_metadata.send_initial_metadata,
+                     &calld->content_type,
+                     GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC));
+    hs_add_error(
         error_name, &error,
-        server_filter_outgoing_metadata(
+        hs_filter_outgoing_metadata(
             elem, op->payload->send_initial_metadata.send_initial_metadata));
     if (error != GRPC_ERROR_NONE) return error;
   }
@@ -343,27 +344,21 @@ static grpc_error* hs_mutate_op(grpc_call_element* elem,
         op->payload->recv_initial_metadata.recv_initial_metadata;
     calld->recv_initial_metadata_flags =
         op->payload->recv_initial_metadata.recv_flags;
-    calld->on_done_recv =
+    calld->original_recv_initial_metadata_ready =
         op->payload->recv_initial_metadata.recv_initial_metadata_ready;
     op->payload->recv_initial_metadata.recv_initial_metadata_ready =
-        &calld->hs_on_recv;
+        &calld->recv_initial_metadata_ready;
   }
 
   if (op->recv_message) {
-    calld->recv_message_ready = op->payload->recv_message.recv_message_ready;
-    calld->pp_recv_message = op->payload->recv_message.recv_message;
-    if (op->payload->recv_message.recv_message_ready) {
-      op->payload->recv_message.recv_message_ready =
-          &calld->hs_recv_message_ready;
-    }
-    if (op->on_complete) {
-      calld->on_complete = op->on_complete;
-      op->on_complete = &calld->hs_on_complete;
-    }
+    calld->recv_message = op->payload->recv_message.recv_message;
+    calld->original_recv_message_ready =
+        op->payload->recv_message.recv_message_ready;
+    op->payload->recv_message.recv_message_ready = &calld->recv_message_ready;
   }
 
   if (op->send_trailing_metadata) {
-    grpc_error* error = server_filter_outgoing_metadata(
+    grpc_error* error = hs_filter_outgoing_metadata(
         elem, op->payload->send_trailing_metadata.send_trailing_metadata);
     if (error != GRPC_ERROR_NONE) return error;
   }
@@ -385,50 +380,47 @@ static void hs_start_transport_stream_op_batch(
 }
 
 /* Constructor for call_data */
-static grpc_error* init_call_elem(grpc_call_element* elem,
-                                  const grpc_call_element_args* args) {
-  /* grab pointers to our data from the call element */
+static grpc_error* hs_init_call_elem(grpc_call_element* elem,
+                                     const grpc_call_element_args* args) {
   call_data* calld = static_cast<call_data*>(elem->call_data);
-  /* initialize members */
   calld->call_combiner = args->call_combiner;
-  GRPC_CLOSURE_INIT(&calld->hs_on_recv, hs_on_recv, elem,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->hs_on_complete, hs_on_complete, elem,
+  GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready,
+                    hs_recv_initial_metadata_ready, elem,
                     grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&calld->hs_recv_message_ready, hs_recv_message_ready, elem,
+  GRPC_CLOSURE_INIT(&calld->recv_message_ready, hs_recv_message_ready, elem,
                     grpc_schedule_on_exec_ctx);
   return GRPC_ERROR_NONE;
 }
 
 /* Destructor for call_data */
-static void destroy_call_elem(grpc_call_element* elem,
-                              const grpc_call_final_info* final_info,
-                              grpc_closure* ignored) {
+static void hs_destroy_call_elem(grpc_call_element* elem,
+                                 const grpc_call_final_info* final_info,
+                                 grpc_closure* ignored) {
   call_data* calld = static_cast<call_data*>(elem->call_data);
-  if (calld->seen_path_with_query && !calld->payload_bin_delivered) {
+  if (calld->have_read_stream) {
     calld->read_stream->Orphan();
   }
 }
 
 /* Constructor for channel_data */
-static grpc_error* init_channel_elem(grpc_channel_element* elem,
-                                     grpc_channel_element_args* args) {
+static grpc_error* hs_init_channel_elem(grpc_channel_element* elem,
+                                        grpc_channel_element_args* args) {
   GPR_ASSERT(!args->is_last);
   return GRPC_ERROR_NONE;
 }
 
 /* Destructor for channel data */
-static void destroy_channel_elem(grpc_channel_element* elem) {}
+static void hs_destroy_channel_elem(grpc_channel_element* elem) {}
 
 const grpc_channel_filter grpc_http_server_filter = {
     hs_start_transport_stream_op_batch,
     grpc_channel_next_op,
     sizeof(call_data),
-    init_call_elem,
+    hs_init_call_elem,
     grpc_call_stack_ignore_set_pollset_or_pollset_set,
-    destroy_call_elem,
-    sizeof(channel_data),
-    init_channel_elem,
-    destroy_channel_elem,
+    hs_destroy_call_elem,
+    0,
+    hs_init_channel_elem,
+    hs_destroy_channel_elem,
     grpc_channel_next_get_info,
     "http-server"};