grpc 1.41.0 → 1.42.0.pre1

data/src/core/lib/surface/call.cc

@@ -50,6 +50,7 @@
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/slice/slice_split.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 #include "src/core/lib/slice/slice_utils.h"
 #include "src/core/lib/surface/api_trace.h"
@@ -107,7 +108,7 @@ struct batch_control {
   grpc_closure start_batch;
   grpc_closure finish_batch;
   std::atomic<intptr_t> steps_to_complete{0};
-  gpr_atm batch_error = reinterpret_cast<gpr_atm>(GRPC_ERROR_NONE);
+  AtomicError batch_error;
   void set_num_steps_to_complete(uintptr_t steps) {
     steps_to_complete.store(steps, std::memory_order_release);
   }
@@ -143,13 +144,7 @@ struct grpc_call {
         cq(args.cq),
         channel(args.channel),
         is_client(args.server_transport_data == nullptr),
-        stream_op_payload(context) {
-    for (int i = 0; i < 2; i++) {
-      for (int j = 0; j < 2; j++) {
-        metadata_batch[i][j].deadline = GRPC_MILLIS_INF_FUTURE;
-      }
-    }
-  }
+        stream_op_payload(context) {}
 
   ~grpc_call() {
     for (int i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
@@ -192,7 +187,10 @@ struct grpc_call {
   grpc_transport_stream_op_batch_payload stream_op_payload;
 
   /* first idx: is_receiving, second idx: is_trailing */
-  grpc_metadata_batch metadata_batch[2][2] = {};
+  grpc_metadata_batch send_initial_metadata{arena};
+  grpc_metadata_batch send_trailing_metadata{arena};
+  grpc_metadata_batch recv_initial_metadata{arena};
+  grpc_metadata_batch recv_trailing_metadata{arena};
 
   /* Buffered read metadata waiting to be returned to the application.
      Element 0 is initial metadata, element 1 is trailing metadata. */
@@ -257,7 +255,7 @@ struct grpc_call {
       grpc_core::Server* core_server;
     } server;
   } final_op;
-  gpr_atm status_error = 0;
+  AtomicError status_error;
 
   /* recv_state can contain one of the following values:
      RECV_NONE :                 :  no initial metadata and messages received
@@ -407,7 +405,7 @@ grpc_error_handle grpc_call_create(const grpc_call_create_args* args,
     GPR_ASSERT(!args->parent->is_client);
 
     if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) {
-      send_deadline = GPR_MIN(send_deadline, args->parent->send_deadline);
+      send_deadline = std::min(send_deadline, args->parent->send_deadline);
     }
     /* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with
      * GRPC_PROPAGATE_STATS_CONTEXT */
@@ -545,31 +543,26 @@ static void release_call(void* call, grpc_error_handle /*error*/) {
 
 static void destroy_call(void* call, grpc_error_handle /*error*/) {
   GPR_TIMER_SCOPE("destroy_call", 0);
-  size_t i;
-  int ii;
   grpc_call* c = static_cast<grpc_call*>(call);
-  for (i = 0; i < 2; i++) {
-    grpc_metadata_batch_destroy(
-        &c->metadata_batch[1 /* is_receiving */][i /* is_initial */]);
-  }
+  c->recv_initial_metadata.Clear();
+  c->recv_trailing_metadata.Clear();
   c->receiving_stream.reset();
   parent_call* pc = get_parent_call(c);
   if (pc != nullptr) {
     pc->~parent_call();
   }
-  for (ii = 0; ii < c->send_extra_metadata_count; ii++) {
-    GRPC_MDELEM_UNREF(c->send_extra_metadata[ii].md);
+  for (int i = 0; i < c->send_extra_metadata_count; i++) {
+    GRPC_MDELEM_UNREF(c->send_extra_metadata[i].md);
   }
   if (c->cq) {
     GRPC_CQ_INTERNAL_UNREF(c->cq, "bind");
   }
 
-  grpc_error_handle status_error =
-      reinterpret_cast<grpc_error_handle>(gpr_atm_acq_load(&c->status_error));
+  grpc_error_handle status_error = c->status_error.get();
   grpc_error_get_status(status_error, c->send_deadline,
                         &c->final_info.final_status, nullptr, nullptr,
                         &(c->final_info.error_string));
-  GRPC_ERROR_UNREF(status_error);
+  c->status_error.set(GRPC_ERROR_NONE);
   c->final_info.stats.latency =
       gpr_cycle_counter_sub(gpr_get_cycle_counter(), c->start_time);
   grpc_call_stack_destroy(CALL_STACK_FROM_CALL(c), &c->final_info,
@@ -734,8 +727,7 @@ static grpc_error_handle error_from_status(grpc_status_code status,
   // guarantee that can be short-lived.
   return grpc_error_set_int(
       grpc_error_set_str(GRPC_ERROR_CREATE_FROM_COPIED_STRING(description),
-                         GRPC_ERROR_STR_GRPC_MESSAGE,
-                         grpc_slice_from_copied_string(description)),
+                         GRPC_ERROR_STR_GRPC_MESSAGE, description),
       GRPC_ERROR_INT_GRPC_STATUS, status);
 }
 
@@ -750,13 +742,14 @@ static void set_final_status(grpc_call* call, grpc_error_handle error) {
     gpr_log(GPR_DEBUG, "%s", grpc_error_std_string(error).c_str());
   }
   if (call->is_client) {
+    std::string status_details;
     grpc_error_get_status(error, call->send_deadline,
-                          call->final_op.client.status,
-                          call->final_op.client.status_details, nullptr,
-                          call->final_op.client.error_string);
-    // explicitly take a ref
-    grpc_slice_ref_internal(*call->final_op.client.status_details);
-    gpr_atm_rel_store(&call->status_error, reinterpret_cast<gpr_atm>(error));
+                          call->final_op.client.status, &status_details,
+                          nullptr, call->final_op.client.error_string);
+    *call->final_op.client.status_details =
+        grpc_slice_from_cpp_string(std::move(status_details));
+    call->status_error.set(error);
+    GRPC_ERROR_UNREF(error);
     grpc_core::channelz::ChannelNode* channelz_channel =
         grpc_channel_get_channelz_node(call->channel);
     if (channelz_channel != nullptr) {
@@ -772,9 +765,7 @@ static void set_final_status(grpc_call* call, grpc_error_handle error) {
     grpc_core::channelz::ServerNode* channelz_node =
         call->final_op.server.core_server->channelz_node();
     if (channelz_node != nullptr) {
-      if (*call->final_op.server.cancelled ||
-          reinterpret_cast<grpc_error_handle>(
-              gpr_atm_acq_load(&call->status_error)) != GRPC_ERROR_NONE) {
+      if (*call->final_op.server.cancelled || !call->status_error.ok()) {
         channelz_node->RecordCallFailed();
       } else {
         channelz_node->RecordCallSucceeded();
@@ -848,7 +839,7 @@ static void set_encodings_accepted_by_peer(grpc_call* /*call*/,
   grpc_slice_split_without_space(accept_encoding_slice, ",",
                                  &accept_encoding_parts);
 
-  GPR_BITSET(encodings_accepted_by_peer, GRPC_COMPRESS_NONE);
+  grpc_core::SetBit(encodings_accepted_by_peer, GRPC_COMPRESS_NONE);
   for (i = 0; i < accept_encoding_parts.count; i++) {
     int r;
     grpc_slice accept_encoding_entry_slice = accept_encoding_parts.slices[i];
@@ -862,7 +853,7 @@ static void set_encodings_accepted_by_peer(grpc_call* /*call*/,
           reinterpret_cast<grpc_stream_compression_algorithm*>(&algorithm));
     }
     if (r) {
-      GPR_BITSET(encodings_accepted_by_peer, algorithm);
+      grpc_core::SetBit(encodings_accepted_by_peer, algorithm);
     } else {
       char* accept_encoding_entry_str =
           grpc_slice_to_c_string(accept_encoding_entry_slice);
@@ -913,8 +904,8 @@ static int prepare_application_metadata(grpc_call* call, int count,
                                         int additional_metadata_count) {
   int total_count = count + additional_metadata_count;
   int i;
-  grpc_metadata_batch* batch =
-      &call->metadata_batch[0 /* is_receiving */][is_trailing];
+  grpc_metadata_batch* batch = is_trailing ? &call->send_trailing_metadata
+                                           : &call->send_initial_metadata;
   for (i = 0; i < total_count; i++) {
     grpc_metadata* md = get_md_elem(metadata, additional_metadata, i, count);
     grpc_linked_mdelem* l = linked_from_md(md);
@@ -947,15 +938,14 @@ static int prepare_application_metadata(grpc_call* call, int count,
     } else {
       for (i = 0; i < call->send_extra_metadata_count; i++) {
         GRPC_LOG_IF_ERROR("prepare_application_metadata",
-                          grpc_metadata_batch_link_tail(
-                              batch, &call->send_extra_metadata[i]));
+                          batch->LinkTail(&call->send_extra_metadata[i]));
       }
     }
   }
   for (i = 0; i < total_count; i++) {
     grpc_metadata* md = get_md_elem(metadata, additional_metadata, i, count);
     grpc_linked_mdelem* l = linked_from_md(md);
-    grpc_error_handle error = grpc_metadata_batch_link_tail(batch, l);
+    grpc_error_handle error = batch->LinkTail(l);
     if (error != GRPC_ERROR_NONE) {
       GRPC_MDELEM_UNREF(l->md);
     }
@@ -1000,53 +990,57 @@ static grpc_stream_compression_algorithm decode_stream_compression(
 
 static void publish_app_metadata(grpc_call* call, grpc_metadata_batch* b,
                                  int is_trailing) {
-  if (b->list.count == 0) return;
+  if (b->non_deadline_count() == 0) return;
   if (!call->is_client && is_trailing) return;
   if (is_trailing && call->buffered_metadata[1] == nullptr) return;
   GPR_TIMER_SCOPE("publish_app_metadata", 0);
   grpc_metadata_array* dest;
   grpc_metadata* mdusr;
   dest = call->buffered_metadata[is_trailing];
-  if (dest->count + b->list.count > dest->capacity) {
-    dest->capacity =
-        GPR_MAX(dest->capacity + b->list.count, dest->capacity * 3 / 2);
+  if (dest->count + b->non_deadline_count() > dest->capacity) {
+    dest->capacity = std::max(dest->capacity + b->non_deadline_count(),
+                              dest->capacity * 3 / 2);
     dest->metadata = static_cast<grpc_metadata*>(
         gpr_realloc(dest->metadata, sizeof(grpc_metadata) * dest->capacity));
   }
-  for (grpc_linked_mdelem* l = b->list.head; l != nullptr; l = l->next) {
+  b->ForEach([&](grpc_mdelem md) {
     mdusr = &dest->metadata[dest->count++];
     /* we pass back borrowed slices that are valid whilst the call is valid */
-    mdusr->key = GRPC_MDKEY(l->md);
-    mdusr->value = GRPC_MDVALUE(l->md);
-  }
+    mdusr->key = GRPC_MDKEY(md);
+    mdusr->value = GRPC_MDVALUE(md);
+  });
 }
 
 static void recv_initial_filter(grpc_call* call, grpc_metadata_batch* b) {
-  if (b->idx.named.content_encoding != nullptr) {
+  if (b->legacy_index()->named.content_encoding != nullptr) {
     GPR_TIMER_SCOPE("incoming_stream_compression_algorithm", 0);
     set_incoming_stream_compression_algorithm(
-        call, decode_stream_compression(b->idx.named.content_encoding->md));
-    grpc_metadata_batch_remove(b, GRPC_BATCH_CONTENT_ENCODING);
+        call, decode_stream_compression(
+                  b->legacy_index()->named.content_encoding->md));
+    b->Remove(GRPC_BATCH_CONTENT_ENCODING);
   }
-  if (b->idx.named.grpc_encoding != nullptr) {
+  if (b->legacy_index()->named.grpc_encoding != nullptr) {
     GPR_TIMER_SCOPE("incoming_message_compression_algorithm", 0);
     set_incoming_message_compression_algorithm(
-        call, decode_message_compression(b->idx.named.grpc_encoding->md));
-    grpc_metadata_batch_remove(b, GRPC_BATCH_GRPC_ENCODING);
+        call,
+        decode_message_compression(b->legacy_index()->named.grpc_encoding->md));
+    b->Remove(GRPC_BATCH_GRPC_ENCODING);
   }
   uint32_t message_encodings_accepted_by_peer = 1u;
   uint32_t stream_encodings_accepted_by_peer = 1u;
-  if (b->idx.named.grpc_accept_encoding != nullptr) {
+  if (b->legacy_index()->named.grpc_accept_encoding != nullptr) {
     GPR_TIMER_SCOPE("encodings_accepted_by_peer", 0);
-    set_encodings_accepted_by_peer(call, b->idx.named.grpc_accept_encoding->md,
-                                   &message_encodings_accepted_by_peer, false);
-    grpc_metadata_batch_remove(b, GRPC_BATCH_GRPC_ACCEPT_ENCODING);
+    set_encodings_accepted_by_peer(
+        call, b->legacy_index()->named.grpc_accept_encoding->md,
+        &message_encodings_accepted_by_peer, false);
+    b->Remove(GRPC_BATCH_GRPC_ACCEPT_ENCODING);
   }
-  if (b->idx.named.accept_encoding != nullptr) {
+  if (b->legacy_index()->named.accept_encoding != nullptr) {
     GPR_TIMER_SCOPE("stream_encodings_accepted_by_peer", 0);
-    set_encodings_accepted_by_peer(call, b->idx.named.accept_encoding->md,
+    set_encodings_accepted_by_peer(call,
+                                   b->legacy_index()->named.accept_encoding->md,
                                    &stream_encodings_accepted_by_peer, true);
-    grpc_metadata_batch_remove(b, GRPC_BATCH_ACCEPT_ENCODING);
+    b->Remove(GRPC_BATCH_ACCEPT_ENCODING);
   }
   call->encodings_accepted_by_peer =
       grpc_compression_bitset_from_message_stream_compression_bitset(
@@ -1060,9 +1054,9 @@ static void recv_trailing_filter(void* args, grpc_metadata_batch* b,
   grpc_call* call = static_cast<grpc_call*>(args);
   if (batch_error != GRPC_ERROR_NONE) {
     set_final_status(call, batch_error);
-  } else if (b->idx.named.grpc_status != nullptr) {
-    grpc_status_code status_code =
-        grpc_get_status_code_from_metadata(b->idx.named.grpc_status->md);
+  } else if (b->legacy_index()->named.grpc_status != nullptr) {
+    grpc_status_code status_code = grpc_get_status_code_from_metadata(
+        b->legacy_index()->named.grpc_status->md);
     grpc_error_handle error = GRPC_ERROR_NONE;
     if (status_code != GRPC_STATUS_OK) {
       char* peer = grpc_call_get_peer(call);
@@ -1072,17 +1066,17 @@ static void recv_trailing_filter(void* args, grpc_metadata_batch* b,
                                  static_cast<intptr_t>(status_code));
       gpr_free(peer);
     }
-    if (b->idx.named.grpc_message != nullptr) {
+    if (b->legacy_index()->named.grpc_message != nullptr) {
       error = grpc_error_set_str(
           error, GRPC_ERROR_STR_GRPC_MESSAGE,
-          grpc_slice_ref_internal(GRPC_MDVALUE(b->idx.named.grpc_message->md)));
-      grpc_metadata_batch_remove(b, GRPC_BATCH_GRPC_MESSAGE);
+          grpc_core::StringViewFromSlice(
+              GRPC_MDVALUE(b->legacy_index()->named.grpc_message->md)));
+      b->Remove(GRPC_BATCH_GRPC_MESSAGE);
     } else if (error != GRPC_ERROR_NONE) {
-      error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE,
-                                 grpc_empty_slice());
+      error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "");
     }
     set_final_status(call, GRPC_ERROR_REF(error));
-    grpc_metadata_batch_remove(b, GRPC_BATCH_GRPC_STATUS);
+    b->Remove(GRPC_BATCH_GRPC_STATUS);
     GRPC_ERROR_UNREF(error);
   } else if (!call->is_client) {
     set_final_status(call, GRPC_ERROR_NONE);
@@ -1156,6 +1150,7 @@ static batch_control* reuse_or_allocate_batch_control(grpc_call* call,
     }
     bctl->~batch_control();
     bctl->op = {};
+    new (&bctl->batch_error) AtomicError();
   } else {
     bctl = call->arena->New<batch_control>();
     *pslot = bctl;
@@ -1174,21 +1169,16 @@ static void finish_batch_completion(void* user_data,
 }
 
 static void reset_batch_errors(batch_control* bctl) {
-  GRPC_ERROR_UNREF(reinterpret_cast<grpc_error_handle>(
-      gpr_atm_acq_load(&bctl->batch_error)));
-  gpr_atm_rel_store(&bctl->batch_error,
-                    reinterpret_cast<gpr_atm>(GRPC_ERROR_NONE));
+  bctl->batch_error.set(GRPC_ERROR_NONE);
 }
 
 static void post_batch_completion(batch_control* bctl) {
   grpc_call* next_child_call;
   grpc_call* call = bctl->call;
-  grpc_error_handle error = GRPC_ERROR_REF(reinterpret_cast<grpc_error_handle>(
-      gpr_atm_acq_load(&bctl->batch_error)));
+  grpc_error_handle error = GRPC_ERROR_REF(bctl->batch_error.get());
 
   if (bctl->op.send_initial_metadata) {
-    grpc_metadata_batch_destroy(
-        &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */]);
+    call->send_initial_metadata.Clear();
   }
   if (bctl->op.send_message) {
     if (bctl->op.payload->send_message.stream_write_closed &&
@@ -1200,8 +1190,7 @@ static void post_batch_completion(batch_control* bctl) {
     call->sending_message = false;
   }
   if (bctl->op.send_trailing_metadata) {
-    grpc_metadata_batch_destroy(
-        &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]);
+    call->send_trailing_metadata.Clear();
   }
   if (bctl->op.recv_trailing_metadata) {
     /* propagate cancellation to any interested children */
@@ -1353,10 +1342,8 @@ static void receiving_stream_ready(void* bctlp, grpc_error_handle error) {
   grpc_call* call = bctl->call;
   if (error != GRPC_ERROR_NONE) {
     call->receiving_stream.reset();
-    if (reinterpret_cast<grpc_error_handle>(
-            gpr_atm_acq_load(&bctl->batch_error)) == GRPC_ERROR_NONE) {
-      gpr_atm_rel_store(&bctl->batch_error,
-                        reinterpret_cast<gpr_atm>(GRPC_ERROR_REF(error)));
+    if (bctl->batch_error.ok()) {
+      bctl->batch_error.set(error);
     }
     cancel_with_error(call, GRPC_ERROR_REF(error));
   }
@@ -1458,8 +1445,8 @@ static void validate_filtered_metadata(batch_control* bctl) {
     }
     /* GRPC_COMPRESS_NONE is always set. */
     GPR_DEBUG_ASSERT(call->encodings_accepted_by_peer != 0);
-    if (GPR_UNLIKELY(!GPR_BITGET(call->encodings_accepted_by_peer,
-                                 compression_algorithm))) {
+    if (GPR_UNLIKELY(!grpc_core::GetBit(call->encodings_accepted_by_peer,
+                                        compression_algorithm))) {
       if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
         handle_compression_algorithm_not_accepted(call, compression_algorithm);
       }
@@ -1475,22 +1462,21 @@ static void receiving_initial_metadata_ready(void* bctlp,
   GRPC_CALL_COMBINER_STOP(&call->call_combiner, "recv_initial_metadata_ready");
 
   if (error == GRPC_ERROR_NONE) {
-    grpc_metadata_batch* md =
-        &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
+    grpc_metadata_batch* md = &call->recv_initial_metadata;
     recv_initial_filter(call, md);
 
     /* TODO(ctiller): this could be moved into recv_initial_filter now */
     GPR_TIMER_SCOPE("validate_filtered_metadata", 0);
     validate_filtered_metadata(bctl);
 
-    if (md->deadline != GRPC_MILLIS_INF_FUTURE && !call->is_client) {
-      call->send_deadline = md->deadline;
+    absl::optional<grpc_millis> deadline =
+        md->get(grpc_core::GrpcTimeoutMetadata());
+    if (deadline.has_value() && !call->is_client) {
+      call->send_deadline = *deadline;
     }
   } else {
-    if (reinterpret_cast<grpc_error_handle>(
-            gpr_atm_acq_load(&bctl->batch_error)) == GRPC_ERROR_NONE) {
-      gpr_atm_rel_store(&bctl->batch_error,
-                        reinterpret_cast<gpr_atm>(GRPC_ERROR_REF(error)));
+    if (bctl->batch_error.ok()) {
+      bctl->batch_error.set(error);
     }
     cancel_with_error(call, GRPC_ERROR_REF(error));
   }
@@ -1532,8 +1518,7 @@ static void receiving_trailing_metadata_ready(void* bctlp,
   batch_control* bctl = static_cast<batch_control*>(bctlp);
   grpc_call* call = bctl->call;
   GRPC_CALL_COMBINER_STOP(&call->call_combiner, "recv_trailing_metadata_ready");
-  grpc_metadata_batch* md =
-      &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
+  grpc_metadata_batch* md = &call->recv_trailing_metadata;
   recv_trailing_filter(call, md, GRPC_ERROR_REF(error));
   finish_batch_step(bctl);
 }
@@ -1542,10 +1527,8 @@ static void finish_batch(void* bctlp, grpc_error_handle error) {
   batch_control* bctl = static_cast<batch_control*>(bctlp);
   grpc_call* call = bctl->call;
   GRPC_CALL_COMBINER_STOP(&call->call_combiner, "on_complete");
-  if (reinterpret_cast<grpc_error_handle>(
-          gpr_atm_acq_load(&bctl->batch_error)) == GRPC_ERROR_NONE) {
-    gpr_atm_rel_store(&bctl->batch_error,
-                      reinterpret_cast<gpr_atm>(GRPC_ERROR_REF(error)));
+  if (bctl->batch_error.ok()) {
+    bctl->batch_error.set(error);
   }
   if (error != GRPC_ERROR_NONE) {
     cancel_with_error(call, GRPC_ERROR_REF(error));
@@ -1669,11 +1652,12 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
           goto done_with_error;
         }
         /* TODO(ctiller): just make these the same variable? */
-        if (call->is_client) {
-          call->metadata_batch[0][0].deadline = call->send_deadline;
+        if (call->is_client && call->send_deadline != GRPC_MILLIS_INF_FUTURE) {
+          call->send_initial_metadata.Set(grpc_core::GrpcTimeoutMetadata(),
+                                          call->send_deadline);
         }
         stream_op_payload->send_initial_metadata.send_initial_metadata =
-            &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */];
+            &call->send_initial_metadata;
         stream_op_payload->send_initial_metadata.send_initial_metadata_flags =
             op->flags;
         if (call->is_client) {
@@ -1730,7 +1714,7 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
         stream_op->send_trailing_metadata = true;
         call->sent_final_op = true;
         stream_op_payload->send_trailing_metadata.send_trailing_metadata =
-            &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */];
+            &call->send_trailing_metadata;
         has_send_ops = true;
         break;
       }
@@ -1771,21 +1755,21 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
         if (op->data.send_status_from_server.status_details != nullptr) {
           call->send_extra_metadata[1].md = grpc_mdelem_from_slices(
               GRPC_MDSTR_GRPC_MESSAGE,
-              grpc_slice_ref_internal(
+              grpc_slice_copy(
                   *op->data.send_status_from_server.status_details));
           call->send_extra_metadata_count++;
           if (status_error != GRPC_ERROR_NONE) {
             char* msg = grpc_slice_to_c_string(
                 GRPC_MDVALUE(call->send_extra_metadata[1].md));
-            status_error =
-                grpc_error_set_str(status_error, GRPC_ERROR_STR_GRPC_MESSAGE,
-                                   grpc_slice_from_copied_string(msg));
+            status_error = grpc_error_set_str(status_error,
+                                              GRPC_ERROR_STR_GRPC_MESSAGE, msg);
             gpr_free(msg);
           }
         }
 
-        gpr_atm_rel_store(&call->status_error,
-                          reinterpret_cast<gpr_atm>(status_error));
+        call->status_error.set(status_error);
+        GRPC_ERROR_UNREF(status_error);
+
         if (!prepare_application_metadata(
                 call,
                 static_cast<int>(
@@ -1800,7 +1784,7 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
           goto done_with_error;
         }
         stream_op_payload->send_trailing_metadata.send_trailing_metadata =
-            &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */];
+            &call->send_trailing_metadata;
         stream_op_payload->send_trailing_metadata.sent =
             &call->sent_server_trailing_metadata;
         has_send_ops = true;
@@ -1824,7 +1808,7 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
                           grpc_schedule_on_exec_ctx);
         stream_op->recv_initial_metadata = true;
         stream_op_payload->recv_initial_metadata.recv_initial_metadata =
-            &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
+            &call->recv_initial_metadata;
         stream_op_payload->recv_initial_metadata.recv_initial_metadata_ready =
             &call->receiving_initial_metadata_ready;
         if (call->is_client) {
@@ -1885,7 +1869,7 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
             op->data.recv_status_on_client.error_string;
         stream_op->recv_trailing_metadata = true;
         stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
-            &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
+            &call->recv_trailing_metadata;
         stream_op_payload->recv_trailing_metadata.collect_stats =
             &call->final_info.stats.transport_stream_stats;
         GRPC_CLOSURE_INIT(&call->receiving_trailing_metadata_ready,
@@ -1915,7 +1899,7 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
             op->data.recv_close_on_server.cancelled;
         stream_op->recv_trailing_metadata = true;
         stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
-            &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
+            &call->recv_trailing_metadata;
         stream_op_payload->recv_trailing_metadata.collect_stats =
             &call->final_info.stats.transport_stream_stats;
         GRPC_CLOSURE_INIT(&call->receiving_trailing_metadata_ready,
@@ -1951,15 +1935,16 @@ done_with_error:
   /* reverse any mutations that occurred */
   if (stream_op->send_initial_metadata) {
     call->sent_initial_metadata = false;
-    grpc_metadata_batch_clear(&call->metadata_batch[0][0]);
+    call->send_initial_metadata.Clear();
   }
   if (stream_op->send_message) {
     call->sending_message = false;
     call->sending_stream->Orphan();
+    stream_op_payload->send_message.send_message.reset();
   }
   if (stream_op->send_trailing_metadata) {
     call->sent_final_op = false;
-    grpc_metadata_batch_clear(&call->metadata_batch[0][1]);
+    call->send_trailing_metadata.Clear();
   }
   if (stream_op->recv_initial_metadata) {
     call->received_initial_metadata = false;
@@ -2024,13 +2009,11 @@ grpc_compression_algorithm grpc_call_compression_for_level(
 
 bool grpc_call_is_trailers_only(const grpc_call* call) {
   bool result = call->is_trailers_only;
-  GPR_DEBUG_ASSERT(
-      !result || call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]
-                         .list.count == 0);
+  GPR_DEBUG_ASSERT(!result || call->recv_initial_metadata.empty());
   return result;
 }
 
-bool grpc_call_failed_before_recv_message(const grpc_call* c) {
+int grpc_call_failed_before_recv_message(const grpc_call* c) {
   return c->call_failed_before_recv_message;
 }
 

data/src/core/lib/surface/call.h

@@ -125,12 +125,6 @@ grpc_compression_algorithm grpc_call_compression_for_level(
                     Move to surface API if requested by other languages. */
 bool grpc_call_is_trailers_only(const grpc_call* call);
 
-/* Returns whether or not the call's receive message operation failed because of
- * an error (as opposed to a graceful end-of-stream) */
-/* TODO(markdroth): This is currently available only to the C++ API.
-                    Move to surface API if requested by other languages. */
-bool grpc_call_failed_before_recv_message(const grpc_call* c);
-
 extern grpc_core::TraceFlag grpc_call_error_trace;
 extern grpc_core::TraceFlag grpc_compression_trace;
 

data/src/core/lib/surface/channel.cc

@@ -34,6 +34,7 @@
 #include "src/core/lib/channel/channel_trace.h"
 #include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/channel/channelz_registry.h"
+#include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
@@ -44,7 +45,6 @@
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
-#include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/transport/static_metadata.h"
 
 /** Cache grpc-status: X mdelems for X = 0..NUM_CACHED_STATUS_ELEMS.
@@ -187,7 +187,9 @@ void channelz_node_destroy(void* p) {
       static_cast<grpc_core::channelz::ChannelNode*>(p);
   node->Unref();
 }
-int channelz_node_cmp(void* p1, void* p2) { return GPR_ICMP(p1, p2); }
+int channelz_node_cmp(void* p1, void* p2) {
+  return grpc_core::QsortCompare(p1, p2);
+}
 const grpc_arg_pointer_vtable channelz_node_arg_vtable = {
     channelz_node_copy, channelz_node_destroy, channelz_node_cmp};
 
@@ -267,7 +269,8 @@ grpc_channel* grpc_channel_create(const char* target,
   grpc_channel_args_destroy(args);
   grpc_channel_stack_builder_set_target(builder, target);
   grpc_channel_stack_builder_set_transport(builder, optional_transport);
-  if (!grpc_channel_init_create_stack(builder, channel_stack_type)) {
+  if (!grpc_core::CoreConfiguration::Get().channel_init().CreateStack(
+          builder, channel_stack_type)) {
     grpc_channel_stack_builder_destroy(builder);
     if (resource_user != nullptr) {
       if (preallocated_bytes > 0) {
@@ -321,7 +324,7 @@ void grpc_channel_update_call_size_estimate(grpc_channel* channel,
     /* size shrank: decrease estimate */
     gpr_atm_no_barrier_cas(
         &channel->call_size_estimate, static_cast<gpr_atm>(cur),
-        static_cast<gpr_atm>(GPR_MIN(cur - 1, (255 * cur + size) / 256)));
+        static_cast<gpr_atm>(std::min(cur - 1, (255 * cur + size) / 256)));
     /* if we lose: never mind, something else will likely update soon enough */
   }
 }
@@ -420,35 +423,19 @@ grpc_call* grpc_channel_create_pollset_set_call(
 namespace grpc_core {
 
 RegisteredCall::RegisteredCall(const char* method_arg, const char* host_arg)
-    : method(method_arg != nullptr ? method_arg : ""),
-      host(host_arg != nullptr ? host_arg : ""),
-      path(grpc_mdelem_from_slices(
-          GRPC_MDSTR_PATH, grpc_core::ExternallyManagedSlice(method.c_str()))),
-      authority(!host.empty()
-                    ? grpc_mdelem_from_slices(
-                          GRPC_MDSTR_AUTHORITY,
-                          grpc_core::ExternallyManagedSlice(host.c_str()))
-                    : GRPC_MDNULL) {}
-
-// TODO(vjpai): Delete copy-constructor when allowed by all supported compilers.
+    : path(method_arg != nullptr && method_arg[0] != 0
+               ? grpc_mdelem_from_slices(
+                     GRPC_MDSTR_PATH, grpc_slice_from_copied_string(method_arg))
+               : GRPC_MDNULL),
+      authority(
+          host_arg != nullptr && host_arg[0] != 0
+              ? grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY,
+                                        grpc_slice_from_copied_string(host_arg))
+              : GRPC_MDNULL) {}
+
 RegisteredCall::RegisteredCall(const RegisteredCall& other)
-    : RegisteredCall(other.method.c_str(), other.host.c_str()) {}
-
-RegisteredCall::RegisteredCall(RegisteredCall&& other) noexcept
-    : method(std::move(other.method)),
-      host(std::move(other.host)),
-      path(grpc_mdelem_from_slices(
-          GRPC_MDSTR_PATH, grpc_core::ExternallyManagedSlice(method.c_str()))),
-      authority(!host.empty()
-                    ? grpc_mdelem_from_slices(
-                          GRPC_MDSTR_AUTHORITY,
-                          grpc_core::ExternallyManagedSlice(host.c_str()))
-                    : GRPC_MDNULL) {
-  GRPC_MDELEM_UNREF(other.path);
-  GRPC_MDELEM_UNREF(other.authority);
-  other.path = GRPC_MDNULL;
-  other.authority = GRPC_MDNULL;
-}
+    : path(GRPC_MDELEM_REF(other.path)),
+      authority(GRPC_MDELEM_REF(other.authority)) {}
 
 RegisteredCall::~RegisteredCall() {
   GRPC_MDELEM_UNREF(path);