grpc 1.38.0 → 1.39.0.pre1

data/src/core/ext/filters/client_channel/retry_service_config.cc

@@ -158,11 +158,11 @@ RetryServiceConfigParser::ParseGlobalParams(const grpc_channel_args* /*args*/,
 
 namespace {
 
-grpc_error_handle ParseRetryPolicy(const Json& json, int* max_attempts,
-                                   grpc_millis* initial_backoff,
-                                   grpc_millis* max_backoff,
-                                   float* backoff_multiplier,
-                                   StatusCodeSet* retryable_status_codes) {
+grpc_error_handle ParseRetryPolicy(
+    const Json& json, int* max_attempts, grpc_millis* initial_backoff,
+    grpc_millis* max_backoff, float* backoff_multiplier,
+    StatusCodeSet* retryable_status_codes,
+    absl::optional<grpc_millis>* per_attempt_recv_timeout) {
   if (json.type() != Json::Type::OBJECT) {
     return GRPC_ERROR_CREATE_FROM_STATIC_STRING(
         "field:retryPolicy error:should be of type object");
@@ -170,7 +170,10 @@ grpc_error_handle ParseRetryPolicy(const Json& json, int* max_attempts,
   std::vector<grpc_error_handle> error_list;
   // Parse maxAttempts.
   auto it = json.object_value().find("maxAttempts");
-  if (it != json.object_value().end()) {
+  if (it == json.object_value().end()) {
+    error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "field:maxAttempts error:required field missing"));
+  } else {
     if (it->second.type() != Json::Type::NUMBER) {
       error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
           "field:maxAttempts error:should be of type number"));
@@ -200,11 +203,14 @@ grpc_error_handle ParseRetryPolicy(const Json& json, int* max_attempts,
                                      max_backoff, &error_list) &&
       *max_backoff == 0) {
     error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-        "field:maxBackoff error:should be greater than 0"));
+        "field:maxBackoff error:must be greater than 0"));
   }
   // Parse backoffMultiplier.
   it = json.object_value().find("backoffMultiplier");
-  if (it != json.object_value().end()) {
+  if (it == json.object_value().end()) {
+    error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "field:backoffMultiplier error:required field missing"));
+  } else {
     if (it->second.type() != Json::Type::NUMBER) {
       error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
           "field:backoffMultiplier error:should be of type number"));
@@ -215,7 +221,7 @@ grpc_error_handle ParseRetryPolicy(const Json& json, int* max_attempts,
             "field:backoffMultiplier error:failed to parse"));
       } else if (*backoff_multiplier <= 0) {
         error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-            "field:backoffMultiplier error:should be greater than 0"));
+            "field:backoffMultiplier error:must be greater than 0"));
       }
     }
   }
@@ -224,7 +230,7 @@ grpc_error_handle ParseRetryPolicy(const Json& json, int* max_attempts,
   if (it != json.object_value().end()) {
     if (it->second.type() != Json::Type::ARRAY) {
       error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-          "field:retryableStatusCodes error:should be of type array"));
+          "field:retryableStatusCodes error:must be of type array"));
     } else {
       for (const Json& element : it->second.array_value()) {
         if (element.type() != Json::Type::STRING) {
@@ -242,19 +248,31 @@ grpc_error_handle ParseRetryPolicy(const Json& json, int* max_attempts,
         }
         retryable_status_codes->Add(status);
       }
-      if (retryable_status_codes->Empty()) {
-        error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-            "field:retryableStatusCodes error:should be non-empty"));
-      };
     }
   }
-  // Make sure required fields are set.
-  if (error_list.empty()) {
-    if (*max_attempts == 0 || *initial_backoff == 0 || *max_backoff == 0 ||
-        *backoff_multiplier == 0 || retryable_status_codes->Empty()) {
-      return GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-          "field:retryPolicy error:Missing required field(s)");
+  // Parse perAttemptRecvTimeout.
+  it = json.object_value().find("perAttemptRecvTimeout");
+  if (it != json.object_value().end()) {
+    grpc_millis per_attempt_recv_timeout_value;
+    if (!ParseDurationFromJson(it->second, &per_attempt_recv_timeout_value)) {
+      error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "field:perAttemptRecvTimeout error:type must be STRING of the "
+          "form given by google.proto.Duration."));
+    } else {
+      *per_attempt_recv_timeout = per_attempt_recv_timeout_value;
+      // TODO(roth): As part of implementing hedging, relax this check such
+      // that we allow a value of 0 if a hedging policy is specified.
+      if (per_attempt_recv_timeout_value == 0) {
+        error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "field:perAttemptRecvTimeout error:must be greater than 0"));
+      }
     }
+  } else if (retryable_status_codes->Empty()) {
+    // If perAttemptRecvTimeout not present, retryableStatusCodes must be
+    // non-empty.
+    error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "field:retryableStatusCodes error:must be non-empty if "
+        "perAttemptRecvTimeout not present"));
   }
   return GRPC_ERROR_CREATE_FROM_VECTOR("retryPolicy", &error_list);
 }
@@ -274,13 +292,14 @@ RetryServiceConfigParser::ParsePerMethodParams(
   grpc_millis max_backoff = 0;
   float backoff_multiplier = 0;
   StatusCodeSet retryable_status_codes;
+  absl::optional<grpc_millis> per_attempt_recv_timeout;
   *error = ParseRetryPolicy(it->second, &max_attempts, &initial_backoff,
                             &max_backoff, &backoff_multiplier,
-                            &retryable_status_codes);
+                            &retryable_status_codes, &per_attempt_recv_timeout);
   if (*error != GRPC_ERROR_NONE) return nullptr;
-  return absl::make_unique<RetryMethodConfig>(max_attempts, initial_backoff,
-                                              max_backoff, backoff_multiplier,
-                                              retryable_status_codes);
+  return absl::make_unique<RetryMethodConfig>(
+      max_attempts, initial_backoff, max_backoff, backoff_multiplier,
+      retryable_status_codes, per_attempt_recv_timeout);
 }
 
 }  // namespace internal

data/src/core/ext/filters/client_channel/retry_service_config.h

@@ -47,12 +47,14 @@ class RetryMethodConfig : public ServiceConfigParser::ParsedConfig {
  public:
   RetryMethodConfig(int max_attempts, grpc_millis initial_backoff,
                     grpc_millis max_backoff, float backoff_multiplier,
-                    StatusCodeSet retryable_status_codes)
+                    StatusCodeSet retryable_status_codes,
+                    absl::optional<grpc_millis> per_attempt_recv_timeout)
       : max_attempts_(max_attempts),
         initial_backoff_(initial_backoff),
         max_backoff_(max_backoff),
         backoff_multiplier_(backoff_multiplier),
-        retryable_status_codes_(retryable_status_codes) {}
+        retryable_status_codes_(retryable_status_codes),
+        per_attempt_recv_timeout_(per_attempt_recv_timeout) {}
 
   int max_attempts() const { return max_attempts_; }
   grpc_millis initial_backoff() const { return initial_backoff_; }
@@ -61,6 +63,9 @@ class RetryMethodConfig : public ServiceConfigParser::ParsedConfig {
   StatusCodeSet retryable_status_codes() const {
     return retryable_status_codes_;
   }
+  absl::optional<grpc_millis> per_attempt_recv_timeout() const {
+    return per_attempt_recv_timeout_;
+  }
 
  private:
   int max_attempts_ = 0;
@@ -68,6 +73,7 @@ class RetryMethodConfig : public ServiceConfigParser::ParsedConfig {
   grpc_millis max_backoff_ = 0;
   float backoff_multiplier_ = 0;
   StatusCodeSet retryable_status_codes_;
+  absl::optional<grpc_millis> per_attempt_recv_timeout_;
 };
 
 class RetryServiceConfigParser : public ServiceConfigParser::Parser {

data/src/core/ext/filters/client_idle/client_idle_filter.cc

@@ -187,7 +187,7 @@ void ChannelData::StartTransportOp(grpc_channel_element* elem,
                                    grpc_transport_op* op) {
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   // Catch the disconnect_with_error transport op.
-  if (op->disconnect_with_error != nullptr) {
+  if (op->disconnect_with_error != GRPC_ERROR_NONE) {
     // IncreaseCallCount() introduces a phony call and prevent the timer from
     // being reset by other threads.
     chand->IncreaseCallCount();

data/src/core/ext/filters/fault_injection/fault_injection_filter.cc

@@ -428,6 +428,12 @@ void CallData::DelayBatch(grpc_call_element* elem,
   MutexLock lock(&delay_mu_);
   delayed_batch_ = batch;
   resume_batch_canceller_ = new ResumeBatchCanceller(elem);
+  // Without this line, ExecCtx::Get()->Now() will return a cached timestamp. If
+  // there are thousands of RPCs happen on one thread, we might observe ms-level
+  // error in Now(). This could mean the construction of RPC object is
+  // microseconds earlier than the filter execution. But we still haven't found
+  // the root cause. Read more: https://github.com/grpc/grpc/pull/25738.
+  ExecCtx::Get()->InvalidateNow();
   grpc_millis resume_time = ExecCtx::Get()->Now() + fi_policy_->delay;
   GRPC_CLOSURE_INIT(&batch->handler_private.closure, ResumeBatch, elem,
                     grpc_schedule_on_exec_ctx);

data/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc

@@ -82,7 +82,8 @@ grpc_channel* grpc_insecure_channel_create_from_fd(
 #else  // !GPR_SUPPORT_CHANNELS_FROM_FD
 
 grpc_channel* grpc_insecure_channel_create_from_fd(
-    const char* target, int fd, const grpc_channel_args* args) {
+    const char* /* target */, int /* fd */,
+    const grpc_channel_args* /* args */) {
   GPR_ASSERT(0);
   return nullptr;
 }

data/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc

@@ -68,8 +68,9 @@ void grpc_server_add_insecure_channel_from_fd(grpc_server* server,
 
 #else  // !GPR_SUPPORT_CHANNELS_FROM_FD
 
-void grpc_server_add_insecure_channel_from_fd(grpc_server* server,
-                                              void* reserved, int fd) {
+void grpc_server_add_insecure_channel_from_fd(grpc_server* /* server */,
+                                              void* /* reserved */,
+                                              int /* fd */) {
   GPR_ASSERT(0);
 }
 

data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc

@@ -564,7 +564,7 @@ static void close_transport_locked(grpc_chttp2_transport* t,
                                  GRPC_STATUS_UNAVAILABLE);
     }
     if (t->write_state != GRPC_CHTTP2_WRITE_STATE_IDLE) {
-      if (t->close_transport_on_writes_finished == nullptr) {
+      if (t->close_transport_on_writes_finished == GRPC_ERROR_NONE) {
         t->close_transport_on_writes_finished =
             GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                 "Delayed close due to in-progress write");
@@ -821,9 +821,9 @@ static void set_write_state(grpc_chttp2_transport* t,
   // from peer while we had some pending writes)
   if (st == GRPC_CHTTP2_WRITE_STATE_IDLE) {
     grpc_core::ExecCtx::RunList(DEBUG_LOCATION, &t->run_after_write);
-    if (t->close_transport_on_writes_finished != nullptr) {
+    if (t->close_transport_on_writes_finished != GRPC_ERROR_NONE) {
       grpc_error_handle err = t->close_transport_on_writes_finished;
-      t->close_transport_on_writes_finished = nullptr;
+      t->close_transport_on_writes_finished = GRPC_ERROR_NONE;
       close_transport_locked(t, err);
     }
   }
@@ -1582,6 +1582,8 @@ static void perform_stream_op_locked(void* stream_op,
     GPR_ASSERT(!s->pending_byte_stream);
     s->recv_message_ready = op_payload->recv_message.recv_message_ready;
     s->recv_message = op_payload->recv_message.recv_message;
+    s->call_failed_before_recv_message =
+        op_payload->recv_message.call_failed_before_recv_message;
     if (s->id != 0) {
       if (!s->read_closed) {
         before = s->frame_storage.length +
@@ -1795,7 +1797,7 @@ static void perform_transport_op_locked(void* stream_op,
   grpc_chttp2_transport* t =
       static_cast<grpc_chttp2_transport*>(op->handler_private.extra_arg);
 
-  if (op->goaway_error) {
+  if (op->goaway_error != GRPC_ERROR_NONE) {
     send_goaway(t, op->goaway_error);
   }
 
@@ -1947,6 +1949,10 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* /*t*/,
       null_then_sched_closure(&s->recv_message_ready);
     } else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) {
       *s->recv_message = nullptr;
+      if (s->call_failed_before_recv_message != nullptr) {
+        *s->call_failed_before_recv_message =
+            (s->published_metadata[1] != GRPC_METADATA_PUBLISHED_AT_CLOSE);
+      }
       null_then_sched_closure(&s->recv_message_ready);
     }
     GRPC_ERROR_UNREF(error);

data/src/core/ext/transport/chttp2/transport/internal.h

@@ -556,6 +556,7 @@ struct grpc_chttp2_stream {
   grpc_closure* recv_initial_metadata_ready = nullptr;
   bool* trailing_metadata_available = nullptr;
   grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message;
+  bool* call_failed_before_recv_message = nullptr;
   grpc_closure* recv_message_ready = nullptr;
   grpc_metadata_batch* recv_trailing_metadata;
   grpc_closure* recv_trailing_metadata_finished = nullptr;

data/src/core/ext/transport/chttp2/transport/parsing.cc

@@ -246,10 +246,10 @@ grpc_error_handle grpc_chttp2_perform_read(grpc_chttp2_transport* t,
         t->incoming_frame_size -= static_cast<uint32_t>(end - cur);
         return GRPC_ERROR_NONE;
       }
-      GPR_UNREACHABLE_CODE(return nullptr);
+      GPR_UNREACHABLE_CODE(return GRPC_ERROR_NONE);
   }
 
-  GPR_UNREACHABLE_CODE(return nullptr);
+  GPR_UNREACHABLE_CODE(return GRPC_ERROR_NONE);
 }
 
 static grpc_error_handle init_frame_parser(grpc_chttp2_transport* t) {

data/src/core/ext/transport/inproc/inproc_transport.cc

@@ -388,7 +388,8 @@ void complete_if_batch_end_locked(inproc_stream* s, grpc_error_handle error,
   int is_rtm = static_cast<int>(op == s->recv_trailing_md_op);
 
   if ((is_sm + is_stm + is_rim + is_rm + is_rtm) == 1) {
-    INPROC_LOG(GPR_INFO, "%s %p %p %p", msg, s, op, error);
+    INPROC_LOG(GPR_INFO, "%s %p %p %s", msg, s, op,
+               grpc_error_std_string(error).c_str());
     grpc_core::ExecCtx::Run(DEBUG_LOCATION, op->on_complete,
                             GRPC_ERROR_REF(error));
   }
@@ -468,8 +469,9 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) {
            .trailing_metadata_available = true;
     }
     INPROC_LOG(GPR_INFO,
-               "fail_helper %p scheduling initial-metadata-ready %p %p", s,
-               error, err);
+               "fail_helper %p scheduling initial-metadata-ready %s %s", s,
+               grpc_error_std_string(error).c_str(),
+               grpc_error_std_string(err).c_str());
     grpc_core::ExecCtx::Run(
         DEBUG_LOCATION,
         s->recv_initial_md_op->payload->recv_initial_metadata
@@ -483,8 +485,13 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) {
     s->recv_initial_md_op = nullptr;
   }
   if (s->recv_message_op) {
-    INPROC_LOG(GPR_INFO, "fail_helper %p scheduling message-ready %p", s,
-               error);
+    INPROC_LOG(GPR_INFO, "fail_helper %p scheduling message-ready %s", s,
+               grpc_error_std_string(error).c_str());
+    if (s->recv_message_op->payload->recv_message
+            .call_failed_before_recv_message != nullptr) {
+      *s->recv_message_op->payload->recv_message
+           .call_failed_before_recv_message = true;
+    }
     grpc_core::ExecCtx::Run(
         DEBUG_LOCATION,
         s->recv_message_op->payload->recv_message.recv_message_ready,
@@ -508,15 +515,15 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) {
     s->send_trailing_md_op = nullptr;
   }
   if (s->recv_trailing_md_op) {
-    INPROC_LOG(GPR_INFO, "fail_helper %p scheduling trailing-metadata-ready %p",
-               s, error);
+    INPROC_LOG(GPR_INFO, "fail_helper %p scheduling trailing-metadata-ready %s",
+               s, grpc_error_std_string(error).c_str());
     grpc_core::ExecCtx::Run(
         DEBUG_LOCATION,
         s->recv_trailing_md_op->payload->recv_trailing_metadata
             .recv_trailing_metadata_ready,
         GRPC_ERROR_REF(error));
-    INPROC_LOG(GPR_INFO, "fail_helper %p scheduling trailing-md-on-complete %p",
-               s, error);
+    INPROC_LOG(GPR_INFO, "fail_helper %p scheduling trailing-md-on-complete %s",
+               s, grpc_error_std_string(error).c_str());
     complete_if_batch_end_locked(
         s, error, s->recv_trailing_md_op,
         "fail_helper scheduling recv-trailing-metadata-on-complete");
@@ -685,8 +692,8 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
       INPROC_LOG(
           GPR_INFO,
           "op_state_machine %p scheduling on_complete errors for already "
-          "recvd initial md %p",
-          s, new_err);
+          "recvd initial md %s",
+          s, grpc_error_std_string(new_err).c_str());
       fail_helper_locked(s, GRPC_ERROR_REF(new_err));
       goto done;
     }
@@ -710,8 +717,8 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
       grpc_metadata_batch_clear(&s->to_read_initial_md);
       s->to_read_initial_md_filled = false;
       INPROC_LOG(GPR_INFO,
-                 "op_state_machine %p scheduling initial-metadata-ready %p", s,
-                 new_err);
+                 "op_state_machine %p scheduling initial-metadata-ready %s", s,
+                 grpc_error_std_string(new_err).c_str());
       grpc_core::ExecCtx::Run(
           DEBUG_LOCATION,
           s->recv_initial_md_op->payload->recv_initial_metadata
@@ -724,8 +731,8 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
 
       if (new_err != GRPC_ERROR_NONE) {
         INPROC_LOG(GPR_INFO,
-                   "op_state_machine %p scheduling on_complete errors2 %p", s,
-                   new_err);
+                   "op_state_machine %p scheduling on_complete errors2 %s", s,
+                   grpc_error_std_string(new_err).c_str());
         fail_helper_locked(s, GRPC_ERROR_REF(new_err));
         goto done;
       }
@@ -753,8 +760,8 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
         INPROC_LOG(
             GPR_INFO,
             "op_state_machine %p scheduling on_complete errors for already "
-            "recvd trailing md %p",
-            s, new_err);
+            "recvd trailing md %s",
+            s, grpc_error_std_string(new_err).c_str());
         fail_helper_locked(s, GRPC_ERROR_REF(new_err));
         goto done;
       }
@@ -801,8 +808,8 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
       //     a final status, so don't mark this op complete)
       if (s->t->is_client || s->trailing_md_sent) {
         INPROC_LOG(GPR_INFO,
-                   "op_state_machine %p scheduling trailing-md-on-complete %p",
-                   s, new_err);
+                   "op_state_machine %p scheduling trailing-md-on-complete %s",
+                   s, grpc_error_std_string(new_err).c_str());
         grpc_core::ExecCtx::Run(
             DEBUG_LOCATION,
             s->recv_trailing_md_op->payload->recv_trailing_metadata
@@ -816,8 +823,8 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
       } else {
         INPROC_LOG(GPR_INFO,
                    "op_state_machine %p server needs to delay handling "
-                   "trailing-md-on-complete %p",
-                   s, new_err);
+                   "trailing-md-on-complete %s",
+                   s, grpc_error_std_string(new_err).c_str());
       }
     } else if (!s->trailing_md_recvd) {
       INPROC_LOG(
@@ -830,8 +837,8 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
     // In this case, we don't care to receive the write-close from the client
     // because we have already sent status and the RPC is over as far as we
     // are concerned.
-    INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling trailing-md-ready %p",
-               s, new_err);
+    INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling trailing-md-ready %s",
+               s, grpc_error_std_string(new_err).c_str());
     grpc_core::ExecCtx::Run(
         DEBUG_LOCATION,
         s->recv_trailing_md_op->payload->recv_trailing_metadata
@@ -1091,8 +1098,8 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
         }
         INPROC_LOG(
             GPR_INFO,
-            "perform_stream_op error %p scheduling initial-metadata-ready %p",
-            s, error);
+            "perform_stream_op error %p scheduling initial-metadata-ready %s",
+            s, grpc_error_std_string(error).c_str());
         grpc_core::ExecCtx::Run(
             DEBUG_LOCATION,
             op->payload->recv_initial_metadata.recv_initial_metadata_ready,
@@ -1101,8 +1108,12 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
       if (op->recv_message) {
         INPROC_LOG(
             GPR_INFO,
-            "perform_stream_op error %p scheduling recv message-ready %p", s,
-            error);
+            "perform_stream_op error %p scheduling recv message-ready %s", s,
+            grpc_error_std_string(error).c_str());
+        if (op->payload->recv_message.call_failed_before_recv_message !=
+            nullptr) {
+          *op->payload->recv_message.call_failed_before_recv_message = true;
+        }
         grpc_core::ExecCtx::Run(DEBUG_LOCATION,
                                 op->payload->recv_message.recv_message_ready,
                                 GRPC_ERROR_REF(error));
@@ -1110,16 +1121,16 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
       if (op->recv_trailing_metadata) {
         INPROC_LOG(
             GPR_INFO,
-            "perform_stream_op error %p scheduling trailing-metadata-ready %p",
-            s, error);
+            "perform_stream_op error %p scheduling trailing-metadata-ready %s",
+            s, grpc_error_std_string(error).c_str());
         grpc_core::ExecCtx::Run(
             DEBUG_LOCATION,
             op->payload->recv_trailing_metadata.recv_trailing_metadata_ready,
             GRPC_ERROR_REF(error));
       }
     }
-    INPROC_LOG(GPR_INFO, "perform_stream_op %p scheduling on_complete %p", s,
-               error);
+    INPROC_LOG(GPR_INFO, "perform_stream_op %p scheduling on_complete %s", s,
+               grpc_error_std_string(error).c_str());
     grpc_core::ExecCtx::Run(DEBUG_LOCATION, on_complete, GRPC_ERROR_REF(error));
   }
   gpr_mu_unlock(mu);