grpc 1.66.0.pre5 → 1.67.0.pre1

data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h

@@ -24,6 +24,7 @@
 #include <utility>
 
 #include "absl/status/statusor.h"
+#include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
 #include <grpc/credentials.h>
@@ -43,6 +44,7 @@
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/arena_promise.h"
 #include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/credentials/token_fetcher/token_fetcher_credentials.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/transport/transport.h"
 #include "src/core/lib/uri/uri_parser.h"
@@ -80,73 +82,45 @@ grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
 /// Destructs the object.
 void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token* refresh_token);
 
-// -- Credentials Metadata Request. --
-
-struct grpc_credentials_metadata_request {
-  explicit grpc_credentials_metadata_request(
-      grpc_core::RefCountedPtr<grpc_call_credentials> creds)
-      : creds(std::move(creds)) {}
-  ~grpc_credentials_metadata_request() {
-    grpc_http_response_destroy(&response);
-  }
-
-  grpc_core::RefCountedPtr<grpc_call_credentials> creds;
-  grpc_http_response response;
-};
-
-struct grpc_oauth2_pending_get_request_metadata
-    : public grpc_core::RefCounted<grpc_oauth2_pending_get_request_metadata> {
-  std::atomic<bool> done{false};
-  grpc_core::Waker waker;
-  grpc_polling_entity* pollent;
-  grpc_core::ClientMetadataHandle md;
-  struct grpc_oauth2_pending_get_request_metadata* next;
-  absl::StatusOr<grpc_core::Slice> result;
-};
-
 // -- Oauth2 Token Fetcher credentials --
 //
 //  This object is a base for credentials that need to acquire an oauth2 token
 //  from an http service.
 
-class grpc_oauth2_token_fetcher_credentials : public grpc_call_credentials {
- public:
-  grpc_oauth2_token_fetcher_credentials();
-  ~grpc_oauth2_token_fetcher_credentials() override;
-
-  grpc_core::ArenaPromise<absl::StatusOr<grpc_core::ClientMetadataHandle>>
-  GetRequestMetadata(grpc_core::ClientMetadataHandle initial_metadata,
-                     const GetRequestMetadataArgs* args) override;
+namespace grpc_core {
 
-  void on_http_response(grpc_credentials_metadata_request* r,
-                        grpc_error_handle error);
+// A base class for oauth2 token fetching credentials.
+// Subclasses must implement StartHttpRequest().
+class Oauth2TokenFetcherCredentials : public TokenFetcherCredentials {
+ public:
   std::string debug_string() override;
 
-  grpc_core::UniqueTypeName type() const override;
+  UniqueTypeName type() const override;
+
+  OrphanablePtr<FetchRequest> FetchToken(
+      Timestamp deadline,
+      absl::AnyInvocable<
+          void(absl::StatusOr<RefCountedPtr<TokenFetcherCredentials::Token>>)>
+          on_done) final;
 
- protected:
-  virtual void fetch_oauth2(grpc_credentials_metadata_request* req,
-                            grpc_polling_entity* pollent, grpc_iomgr_cb_func cb,
-                            grpc_core::Timestamp deadline) = 0;
+  virtual OrphanablePtr<HttpRequest> StartHttpRequest(
+      grpc_polling_entity* pollent, Timestamp deadline,
+      grpc_http_response* response, grpc_closure* on_complete) = 0;
 
  private:
+  class HttpFetchRequest;
+
   int cmp_impl(const grpc_call_credentials* other) const override {
     // TODO(yashykt): Check if we can do something better here
-    return grpc_core::QsortCompare(
-        static_cast<const grpc_call_credentials*>(this), other);
+    return QsortCompare(static_cast<const grpc_call_credentials*>(this), other);
   }
-
-  gpr_mu mu_;
-  absl::optional<grpc_core::Slice> access_token_value_;
-  gpr_timespec token_expiration_;
-  bool token_fetch_pending_ = false;
-  grpc_oauth2_pending_get_request_metadata* pending_requests_ = nullptr;
-  grpc_polling_entity pollent_;
 };
 
+}  // namespace grpc_core
+
 // Google refresh token credentials.
 class grpc_google_refresh_token_credentials final
-    : public grpc_oauth2_token_fetcher_credentials {
+    : public grpc_core::Oauth2TokenFetcherCredentials {
  public:
   explicit grpc_google_refresh_token_credentials(
       grpc_auth_refresh_token refresh_token);
@@ -160,15 +134,12 @@ class grpc_google_refresh_token_credentials final
 
   grpc_core::UniqueTypeName type() const override;
 
- protected:
-  void fetch_oauth2(grpc_credentials_metadata_request* req,
-                    grpc_polling_entity* pollent, grpc_iomgr_cb_func cb,
-                    grpc_core::Timestamp deadline) override;
-
  private:
+  grpc_core::OrphanablePtr<grpc_core::HttpRequest> StartHttpRequest(
+      grpc_polling_entity* pollent, grpc_core::Timestamp deadline,
+      grpc_http_response* response, grpc_closure* on_complete) override;
+
   grpc_auth_refresh_token refresh_token_;
-  grpc_closure http_post_cb_closure_;
-  grpc_core::OrphanablePtr<grpc_core::HttpRequest> http_request_;
 };
 
 // Access token credentials.
@@ -176,6 +147,8 @@ class grpc_access_token_credentials final : public grpc_call_credentials {
  public:
   explicit grpc_access_token_credentials(const char* access_token);
 
+  void Orphaned() override {}
+
   grpc_core::ArenaPromise<absl::StatusOr<grpc_core::ClientMetadataHandle>>
   GetRequestMetadata(grpc_core::ClientMetadataHandle initial_metadata,
                      const GetRequestMetadataArgs* args) override;
@@ -202,6 +175,11 @@ grpc_core::RefCountedPtr<grpc_call_credentials>
 grpc_refresh_token_credentials_create_from_auth_refresh_token(
     grpc_auth_refresh_token token);
 
+grpc_credentials_status
+grpc_oauth2_token_fetcher_credentials_parse_server_response_body(
+    absl::string_view body, absl::optional<grpc_core::Slice>* token_value,
+    grpc_core::Duration* token_lifetime);
+
 // Exposed for testing only.
 grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(

data/src/core/lib/security/credentials/plugin/plugin_credentials.cc

@@ -121,10 +121,9 @@ void grpc_plugin_credentials::PendingRequest::RequestMetadataReady(
                               GRPC_EXEC_CTX_FLAG_THREAD_RESOURCE_LOOP);
   grpc_core::RefCountedPtr<grpc_plugin_credentials::PendingRequest> r(
       static_cast<grpc_plugin_credentials::PendingRequest*>(request));
-  if (GRPC_TRACE_FLAG_ENABLED(plugin_credentials)) {
-    LOG(INFO) << "plugin_credentials[" << r->creds() << "]: request " << r.get()
-              << ": plugin returned asynchronously";
-  }
+  GRPC_TRACE_LOG(plugin_credentials, INFO)
+      << "plugin_credentials[" << r->creds() << "]: request " << r.get()
+      << ": plugin returned asynchronously";
   for (size_t i = 0; i < num_md; ++i) {
     grpc_metadata p;
     p.key = grpc_core::CSliceRef(md[i].key);
@@ -150,10 +149,9 @@ grpc_plugin_credentials::GetRequestMetadata(
       RefAsSubclass<grpc_plugin_credentials>(), std::move(initial_metadata),
       args);
   // Invoke the plugin.  The callback holds a ref to us.
-  if (GRPC_TRACE_FLAG_ENABLED(plugin_credentials)) {
-    LOG(INFO) << "plugin_credentials[" << this << "]: request " << request.get()
-              << ": invoking plugin";
-  }
+  GRPC_TRACE_LOG(plugin_credentials, INFO)
+      << "plugin_credentials[" << this << "]: request " << request.get()
+      << ": invoking plugin";
   grpc_metadata creds_md[GRPC_METADATA_CREDENTIALS_PLUGIN_SYNC_MAX];
   size_t num_creds_md = 0;
   grpc_status_code status = GRPC_STATUS_OK;
@@ -168,17 +166,15 @@ grpc_plugin_credentials::GetRequestMetadata(
                             child_request.get(), creds_md, &num_creds_md,
                             &status, &error_details)) {
     child_request.release();
-    if (GRPC_TRACE_FLAG_ENABLED(plugin_credentials)) {
-      LOG(INFO) << "plugin_credentials[" << this << "]: request "
-                << request.get() << ": plugin will return asynchronously";
-    }
+    GRPC_TRACE_LOG(plugin_credentials, INFO)
+        << "plugin_credentials[" << this << "]: request " << request.get()
+        << ": plugin will return asynchronously";
     return [request] { return request->PollAsyncResult(); };
   }
   // Synchronous return.
-  if (GRPC_TRACE_FLAG_ENABLED(plugin_credentials)) {
-    LOG(INFO) << "plugin_credentials[" << this << "]: request " << request.get()
-              << ": plugin returned synchronously";
-  }
+  GRPC_TRACE_LOG(plugin_credentials, INFO)
+      << "plugin_credentials[" << this << "]: request " << request.get()
+      << ": plugin returned synchronously";
   auto result = request->ProcessPluginResult(creds_md, num_creds_md, status,
                                              error_details);
   // Clean up.

data/src/core/lib/security/credentials/plugin/plugin_credentials.h

@@ -57,6 +57,8 @@ struct grpc_plugin_credentials final : public grpc_call_credentials {
                                    grpc_security_level min_security_level);
   ~grpc_plugin_credentials() override;
 
+  void Orphaned() override {}
+
   grpc_core::ArenaPromise<absl::StatusOr<grpc_core::ClientMetadataHandle>>
   GetRequestMetadata(grpc_core::ClientMetadataHandle initial_metadata,
                      const GetRequestMetadataArgs* args) override;

data/src/core/lib/security/credentials/ssl/ssl_credentials.h

@@ -26,7 +26,6 @@
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
 #include <grpc/grpc_security_constants.h>
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/channel/channel_args.h"

data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc

@@ -21,7 +21,6 @@
 
 #include <grpc/credentials.h>
 #include <grpc/grpc_security.h>
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 void grpc_tls_certificate_distributor::SetKeyMaterials(

data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h

@@ -29,7 +29,6 @@
 #include "absl/types/optional.h"
 
 #include <grpc/grpc_security.h>
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>
 

data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc

@@ -25,7 +25,6 @@
 #include "absl/strings/string_view.h"
 
 #include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/string_util.h>
 

data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h

@@ -27,7 +27,6 @@
 #include <grpc/credentials.h>
 #include <grpc/grpc_security.h>
 #include <grpc/status.h>
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/gprpp/ref_counted.h"

data/src/core/lib/security/credentials/token_fetcher/token_fetcher_credentials.cc

@@ -0,0 +1,298 @@
+//
+//
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+//
+
+#include "src/core/lib/security/credentials/token_fetcher/token_fetcher_credentials.h"
+
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/event_engine/default_event_engine.h"
+#include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/promise/context.h"
+#include "src/core/lib/promise/poll.h"
+#include "src/core/lib/promise/promise.h"
+
+namespace grpc_core {
+
+namespace {
+
+// Amount of time before the token's expiration that we consider it
+// invalid to account for server processing time and clock skew.
+constexpr Duration kTokenExpirationAdjustmentDuration = Duration::Seconds(30);
+
+// Amount of time before the token's expiration that we pre-fetch a new
+// token.  Also determines the timeout for the fetch request.
+constexpr Duration kTokenRefreshDuration = Duration::Seconds(60);
+
+}  // namespace
+
+//
+// TokenFetcherCredentials::Token
+//
+
+TokenFetcherCredentials::Token::Token(Slice token, Timestamp expiration)
+    : token_(std::move(token)),
+      expiration_(expiration - kTokenExpirationAdjustmentDuration) {}
+
+void TokenFetcherCredentials::Token::AddTokenToClientInitialMetadata(
+    ClientMetadata& metadata) const {
+  metadata.Append(GRPC_AUTHORIZATION_METADATA_KEY, token_.Ref(),
+                  [](absl::string_view, const Slice&) { abort(); });
+}
+
+//
+// TokenFetcherCredentials::FetchState::BackoffTimer
+//
+
+TokenFetcherCredentials::FetchState::BackoffTimer::BackoffTimer(
+    RefCountedPtr<FetchState> fetch_state)
+    : fetch_state_(std::move(fetch_state)) {
+  const Duration delay = fetch_state_->backoff_.NextAttemptDelay();
+  GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+      << "[TokenFetcherCredentials " << fetch_state_->creds_.get()
+      << "]: fetch_state=" << fetch_state_.get() << " backoff_timer=" << this
+      << ": starting backoff timer for " << delay;
+  timer_handle_ = fetch_state_->creds_->event_engine().RunAfter(
+      delay, [self = Ref()]() mutable {
+        ApplicationCallbackExecCtx callback_exec_ctx;
+        ExecCtx exec_ctx;
+        self->OnTimer();
+        self.reset();
+      });
+}
+
+void TokenFetcherCredentials::FetchState::BackoffTimer::Orphan() {
+  GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+      << "[TokenFetcherCredentials " << fetch_state_->creds_.get()
+      << "]: fetch_state=" << fetch_state_.get() << " backoff_timer=" << this
+      << ": backoff timer shut down";
+  if (timer_handle_.has_value()) {
+    GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+        << "[TokenFetcherCredentials " << fetch_state_->creds_.get()
+        << "]: fetch_state=" << fetch_state_.get() << " backoff_timer=" << this
+        << ": cancelling timer";
+    fetch_state_->creds_->event_engine().Cancel(*timer_handle_);
+    timer_handle_.reset();
+    fetch_state_->ResumeQueuedCalls(
+        absl::CancelledError("credentials shutdown"));
+  }
+  Unref();
+}
+
+void TokenFetcherCredentials::FetchState::BackoffTimer::OnTimer() {
+  MutexLock lock(&fetch_state_->creds_->mu_);
+  if (!timer_handle_.has_value()) return;
+  timer_handle_.reset();
+  GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+      << "[TokenFetcherCredentials " << fetch_state_->creds_.get()
+      << "]: fetch_state=" << fetch_state_.get() << " backoff_timer=" << this
+      << ": backoff timer fired";
+  if (fetch_state_->queued_calls_.empty()) {
+    // If there are no pending calls when the timer fires, then orphan
+    // the FetchState object.  Note that this drops the backoff state,
+    // but that's probably okay, because if we didn't have any pending
+    // calls during the backoff period, we probably won't see any
+    // immediately now either.
+    GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+        << "[TokenFetcherCredentials " << fetch_state_->creds_.get()
+        << "]: fetch_state=" << fetch_state_.get() << " backoff_timer=" << this
+        << ": no pending calls, clearing state";
+    fetch_state_->creds_->fetch_state_.reset();
+  } else {
+    // If there are pending calls, then start a new fetch attempt.
+    GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+        << "[TokenFetcherCredentials " << fetch_state_->creds_.get()
+        << "]: fetch_state=" << fetch_state_.get() << " backoff_timer=" << this
+        << ": starting new fetch attempt";
+    fetch_state_->StartFetchAttempt();
+  }
+}
+
+//
+// TokenFetcherCredentials::FetchState
+//
+
+TokenFetcherCredentials::FetchState::FetchState(
+    WeakRefCountedPtr<TokenFetcherCredentials> creds)
+    : creds_(std::move(creds)),
+      backoff_(BackOff::Options()
+                   .set_initial_backoff(Duration::Seconds(1))
+                   .set_multiplier(1.6)
+                   .set_jitter(creds_->test_only_use_backoff_jitter_ ? 0.2 : 0)
+                   .set_max_backoff(Duration::Seconds(120))) {
+  StartFetchAttempt();
+}
+
+void TokenFetcherCredentials::FetchState::Orphan() {
+  GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+      << "[TokenFetcherCredentials " << creds_.get()
+      << "]: fetch_state=" << this << ": shutting down";
+  // Cancels fetch or backoff timer, if any.
+  state_ = Shutdown{};
+  Unref();
+}
+
+void TokenFetcherCredentials::FetchState::StartFetchAttempt() {
+  GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+      << "[TokenFetcherCredentials " << creds_.get()
+      << "]: fetch_state=" << this << ": starting fetch";
+  state_ = creds_->FetchToken(
+      /*deadline=*/Timestamp::Now() + kTokenRefreshDuration,
+      [self = Ref()](absl::StatusOr<RefCountedPtr<Token>> token) mutable {
+        self->TokenFetchComplete(std::move(token));
+      });
+}
+
+void TokenFetcherCredentials::FetchState::TokenFetchComplete(
+    absl::StatusOr<RefCountedPtr<Token>> token) {
+  MutexLock lock(&creds_->mu_);
+  // If we were shut down, clean up.
+  if (absl::holds_alternative<Shutdown>(state_)) {
+    if (token.ok()) token = absl::CancelledError("credentials shutdown");
+    GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+        << "[TokenFetcherCredentials " << creds_.get()
+        << "]: fetch_state=" << this
+        << ": shut down before fetch completed: " << token.status();
+    ResumeQueuedCalls(std::move(token));
+    return;
+  }
+  // If succeeded, update cache in creds object.
+  if (token.ok()) {
+    GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+        << "[TokenFetcherCredentials " << creds_.get()
+        << "]: fetch_state=" << this << ": token fetch succeeded";
+    creds_->token_ = *token;
+    creds_->fetch_state_.reset();  // Orphan ourselves.
+  } else {
+    GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+        << "[TokenFetcherCredentials " << creds_.get()
+        << "]: fetch_state=" << this
+        << ": token fetch failed: " << token.status();
+    // If failed, start backoff timer.
+    state_ = OrphanablePtr<BackoffTimer>(new BackoffTimer(Ref()));
+  }
+  ResumeQueuedCalls(std::move(token));
+}
+
+void TokenFetcherCredentials::FetchState::ResumeQueuedCalls(
+    absl::StatusOr<RefCountedPtr<Token>> token) {
+  // Invoke callbacks for all pending requests.
+  for (auto& queued_call : queued_calls_) {
+    queued_call->result = token;
+    queued_call->done.store(true, std::memory_order_release);
+    queued_call->waker.Wakeup();
+    grpc_polling_entity_del_from_pollset_set(
+        queued_call->pollent,
+        grpc_polling_entity_pollset_set(&creds_->pollent_));
+  }
+  queued_calls_.clear();
+}
+
+RefCountedPtr<TokenFetcherCredentials::QueuedCall>
+TokenFetcherCredentials::FetchState::QueueCall(
+    ClientMetadataHandle initial_metadata) {
+  // Add call to pending list.
+  auto queued_call = MakeRefCounted<QueuedCall>();
+  queued_call->waker = GetContext<Activity>()->MakeNonOwningWaker();
+  queued_call->pollent = GetContext<grpc_polling_entity>();
+  grpc_polling_entity_add_to_pollset_set(
+      queued_call->pollent, grpc_polling_entity_pollset_set(&creds_->pollent_));
+  queued_call->md = std::move(initial_metadata);
+  queued_calls_.insert(queued_call);
+  return queued_call;
+}
+
+//
+// TokenFetcherCredentials
+//
+
+TokenFetcherCredentials::TokenFetcherCredentials(
+    std::shared_ptr<grpc_event_engine::experimental::EventEngine> event_engine,
+    bool test_only_use_backoff_jitter)
+    : event_engine_(
+          event_engine == nullptr
+              ? grpc_event_engine::experimental::GetDefaultEventEngine()
+              : std::move(event_engine)),
+      test_only_use_backoff_jitter_(test_only_use_backoff_jitter),
+      pollent_(grpc_polling_entity_create_from_pollset_set(
+          grpc_pollset_set_create())) {}
+
+TokenFetcherCredentials::~TokenFetcherCredentials() {
+  grpc_pollset_set_destroy(grpc_polling_entity_pollset_set(&pollent_));
+}
+
+void TokenFetcherCredentials::Orphaned() {
+  MutexLock lock(&mu_);
+  fetch_state_.reset();
+}
+
+ArenaPromise<absl::StatusOr<ClientMetadataHandle>>
+TokenFetcherCredentials::GetRequestMetadata(
+    ClientMetadataHandle initial_metadata, const GetRequestMetadataArgs*) {
+  RefCountedPtr<QueuedCall> queued_call;
+  {
+    MutexLock lock(&mu_);
+    // If we don't have a cached token or the token is within the
+    // refresh duration, start a new fetch if there isn't a pending one.
+    if ((token_ == nullptr || (token_->ExpirationTime() - Timestamp::Now()) <=
+                                  kTokenRefreshDuration) &&
+        fetch_state_ == nullptr) {
+      GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+          << "[TokenFetcherCredentials " << this
+          << "]: " << GetContext<Activity>()->DebugTag()
+          << " triggering new token fetch";
+      fetch_state_ = OrphanablePtr<FetchState>(
+          new FetchState(WeakRefAsSubclass<TokenFetcherCredentials>()));
+    }
+    // If we have a cached non-expired token, use it.
+    if (token_ != nullptr &&
+        (token_->ExpirationTime() - Timestamp::Now()) > Duration::Zero()) {
+      GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+          << "[TokenFetcherCredentials " << this
+          << "]: " << GetContext<Activity>()->DebugTag()
+          << " using cached token";
+      token_->AddTokenToClientInitialMetadata(*initial_metadata);
+      return Immediate(std::move(initial_metadata));
+    }
+    // If we don't have a cached token, this call will need to be queued.
+    GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+        << "[TokenFetcherCredentials " << this
+        << "]: " << GetContext<Activity>()->DebugTag()
+        << " no cached token; queuing call";
+    queued_call = fetch_state_->QueueCall(std::move(initial_metadata));
+  }
+  return [this, queued_call = std::move(queued_call)]()
+             -> Poll<absl::StatusOr<ClientMetadataHandle>> {
+    if (!queued_call->done.load(std::memory_order_acquire)) {
+      return Pending{};
+    }
+    if (!queued_call->result.ok()) {
+      GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+          << "[TokenFetcherCredentials " << this
+          << "]: " << GetContext<Activity>()->DebugTag()
+          << " token fetch failed; failing call";
+      return queued_call->result.status();
+    }
+    GRPC_TRACE_LOG(token_fetcher_credentials, INFO)
+        << "[TokenFetcherCredentials " << this
+        << "]: " << GetContext<Activity>()->DebugTag()
+        << " token fetch complete; resuming call";
+    (*queued_call->result)->AddTokenToClientInitialMetadata(*queued_call->md);
+    return std::move(queued_call->md);
+  };
+}
+
+}  // namespace grpc_core