grpc 1.22.0 → 1.23.0

data/src/core/lib/gprpp/string_view.h

@@ -0,0 +1,143 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#ifndef GRPC_CORE_LIB_GPRPP_STRING_VIEW_H
+#define GRPC_CORE_LIB_GPRPP_STRING_VIEW_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/impl/codegen/slice.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <cstring>
+#include <limits>
+
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/memory.h"
+
+namespace grpc_core {
+
+// Provides a light-weight view over a char array or a slice, similar but not
+// identical to absl::string_view.
+//
+// Any method that has the same name as absl::string_view MUST HAVE identical
+// semantics to what absl::string_view provides.
+//
+// Methods that are not part of absl::string_view API, must be clearly
+// annotated.
+//
+// StringView does not own the buffers that back the view. Callers must ensure
+// the buffer stays around while the StringView is accessible.
+//
+// Pass StringView by value in functions, since it is exactly two pointers in
+// size.
+//
+// The interface used here is not identical to absl::string_view. Notably, we
+// need to support slices while we cannot support std::string, and gpr string
+// style functions such as strdup() and cmp(). Once we switch to
+// absl::string_view this class will inherit from absl::string_view and add the
+// gRPC-specific APIs.
+class StringView final {
+ public:
+  static constexpr size_t npos = std::numeric_limits<size_t>::max();
+
+  constexpr StringView(const char* ptr, size_t size) : ptr_(ptr), size_(size) {}
+  constexpr StringView(const char* ptr)
+      : StringView(ptr, ptr == nullptr ? 0 : strlen(ptr)) {}
+  // Not part of absl::string_view API.
+  StringView(const grpc_slice& slice)
+      : StringView(reinterpret_cast<const char*>(GRPC_SLICE_START_PTR(slice)),
+                   GRPC_SLICE_LENGTH(slice)) {}
+  constexpr StringView() : StringView(nullptr, 0) {}
+
+  constexpr const char* data() const { return ptr_; }
+  constexpr size_t size() const { return size_; }
+  constexpr bool empty() const { return size_ == 0; }
+
+  StringView substr(size_t start, size_t size = npos) {
+    GPR_DEBUG_ASSERT(start + size <= size_);
+    return StringView(ptr_ + start, std::min(size, size_ - start));
+  }
+
+  constexpr const char& operator[](size_t i) const { return ptr_[i]; }
+
+  const char& front() const { return ptr_[0]; }
+  const char& back() const { return ptr_[size_ - 1]; }
+
+  void remove_prefix(size_t n) {
+    GPR_DEBUG_ASSERT(n <= size_);
+    ptr_ += n;
+    size_ -= n;
+  }
+
+  void remove_suffix(size_t n) {
+    GPR_DEBUG_ASSERT(n <= size_);
+    size_ -= n;
+  }
+
+  size_t find(char c, size_t pos = 0) const {
+    if (empty() || pos >= size_) return npos;
+    const char* result =
+        static_cast<const char*>(memchr(ptr_ + pos, c, size_ - pos));
+    return result != nullptr ? result - ptr_ : npos;
+  }
+
+  void clear() {
+    ptr_ = nullptr;
+    size_ = 0;
+  }
+
+  // Creates a dup of the string viewed by this class.
+  // Return value is null-terminated and never nullptr.
+  //
+  // Not part of absl::string_view API.
+  grpc_core::UniquePtr<char> dup() const {
+    char* str = static_cast<char*>(gpr_malloc(size_ + 1));
+    if (size_ > 0) memcpy(str, ptr_, size_);
+    str[size_] = '\0';
+    return grpc_core::UniquePtr<char>(str);
+  }
+
+  // Not part of absl::string_view API.
+  int cmp(StringView other) const {
+    const size_t len = GPR_MIN(size(), other.size());
+    const int ret = strncmp(data(), other.data(), len);
+    if (ret != 0) return ret;
+    if (size() == other.size()) return 0;
+    if (size() < other.size()) return -1;
+    return 1;
+  }
+
+ private:
+  const char* ptr_;
+  size_t size_;
+};
+
+inline bool operator==(StringView lhs, StringView rhs) {
+  return lhs.size() == rhs.size() &&
+         strncmp(lhs.data(), rhs.data(), lhs.size()) == 0;
+}
+
+inline bool operator!=(StringView lhs, StringView rhs) { return !(lhs == rhs); }
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_LIB_GPRPP_STRING_VIEW_H */

data/src/core/lib/gprpp/thd.h

@@ -49,7 +49,7 @@ class Thread {
  public:
   class Options {
    public:
-    Options() : joinable_(true), tracked_(true) {}
+    Options() : joinable_(true), tracked_(true), stack_size_(0) {}
     /// Set whether the thread is joinable or detached.
     Options& set_joinable(bool joinable) {
       joinable_ = joinable;
@@ -64,9 +64,18 @@ class Thread {
     }
     bool tracked() const { return tracked_; }
 
+    /// Sets thread stack size (in bytes). Sets to 0 will use the default stack
+    /// size which is 64KB for Windows threads and 2MB for Posix(x86) threads.
+    Options& set_stack_size(size_t bytes) {
+      stack_size_ = bytes;
+      return *this;
+    }
+    size_t stack_size() const { return stack_size_; }
+
    private:
     bool joinable_;
     bool tracked_;
+    size_t stack_size_;
   };
   /// Default constructor only to allow use in structs that lack constructors
   /// Does not produce a validly-constructed thread; must later

data/src/core/lib/gprpp/thd_posix.cc

@@ -31,6 +31,7 @@
 #include <pthread.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/fork.h"
@@ -48,6 +49,25 @@ struct thd_arg {
   bool tracked;
 };
 
+size_t RoundUpToPageSize(size_t size) {
+  // TODO(yunjiaw): Change this variable (page_size) to a function-level static
+  // when possible
+  size_t page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
+  return (size + page_size - 1) & ~(page_size - 1);
+}
+
+// Returns the minimum valid stack size that can be passed to
+// pthread_attr_setstacksize.
+size_t MinValidStackSize(size_t request_size) {
+  if (request_size < _SC_THREAD_STACK_MIN) {
+    request_size = _SC_THREAD_STACK_MIN;
+  }
+
+  // On some systems, pthread_attr_setstacksize() can fail if stacksize is
+  // not a multiple of the system page size.
+  return RoundUpToPageSize(request_size);
+}
+
 class ThreadInternalsPosix : public internal::ThreadInternalsInterface {
  public:
   ThreadInternalsPosix(const char* thd_name, void (*thd_body)(void* arg),
@@ -79,6 +99,11 @@ class ThreadInternalsPosix : public internal::ThreadInternalsInterface {
                  0);
     }
 
+    if (options.stack_size() != 0) {
+      size_t stack_size = MinValidStackSize(options.stack_size());
+      GPR_ASSERT(pthread_attr_setstacksize(&attr, stack_size) == 0);
+    }
+
     *success =
         (pthread_create(&pthread_id_, &attr,
                         [](void* v) -> void* {

data/src/core/lib/gprpp/thd_windows.cc

@@ -75,7 +75,15 @@ class ThreadInternalsWindows
         return;
       }
     }
-    handle = CreateThread(nullptr, 64 * 1024, thread_body, info_, 0, nullptr);
+
+    if (options.stack_size() != 0) {
+      // Windows will round up the given stack_size value to nearest page.
+      handle = CreateThread(nullptr, options.stack_size(), thread_body, info_,
+                            0, nullptr);
+    } else {
+      handle = CreateThread(nullptr, 64 * 1024, thread_body, info_, 0, nullptr);
+    }
+
     if (handle == nullptr) {
       destroy_thread();
       *success = false;

data/src/core/lib/http/httpcli_security_connector.cc

@@ -30,6 +30,7 @@
 #include "src/core/lib/channel/handshaker_registry.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/string_view.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/security/security_connector/ssl_utils.h"
@@ -108,7 +109,8 @@ class grpc_httpcli_ssl_channel_security_connector final
     return strcmp(secure_peer_name_, other->secure_peer_name_);
   }
 
-  bool check_call_host(const char* host, grpc_auth_context* auth_context,
+  bool check_call_host(grpc_core::StringView host,
+                       grpc_auth_context* auth_context,
                        grpc_closure* on_call_host_checked,
                        grpc_error** error) override {
     *error = GRPC_ERROR_NONE;

data/src/core/lib/iomgr/cfstream_handle.cc

@@ -25,6 +25,7 @@
 #import <CoreFoundation/CoreFoundation.h>
 #import "src/core/lib/iomgr/cfstream_handle.h"
 
+#include <grpc/grpc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/sync.h>
 
@@ -35,6 +36,10 @@
 
 extern grpc_core::TraceFlag grpc_tcp_trace;
 
+GrpcLibraryInitHolder::GrpcLibraryInitHolder() { grpc_init(); }
+
+GrpcLibraryInitHolder::~GrpcLibraryInitHolder() { grpc_shutdown(); }
+
 void* CFStreamHandle::Retain(void* info) {
   CFStreamHandle* handle = static_cast<CFStreamHandle*>(info);
   CFSTREAM_HANDLE_REF(handle, "retain");
@@ -184,7 +189,7 @@ void CFStreamHandle::Ref(const char* file, int line, const char* reason) {
 void CFStreamHandle::Unref(const char* file, int line, const char* reason) {
   if (grpc_tcp_trace.enabled()) {
     gpr_atm val = gpr_atm_no_barrier_load(&refcount_.count);
-    gpr_log(GPR_ERROR,
+    gpr_log(GPR_DEBUG,
             "CFStream Handle unref %p : %s %" PRIdPTR " -> %" PRIdPTR, this,
             reason, val, val - 1);
   }

data/src/core/lib/iomgr/cfstream_handle.h

@@ -33,11 +33,17 @@
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/lockfree_event.h"
 
-class CFStreamHandle final {
+class GrpcLibraryInitHolder {
+ public:
+  GrpcLibraryInitHolder();
+  virtual ~GrpcLibraryInitHolder();
+};
+
+class CFStreamHandle : public GrpcLibraryInitHolder {
  public:
   static CFStreamHandle* CreateStreamHandle(CFReadStreamRef read_stream,
                                             CFWriteStreamRef write_stream);
-  ~CFStreamHandle();
+  ~CFStreamHandle() override;
   CFStreamHandle(const CFStreamHandle& ref) = delete;
   CFStreamHandle(CFStreamHandle&& ref) = delete;
   CFStreamHandle& operator=(const CFStreamHandle& rhs) = delete;

data/src/core/lib/iomgr/combiner.cc

@@ -233,11 +233,11 @@ bool grpc_combiner_continue_exec_ctx() {
   // offload only if all the following conditions are true:
   // 1. the combiner is contended and has more than one closure to execute
   // 2. the current execution context needs to finish as soon as possible
-  // 3. the DEFAULT executor is threaded
-  // 4. the current thread is not a worker for any background poller
+  // 3. the current thread is not a worker for any background poller
+  // 4. the DEFAULT executor is threaded
   if (contended && grpc_core::ExecCtx::Get()->IsReadyToFinish() &&
-      grpc_core::Executor::IsThreadedDefault() &&
-      !grpc_iomgr_is_any_background_poller_thread()) {
+      !grpc_iomgr_is_any_background_poller_thread() &&
+      grpc_core::Executor::IsThreadedDefault()) {
     GPR_TIMER_MARK("offload_from_finished_exec_ctx", 0);
     // this execution context wants to move on: schedule remaining work to be
     // picked up on the executor

data/src/core/lib/iomgr/error.cc

@@ -73,6 +73,8 @@ static const char* error_int_name(grpc_error_ints key) {
       return "limit";
     case GRPC_ERROR_INT_OCCURRED_DURING_WRITE:
       return "occurred_during_write";
+    case GRPC_ERROR_INT_CHANNEL_CONNECTIVITY_STATE:
+      return "channel_connectivity_state";
     case GRPC_ERROR_INT_MAX:
       GPR_UNREACHABLE_CODE(return "unknown");
   }
@@ -447,13 +449,17 @@ grpc_error* grpc_error_set_int(grpc_error* src, grpc_error_ints which,
 typedef struct {
   grpc_status_code code;
   const char* msg;
+  size_t len;
 } special_error_status_map;
-static const special_error_status_map error_status_map[] = {
-    {GRPC_STATUS_OK, ""},                               // GRPC_ERROR_NONE
-    {GRPC_STATUS_INVALID_ARGUMENT, ""},                 // GRPC_ERROR_RESERVED_1
-    {GRPC_STATUS_RESOURCE_EXHAUSTED, "Out of memory"},  // GRPC_ERROR_OOM
-    {GRPC_STATUS_INVALID_ARGUMENT, ""},                 // GRPC_ERROR_RESERVED_2
-    {GRPC_STATUS_CANCELLED, "Cancelled"},               // GRPC_ERROR_CANCELLED
+
+const special_error_status_map error_status_map[] = {
+    {GRPC_STATUS_OK, "", 0},                // GRPC_ERROR_NONE
+    {GRPC_STATUS_INVALID_ARGUMENT, "", 0},  // GRPC_ERROR_RESERVED_1
+    {GRPC_STATUS_RESOURCE_EXHAUSTED, "Out of memory",
+     strlen("Out of memory")},              // GRPC_ERROR_OOM
+    {GRPC_STATUS_INVALID_ARGUMENT, "", 0},  // GRPC_ERROR_RESERVED_2
+    {GRPC_STATUS_CANCELLED, "Cancelled",
+     strlen("Cancelled")},  // GRPC_ERROR_CANCELLED
 };
 
 bool grpc_error_get_int(grpc_error* err, grpc_error_ints which, intptr_t* p) {
@@ -483,8 +489,12 @@ bool grpc_error_get_str(grpc_error* err, grpc_error_strs which,
                         grpc_slice* str) {
   if (grpc_error_is_special(err)) {
     if (which != GRPC_ERROR_STR_GRPC_MESSAGE) return false;
-    *str = grpc_slice_from_static_string(
-        error_status_map[reinterpret_cast<size_t>(err)].msg);
+    const special_error_status_map& msg =
+        error_status_map[reinterpret_cast<size_t>(err)];
+    str->refcount = &grpc_core::kNoopRefcount;
+    str->data.refcounted.bytes =
+        reinterpret_cast<uint8_t*>(const_cast<char*>(msg.msg));
+    str->data.refcounted.length = msg.len;
     return true;
   }
   uint8_t slot = err->strs[which];

data/src/core/lib/iomgr/error.h

@@ -73,6 +73,8 @@ typedef enum {
   GRPC_ERROR_INT_LIMIT,
   /// chttp2: did the error occur while a write was in progress
   GRPC_ERROR_INT_OCCURRED_DURING_WRITE,
+  /// channel connectivity state associated with the error
+  GRPC_ERROR_INT_CHANNEL_CONNECTIVITY_STATE,
 
   /// Must always be last
   GRPC_ERROR_INT_MAX,

data/src/core/lib/iomgr/ev_posix.cc

@@ -206,7 +206,8 @@ void grpc_register_event_engine_factory(const char* name,
   GPR_ASSERT(false);
 }
 
-/* Call this only after calling grpc_event_engine_init() */
+/*If grpc_event_engine_init() has been called, returns the poll_strategy_name.
+ * Otherwise, returns nullptr. */
 const char* grpc_get_poll_strategy_name() { return g_poll_strategy_name; }
 
 void grpc_event_engine_init(void) {
@@ -246,7 +247,8 @@ bool grpc_event_engine_can_track_errors(void) {
 }
 
 bool grpc_event_engine_run_in_background(void) {
-  return g_event_engine->run_in_background;
+  // g_event_engine is nullptr when using a custom iomgr.
+  return g_event_engine != nullptr && g_event_engine->run_in_background;
 }
 
 grpc_fd* grpc_fd_create(int fd, const char* name, bool track_err) {

data/src/core/lib/iomgr/executor.cc

@@ -120,7 +120,10 @@ size_t Executor::RunClosures(const char* executor_name,
   // thread itself, but this is the point where we could start seeing
   // application-level callbacks. No need to create a new ExecCtx, though,
   // since there already is one and it is flushed (but not destructed) in this
-  // function itself.
+  // function itself. The ApplicationCallbackExecCtx will have its callbacks
+  // invoked on its destruction, which will be after completing any closures in
+  // the executor's closure list (which were explicitly scheduled onto the
+  // executor).
   grpc_core::ApplicationCallbackExecCtx callback_exec_ctx(
       GRPC_APP_CALLBACK_EXEC_CTX_FLAG_IS_INTERNAL_THREAD);
 

data/src/core/lib/iomgr/executor/mpmcqueue.cc

@@ -0,0 +1,183 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/executor/mpmcqueue.h"
+
+namespace grpc_core {
+
+DebugOnlyTraceFlag grpc_thread_pool_trace(false, "thread_pool");
+
+inline void* InfLenFIFOQueue::PopFront() {
+  // Caller should already check queue is not empty and has already held the
+  // mutex. This function will assume that there is at least one element in the
+  // queue (i.e. queue_head_->content is valid).
+  void* result = queue_head_->content;
+  count_.Store(count_.Load(MemoryOrder::RELAXED) - 1, MemoryOrder::RELAXED);
+
+  // Updates Stats when trace flag turned on.
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_thread_pool_trace)) {
+    gpr_timespec wait_time =
+        gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), queue_head_->insert_time);
+    stats_.num_completed++;
+    stats_.total_queue_time = gpr_time_add(stats_.total_queue_time, wait_time);
+    stats_.max_queue_time = gpr_time_max(
+        gpr_convert_clock_type(stats_.max_queue_time, GPR_TIMESPAN), wait_time);
+
+    if (count_.Load(MemoryOrder::RELAXED) == 0) {
+      stats_.busy_queue_time =
+          gpr_time_add(stats_.busy_queue_time,
+                       gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), busy_time));
+    }
+
+    gpr_log(GPR_INFO,
+            "[InfLenFIFOQueue PopFront] num_completed:        %" PRIu64
+            " total_queue_time: %f max_queue_time:   %f busy_queue_time:   %f",
+            stats_.num_completed,
+            gpr_timespec_to_micros(stats_.total_queue_time),
+            gpr_timespec_to_micros(stats_.max_queue_time),
+            gpr_timespec_to_micros(stats_.busy_queue_time));
+  }
+
+  queue_head_ = queue_head_->next;
+  // Signal waiting thread
+  if (count_.Load(MemoryOrder::RELAXED) > 0) {
+    TopWaiter()->cv.Signal();
+  }
+
+  return result;
+}
+
+InfLenFIFOQueue::Node* InfLenFIFOQueue::AllocateNodes(int num) {
+  num_nodes_ = num_nodes_ + num;
+  Node* new_chunk = static_cast<Node*>(gpr_zalloc(sizeof(Node) * num));
+  new_chunk[0].next = &new_chunk[1];
+  new_chunk[num - 1].prev = &new_chunk[num - 2];
+  for (int i = 1; i < num - 1; ++i) {
+    new_chunk[i].prev = &new_chunk[i - 1];
+    new_chunk[i].next = &new_chunk[i + 1];
+  }
+  return new_chunk;
+}
+
+InfLenFIFOQueue::InfLenFIFOQueue() {
+  delete_list_size_ = kDeleteListInitSize;
+  delete_list_ =
+      static_cast<Node**>(gpr_zalloc(sizeof(Node*) * delete_list_size_));
+
+  Node* new_chunk = AllocateNodes(kQueueInitNumNodes);
+  delete_list_[delete_list_count_++] = new_chunk;
+  queue_head_ = queue_tail_ = new_chunk;
+  new_chunk[0].prev = &new_chunk[kQueueInitNumNodes - 1];
+  new_chunk[kQueueInitNumNodes - 1].next = &new_chunk[0];
+
+  waiters_.next = &waiters_;
+  waiters_.prev = &waiters_;
+}
+
+InfLenFIFOQueue::~InfLenFIFOQueue() {
+  GPR_ASSERT(count_.Load(MemoryOrder::RELAXED) == 0);
+  for (size_t i = 0; i < delete_list_count_; ++i) {
+    gpr_free(delete_list_[i]);
+  }
+  gpr_free(delete_list_);
+}
+
+void InfLenFIFOQueue::Put(void* elem) {
+  MutexLock l(&mu_);
+
+  int curr_count = count_.Load(MemoryOrder::RELAXED);
+
+  if (queue_tail_ == queue_head_ && curr_count != 0) {
+    // List is full. Expands list to double size by inserting new chunk of nodes
+    Node* new_chunk = AllocateNodes(curr_count);
+    delete_list_[delete_list_count_++] = new_chunk;
+    // Expands delete list on full.
+    if (delete_list_count_ == delete_list_size_) {
+      delete_list_size_ = delete_list_size_ * 2;
+      delete_list_ = static_cast<Node**>(
+          gpr_realloc(delete_list_, sizeof(Node*) * delete_list_size_));
+    }
+    new_chunk[0].prev = queue_tail_->prev;
+    new_chunk[curr_count - 1].next = queue_head_;
+    queue_tail_->prev->next = new_chunk;
+    queue_head_->prev = &new_chunk[curr_count - 1];
+    queue_tail_ = new_chunk;
+  }
+  queue_tail_->content = static_cast<void*>(elem);
+
+  // Updates Stats info
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_thread_pool_trace)) {
+    stats_.num_started++;
+    gpr_log(GPR_INFO, "[InfLenFIFOQueue Put] num_started:        %" PRIu64,
+            stats_.num_started);
+    auto current_time = gpr_now(GPR_CLOCK_MONOTONIC);
+    if (curr_count == 0) {
+      busy_time = current_time;
+    }
+    queue_tail_->insert_time = current_time;
+  }
+
+  count_.Store(curr_count + 1, MemoryOrder::RELAXED);
+  queue_tail_ = queue_tail_->next;
+
+  TopWaiter()->cv.Signal();
+}
+
+void* InfLenFIFOQueue::Get(gpr_timespec* wait_time) {
+  MutexLock l(&mu_);
+
+  if (count_.Load(MemoryOrder::RELAXED) == 0) {
+    gpr_timespec start_time;
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_thread_pool_trace) &&
+        wait_time != nullptr) {
+      start_time = gpr_now(GPR_CLOCK_MONOTONIC);
+    }
+
+    Waiter self;
+    PushWaiter(&self);
+    do {
+      self.cv.Wait(&mu_);
+    } while (count_.Load(MemoryOrder::RELAXED) == 0);
+    RemoveWaiter(&self);
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_thread_pool_trace) &&
+        wait_time != nullptr) {
+      *wait_time = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), start_time);
+    }
+  }
+  GPR_DEBUG_ASSERT(count_.Load(MemoryOrder::RELAXED) > 0);
+  return PopFront();
+}
+
+void InfLenFIFOQueue::PushWaiter(Waiter* waiter) {
+  waiter->next = waiters_.next;
+  waiter->prev = &waiters_;
+  waiter->next->prev = waiter;
+  waiter->prev->next = waiter;
+}
+
+void InfLenFIFOQueue::RemoveWaiter(Waiter* waiter) {
+  GPR_DEBUG_ASSERT(waiter != &waiters_);
+  waiter->next->prev = waiter->prev;
+  waiter->prev->next = waiter->next;
+}
+
+InfLenFIFOQueue::Waiter* InfLenFIFOQueue::TopWaiter() { return waiters_.next; }
+
+}  // namespace grpc_core