grpc 1.43.1 → 1.53.0

data/src/core/lib/promise/arena_promise.h

@@ -0,0 +1,231 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_PROMISE_ARENA_PROMISE_H
+#define GRPC_SRC_CORE_LIB_PROMISE_ARENA_PROMISE_H
+
+#include <grpc/support/port_platform.h>
+
+#include <stdlib.h>
+
+#include <memory>
+#include <type_traits>
+#include <utility>
+
+#include "absl/meta/type_traits.h"
+
+#include "src/core/lib/gprpp/construct_destruct.h"
+#include "src/core/lib/promise/context.h"
+#include "src/core/lib/promise/poll.h"
+#include "src/core/lib/resource_quota/arena.h"
+
+namespace grpc_core {
+
+namespace arena_promise_detail {
+
+using ArgType = std::aligned_storage_t<sizeof(void*)>;
+template <typename T>
+T*& ArgAsPtr(ArgType* arg) {
+  static_assert(sizeof(ArgType) >= sizeof(T**),
+                "Must have ArgType of at least one pointer size");
+  return *reinterpret_cast<T**>(arg);
+}
+
+template <typename T>
+struct Vtable {
+  // Poll the promise, once.
+  Poll<T> (*poll_once)(ArgType* arg);
+  // Destroy the underlying callable object if there is one.
+  // Since we don't delete (the arena owns the memory) but we may need to call a
+  // destructor, we expose this for when the ArenaPromise object is destroyed.
+  void (*destroy)(ArgType* arg);
+};
+
+template <typename T>
+struct VtableAndArg {
+  const Vtable<T>* vtable;
+  ArgType arg;
+};
+
+// Implementation of Vtable for an empty object.
+// Used when an empty ArenaPromise is created, or when the ArenaPromise is moved
+// from. Since in either case these objects should not be polled, we simply
+// crash if it is.
+template <typename T>
+struct Null {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType*) {
+    abort();
+    GPR_UNREACHABLE_CODE(return Pending{});
+  }
+
+  static void Destroy(ArgType*) {}
+};
+
+template <typename T>
+const Vtable<T> Null<T>::vtable = {PollOnce, Destroy};
+
+// Implementation of ImplInterface for a callable object.
+template <typename T, typename Callable>
+struct AllocatedCallable {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType* arg) {
+    return poll_cast<T>((*ArgAsPtr<Callable>(arg))());
+  }
+
+  static void Destroy(ArgType* arg) { Destruct(ArgAsPtr<Callable>(arg)); }
+};
+
+template <typename T, typename Callable>
+const Vtable<T> AllocatedCallable<T, Callable>::vtable = {PollOnce, Destroy};
+
+// Implementation of ImplInterface for a small callable object (one that fits
+// within the ArgType arg)
+template <typename T, typename Callable>
+struct Inlined {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType* arg) {
+    return poll_cast<T>((*reinterpret_cast<Callable*>(arg))());
+  }
+
+  static void Destroy(ArgType* arg) {
+    Destruct(reinterpret_cast<Callable*>(arg));
+  }
+};
+
+template <typename T, typename Callable>
+const Vtable<T> Inlined<T, Callable>::vtable = {PollOnce, Destroy};
+
+// If a callable object is empty we can substitute any instance of that callable
+// for the one we call (for how could we tell the difference)?
+// Since this corresponds to a lambda with no fields, and we expect these to be
+// reasonably common, we can elide the arena allocation entirely and simply poll
+// a global shared instance.
+// (this comes up often when the promise only accesses context data from the
+// containing activity).
+template <typename T, typename Callable>
+struct SharedCallable {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType* arg) {
+    return (*reinterpret_cast<Callable*>(arg))();
+  }
+};
+
+template <typename T, typename Callable>
+const Vtable<T> SharedCallable<T, Callable>::vtable = {PollOnce,
+                                                       Null<T>::Destroy};
+
+// Redirector type: given a callable type, expose a Make() function that creates
+// the appropriate underlying implementation.
+template <typename T, typename Callable, typename Ignored = void>
+struct ChooseImplForCallable;
+
+template <typename T, typename Callable>
+struct ChooseImplForCallable<
+    T, Callable,
+    absl::enable_if_t<!std::is_empty<Callable>::value &&
+                      (sizeof(Callable) > sizeof(ArgType))>> {
+  static void Make(Callable&& callable, VtableAndArg<T>* out) {
+    out->vtable = &AllocatedCallable<T, Callable>::vtable;
+    ArgAsPtr<Callable>(&out->arg) = GetContext<Arena>()->template New<Callable>(
+        std::forward<Callable>(callable));
+  }
+};
+
+template <typename T, typename Callable>
+struct ChooseImplForCallable<
+    T, Callable,
+    absl::enable_if_t<!std::is_empty<Callable>::value &&
+                      (sizeof(Callable) <= sizeof(ArgType))>> {
+  static void Make(Callable&& callable, VtableAndArg<T>* out) {
+    out->vtable = &Inlined<T, Callable>::vtable;
+    Construct(reinterpret_cast<Callable*>(&out->arg),
+              std::forward<Callable>(callable));
+  }
+};
+
+template <typename T, typename Callable>
+struct ChooseImplForCallable<
+    T, Callable, absl::enable_if_t<std::is_empty<Callable>::value>> {
+  static void Make(Callable&&, VtableAndArg<T>* out) {
+    out->vtable = &SharedCallable<T, Callable>::vtable;
+  }
+};
+
+// Wrap ChooseImplForCallable with a friend approachable syntax.
+template <typename T, typename Callable>
+void MakeImplForCallable(Callable&& callable, VtableAndArg<T>* out) {
+  ChooseImplForCallable<T, Callable>::Make(std::forward<Callable>(callable),
+                                           out);
+}
+
+}  // namespace arena_promise_detail
+
+// A promise for which the state memory is allocated from an arena.
+template <typename T>
+class ArenaPromise {
+ public:
+  // Construct an empty, uncallable, invalid ArenaPromise.
+  ArenaPromise() = default;
+
+  // Construct an ArenaPromise that will call the given callable when polled.
+  template <typename Callable,
+            typename Ignored =
+                absl::enable_if_t<!std::is_same<Callable, ArenaPromise>::value>>
+  // NOLINTNEXTLINE(google-explicit-constructor)
+  ArenaPromise(Callable&& callable) {
+    arena_promise_detail::MakeImplForCallable(std::forward<Callable>(callable),
+                                              &vtable_and_arg_);
+  }
+
+  // ArenaPromise is not copyable.
+  ArenaPromise(const ArenaPromise&) = delete;
+  ArenaPromise& operator=(const ArenaPromise&) = delete;
+  // ArenaPromise is movable.
+  ArenaPromise(ArenaPromise&& other) noexcept
+      : vtable_and_arg_(other.vtable_and_arg_) {
+    other.vtable_and_arg_.vtable = &arena_promise_detail::Null<T>::vtable;
+  }
+  ArenaPromise& operator=(ArenaPromise&& other) noexcept {
+    vtable_and_arg_.vtable->destroy(&vtable_and_arg_.arg);
+    vtable_and_arg_ = other.vtable_and_arg_;
+    other.vtable_and_arg_.vtable = &arena_promise_detail::Null<T>::vtable;
+    return *this;
+  }
+
+  // Destruction => call Destroy on the underlying impl object.
+  ~ArenaPromise() { vtable_and_arg_.vtable->destroy(&vtable_and_arg_.arg); }
+
+  // Expose the promise interface: a call operator that returns Poll<T>.
+  Poll<T> operator()() {
+    return vtable_and_arg_.vtable->poll_once(&vtable_and_arg_.arg);
+  }
+
+  bool has_value() const {
+    return vtable_and_arg_.vtable != &arena_promise_detail::Null<T>::vtable;
+  }
+
+ private:
+  // Underlying impl object.
+  arena_promise_detail::VtableAndArg<T> vtable_and_arg_ = {
+      &arena_promise_detail::Null<T>::vtable, {}};
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_PROMISE_ARENA_PROMISE_H

data/src/core/lib/promise/context.h

@@ -12,15 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_PROMISE_CONTEXT_H
-#define GRPC_CORE_LIB_PROMISE_CONTEXT_H
+#ifndef GRPC_SRC_CORE_LIB_PROMISE_CONTEXT_H
+#define GRPC_SRC_CORE_LIB_PROMISE_CONTEXT_H
 
 #include <grpc/support/port_platform.h>
 
 #include <type_traits>
 #include <utility>
 
-#include "src/core/lib/gpr/tls.h"
+#include <grpc/support/log.h>
 
 namespace grpc_core {
 
@@ -29,7 +29,7 @@ namespace grpc_core {
 // not contain any members, only exist.
 // The reason for avoiding this is that context types each use a thread local.
 template <typename T>
-struct ContextType;
+struct ContextType;  // IWYU pragma: keep
 
 namespace promise_detail {
 
@@ -45,12 +45,11 @@ class Context : public ContextType<T> {
 
  private:
   T* const old_;
-  static GPR_THREAD_LOCAL(T*) current_;
+  static thread_local T* current_;
 };
 
 template <typename T>
-GPR_THREAD_LOCAL(T*)
-Context<T>::current_;
+thread_local T* Context<T>::current_;
 
 template <typename T, typename F>
 class WithContext {
@@ -69,18 +68,26 @@ class WithContext {
 
 }  // namespace promise_detail
 
-// Retrieve the current value of a context.
+// Return true if a context of type T is currently active.
+template <typename T>
+bool HasContext() {
+  return promise_detail::Context<T>::get() != nullptr;
+}
+
+// Retrieve the current value of a context, or abort if the value is unset.
 template <typename T>
 T* GetContext() {
-  return promise_detail::Context<T>::get();
+  auto* p = promise_detail::Context<T>::get();
+  GPR_ASSERT(p != nullptr);
+  return p;
 }
 
 // Given a promise and a context, return a promise that has that context set.
 template <typename T, typename F>
 promise_detail::WithContext<T, F> WithContext(F f, T* context) {
-  return promise_detail::WithContext<T, F>(f, context);
+  return promise_detail::WithContext<T, F>(std::move(f), context);
 }
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_PROMISE_CONTEXT_H
+#endif  // GRPC_SRC_CORE_LIB_PROMISE_CONTEXT_H

data/src/core/lib/promise/detail/basic_join.h

@@ -0,0 +1,197 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_PROMISE_DETAIL_BASIC_JOIN_H
+#define GRPC_SRC_CORE_LIB_PROMISE_DETAIL_BASIC_JOIN_H
+
+#include <grpc/support/port_platform.h>
+
+#include <assert.h>
+#include <stddef.h>
+
+#include <array>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+#include "absl/utility/utility.h"
+
+#include "src/core/lib/gprpp/bitset.h"
+#include "src/core/lib/gprpp/construct_destruct.h"
+#include "src/core/lib/promise/detail/promise_like.h"
+#include "src/core/lib/promise/poll.h"
+
+namespace grpc_core {
+namespace promise_detail {
+
+// This union can either be a functor, or the result of the functor (after
+// mapping via a trait). Allows us to remember the result of one joined functor
+// until the rest are ready.
+template <typename Traits, typename F>
+union Fused {
+  explicit Fused(F&& f) : f(std::forward<F>(f)) {}
+  explicit Fused(PromiseLike<F>&& f) : f(std::forward<PromiseLike<F>>(f)) {}
+  ~Fused() {}
+  // Wrap the functor in a PromiseLike to handle immediately returning functors
+  // and the like.
+  using Promise = PromiseLike<F>;
+  GPR_NO_UNIQUE_ADDRESS Promise f;
+  // Compute the result type: We take the result of the promise, and pass it via
+  // our traits, so that, for example, TryJoin and take a StatusOr<T> and just
+  // store a T.
+  using Result = typename Traits::template ResultType<typename Promise::Result>;
+  GPR_NO_UNIQUE_ADDRESS Result result;
+};
+
+// A join gets composed of joints... these are just wrappers around a Fused for
+// their data, with some machinery as methods to get the system working.
+template <typename Traits, size_t kRemaining, typename... Fs>
+struct Joint : public Joint<Traits, kRemaining - 1, Fs...> {
+  // The index into Fs for this Joint
+  static constexpr size_t kIdx = sizeof...(Fs) - kRemaining;
+  // The next join (the one we derive from)
+  using NextJoint = Joint<Traits, kRemaining - 1, Fs...>;
+  // From Fs, extract the functor for this joint.
+  using F = typename std::tuple_element<kIdx, std::tuple<Fs...>>::type;
+  // Generate the Fused type for this functor.
+  using Fsd = Fused<Traits, F>;
+  GPR_NO_UNIQUE_ADDRESS Fsd fused;
+  // Figure out what kind of bitmask will be used by the outer join.
+  using Bits = BitSet<sizeof...(Fs)>;
+  // Initialize from a tuple of pointers to Fs
+  explicit Joint(std::tuple<Fs*...> fs)
+      : NextJoint(fs), fused(std::move(*std::get<kIdx>(fs))) {}
+  // Copy: assume that the Fuse is still in the promise state (since it's not
+  // legal to copy after the first poll!)
+  Joint(const Joint& j) : NextJoint(j), fused(j.fused.f) {}
+  // Move: assume that the Fuse is still in the promise state (since it's not
+  // legal to move after the first poll!)
+  Joint(Joint&& j) noexcept
+      : NextJoint(std::forward<NextJoint>(j)), fused(std::move(j.fused.f)) {}
+  // Destruct: check bits to see if we're in promise or result state, and call
+  // the appropriate destructor. Recursively, call up through the join.
+  void DestructAll(const Bits& bits) {
+    if (!bits.is_set(kIdx)) {
+      Destruct(&fused.f);
+    } else {
+      Destruct(&fused.result);
+    }
+    NextJoint::DestructAll(bits);
+  }
+  // Poll all joints up, and then call finally.
+  template <typename F>
+  auto Run(Bits* bits, F finally) -> decltype(finally()) {
+    // If we're still in the promise state...
+    if (!bits->is_set(kIdx)) {
+      // Poll the promise
+      auto r = fused.f();
+      if (auto* p = r.value_if_ready()) {
+        // If it's done, then ask the trait to unwrap it and store that result
+        // in the Fused, and continue the iteration. Note that OnResult could
+        // instead choose to return a value instead of recursing through the
+        // iteration, in that case we continue returning the same result up.
+        // Here is where TryJoin can escape out.
+        return Traits::OnResult(
+            std::move(*p), [this, bits, &finally](typename Fsd::Result result) {
+              bits->set(kIdx);
+              Destruct(&fused.f);
+              Construct(&fused.result, std::move(result));
+              return NextJoint::Run(bits, std::move(finally));
+            });
+      }
+    }
+    // That joint is still pending... we'll still poll the result of the joints.
+    return NextJoint::Run(bits, std::move(finally));
+  }
+};
+
+// Terminating joint... for each of the recursions, do the thing we're supposed
+// to do at the end.
+template <typename Traits, typename... Fs>
+struct Joint<Traits, 0, Fs...> {
+  explicit Joint(std::tuple<Fs*...>) {}
+  Joint(const Joint&) {}
+  Joint(Joint&&) noexcept {}
+  template <typename T>
+  void DestructAll(const T&) {}
+  template <typename F>
+  auto Run(BitSet<sizeof...(Fs)>*, F finally) -> decltype(finally()) {
+    return finally();
+  }
+};
+
+template <typename Traits, typename... Fs>
+class BasicJoin {
+ private:
+  // How many things are we joining?
+  static constexpr size_t N = sizeof...(Fs);
+  // Bitset: if a bit is 0, that joint is still in promise state. If it's 1,
+  // then the joint has a result.
+  GPR_NO_UNIQUE_ADDRESS BitSet<N> state_;
+  // The actual joints, wrapped in an anonymous union to give us control of
+  // construction/destruction.
+  union {
+    GPR_NO_UNIQUE_ADDRESS Joint<Traits, sizeof...(Fs), Fs...> joints_;
+  };
+
+  // Access joint index I
+  template <size_t I>
+  Joint<Traits, sizeof...(Fs) - I, Fs...>* GetJoint() {
+    return static_cast<Joint<Traits, sizeof...(Fs) - I, Fs...>*>(&joints_);
+  }
+
+  // The tuple of results of all our promises
+  using Tuple = std::tuple<typename Fused<Traits, Fs>::Result...>;
+
+  // Collect up all the results and construct a tuple.
+  template <size_t... I>
+  Tuple Finish(absl::index_sequence<I...>) {
+    return Tuple(std::move(GetJoint<I>()->fused.result)...);
+  }
+
+ public:
+  explicit BasicJoin(Fs&&... fs) : joints_(std::tuple<Fs*...>(&fs...)) {}
+  BasicJoin& operator=(const BasicJoin&) = delete;
+  // Copy a join - only available before polling.
+  BasicJoin(const BasicJoin& other) {
+    assert(other.state_.none());
+    Construct(&joints_, other.joints_);
+  }
+  // Move a join - only available before polling.
+  BasicJoin(BasicJoin&& other) noexcept {
+    assert(other.state_.none());
+    Construct(&joints_, std::move(other.joints_));
+  }
+  ~BasicJoin() { joints_.DestructAll(state_); }
+  using Result = decltype(Traits::Wrap(std::declval<Tuple>()));
+  // Poll the join
+  Poll<Result> operator()() {
+    // Poll the joints...
+    return joints_.Run(&state_, [this]() -> Poll<Result> {
+      // If all of them are completed, collect the results, and then ask our
+      // traits to wrap them - allowing for example TryJoin to turn tuple<A,B,C>
+      // into StatusOr<tuple<A,B,C>>.
+      if (state_.all()) {
+        return Traits::Wrap(Finish(absl::make_index_sequence<N>()));
+      } else {
+        return Pending();
+      }
+    });
+  }
+};
+
+}  // namespace promise_detail
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_PROMISE_DETAIL_BASIC_JOIN_H

data/src/core/lib/promise/detail/basic_seq.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_PROMISE_DETAIL_BASIC_SEQ_H
-#define GRPC_CORE_LIB_PROMISE_DETAIL_BASIC_SEQ_H
+#ifndef GRPC_SRC_CORE_LIB_PROMISE_DETAIL_BASIC_SEQ_H
+#define GRPC_SRC_CORE_LIB_PROMISE_DETAIL_BASIC_SEQ_H
 
 #include <grpc/support/port_platform.h>
 
@@ -24,18 +24,16 @@
 #include <utility>
 
 #include "absl/meta/type_traits.h"
-#include "absl/types/variant.h"
 #include "absl/utility/utility.h"
 
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
+#include "src/core/lib/promise/detail/promise_like.h"
 #include "src/core/lib/promise/detail/switch.h"
 #include "src/core/lib/promise/poll.h"
 
 namespace grpc_core {
 namespace promise_detail {
-template <typename F>
-class PromiseLike;
 
 // Helper for SeqState to evaluate some common types to all partial
 // specializations.
@@ -50,7 +48,7 @@ struct SeqStateTypes {
   // Wrap the factory callable in our factory wrapper to deal with common edge
   // cases. We use the 'unwrapped type' from the traits, so for instance, TrySeq
   // can pass back a T from a StatusOr<T>.
-  using Next = promise_detail::PromiseFactory<
+  using Next = promise_detail::OncePromiseFactory<
       typename PromiseResultTraits::UnwrappedType, FNext>;
 };
 
@@ -162,9 +160,9 @@ absl::enable_if_t<I <= J, SeqState<Traits, I, Fs...>*> GetSeqState(
 }
 
 template <template <typename> class Traits, char I, typename... Fs, typename T>
-auto CallNext(SeqState<Traits, I, Fs...>* state, T&& arg) -> decltype(
-    SeqState<Traits, I, Fs...>::Types::PromiseResultTraits::CallFactory(
-        &state->next_factory, std::forward<T>(arg))) {
+auto CallNext(SeqState<Traits, I, Fs...>* state, T&& arg)
+    -> decltype(SeqState<Traits, I, Fs...>::Types::PromiseResultTraits::
+                    CallFactory(&state->next_factory, std::forward<T>(arg))) {
   return SeqState<Traits, I, Fs...>::Types::PromiseResultTraits::CallFactory(
       &state->next_factory, std::forward<T>(arg));
 }
@@ -277,18 +275,15 @@ class BasicSeq {
     // Poll the current promise in this state.
     auto r = s->current_promise();
     // If we are still pending, say so by returning.
-    if (absl::holds_alternative<Pending>(r)) {
-      return Pending();
-    }
+    if (r.pending()) return Pending();
     // Current promise is ready, as the traits to do the next thing.
     // That may be returning - eg if TrySeq sees an error.
     // Or it may be by calling the callable we hand down - RunNext - which
     // will advance the state and call the next promise.
     return Traits<
         typename absl::remove_reference_t<decltype(*s)>::Types::PromiseResult>::
-        template CheckResultAndRunNext<Result>(
-            std::move(absl::get<kPollReadyIdx>(std::move(r))),
-            RunNext<I>{this});
+        template CheckResultAndRunNext<Result>(std::move(r.value()),
+                                               RunNext<I>{this});
   }
 
   // Specialization of RunState to run the final state.
@@ -297,11 +292,9 @@ class BasicSeq {
     // Poll the final promise.
     auto r = final_promise_();
     // If we are still pending, say so by returning.
-    if (absl::holds_alternative<Pending>(r)) {
-      return Pending();
-    }
+    if (r.pending()) return Pending();
     // We are complete, return the (wrapped) result.
-    return Result(std::move(absl::get<kPollReadyIdx>(std::move(r))));
+    return Result(std::move(r.value()));
   }
 
   // For state numbered I, destruct the current promise and the next promise
@@ -401,7 +394,98 @@ class BasicSeq {
   }
 };
 
+// As above, but models a sequence of unknown size
+// At each element, the accumulator A and the current value V is passed to some
+// function of type IterTraits::Factory as f(V, IterTraits::Argument); f is
+// expected to return a promise that resolves to IterTraits::Wrapped.
+template <class IterTraits>
+class BasicSeqIter {
+ private:
+  using Traits = typename IterTraits::Traits;
+  using Iter = typename IterTraits::Iter;
+  using Factory = typename IterTraits::Factory;
+  using Argument = typename IterTraits::Argument;
+  using IterValue = typename IterTraits::IterValue;
+  using StateCreated = typename IterTraits::StateCreated;
+  using State = typename IterTraits::State;
+  using Wrapped = typename IterTraits::Wrapped;
+
+ public:
+  BasicSeqIter(Iter begin, Iter end, Factory f, Argument arg)
+      : cur_(begin), end_(end), f_(std::move(f)) {
+    if (cur_ == end_) {
+      Construct(&result_, std::move(arg));
+    } else {
+      Construct(&state_, f_(*cur_, std::move(arg)));
+    }
+  }
+
+  ~BasicSeqIter() {
+    if (cur_ == end_) {
+      Destruct(&result_);
+    } else {
+      Destruct(&state_);
+    }
+  }
+
+  BasicSeqIter(const BasicSeqIter& other) = delete;
+  BasicSeqIter& operator=(const BasicSeqIter&) = delete;
+
+  BasicSeqIter(BasicSeqIter&& other) noexcept
+      : cur_(other.cur_), end_(other.end_), f_(std::move(other.f_)) {
+    if (cur_ == end_) {
+      Construct(&result_, std::move(other.result_));
+    } else {
+      Construct(&state_, std::move(other.state_));
+    }
+  }
+  BasicSeqIter& operator=(BasicSeqIter&& other) noexcept {
+    cur_ = other.cur_;
+    end_ = other.end_;
+    if (cur_ == end_) {
+      Construct(&result_, std::move(other.result_));
+    } else {
+      Construct(&state_, std::move(other.state_));
+    }
+    return *this;
+  }
+
+  Poll<Wrapped> operator()() {
+    if (cur_ == end_) {
+      return std::move(result_);
+    }
+    return PollNonEmpty();
+  }
+
+ private:
+  Poll<Wrapped> PollNonEmpty() {
+    Poll<Wrapped> r = state_();
+    if (r.pending()) return r;
+    return Traits::template CheckResultAndRunNext<Wrapped>(
+        std::move(r.value()), [this](Wrapped arg) -> Poll<Wrapped> {
+          auto next = cur_;
+          ++next;
+          if (next == end_) {
+            return std::move(arg);
+          }
+          cur_ = next;
+          state_.~State();
+          Construct(&state_,
+                    Traits::template CallSeqFactory(f_, *cur_, std::move(arg)));
+          return PollNonEmpty();
+        });
+  }
+
+  Iter cur_;
+  const Iter end_;
+  GPR_NO_UNIQUE_ADDRESS Factory f_;
+  union {
+    GPR_NO_UNIQUE_ADDRESS State state_;
+    GPR_NO_UNIQUE_ADDRESS Argument result_;
+  };
+};
+
 }  // namespace promise_detail
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_PROMISE_DETAIL_BASIC_SEQ_H
+#endif  // GRPC_SRC_CORE_LIB_PROMISE_DETAIL_BASIC_SEQ_H