grpc 1.44.0.pre2 → 1.46.3

data/src/core/lib/promise/arena_promise.h

@@ -0,0 +1,188 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_PROMISE_ARENA_PROMISE_H
+#define GRPC_CORE_LIB_PROMISE_ARENA_PROMISE_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/support/log.h>
+
+#include "src/core/lib/promise/poll.h"
+#include "src/core/lib/resource_quota/arena.h"
+
+namespace grpc_core {
+
+namespace arena_promise_detail {
+
+// Type erased promise stored in the arena.
+template <typename T>
+class ImplInterface {
+ public:
+  // Poll the promise, once.
+  virtual Poll<T> PollOnce() = 0;
+  // Destroy the underlying callable object if there is one.
+  // Since we don't delete (the arena owns the memory) but we may need to call a
+  // destructor, we expose this for when the ArenaPromise object is destroyed.
+  virtual void Destroy() = 0;
+
+ protected:
+  ~ImplInterface() = default;
+};
+
+// Implementation of ImplInterface for an empty object.
+// Used when an empty ArenaPromise is created, or when the ArenaPromise is moved
+// from. Since in either case these objects should not be polled, we simply
+// crash if it is.
+template <typename T>
+class NullImpl final : public ImplInterface<T> {
+ public:
+  Poll<T> PollOnce() override {
+    abort();
+    GPR_UNREACHABLE_CODE(return Pending{});
+  }
+  void Destroy() override {}
+
+  static ImplInterface<T>* Get() {
+    static NullImpl<T> instance;
+    return &instance;
+  }
+
+ private:
+  ~NullImpl() = default;
+};
+
+// Implementation of ImplInterface for a callable object.
+template <typename T, typename Callable>
+class CallableImpl final : public ImplInterface<T> {
+ public:
+  explicit CallableImpl(Callable&& callable) : callable_(std::move(callable)) {}
+  // Forward polls to the callable object.
+  Poll<T> PollOnce() override { return poll_cast<T>(callable_()); }
+  // Destroy destructs the callable object.
+  void Destroy() override { this->~CallableImpl(); }
+
+ private:
+  // Should only be called by Destroy().
+  ~CallableImpl() = default;
+
+  Callable callable_;
+};
+
+// If a callable object is empty we can substitute any instance of that callable
+// for the one we call (for how could we tell the difference)?
+// Since this corresponds to a lambda with no fields, and we expect these to be
+// reasonably common, we can elide the arena allocation entirely and simply poll
+// a global shared instance.
+// (this comes up often when the promise only accesses context data from the
+// containing activity).
+template <typename T, typename Callable>
+class SharedImpl final : public ImplInterface<T>, private Callable {
+ public:
+  // Call the callable, or at least an exact duplicate of it - if you have no
+  // members, all your instances look the same.
+  Poll<T> PollOnce() override { return Callable::operator()(); }
+  // Nothing to destroy.
+  void Destroy() override {}
+  // Return a pointer to the shared instance - these are singletons, and are
+  // needed just to get the vtable in place.
+  static SharedImpl* Get(Callable&& callable) {
+    static_assert(sizeof(SharedImpl) == sizeof(void*),
+                  "SharedImpl should be pointer sized");
+    static SharedImpl impl(std::forward<Callable>(callable));
+    return &impl;
+  }
+
+ private:
+  explicit SharedImpl(Callable&& callable)
+      : Callable(std::forward<Callable>(callable)) {}
+  ~SharedImpl() = default;
+};
+
+// Redirector type: given a callable type, expose a Make() function that creates
+// the appropriate underlying implementation.
+template <typename T, typename Callable, typename Ignored = void>
+struct ChooseImplForCallable;
+
+template <typename T, typename Callable>
+struct ChooseImplForCallable<
+    T, Callable, absl::enable_if_t<!std::is_empty<Callable>::value>> {
+  static ImplInterface<T>* Make(Callable&& callable) {
+    return GetContext<Arena>()->template New<CallableImpl<T, Callable>>(
+        std::forward<Callable>(callable));
+  }
+};
+
+template <typename T, typename Callable>
+struct ChooseImplForCallable<
+    T, Callable, absl::enable_if_t<std::is_empty<Callable>::value>> {
+  static ImplInterface<T>* Make(Callable&& callable) {
+    return SharedImpl<T, Callable>::Get(std::forward<Callable>(callable));
+  }
+};
+
+// Wrap ChooseImplForCallable with a friend approachable syntax.
+template <typename T, typename Callable>
+ImplInterface<T>* MakeImplForCallable(Callable&& callable) {
+  return ChooseImplForCallable<T, Callable>::Make(
+      std::forward<Callable>(callable));
+}
+
+}  // namespace arena_promise_detail
+
+// A promise for which the state memory is allocated from an arena.
+template <typename T>
+class ArenaPromise {
+ public:
+  // Construct an empty, uncallable, invalid ArenaPromise.
+  ArenaPromise() = default;
+
+  // Construct an ArenaPromise that will call the given callable when polled.
+  template <typename Callable,
+            typename Ignored =
+                absl::enable_if_t<!std::is_same<Callable, ArenaPromise>::value>>
+  // NOLINTNEXTLINE(google-explicit-constructor)
+  ArenaPromise(Callable&& callable)
+      : impl_(arena_promise_detail::MakeImplForCallable<T>(
+            std::forward<Callable>(callable))) {}
+
+  // ArenaPromise is not copyable.
+  ArenaPromise(const ArenaPromise&) = delete;
+  ArenaPromise& operator=(const ArenaPromise&) = delete;
+  // ArenaPromise is movable.
+  ArenaPromise(ArenaPromise&& other) noexcept : impl_(other.impl_) {
+    other.impl_ = arena_promise_detail::NullImpl<T>::Get();
+  }
+  ArenaPromise& operator=(ArenaPromise&& other) noexcept {
+    impl_->Destroy();
+    impl_ = other.impl_;
+    other.impl_ = arena_promise_detail::NullImpl<T>::Get();
+    return *this;
+  }
+
+  // Destruction => call Destroy on the underlying impl object.
+  ~ArenaPromise() { impl_->Destroy(); }
+
+  // Expose the promise interface: a call operator that returns Poll<T>.
+  Poll<T> operator()() { return impl_->PollOnce(); }
+
+ private:
+  // Underlying impl object.
+  arena_promise_detail::ImplInterface<T>* impl_ =
+      arena_promise_detail::NullImpl<T>::Get();
+};
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_LIB_PROMISE_ARENA_PROMISE_H */

data/src/core/lib/promise/call_push_pull.h

@@ -0,0 +1,144 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_PROMISE_CALL_PUSH_PULL_H
+#define GRPC_CORE_LIB_PROMISE_CALL_PUSH_PULL_H
+
+#include <grpc/support/port_platform.h>
+
+#include <assert.h>
+
+#include "src/core/lib/gprpp/bitset.h"
+#include "src/core/lib/gprpp/construct_destruct.h"
+#include "src/core/lib/promise/detail/promise_like.h"
+#include "src/core/lib/promise/detail/status.h"
+#include "src/core/lib/promise/poll.h"
+
+namespace grpc_core {
+
+namespace promise_detail {
+
+template <typename FMain, typename FPush, typename FPull>
+class CallPushPull {
+ public:
+  CallPushPull(FMain f_main, FPush f_push, FPull f_pull)
+      : push_(std::move(f_push)), pull_(std::move(f_pull)) {
+    Construct(&main_, std::move(f_main));
+  }
+
+  CallPushPull(const CallPushPull&) = delete;
+  CallPushPull& operator=(const CallPushPull&) = delete;
+  CallPushPull(CallPushPull&& other) noexcept
+      : done_(other.done_),
+        push_(std::move(other.push_)),
+        pull_(std::move(other.pull_)) {
+    assert(!done_.is_set(kDoneMain));
+    Construct(&main_, std::move(other.main_));
+  }
+
+  CallPushPull& operator=(CallPushPull&& other) noexcept {
+    assert(!done_.is_set(kDoneMain));
+    done_ = other.done_;
+    assert(!done_.is_set(kDoneMain));
+    push_ = std::move(other.push_);
+    main_ = std::move(other.main_);
+    pull_ = std::move(other.pull_);
+    return *this;
+  }
+
+  ~CallPushPull() {
+    if (done_.is_set(kDoneMain)) {
+      Destruct(&result_);
+    } else {
+      Destruct(&main_);
+    }
+  }
+
+  using Result =
+      typename PollTraits<decltype(std::declval<PromiseLike<FMain>>()())>::Type;
+
+  Poll<Result> operator()() {
+    if (!done_.is_set(kDonePush)) {
+      auto p = push_();
+      if (auto* status = absl::get_if<kPollReadyIdx>(&p)) {
+        if (IsStatusOk(*status)) {
+          done_.set(kDonePush);
+        } else {
+          return Result(std::move(*status));
+        }
+      }
+    }
+    if (!done_.is_set(kDoneMain)) {
+      auto p = main_();
+      if (auto* status = absl::get_if<kPollReadyIdx>(&p)) {
+        if (IsStatusOk(*status)) {
+          done_.set(kDoneMain);
+          Destruct(&main_);
+          Construct(&result_, std::move(*status));
+        } else {
+          return std::move(*status);
+        }
+      }
+    }
+    if (!done_.is_set(kDonePull)) {
+      auto p = pull_();
+      if (auto* status = absl::get_if<kPollReadyIdx>(&p)) {
+        if (IsStatusOk(*status)) {
+          done_.set(kDonePull);
+        } else {
+          return Result(std::move(*status));
+        }
+      }
+    }
+    if (done_.all()) return std::move(result_);
+    return Pending{};
+  }
+
+ private:
+  enum { kDonePull = 0, kDoneMain = 1, kDonePush = 2 };
+  BitSet<3> done_;
+  GPR_NO_UNIQUE_ADDRESS PromiseLike<FPush> push_;
+  union {
+    PromiseLike<FMain> main_;
+    Result result_;
+  };
+  GPR_NO_UNIQUE_ADDRESS PromiseLike<FPull> pull_;
+};
+
+}  // namespace promise_detail
+
+// For promises representing calls a common pattern emerges:
+// There's a process pushing data down the stack, a process handling the main
+// call part, and a process pulling data back up the stack.
+//
+// This can reasonably be represented by the right combinations of TryJoins and
+// Maps, but since the structure is fundamental to the domain we introduce
+// this simple helper to make it easier to write the common case.
+//
+// It takes three promises: the main call, the push and the pull.
+// When polling, the push is polled first, then the main call (descending the
+// stack), then the pull (as we ascend once more).
+//
+// This strategy minimizes repolls.
+template <typename FMain, typename FPush, typename FPull>
+promise_detail::CallPushPull<FMain, FPush, FPull> CallPushPull(FMain f_main,
+                                                               FPush f_push,
+                                                               FPull f_pull) {
+  return promise_detail::CallPushPull<FMain, FPush, FPull>(
+      std::move(f_main), std::move(f_push), std::move(f_pull));
+}
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_LIB_PROMISE_CALL_PUSH_PULL_H

data/src/core/lib/promise/detail/basic_seq.h

@@ -162,9 +162,9 @@ absl::enable_if_t<I <= J, SeqState<Traits, I, Fs...>*> GetSeqState(
 }
 
 template <template <typename> class Traits, char I, typename... Fs, typename T>
-auto CallNext(SeqState<Traits, I, Fs...>* state, T&& arg) -> decltype(
-    SeqState<Traits, I, Fs...>::Types::PromiseResultTraits::CallFactory(
-        &state->next_factory, std::forward<T>(arg))) {
+auto CallNext(SeqState<Traits, I, Fs...>* state, T&& arg)
+    -> decltype(SeqState<Traits, I, Fs...>::Types::PromiseResultTraits::
+                    CallFactory(&state->next_factory, std::forward<T>(arg))) {
   return SeqState<Traits, I, Fs...>::Types::PromiseResultTraits::CallFactory(
       &state->next_factory, std::forward<T>(arg));
 }
@@ -401,6 +401,95 @@ class BasicSeq {
   }
 };
 
+// As above, but models a sequence of unknown size
+// At each element, the accumulator A and the current value V is passed to some
+// function of type F as f(V, A); f is expected to return a promise that
+// resolves to Traits::WrappedType.
+template <template <typename Wrapped> class Traits, typename F, typename Arg,
+          typename Iter>
+class BasicSeqIter {
+ private:
+  using IterValue = decltype(*std::declval<Iter>());
+  using StateCreated = decltype(std::declval<F>()(std::declval<IterValue>(),
+                                                  std::declval<Arg>()));
+  using State = PromiseLike<StateCreated>;
+  using Wrapped = typename State::Result;
+
+ public:
+  BasicSeqIter(Iter begin, Iter end, F f, Arg arg)
+      : cur_(begin), end_(end), f_(std::move(f)) {
+    if (cur_ == end_) {
+      Construct(&result_, std::move(arg));
+    } else {
+      Construct(&state_, f_(*cur_, std::move(arg)));
+    }
+  }
+
+  ~BasicSeqIter() {
+    if (cur_ == end_) {
+      Destruct(&result_);
+    } else {
+      Destruct(&state_);
+    }
+  }
+
+  BasicSeqIter(const BasicSeqIter& other) = delete;
+  BasicSeqIter& operator=(const BasicSeqIter&) = delete;
+
+  BasicSeqIter(BasicSeqIter&& other) noexcept
+      : cur_(other.cur_), end_(other.end_), f_(std::move(other.f_)) {
+    if (cur_ == end_) {
+      Construct(&result_, std::move(other.result_));
+    } else {
+      Construct(&state_, std::move(other.state_));
+    }
+  }
+  BasicSeqIter& operator=(BasicSeqIter&& other) noexcept {
+    cur_ = other.cur_;
+    end_ = other.end_;
+    if (cur_ == end_) {
+      Construct(&result_, std::move(other.result_));
+    } else {
+      Construct(&state_, std::move(other.state_));
+    }
+    return *this;
+  }
+
+  Poll<Wrapped> operator()() {
+    if (cur_ == end_) {
+      return std::move(result_);
+    }
+    return PollNonEmpty();
+  }
+
+ private:
+  Poll<Wrapped> PollNonEmpty() {
+    Poll<Wrapped> r = state_();
+    if (absl::holds_alternative<Pending>(r)) return r;
+    return Traits<Wrapped>::template CheckResultAndRunNext<Wrapped>(
+        std::move(absl::get<Wrapped>(r)), [this](Wrapped arg) -> Poll<Wrapped> {
+          auto next = cur_;
+          ++next;
+          if (next == end_) {
+            return std::move(arg);
+          }
+          cur_ = next;
+          state_.~State();
+          Construct(&state_,
+                    Traits<Wrapped>::CallSeqFactory(f_, *cur_, std::move(arg)));
+          return PollNonEmpty();
+        });
+  }
+
+  Iter cur_;
+  const Iter end_;
+  GPR_NO_UNIQUE_ADDRESS F f_;
+  union {
+    GPR_NO_UNIQUE_ADDRESS State state_;
+    GPR_NO_UNIQUE_ADDRESS Arg result_;
+  };
+};
+
 }  // namespace promise_detail
 }  // namespace grpc_core
 

data/src/core/lib/promise/detail/promise_factory.h

@@ -88,7 +88,7 @@ class Curried {
       : f_(std::forward<F>(f)), arg_(std::forward<Arg>(arg)) {}
   Curried(const F& f, Arg&& arg) : f_(f), arg_(std::forward<Arg>(arg)) {}
   using Result = decltype(std::declval<F>()(std::declval<Arg>()));
-  Result operator()() { return f_(arg_); }
+  Result operator()() { return f_(std::move(arg_)); }
 
  private:
   GPR_NO_UNIQUE_ADDRESS F f_;

data/src/core/lib/promise/detail/status.h

@@ -39,11 +39,12 @@ inline absl::Status IntoStatus(absl::Status* status) {
 }
 
 }  // namespace promise_detail
-}  // namespace grpc_core
 
 // Return true if the status represented by the argument is ok, false if not.
 // By implementing this function for other, non-absl::Status types, those types
 // can participate in TrySeq as result types that affect control flow.
 inline bool IsStatusOk(const absl::Status& status) { return status.ok(); }
 
+}  // namespace grpc_core
+
 #endif  // GRPC_CORE_LIB_PROMISE_DETAIL_STATUS_H

data/src/core/lib/promise/intra_activity_waiter.h

@@ -0,0 +1,49 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_PROMISE_INTRA_ACTIVITY_WAITER_H
+#define GRPC_CORE_LIB_PROMISE_INTRA_ACTIVITY_WAITER_H
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/poll.h"
+
+namespace grpc_core {
+
+// Helper type to track wakeups between objects in the same activity.
+// Can be fairly fast as no ref counting or locking needs to occur.
+class IntraActivityWaiter {
+ public:
+  // Register for wakeup, return Pending(). If state is not ready to proceed,
+  // Promises should bottom out here.
+  Pending pending() {
+    waiting_ = true;
+    return Pending();
+  }
+  // Wake the activity
+  void Wake() {
+    if (waiting_) {
+      waiting_ = false;
+      Activity::current()->ForceImmediateRepoll();
+    }
+  }
+
+ private:
+  bool waiting_ = false;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_LIB_PROMISE_INTRA_ACTIVITY_WAITER_H

data/src/core/lib/promise/latch.h

@@ -0,0 +1,104 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_PROMISE_LATCH_H
+#define GRPC_CORE_LIB_PROMISE_LATCH_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/support/log.h>
+
+#include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/intra_activity_waiter.h"
+#include "src/core/lib/promise/poll.h"
+
+namespace grpc_core {
+
+// Latch provides a single set waitable object.
+// Initially the Latch is unset.
+// It can be waited upon by the Wait method, which produces a Promise that
+// resolves when the Latch is Set to a value of type T.
+template <typename T>
+class Latch {
+ public:
+  // This is the type of the promise returned by Wait.
+  class WaitPromise {
+   public:
+    Poll<T*> operator()() const {
+      if (latch_->has_value_) {
+        return &latch_->value_;
+      } else {
+        return latch_->waiter_.pending();
+      }
+    }
+
+   private:
+    friend class Latch;
+    explicit WaitPromise(Latch* latch) : latch_(latch) {}
+    Latch* latch_;
+  };
+
+  Latch() = default;
+  Latch(const Latch&) = delete;
+  Latch& operator=(const Latch&) = delete;
+  Latch(Latch&& other) noexcept
+      : value_(std::move(other.value_)), has_value_(other.has_value_) {
+#ifndef NDEBUG
+    GPR_DEBUG_ASSERT(!other.has_had_waiters_);
+#endif
+  }
+  Latch& operator=(Latch&& other) noexcept {
+#ifndef NDEBUG
+    GPR_DEBUG_ASSERT(!other.has_had_waiters_);
+#endif
+    value_ = std::move(other.value_);
+    has_value_ = other.has_value_;
+    return *this;
+  }
+
+  // Produce a promise to wait for a value from this latch.
+  WaitPromise Wait() {
+#ifndef NDEBUG
+    has_had_waiters_ = true;
+#endif
+    return WaitPromise(this);
+  }
+
+  // Set the value of the latch. Can only be called once.
+  void Set(T value) {
+    GPR_DEBUG_ASSERT(!has_value_);
+    value_ = std::move(value);
+    has_value_ = true;
+    waiter_.Wake();
+  }
+
+ private:
+  // The value stored (if has_value_ is true), otherwise some random value, we
+  // don't care.
+  // Why not absl::optional<>? Writing things this way lets us compress
+  // has_value_ with waiter_ and leads to some significant memory savings for
+  // some scenarios.
+  GPR_NO_UNIQUE_ADDRESS T value_;
+  // True if we have a value set, false otherwise.
+  bool has_value_ = false;
+#ifndef NDEBUG
+  // Has this latch ever had waiters.
+  bool has_had_waiters_ = false;
+#endif
+  IntraActivityWaiter waiter_;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_LIB_PROMISE_LATCH_H