grpc 1.50.0-x86_64-linux → 1.51.0-x86_64-linux

data/src/core/lib/event_engine/posix_engine/lockfree_event.cc

@@ -0,0 +1,267 @@
+// Copyright 2022 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/event_engine/posix_engine/lockfree_event.h"
+
+#include <stdlib.h>
+
+#include <atomic>
+#include <cstdint>
+
+#include "absl/status/status.h"
+
+#include <grpc/support/atm.h>
+#include <grpc/support/log.h>
+
+#include "src/core/lib/event_engine/posix_engine/event_poller.h"
+#include "src/core/lib/event_engine/posix_engine/posix_engine_closure.h"
+#include "src/core/lib/gprpp/status_helper.h"
+
+//  'state' holds the to call when the fd is readable or writable respectively.
+//    It can contain one of the following values:
+//      kClosureReady     : The fd has an I/O event of interest but there is no
+//                          closure yet to execute
+
+//      kClosureNotReady : The fd has no I/O event of interest
+
+//      closure ptr       : The closure to be executed when the fd has an I/O
+//                          event of interest
+
+//      shutdown_error | kShutdownBit :
+//                         'shutdown_error' field ORed with kShutdownBit.
+//                          This indicates that the fd is shutdown. Since all
+//                          memory allocations are word-aligned, the lower two
+//                          bits of the shutdown_error pointer are always 0. So
+//                          it is safe to OR these with kShutdownBit
+
+//    Valid state transitions:
+
+//    <closure ptr> <-----3------ kClosureNotReady -----1------->  kClosureReady
+//        |  |                         ^   |    ^                         |  |
+//        |  |                         |   |    |                         |  |
+//        |  +--------------4----------+   6    +---------2---------------+  |
+//        |                                |                                 |
+//        |                                v                                 |
+//        +-----5------->  [shutdown_error | kShutdownBit] <-------7---------+
+
+//     For 1, 4 : See SetReady() function
+//     For 2, 3 : See NotifyOn() function
+//     For 5,6,7: See SetShutdown() function
+
+namespace grpc_event_engine {
+namespace posix_engine {
+
+void LockfreeEvent::InitEvent() {
+  // Perform an atomic store to start the state machine.
+
+  // Note carefully that LockfreeEvent *MAY* be used whilst in a destroyed
+  // state, while a file descriptor is on a freelist. In such a state it may
+  // be SetReady'd, and so we need to perform an atomic operation here to
+  // ensure no races
+  state_.store(kClosureNotReady, std::memory_order_relaxed);
+}
+
+void LockfreeEvent::DestroyEvent() {
+  intptr_t curr;
+  do {
+    curr = state_.load(std::memory_order_relaxed);
+    if (curr & kShutdownBit) {
+      grpc_core::internal::StatusFreeHeapPtr(curr & ~kShutdownBit);
+    } else {
+      GPR_ASSERT(curr == kClosureNotReady || curr == kClosureReady);
+    }
+    // we CAS in a shutdown, no error value here. If this event is interacted
+    // with post-deletion (see the note in the constructor) we want the bit
+    // pattern to prevent error retention in a deleted object
+  } while (!state_.compare_exchange_strong(curr, kShutdownBit,
+                                           std::memory_order_relaxed,
+                                           std::memory_order_relaxed));
+}
+
+void LockfreeEvent::NotifyOn(PosixEngineClosure* closure) {
+  // This load needs to be an acquire load because this can be a shutdown
+  // error that we might need to reference. Adding acquire semantics makes
+  // sure that the shutdown error has been initialized properly before us
+  // referencing it. The load() needs to be performed only once before entry
+  // into the loop. This is because if any of the compare_exchange_strong
+  // operations inside the loop return false, they automatically update curr
+  // with the new value. So it doesn't need to be loaded again.
+  intptr_t curr = state_.load(std::memory_order_acquire);
+  while (true) {
+    switch (curr) {
+      case kClosureNotReady: {
+        // kClosureNotReady -> <closure>.
+
+        // We're guaranteed by API that there's an acquire barrier before here,
+        // so there's no need to double-dip and this can be a release-only.
+
+        // The release itself pairs with the acquire half of a set_ready full
+        // barrier.
+        if (state_.compare_exchange_strong(
+                curr, reinterpret_cast<intptr_t>(closure),
+                std::memory_order_release, std::memory_order_relaxed)) {
+          return;  // Successful. Return
+        }
+
+        break;  // retry
+      }
+
+      case kClosureReady: {
+        // Change the state to kClosureNotReady. Schedule the closure if
+        // successful. If not, the state most likely transitioned to shutdown.
+        // We should retry.
+
+        // This can be a no-barrier cas since the state is being transitioned to
+        // kClosureNotReady; set_ready and set_shutdown do not schedule any
+        // closure when transitioning out of CLOSURE_NO_READY state (i.e there
+        // is no other code that needs to 'happen-after' this)
+        if (state_.compare_exchange_strong(curr, kClosureNotReady,
+                                           std::memory_order_relaxed,
+                                           std::memory_order_relaxed)) {
+          scheduler_->Run(closure);
+          return;  // Successful. Return.
+        }
+        break;  // retry
+      }
+
+      default: {
+        // 'curr' is either a closure or the fd is shutdown(in which case 'curr'
+        // contains a pointer to the shutdown-error). If the fd is shutdown,
+        // schedule the closure with the shutdown error
+        if ((curr & kShutdownBit) > 0) {
+          absl::Status shutdown_err =
+              grpc_core::internal::StatusGetFromHeapPtr(curr & ~kShutdownBit);
+          closure->SetStatus(shutdown_err);
+          scheduler_->Run(closure);
+          return;
+        }
+
+        // There is already a closure!. This indicates a bug in the code.
+        gpr_log(GPR_ERROR,
+                "LockfreeEvent::NotifyOn: notify_on called with a previous "
+                "callback still pending");
+        abort();
+      }
+    }
+  }
+
+  GPR_UNREACHABLE_CODE(return);
+}
+
+bool LockfreeEvent::SetShutdown(absl::Status shutdown_error) {
+  intptr_t status_ptr = grpc_core::internal::StatusAllocHeapPtr(shutdown_error);
+  gpr_atm new_state = status_ptr | kShutdownBit;
+  // The load() needs to be performed only once before entry
+  // into the loop. This is because if any of the compare_exchange_strong
+  // operations inside the loop return false, they automatically update curr
+  // with the new value. So it doesn't need to be loaded again.
+  intptr_t curr = state_.load(std::memory_order_acquire);
+
+  while (true) {
+    switch (curr) {
+      case kClosureReady:
+      case kClosureNotReady:
+        // Need a full barrier here so that the initial load in notify_on
+        // doesn't need a barrier
+        if (state_.compare_exchange_strong(curr, new_state,
+                                           std::memory_order_acq_rel,
+                                           std::memory_order_relaxed)) {
+          return true;  // early out
+        }
+        break;  // retry
+
+      default: {
+        // 'curr' is either a closure or the fd is already shutdown
+
+        // If fd is already shutdown, we are done.
+        if ((curr & kShutdownBit) > 0) {
+          grpc_core::internal::StatusFreeHeapPtr(status_ptr);
+          return false;
+        }
+
+        // Fd is not shutdown. Schedule the closure and move the state to
+        // shutdown state.
+        // Needs an acquire to pair with setting the closure (and get a
+        // happens-after on that edge), and a release to pair with anything
+        // loading the shutdown state.
+        if (state_.compare_exchange_strong(curr, new_state,
+                                           std::memory_order_acq_rel,
+                                           std::memory_order_relaxed)) {
+          auto closure = reinterpret_cast<PosixEngineClosure*>(curr);
+          closure->SetStatus(shutdown_error);
+          scheduler_->Run(closure);
+          return true;
+        }
+        // 'curr' was a closure but now changed to a different state. We will
+        // have to retry
+        break;
+      }
+    }
+  }
+  GPR_UNREACHABLE_CODE(return false);
+}
+
+void LockfreeEvent::SetReady() {
+  // The load() needs to be performed only once before entry
+  // into the loop. This is because if any of the compare_exchange_strong
+  // operations inside the loop return false, they automatically update curr
+  // with the new value. So it doesn't need to be loaded again.
+  intptr_t curr = state_.load(std::memory_order_acquire);
+  while (true) {
+    switch (curr) {
+      case kClosureReady: {
+        // Already ready. We are done here.
+        return;
+      }
+
+      case kClosureNotReady: {
+        // No barrier required as we're transitioning to a state that does not
+        // involve a closure
+        if (state_.compare_exchange_strong(curr, kClosureReady,
+                                           std::memory_order_relaxed,
+                                           std::memory_order_relaxed)) {
+          return;  // early out
+        }
+        break;  // retry
+      }
+
+      default: {
+        // 'curr' is either a closure or the fd is shutdown
+        if ((curr & kShutdownBit) > 0) {
+          // The fd is shutdown. Do nothing.
+          return;
+        } else if (state_.compare_exchange_strong(curr, kClosureNotReady,
+                                                  std::memory_order_acq_rel,
+                                                  std::memory_order_relaxed)) {
+          // Full cas: acquire pairs with this cas' release in the event of a
+          // spurious set_ready; release pairs with this or the acquire in
+          // notify_on (or set_shutdown)
+          auto closure = reinterpret_cast<PosixEngineClosure*>(curr);
+          closure->SetStatus(absl::OkStatus());
+          scheduler_->Run(closure);
+          return;
+        }
+        // else the state changed again (only possible by either a racing
+        // set_ready or set_shutdown functions. In both these cases, the
+        // closure would have been scheduled for execution. So we are done
+        // here
+        return;
+      }
+    }
+  }
+}
+
+}  // namespace posix_engine
+}  // namespace grpc_event_engine

data/src/core/lib/event_engine/posix_engine/lockfree_event.h

@@ -0,0 +1,73 @@
+// Copyright 2022 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_EVENT_ENGINE_POSIX_ENGINE_LOCKFREE_EVENT_H
+#define GRPC_CORE_LIB_EVENT_ENGINE_POSIX_ENGINE_LOCKFREE_EVENT_H
+#include <grpc/support/port_platform.h>
+
+#include <atomic>
+#include <cstdint>
+
+#include "absl/status/status.h"
+
+#include "src/core/lib/event_engine/posix_engine/posix_engine_closure.h"
+
+namespace grpc_event_engine {
+namespace posix_engine {
+
+class Scheduler;
+
+class LockfreeEvent {
+ public:
+  explicit LockfreeEvent(Scheduler* scheduler) : scheduler_(scheduler) {}
+
+  LockfreeEvent(const LockfreeEvent&) = delete;
+  LockfreeEvent& operator=(const LockfreeEvent&) = delete;
+
+  // These methods are used to initialize and destroy the internal state. These
+  // cannot be done in constructor and destructor because SetReady may be called
+  // when the event is destroyed and put in a freelist.
+  void InitEvent();
+  void DestroyEvent();
+
+  // Returns true if fd has been shutdown, false otherwise.
+  bool IsShutdown() const {
+    return (state_.load(std::memory_order_relaxed) & kShutdownBit) != 0;
+  }
+
+  // Schedules \a closure when the event is received (see SetReady()) or the
+  // shutdown state has been set. Note that the event may have already been
+  // received, in which case the closure would be scheduled immediately.
+  // If the shutdown state has already been set, then \a closure is scheduled
+  // with the shutdown error.
+  void NotifyOn(PosixEngineClosure* closure);
+
+  // Sets the shutdown state. If a closure had been provided by NotifyOn and has
+  // not yet been scheduled, it will be scheduled with \a shutdown_error.
+  bool SetShutdown(absl::Status shutdown_error);
+
+  // Signals that the event has been received.
+  void SetReady();
+
+ private:
+  enum State { kClosureNotReady = 0, kClosureReady = 2, kShutdownBit = 1 };
+
+  std::atomic<intptr_t> state_;
+  Scheduler* scheduler_;
+};
+
+}  // namespace posix_engine
+}  // namespace grpc_event_engine
+
+#endif  // GRPC_CORE_LIB_EVENT_ENGINE_POSIX_ENGINE_LOCKFREE_EVENT_H