libv8-node 18.13.0.0-aarch64-linux-musl → 19.9.0.0-aarch64-linux-musl

data/vendor/v8/include/cppgc/internal/write-barrier.h

@@ -8,9 +8,12 @@
 #include <cstddef>
 #include <cstdint>
 
+#include "cppgc/heap-handle.h"
 #include "cppgc/heap-state.h"
 #include "cppgc/internal/api-constants.h"
 #include "cppgc/internal/atomic-entry-flag.h"
+#include "cppgc/internal/base-page-handle.h"
+#include "cppgc/internal/member-storage.h"
 #include "cppgc/platform.h"
 #include "cppgc/sentinel-pointer.h"
 #include "cppgc/trace-trait.h"
@@ -18,6 +21,7 @@
 
 #if defined(CPPGC_CAGED_HEAP)
 #include "cppgc/internal/caged-heap-local-data.h"
+#include "cppgc/internal/caged-heap.h"
 #endif
 
 namespace cppgc {
@@ -40,16 +44,18 @@ class V8_EXPORT WriteBarrier final {
     kGenerational,
   };
 
+  enum class GenerationalBarrierType : uint8_t {
+    kPreciseSlot,
+    kPreciseUncompressedSlot,
+    kImpreciseSlot,
+  };
+
   struct Params {
     HeapHandle* heap = nullptr;
 #if V8_ENABLE_CHECKS
     Type type = Type::kNone;
 #endif  // !V8_ENABLE_CHECKS
 #if defined(CPPGC_CAGED_HEAP)
-    uintptr_t start = 0;
-    CagedHeapLocalData& caged_heap() const {
-      return *reinterpret_cast<CagedHeapLocalData*>(start);
-    }
     uintptr_t slot_offset = 0;
     uintptr_t value_offset = 0;
 #endif  // CPPGC_CAGED_HEAP
@@ -63,6 +69,9 @@ class V8_EXPORT WriteBarrier final {
   // Returns the required write barrier for a given `slot` and `value`.
   static V8_INLINE Type GetWriteBarrierType(const void* slot, const void* value,
                                             Params& params);
+  // Returns the required write barrier for a given `slot` and `value`.
+  static V8_INLINE Type GetWriteBarrierType(const void* slot, MemberStorage,
+                                            Params& params);
   // Returns the required write barrier for a given `slot`.
   template <typename HeapHandleCallback>
   static V8_INLINE Type GetWriteBarrierType(const void* slot, Params& params,
@@ -78,15 +87,13 @@ class V8_EXPORT WriteBarrier final {
   static V8_INLINE void SteeleMarkingBarrier(const Params& params,
                                              const void* object);
 #if defined(CPPGC_YOUNG_GENERATION)
+  template <GenerationalBarrierType>
   static V8_INLINE void GenerationalBarrier(const Params& params,
                                             const void* slot);
-  static V8_INLINE void GenerationalBarrierForSourceObject(
-      const Params& params, const void* inner_pointer);
 #else  // !CPPGC_YOUNG_GENERATION
+  template <GenerationalBarrierType>
   static V8_INLINE void GenerationalBarrier(const Params& params,
-                                            const void* slot) {}
-  static V8_INLINE void GenerationalBarrierForSourceObject(
-      const Params& params, const void* inner_pointer) {}
+                                            const void* slot){}
 #endif  // CPPGC_YOUNG_GENERATION
 
 #if V8_ENABLE_CHECKS
@@ -95,12 +102,10 @@ class V8_EXPORT WriteBarrier final {
   static void CheckParams(Type expected_type, const Params& params) {}
 #endif  // !V8_ENABLE_CHECKS
 
-  // The IncrementalOrConcurrentUpdater class allows cppgc internal to update
-  // |incremental_or_concurrent_marking_flag_|.
-  class IncrementalOrConcurrentMarkingFlagUpdater;
-  static bool IsAnyIncrementalOrConcurrentMarking() {
-    return incremental_or_concurrent_marking_flag_.MightBeEntered();
-  }
+  // The FlagUpdater class allows cppgc internal to update
+  // |write_barrier_enabled_|.
+  class FlagUpdater;
+  static bool IsEnabled() { return write_barrier_enabled_.MightBeEntered(); }
 
  private:
   WriteBarrier() = delete;
@@ -125,17 +130,23 @@ class V8_EXPORT WriteBarrier final {
   static CagedHeapLocalData& GetLocalData(HeapHandle&);
   static void GenerationalBarrierSlow(const CagedHeapLocalData& local_data,
                                       const AgeTable& age_table,
-                                      const void* slot, uintptr_t value_offset);
+                                      const void* slot, uintptr_t value_offset,
+                                      HeapHandle* heap_handle);
+  static void GenerationalBarrierForUncompressedSlotSlow(
+      const CagedHeapLocalData& local_data, const AgeTable& age_table,
+      const void* slot, uintptr_t value_offset, HeapHandle* heap_handle);
   static void GenerationalBarrierForSourceObjectSlow(
-      const CagedHeapLocalData& local_data, const void* object);
+      const CagedHeapLocalData& local_data, const void* object,
+      HeapHandle* heap_handle);
 #endif  // CPPGC_YOUNG_GENERATION
 
-  static AtomicEntryFlag incremental_or_concurrent_marking_flag_;
+  static AtomicEntryFlag write_barrier_enabled_;
 };
 
 template <WriteBarrier::Type type>
 V8_INLINE WriteBarrier::Type SetAndReturnType(WriteBarrier::Params& params) {
-  if (type == WriteBarrier::Type::kNone) return WriteBarrier::Type::kNone;
+  if constexpr (type == WriteBarrier::Type::kNone)
+    return WriteBarrier::Type::kNone;
 #if V8_ENABLE_CHECKS
   params.type = type;
 #endif  // !V8_ENABLE_CHECKS
@@ -152,6 +163,13 @@ class V8_EXPORT WriteBarrierTypeForCagedHeapPolicy final {
     return ValueModeDispatch<value_mode>::Get(slot, value, params, callback);
   }
 
+  template <WriteBarrier::ValueMode value_mode, typename HeapHandleCallback>
+  static V8_INLINE WriteBarrier::Type Get(const void* slot, MemberStorage value,
+                                          WriteBarrier::Params& params,
+                                          HeapHandleCallback callback) {
+    return ValueModeDispatch<value_mode>::Get(slot, value, params, callback);
+  }
+
   template <WriteBarrier::ValueMode value_mode, typename HeapHandleCallback>
   static V8_INLINE WriteBarrier::Type Get(const void* value,
                                           WriteBarrier::Params& params,
@@ -166,69 +184,77 @@ class V8_EXPORT WriteBarrierTypeForCagedHeapPolicy final {
   static V8_INLINE WriteBarrier::Type GetNoSlot(const void* value,
                                                 WriteBarrier::Params& params,
                                                 HeapHandleCallback) {
-    if (!TryGetCagedHeap(value, value, params)) {
-      return WriteBarrier::Type::kNone;
-    }
-    if (V8_UNLIKELY(params.caged_heap().is_incremental_marking_in_progress)) {
+    const bool within_cage = CagedHeapBase::IsWithinCage(value);
+    if (!within_cage) return WriteBarrier::Type::kNone;
+
+    // We know that |value| points either within the normal page or to the
+    // beginning of large-page, so extract the page header by bitmasking.
+    BasePageHandle* page =
+        BasePageHandle::FromPayload(const_cast<void*>(value));
+
+    HeapHandle& heap_handle = page->heap_handle();
+    if (V8_UNLIKELY(heap_handle.is_incremental_marking_in_progress())) {
       return SetAndReturnType<WriteBarrier::Type::kMarking>(params);
     }
+
     return SetAndReturnType<WriteBarrier::Type::kNone>(params);
   }
 
   template <WriteBarrier::ValueMode value_mode>
   struct ValueModeDispatch;
-
-  static V8_INLINE bool TryGetCagedHeap(const void* slot, const void* value,
-                                        WriteBarrier::Params& params) {
-    // TODO(chromium:1056170): Check if the null check can be folded in with
-    // the rest of the write barrier.
-    if (!value) return false;
-    params.start = reinterpret_cast<uintptr_t>(value) &
-                   ~(api_constants::kCagedHeapReservationAlignment - 1);
-    const uintptr_t slot_offset =
-        reinterpret_cast<uintptr_t>(slot) - params.start;
-    if (slot_offset > api_constants::kCagedHeapReservationSize) {
-      // Check if slot is on stack or value is sentinel or nullptr. This relies
-      // on the fact that kSentinelPointer is encoded as 0x1.
-      return false;
-    }
-    return true;
-  }
-
-  // Returns whether marking is in progress. If marking is not in progress
-  // sets the start of the cage accordingly.
-  //
-  // TODO(chromium:1056170): Create fast path on API.
-  static bool IsMarking(const HeapHandle&, WriteBarrier::Params&);
 };
 
 template <>
 struct WriteBarrierTypeForCagedHeapPolicy::ValueModeDispatch<
     WriteBarrier::ValueMode::kValuePresent> {
+  template <typename HeapHandleCallback>
+  static V8_INLINE WriteBarrier::Type Get(const void* slot,
+                                          MemberStorage storage,
+                                          WriteBarrier::Params& params,
+                                          HeapHandleCallback) {
+    if (V8_LIKELY(!WriteBarrier::IsEnabled()))
+      return SetAndReturnType<WriteBarrier::Type::kNone>(params);
+
+    return BarrierEnabledGet(slot, storage.Load(), params);
+  }
+
   template <typename HeapHandleCallback>
   static V8_INLINE WriteBarrier::Type Get(const void* slot, const void* value,
                                           WriteBarrier::Params& params,
                                           HeapHandleCallback) {
-#if !defined(CPPGC_YOUNG_GENERATION)
-    if (V8_LIKELY(!WriteBarrier::IsAnyIncrementalOrConcurrentMarking())) {
+    if (V8_LIKELY(!WriteBarrier::IsEnabled()))
       return SetAndReturnType<WriteBarrier::Type::kNone>(params);
-    }
-#endif  // !CPPGC_YOUNG_GENERATION
-    bool within_cage = TryGetCagedHeap(slot, value, params);
-    if (!within_cage) {
-      return WriteBarrier::Type::kNone;
-    }
-    if (V8_LIKELY(!params.caged_heap().is_incremental_marking_in_progress)) {
+
+    return BarrierEnabledGet(slot, value, params);
+  }
+
+ private:
+  static V8_INLINE WriteBarrier::Type BarrierEnabledGet(
+      const void* slot, const void* value, WriteBarrier::Params& params) {
+    const bool within_cage = CagedHeapBase::AreWithinCage(slot, value);
+    if (!within_cage) return WriteBarrier::Type::kNone;
+
+    // We know that |value| points either within the normal page or to the
+    // beginning of large-page, so extract the page header by bitmasking.
+    BasePageHandle* page =
+        BasePageHandle::FromPayload(const_cast<void*>(value));
+
+    HeapHandle& heap_handle = page->heap_handle();
+    if (V8_LIKELY(!heap_handle.is_incremental_marking_in_progress())) {
 #if defined(CPPGC_YOUNG_GENERATION)
-      params.heap = reinterpret_cast<HeapHandle*>(params.start);
-      params.slot_offset = reinterpret_cast<uintptr_t>(slot) - params.start;
-      params.value_offset = reinterpret_cast<uintptr_t>(value) - params.start;
+      if (!heap_handle.is_young_generation_enabled())
+        return WriteBarrier::Type::kNone;
+      params.heap = &heap_handle;
+      params.slot_offset = CagedHeapBase::OffsetFromAddress(slot);
+      params.value_offset = CagedHeapBase::OffsetFromAddress(value);
       return SetAndReturnType<WriteBarrier::Type::kGenerational>(params);
 #else   // !CPPGC_YOUNG_GENERATION
       return SetAndReturnType<WriteBarrier::Type::kNone>(params);
 #endif  // !CPPGC_YOUNG_GENERATION
     }
-    params.heap = reinterpret_cast<HeapHandle*>(params.start);
+
+    // Use marking barrier.
+    params.heap = &heap_handle;
     return SetAndReturnType<WriteBarrier::Type::kMarking>(params);
   }
 };
@@ -240,28 +266,28 @@ struct WriteBarrierTypeForCagedHeapPolicy::ValueModeDispatch<
   static V8_INLINE WriteBarrier::Type Get(const void* slot, const void*,
                                           WriteBarrier::Params& params,
                                           HeapHandleCallback callback) {
-#if defined(CPPGC_YOUNG_GENERATION)
+    if (V8_LIKELY(!WriteBarrier::IsEnabled()))
+      return SetAndReturnType<WriteBarrier::Type::kNone>(params);
+
     HeapHandle& handle = callback();
-    if (V8_LIKELY(!IsMarking(handle, params))) {
-      // params.start is populated by IsMarking().
+#if defined(CPPGC_YOUNG_GENERATION)
+    if (V8_LIKELY(!handle.is_incremental_marking_in_progress())) {
+      if (!handle.is_young_generation_enabled()) {
+        return WriteBarrier::Type::kNone;
+      }
       params.heap = &handle;
-      params.slot_offset = reinterpret_cast<uintptr_t>(slot) - params.start;
-      // params.value_offset stays 0.
-      if (params.slot_offset > api_constants::kCagedHeapReservationSize) {
-        // Check if slot is on stack.
+      // Check if slot is on stack.
+      if (V8_UNLIKELY(!CagedHeapBase::IsWithinCage(slot))) {
         return SetAndReturnType<WriteBarrier::Type::kNone>(params);
       }
+      params.slot_offset = CagedHeapBase::OffsetFromAddress(slot);
       return SetAndReturnType<WriteBarrier::Type::kGenerational>(params);
     }
-#else   // !CPPGC_YOUNG_GENERATION
-    if (V8_LIKELY(!WriteBarrier::IsAnyIncrementalOrConcurrentMarking())) {
-      return SetAndReturnType<WriteBarrier::Type::kNone>(params);
-    }
-    HeapHandle& handle = callback();
-    if (V8_UNLIKELY(!subtle::HeapState::IsMarking(handle))) {
+#else   // !defined(CPPGC_YOUNG_GENERATION)
+    if (V8_UNLIKELY(!handle.is_incremental_marking_in_progress())) {
       return SetAndReturnType<WriteBarrier::Type::kNone>(params);
     }
-#endif  // !CPPGC_YOUNG_GENERATION
+#endif  // !defined(CPPGC_YOUNG_GENERATION)
     params.heap = &handle;
     return SetAndReturnType<WriteBarrier::Type::kMarking>(params);
   }
@@ -278,6 +304,16 @@ class V8_EXPORT WriteBarrierTypeForNonCagedHeapPolicy final {
     return ValueModeDispatch<value_mode>::Get(slot, value, params, callback);
   }
 
+  template <WriteBarrier::ValueMode value_mode, typename HeapHandleCallback>
+  static V8_INLINE WriteBarrier::Type Get(const void* slot, MemberStorage value,
+                                          WriteBarrier::Params& params,
+                                          HeapHandleCallback callback) {
+    // `MemberStorage` will always be `RawPointer` for non-caged heap builds.
+    // Just convert to `void*` in this case.
+    return ValueModeDispatch<value_mode>::Get(slot, value.Load(), params,
+                                              callback);
+  }
+
   template <WriteBarrier::ValueMode value_mode, typename HeapHandleCallback>
   static V8_INLINE WriteBarrier::Type Get(const void* value,
                                           WriteBarrier::Params& params,
@@ -291,11 +327,6 @@ class V8_EXPORT WriteBarrierTypeForNonCagedHeapPolicy final {
   template <WriteBarrier::ValueMode value_mode>
   struct ValueModeDispatch;
 
-  // TODO(chromium:1056170): Create fast path on API.
-  static bool IsMarking(const void*, HeapHandle**);
-  // TODO(chromium:1056170): Create fast path on API.
-  static bool IsMarking(HeapHandle&);
-
   WriteBarrierTypeForNonCagedHeapPolicy() = delete;
 };
 
@@ -310,10 +341,16 @@ struct WriteBarrierTypeForNonCagedHeapPolicy::ValueModeDispatch<
     if (object <= static_cast<void*>(kSentinelPointer)) {
       return SetAndReturnType<WriteBarrier::Type::kNone>(params);
     }
-    if (V8_LIKELY(!WriteBarrier::IsAnyIncrementalOrConcurrentMarking())) {
+    if (V8_LIKELY(!WriteBarrier::IsEnabled())) {
       return SetAndReturnType<WriteBarrier::Type::kNone>(params);
     }
-    if (IsMarking(object, &params.heap)) {
+    // We know that |object| is within the normal page or in the beginning of a
+    // large page, so extract the page header by bitmasking.
+    BasePageHandle* page =
+        BasePageHandle::FromPayload(const_cast<void*>(object));
+
+    HeapHandle& heap_handle = page->heap_handle();
+    if (V8_LIKELY(heap_handle.is_incremental_marking_in_progress())) {
       return SetAndReturnType<WriteBarrier::Type::kMarking>(params);
     }
     return SetAndReturnType<WriteBarrier::Type::kNone>(params);
@@ -327,9 +364,9 @@ struct WriteBarrierTypeForNonCagedHeapPolicy::ValueModeDispatch<
   static V8_INLINE WriteBarrier::Type Get(const void*, const void*,
                                           WriteBarrier::Params& params,
                                           HeapHandleCallback callback) {
-    if (V8_UNLIKELY(WriteBarrier::IsAnyIncrementalOrConcurrentMarking())) {
+    if (V8_UNLIKELY(WriteBarrier::IsEnabled())) {
       HeapHandle& handle = callback();
-      if (IsMarking(handle)) {
+      if (V8_LIKELY(handle.is_incremental_marking_in_progress())) {
         params.heap = &handle;
         return SetAndReturnType<WriteBarrier::Type::kMarking>(params);
       }
@@ -345,6 +382,13 @@ WriteBarrier::Type WriteBarrier::GetWriteBarrierType(
                                                                params, []() {});
 }
 
+// static
+WriteBarrier::Type WriteBarrier::GetWriteBarrierType(
+    const void* slot, MemberStorage value, WriteBarrier::Params& params) {
+  return WriteBarrierTypePolicy::Get<ValueMode::kValuePresent>(slot, value,
+                                                               params, []() {});
+}
+
 // static
 template <typename HeapHandleCallback>
 WriteBarrier::Type WriteBarrier::GetWriteBarrierType(
@@ -397,34 +441,32 @@ void WriteBarrier::SteeleMarkingBarrier(const Params& params,
 }
 
 #if defined(CPPGC_YOUNG_GENERATION)
-// static
-void WriteBarrier::GenerationalBarrier(const Params& params, const void* slot) {
-  CheckParams(Type::kGenerational, params);
-
-  const CagedHeapLocalData& local_data = params.caged_heap();
-  const AgeTable& age_table = local_data.age_table;
-
-  // Bail out if the slot is in young generation.
-  if (V8_LIKELY(age_table.GetAge(params.slot_offset) == AgeTable::Age::kYoung))
-    return;
-
-  GenerationalBarrierSlow(local_data, age_table, slot, params.value_offset);
-}
 
 // static
-void WriteBarrier::GenerationalBarrierForSourceObject(
-    const Params& params, const void* inner_pointer) {
+template <WriteBarrier::GenerationalBarrierType type>
+void WriteBarrier::GenerationalBarrier(const Params& params, const void* slot) {
   CheckParams(Type::kGenerational, params);
 
-  const CagedHeapLocalData& local_data = params.caged_heap();
+  const CagedHeapLocalData& local_data = CagedHeapLocalData::Get();
   const AgeTable& age_table = local_data.age_table;
 
-  // Assume that if the first element is in young generation, the whole range is
-  // in young generation.
+  // Bail out if the slot (precise or imprecise) is in young generation.
   if (V8_LIKELY(age_table.GetAge(params.slot_offset) == AgeTable::Age::kYoung))
     return;
 
-  GenerationalBarrierForSourceObjectSlow(local_data, inner_pointer);
+  // Dispatch between different types of barriers.
+  // TODO(chromium:1029379): Consider reload local_data in the slow path to
+  // reduce register pressure.
+  if constexpr (type == GenerationalBarrierType::kPreciseSlot) {
+    GenerationalBarrierSlow(local_data, age_table, slot, params.value_offset,
+                            params.heap);
+  } else if constexpr (type ==
+                       GenerationalBarrierType::kPreciseUncompressedSlot) {
+    GenerationalBarrierForUncompressedSlotSlow(
+        local_data, age_table, slot, params.value_offset, params.heap);
+  } else {
+    GenerationalBarrierForSourceObjectSlow(local_data, slot, params.heap);
+  }
 }
 
 #endif  // !CPPGC_YOUNG_GENERATION

data/vendor/v8/include/cppgc/liveness-broker.h

@@ -7,6 +7,7 @@
 
 #include "cppgc/heap.h"
 #include "cppgc/member.h"
+#include "cppgc/sentinel-pointer.h"
 #include "cppgc/trace-trait.h"
 #include "v8config.h"  // NOLINT(build/include_directory)
 
@@ -44,24 +45,24 @@ class V8_EXPORT LivenessBroker final {
  public:
   template <typename T>
   bool IsHeapObjectAlive(const T* object) const {
-    // nullptr objects are considered alive to allow weakness to be used from
+    // - nullptr objects are considered alive to allow weakness to be used from
     // stack while running into a conservative GC. Treating nullptr as dead
-    // would mean that e.g. custom collectins could not be strongified on stack.
-    return !object ||
+    // would mean that e.g. custom collections could not be strongified on
+    // stack.
+    // - Sentinel pointers are also preserved in weakness and not cleared.
+    return !object || object == kSentinelPointer ||
            IsHeapObjectAliveImpl(
                TraceTrait<T>::GetTraceDescriptor(object).base_object_payload);
   }
 
   template <typename T>
   bool IsHeapObjectAlive(const WeakMember<T>& weak_member) const {
-    return (weak_member != kSentinelPointer) &&
-           IsHeapObjectAlive<T>(weak_member.Get());
+    return IsHeapObjectAlive<T>(weak_member.Get());
   }
 
   template <typename T>
   bool IsHeapObjectAlive(const UntracedMember<T>& untraced_member) const {
-    return (untraced_member != kSentinelPointer) &&
-           IsHeapObjectAlive<T>(untraced_member.Get());
+    return IsHeapObjectAlive<T>(untraced_member.Get());
   }
 
  private: