@nxtedition/rocksdb 13.5.12 → 14.0.0

package/deps/rocksdb/rocksdb/include/rocksdb/cache.h

@@ -210,7 +210,15 @@ struct ShardedCacheOptions {
 // shard has its own LRU list for evictions. Each shard also has a mutex for
 // exclusive access during operations; even read operations need exclusive
 // access in order to update the LRU list. Mutex contention is usually low
-// with enough shards.
+// with enough shards. However,
+// * For a single hot block, there will be mutex contention even for reads
+// regardless of the number of shards.
+// * LRUCaches in the size of MBs instead of GBs can have shards small enough
+// that there is a random probability of some modest number of large blocks
+// (especially non-partitioned filters) thrashing a single cache shard.
+//
+// HYPERCLOCKCACHE IS NOW GENERALLY RECOMMENDED OVER LRUCACHE. See
+// HyperClockCacheOptions below.
 struct LRUCacheOptions : public ShardedCacheOptions {
   // Ratio of cache reserved for high-priority and low-priority entries,
   // respectively. (See Cache::Priority below more information on the levels.)
@@ -371,64 +379,50 @@ inline std::shared_ptr<SecondaryCache> NewCompressedSecondaryCache(
   return opts.MakeSharedSecondaryCache();
 }
 
-// HyperClockCache - A lock-free Cache alternative for RocksDB block cache
-// that offers much improved CPU efficiency vs. LRUCache under high parallel
-// load or high contention, with some caveats:
+// HyperClockCache (also known as HCC) - A lock-free Cache alternative for
+// RocksDB block cache that offers much improved CPU efficiency vs. LRUCache
+// under high parallel load or high contention. Additionally, HCC only uses
+// sharding for a modest performance boost, so can use much larger cache shards
+// than LRUCache, dramatically reducing the risk of thrashing in configurations
+// or work loads with some large blocks.
+//
+// HYPERCLOCKCACHE IS NOW GENERALLY RECOMMENDED OVER LRUCACHE
+//
+// Some caveats:
 // * Not a general Cache implementation: can only be used for
 // BlockBasedTableOptions::block_cache, which RocksDB uses in a way that is
 // compatible with HyperClockCache.
-// * Requires an extra tuning parameter: see estimated_entry_charge below.
-// Similarly, substantially changing the capacity with SetCapacity could
-// harm efficiency. -> EXPERIMENTAL: the tuning parameter can be set to 0
-// to find the appropriate balance automatically.
 // * Cache priorities are less aggressively enforced, which could cause
 // cache dilution from long range scans (unless they use fill_cache=false).
+// * In some configurations, depends on anonymous mmap support, available in
+// Linux, Windows and more.
+// * May have slightly lower (or slightly higher) cache hit rate vs. LRUCache,
+// because of the bounded counting-CLOCK eviction algorithm.
 //
 // See internal cache/clock_cache.h for full description.
 struct HyperClockCacheOptions : public ShardedCacheOptions {
-  // The estimated average `charge` associated with cache entries.
-  //
-  // EXPERIMENTAL: the field can be set to 0 to size the table dynamically
-  // and automatically. See also min_avg_entry_charge. This feature requires
-  // platform support for lazy anonymous memory mappings (incl Linux, Windows).
-  // Performance is very similar to choosing the best configuration parameter.
-  //
-  // PRODUCTION-TESTED: This is a critical configuration parameter for good
-  // performance, because having a table size that is fixed at creation time
-  // greatly reduces the required synchronization between threads.
-  // * If the estimate is substantially too low (e.g. less than half the true
-  // average) then metadata space overhead with be substantially higher (e.g.
-  // 200 bytes per entry rather than 100). With kFullChargeCacheMetadata, this
-  // can slightly reduce cache hit rates, and slightly reduce access times due
-  // to the larger working memory size.
-  // * If the estimate is substantially too high (e.g. 25% higher than the true
-  // average) then there might not be sufficient slots in the hash table for
-  // both efficient operation and capacity utilization (hit rate). The hyper
-  // cache will evict entries to prevent load factors that could dramatically
-  // affect lookup times, instead letting the hit rate suffer by not utilizing
-  // the full capacity.
+  // OPTIONAL: The estimated average `charge` associated with cache entries.
   //
-  // A reasonable choice is the larger of block_size and metadata_block_size.
-  // When WriteBufferManager (and similar) charge memory usage to the block
-  // cache, this can lead to the same effect as estimate being too low, which
-  // is better than the opposite. Therefore, the general recommendation is to
-  // assume that other memory charged to block cache could be negligible, and
-  // ignore it in making the estimate.
+  // When not provided (== 0, recommended and default), an HCC variant with a
+  // dynamically-growing table and generally good performance is used. This
+  // variant depends on anonymous mmaps so might not be available on all
+  // platforms.
   //
-  // The best parameter choice based on a cache in use is given by
-  // GetUsage() / GetOccupancyCount(), ignoring metadata overheads such as
-  // with kDontChargeCacheMetadata. More precisely with
-  // kFullChargeCacheMetadata is (GetUsage() - 64 * GetTableAddressCount()) /
-  // GetOccupancyCount(). However, when the average value size might vary
-  // (e.g. balance between metadata and data blocks in cache), it is better
-  // to estimate toward the lower side than the higher side.
+  // If the average "charge" (uncompressed block size) of block cache entries
+  // is reasonably predicted and provided here, the most efficient variant of
+  // HCC is used. Performance is degraded if the prediction is inaccurate.
+  // Prediction could be difficult or impossible with cache-charging features
+  // such as WriteBufferManager. The best parameter choice based on a cache
+  // in use is roughly given by GetUsage() / GetOccupancyCount(), though it is
+  // better to estimate toward the lower side than the higher side when the
+  // ratio might vary.
   size_t estimated_entry_charge;
 
-  // EXPERIMENTAL: When estimated_entry_charge == 0, this parameter establishes
-  // a promised lower bound on the average charge of all entries in the table,
-  // which is roughly the average uncompressed SST block size of block cache
-  // entries, typically > 4KB. The default should generally suffice with almost
-  // no cost. (This option is ignored for estimated_entry_charge > 0.)
+  // When estimated_entry_charge == 0, this parameter establishes a promised
+  // lower bound on the average charge of all entries in the table, which is
+  // roughly the average uncompressed SST block size of block cache entries,
+  // typically > 4KB. The default should generally suffice with almost no cost.
+  // (This option is ignored for estimated_entry_charge > 0.)
   //
   // More detail: The table for indexing cache entries will grow automatically
   // as needed, but a hard upper bound on that size is needed at creation time.
@@ -478,8 +472,8 @@ struct HyperClockCacheOptions : public ShardedCacheOptions {
   // keep operations very fast.
   int eviction_effort_cap = 30;
 
-  HyperClockCacheOptions(
-      size_t _capacity, size_t _estimated_entry_charge,
+  explicit HyperClockCacheOptions(
+      size_t _capacity, size_t _estimated_entry_charge = 0,
       int _num_shard_bits = -1, bool _strict_capacity_limit = false,
       std::shared_ptr<MemoryAllocator> _memory_allocator = nullptr,
       CacheMetadataChargePolicy _metadata_charge_policy =

package/deps/rocksdb/rocksdb/include/rocksdb/compaction_job_stats.h

@@ -24,9 +24,10 @@ struct CompactionJobStats {
   // the elapsed CPU time of this compaction in microseconds.
   uint64_t cpu_micros = 0;
 
-  // Used internally indicating whether a subcompaction's
-  // `num_input_records` is accurate.
-  bool has_num_input_records = false;
+  // True if `num_input_records` is accurate across all subcompactions.
+  // See CompactionIterator::must_count_input_entries for some implementation
+  // details why `num_input_records` may not be accurate.
+  bool has_accurate_num_input_records = true;
   // the number of compaction input records.
   uint64_t num_input_records = 0;
   // the number of blobs read from blob files

package/deps/rocksdb/rocksdb/include/rocksdb/compression_type.h

@@ -226,11 +226,15 @@ struct CompressionOptions {
   // The training data will be used to generate a dictionary of max_dict_bytes.
   uint32_t zstd_max_train_bytes = 0;
 
-  // Number of threads for parallel compression.
-  // Parallel compression is enabled only if threads > 1.
-  // THE FEATURE IS STILL EXPERIMENTAL
+  // Number of threads for parallel compression for each running flush or
+  // compaction job. Parallel compression is enabled only if threads > 1. Not
+  // recommended for lightweight compression algorithms such as Snappy, LZ4, and
+  // obviously kNoCompression because there is unlikely to be a throughput gain.
   //
-  // This option is valid only when BlockBasedTable is used.
+  // This option is valid only when BlockBasedTable is used and is disabled
+  // (sanitized to 1) with any of these:
+  // * User-defined index (UserDefinedIndexFactory)
+  // * partition_filters == true && decouple_partitioned_filters == false
   //
   // When parallel compression is enabled, SST size file sizes might be
   // more inflated compared to the target size, because more data of unknown
@@ -309,9 +313,7 @@ struct CompressionOptions {
     max_compressed_bytes_per_kb = static_cast<int>(1024.0 / min_ratio + 0.5);
   }
 
-#if __cplusplus >= 202002L
   bool operator==(const CompressionOptions& rhs) const = default;
-#endif
 };
 
 // See advanced_compression.h

package/deps/rocksdb/rocksdb/include/rocksdb/data_structure.h

@@ -10,8 +10,13 @@
 #include <array>
 #include <cstddef>
 #include <cstdint>
+#include <functional>
+#include <set>
+#include <variant>
 
+#include "rocksdb/comparator.h"
 #include "rocksdb/rocksdb_namespace.h"
+#include "rocksdb/slice.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -281,4 +286,486 @@ class ManagedPtr {
   Owner* owner_ = nullptr;
 };
 
+template <typename T, typename comp>
+class Interval;
+
+// The Interval Class is a generic class for holding a range, for example [2,
+// 4]. It can be used within the IntervalSet class, which is able to keep an
+// ordered, non-intersecting set of intervals within it.  Intervals can have
+// open-ended end points, (i.e., to infinity) for example [2,).
+template <typename T, typename comp = std::less<T>>
+class Interval {
+ public:
+  enum class End { INF };
+  struct CompareVariant {
+    comp comparator;
+    bool operator()(const std::variant<T, End>& a,
+                    const std::variant<T, End>& b) const {
+      if (std::holds_alternative<T>(a) && std::holds_alternative<T>(b)) {
+        return comparator(std::get<T>(a), std::get<T>(b));
+      }
+      if (std::holds_alternative<End>(a) && std::holds_alternative<End>(b)) {
+        return false;
+      }
+      if (std::holds_alternative<T>(a) && std::holds_alternative<End>(b)) {
+        return false;
+      }
+      return true;  // std::holds_alternative<End>(a) &&
+                    // std::holds_alternative<T>(b)
+    }
+  };
+
+  /* implicit */ Interval(const T& start, const T& end)
+      : start_(start), end_(end) {}
+  /* implicit */ Interval(const T& start) : start_(start), end_(End::INF) {}
+
+  // Add constructor that takes a pair
+  /* implicit */ Interval(const std::pair<T, T>& p)
+      : start_(p.first), end_(p.second) {}
+
+  T& start() { return start_; }
+
+  const T& start() const { return start_; }
+
+  bool has_end() const { return std::holds_alternative<T>(end_); }
+
+  T& end() { return std::get<T>(end_); }
+
+  const T& end() const { return std::get<T>(end_); }
+
+  // Support comparison with std::pair
+  bool operator==(const std::pair<T, T>& p) const {
+    return start_ == p.first && has_end() && end() == p.second;
+  }
+
+  // Support comparison with another Interval
+  bool operator==(const Interval& other) const {
+    if (start_ != other.start_) {
+      return false;
+    }
+
+    // Both have infinite end
+    if (!has_end() && !other.has_end()) {
+      return true;
+    }
+
+    // One has infinite end, the other doesn't
+    if (has_end() != other.has_end()) {
+      return false;
+    }
+
+    // Both have finite end
+    return end() == other.end();
+  }
+
+  // Support comparison with another Interval
+  bool operator<(const Interval& other) const {
+    return comparator(start_, other.start_);
+  }
+
+  bool Compare(const Interval& other) const {
+    return comparator(start_, other.start_);
+  }
+
+ private:
+  T start_;
+  std::variant<T, End> end_;
+  comp comparator;
+};
+
+// Specialized version of Interval for Slice
+template <>
+class Interval<Slice, Comparator> {
+ public:
+  enum class End { INF };
+
+  // Constructors that take a Comparator
+  /* implicit */ Interval(const Comparator* c, const Slice& start,
+                          const Slice& end)
+      : start_(start), end_(end), comparator_(c) {}
+
+  /* implicit */ Interval(const Comparator* c, const Slice& start)
+      : start_(start), end_(End::INF), comparator_(c) {}
+
+  // Constructor that takes a pair
+  /* implicit */ Interval(const Comparator* c, const std::pair<Slice, Slice>& p)
+      : start_(p.first), end_(p.second), comparator_(c) {}
+
+  Slice& start() { return start_; }
+
+  const Slice& start() const { return start_; }
+
+  bool has_end() const { return std::holds_alternative<Slice>(end_); }
+
+  Slice& end() { return std::get<Slice>(end_); }
+
+  const Slice& end() const { return std::get<Slice>(end_); }
+
+  // Support comparison with std::pair
+  bool operator==(const std::pair<Slice, Slice>& p) const {
+    return start_ == p.first && has_end() && end() == p.second;
+  }
+
+  // Support comparison with another Interval
+  bool operator==(const Interval& other) const {
+    if (comparator_->Compare(start_, other.start_) != 0) {
+      return false;
+    }
+
+    // Both have infinite end
+    if (!has_end() && !other.has_end()) {
+      return true;
+    }
+
+    // One has infinite end, the other doesn't
+    if (has_end() != other.has_end()) {
+      return false;
+    }
+
+    // Both have finite end
+    return comparator_->Compare(end(), other.end()) == 0;
+  }
+
+  // Support comparison with another Interval
+  bool operator<(const Interval& other) const {
+    return comparator_->Compare(start_, other.start_) < 0;
+  }
+
+  bool Compare(const Interval& other) const {
+    return comparator_->Compare(start_, other.start_) < 0;
+  }
+
+  const Comparator* GetComparator() const { return comparator_; }
+
+ private:
+  Slice start_;
+  std::variant<Slice, End> end_;
+  const Comparator* comparator_;
+
+  std::unordered_map<std::string, std::string> property_bag;
+};
+
+template <typename T, typename Compare = std::less<T>>
+struct CompareInterval {
+  bool operator()(const Interval<T, Compare>& a,
+                  const Interval<T, Compare>& b) const {
+    return a.Compare(b);
+  }
+};
+
+// IntervalSet will be used to represent a set of intervals (including unbounded
+// ones). The intervals are unique and disjoint. Intervals that are inserted
+// will merge with any range they intersect with.
+template <typename T, typename Compare = typename Interval<T>::CompareVariant>
+class IntervalSet {
+ public:
+  IntervalSet(Compare c = Compare()) : comp_(c) {}
+
+  void insert(Interval<T>&& i) { insertImpl(i); }
+
+  void insert(const T& start, const T& end) {
+    insertImpl(Interval<T>(start, end));
+  }
+
+  void insert(const T& start) { insertImpl(Interval<T>(start)); }
+
+  bool empty() const { return intervals_.empty(); }
+  void clear() { intervals_.clear(); }
+
+  auto begin() { return intervals_.begin(); }
+  auto end() { return intervals_.end(); }
+
+  auto cbegin() const { return intervals_.cbegin(); }
+  auto cend() const { return intervals_.cend(); }
+
+  size_t size() const { return intervals_.size(); }
+
+ private:
+  void insertImpl(const Interval<T>& i) {
+    // Skip empty intervals
+    if (i.has_end() && !comp_(i.start(), i.end()) &&
+        !comp_(i.end(), i.start())) {
+      return;
+    }
+
+    // First, check if there's any infinite interval that would contain this one
+    for (auto it = intervals_.begin(); it != intervals_.end(); ++it) {
+      if (!it->has_end() && !comp_(i.start(), it->start())) {
+        // This interval starts at or after an infinite interval
+        return;
+      }
+    }
+
+    // Find the position where the interval should be inserted
+    auto it = intervals_.begin();
+    while (it != intervals_.end() && comp_(it->start(), i.start())) {
+      ++it;
+    }
+
+    // Check if we need to consider the previous interval
+    if (it != intervals_.begin()) {
+      --it;
+      if (it->has_end() && comp_(it->end(), i.start())) {
+        ++it;
+      }
+    }
+
+    T new_start = i.start();
+    T new_end;
+    bool inf_end = false;
+    if (i.has_end()) {
+      new_end = i.end();
+    } else {
+      // For infinite end intervals, we need to merge all intervals that start
+      // after new_start
+      std::vector<decltype(it)> to_erase;
+      while (it != intervals_.end()) {
+        new_start = comp_(it->start(), new_start) ? it->start() : new_start;
+        to_erase.push_back(it++);
+      }
+
+      for (auto& eit : to_erase) {
+        intervals_.erase(eit);
+      }
+
+      // Insert the new interval with infinite end
+      intervals_.insert(Interval<T>(new_start));
+      return;
+    }
+
+    // For finite end intervals, proceed as before
+    std::vector<decltype(it)> to_erase;
+    while (it != intervals_.end() && !comp_(new_end, it->start())) {
+      if (it->has_end() && comp_(it->end(), new_start)) {
+        ++it;
+        continue;
+      }
+      new_start = comp_(it->start(), new_start) ? it->start() : new_start;
+      if (it->has_end()) {
+        new_end = comp_(new_end, it->end()) ? it->end() : new_end;
+      } else {
+        // If we encounter an interval with infinite end, our new interval also
+        // becomes infinite
+        inf_end = true;
+        break;
+      }
+      to_erase.push_back(it++);
+    }
+
+    // Check for any infinite intervals that start after this one
+    auto check_it = it;
+    while (check_it != intervals_.end()) {
+      if (!check_it->has_end()) {
+        inf_end = true;
+        to_erase.push_back(check_it);
+      }
+      ++check_it;
+    }
+
+    for (auto& eit : to_erase) {
+      intervals_.erase(eit);
+    }
+
+    if (inf_end) {
+      intervals_.insert(Interval<T>(new_start));
+    } else {
+      intervals_.insert(Interval<T>(new_start, new_end));
+    }
+  }
+
+  std::set<Interval<T>, CompareInterval<T>> intervals_;
+  Compare comp_;
+};
+
+// Specialization of IntervalSet for Slices.
+// Slice based intervals can have properties attached to them. This is used to
+// push down properties in the MultiScan API.  We accept two modes with
+// IntervalSet, fail_on_intersect, which imposes a restriction that inserted
+// ranges will be disjoint, this is needed when using properties. Insert will
+// fail if a range is found to not be disjoint. When fail_on_instersect is
+// false, the ranges will be merged.
+template <>
+class IntervalSet<Slice, Comparator> {
+ public:
+  explicit IntervalSet(const Comparator* c, bool fail_on_intersect = false)
+      : comp_(c), prop_(fail_on_intersect) {}
+
+  // Insert returns true if the interval was inserted. False indicates that the
+  // interval was not inserted, this could be do to an empty range OR that the
+  // IntervalSet is in with_properties mode and the interval overlaps with an
+  // existing interval.
+  bool insert(const Slice& start, const Slice& end) {
+    return insertImpl(Interval<Slice, Comparator>(comp_, start, end));
+  }
+
+  // Insert returns true if the interval was inserted. False indicates that the
+  // interval was not inserted, this could be do to an empty range OR that the
+  // IntervalSet is in with_properties mode and the interval overlaps with an
+  // existing interval.
+  bool insert(const Slice& start) {
+    // Create an interval with infinite end
+    Interval<Slice, Comparator> interval(comp_, start);
+    return insertImpl(interval);
+  }
+
+  bool insert(Interval<Slice, Comparator>&& i) { return insertImpl(i); }
+
+  bool empty() const { return intervals_.empty(); }
+  void clear() { intervals_.clear(); }
+
+  auto begin() { return intervals_.begin(); }
+  auto end() { return intervals_.end(); }
+
+  auto cbegin() const { return intervals_.cbegin(); }
+  auto cend() const { return intervals_.cend(); }
+
+  size_t size() const { return intervals_.size(); }
+
+ private:
+  // Custom comparator for finding intervals in the vector
+  struct IntervalComparator {
+    explicit IntervalComparator(const Comparator* comp) : comp_(comp) {}
+
+    bool operator()(const Interval<Slice, Comparator>& a,
+                    const Interval<Slice, Comparator>& b) const {
+      return comp_->Compare(a.start(), b.start()) < 0;
+    }
+
+    const Comparator* comp_;
+  };
+
+  typename std::vector<Interval<Slice, Comparator>>::iterator findPosition(
+      const Interval<Slice, Comparator>& interval) {
+    // Find the position where the new interval should be inserted
+    for (auto it = intervals_.begin(); it != intervals_.end(); ++it) {
+      if (comp_->Compare(it->start(), interval.start()) >= 0) {
+        return it;
+      }
+    }
+    return intervals_.end();
+  }
+
+  bool insertImpl(const Interval<Slice, Comparator>& i) {
+    // Skip empty intervals
+    if (i.has_end() && comp_->Compare(i.start(), i.end()) >= 0) {
+      return false;
+    }
+
+    // Find the position where this interval would be inserted
+    // This also checks if the interval is completely contained within an
+    // existing one
+    auto it = findPosition(i);
+
+    // Check if we need to merge with previous interval
+    if (it != intervals_.begin()) {
+      auto prev = it - 1;
+      if (prev->has_end() && comp_->Compare(prev->end(), i.start()) < 0) {
+        // No overlap with previous interval
+      } else {
+        // There is overlap, adjust iterator to include previous interval
+        if (prop_) {
+          return false;
+        }
+        it = prev;
+      }
+    }
+
+    Slice new_start = i.start();
+    Slice new_end;
+    bool inf_end = false;
+
+    if (i.has_end()) {
+      new_end = i.end();
+    } else {
+      // For infinite end intervals, we need to merge all intervals that start
+      // after new_start
+      auto erase_start = it;
+      while (it != intervals_.end()) {
+        if (comp_->Compare(it->start(), new_start) < 0) {
+          if (prop_) {
+            return false;
+          }
+          new_start = it->start();
+        }
+        ++it;
+      }
+
+      // Erase all intervals from erase_start to end
+      if (erase_start != intervals_.end()) {
+        if (prop_) {
+          return false;
+        }
+        intervals_.erase(erase_start, intervals_.end());
+      }
+
+      // Insert the new interval with infinite end
+      Interval<Slice, Comparator> new_interval(comp_, new_start);
+      auto pos = findPosition(new_interval);
+      intervals_.insert(pos, new_interval);
+      return true;
+    }
+
+    // For finite end intervals, find all overlapping intervals
+    auto erase_start = it;
+    auto erase_end = it;
+
+    while (it != intervals_.end() &&
+           comp_->Compare(new_end, it->start()) >= 0) {
+      if (it->has_end() && comp_->Compare(it->end(), new_start) < 0) {
+        // No overlap
+        ++it;
+        erase_end = it;
+        continue;
+      }
+
+      if (comp_->Compare(it->start(), new_start) < 0) {
+        new_start = it->start();
+      }
+
+      if (it->has_end()) {
+        if (comp_->Compare(new_end, it->end()) < 0) {
+          new_end = it->end();
+        }
+      } else {
+        // If we encounter an interval with infinite end, our new interval also
+        // becomes infinite
+        inf_end = true;
+        erase_end = intervals_.end();
+        break;
+      }
+
+      ++it;
+      erase_end = it;
+    }
+
+    // Check for any infinite intervals that start after this one
+    while (it != intervals_.end()) {
+      if (!it->has_end()) {
+        inf_end = true;
+        erase_end = intervals_.end();
+        break;
+      }
+      ++it;
+    }
+
+    // Erase all merged intervals
+    if (erase_start != erase_end) {
+      intervals_.erase(erase_start, erase_end);
+    }
+
+    // Insert the new merged interval
+    Interval<Slice, Comparator> new_interval =
+        inf_end ? Interval<Slice, Comparator>(comp_, new_start)
+                : Interval<Slice, Comparator>(comp_, new_start, new_end);
+
+    auto pos = findPosition(new_interval);
+    intervals_.insert(pos, new_interval);
+    return true;
+  }
+
+  const Comparator* comp_;
+  std::vector<Interval<Slice, Comparator>> intervals_;
+  bool prop_;
+};
+
 }  // namespace ROCKSDB_NAMESPACE