@nxtedition/rocksdb 15.4.1 → 16.0.0

package/deps/rocksdb/rocksdb/table/block_based/block.h

@@ -155,8 +155,11 @@ class BlockReadAmpBitmap {
 class Block {
  public:
   // Initialize the block with the specified contents.
+  // If restart_interval is provided (non-zero), it will be stored directly
+  // instead of being calculated later.
   explicit Block(BlockContents&& contents, size_t read_amp_bytes_per_bit = 0,
-                 Statistics* statistics = nullptr);
+                 Statistics* statistics = nullptr,
+                 uint32_t restart_interval = 1);
   // No copying allowed
   Block(const Block&) = delete;
   void operator=(const Block&) = delete;
@@ -167,8 +170,9 @@ class Block {
   const char* data() const { return contents_.data.data(); }
   // The additional memory space taken by the block data.
   size_t usable_size() const { return contents_.usable_size(); }
-  uint32_t NumRestarts() const;
+  uint32_t NumRestarts() const { return num_restarts_; }
   bool own_bytes() const { return contents_.own_bytes(); }
+  bool IsUniform() const { return is_uniform_; }
 
   BlockBasedTableOptions::DataBlockIndexType IndexType() const;
 
@@ -233,13 +237,19 @@ class Block {
   // It is determined by IndexType property of the table.
   // `user_defined_timestamps_persisted` controls whether a min timestamp is
   // padded while key is being parsed from the block.
+  // `index_block_search_type` controls which search algorithm to use when
+  // reading the index block. kBinary uses binary search, while
+  // kInterpolation uses interpolation search which can be faster
+  // for uniformly distributed keys.
   IndexBlockIter* NewIndexIterator(
       const Comparator* raw_ucmp, SequenceNumber global_seqno,
       IndexBlockIter* iter, Statistics* stats, bool total_order_seek,
       bool have_first_key, bool key_includes_seq, bool value_is_full,
       bool block_contents_pinned = false,
       bool user_defined_timestamps_persisted = true,
-      BlockPrefixIndex* prefix_index = nullptr);
+      BlockPrefixIndex* prefix_index = nullptr,
+      BlockBasedTableOptions::BlockSearchType index_block_search_type =
+          BlockBasedTableOptions::kBinary);
 
   // Report an approximation of how much memory has been used.
   size_t ApproximateMemoryUsage() const;
@@ -273,12 +283,22 @@ class Block {
     ProtectionInfo64().ProtectKV(key, value).Encode(checksum_len, checksum_ptr);
   }
 
+  bool HasSeparatedKV() const { return values_section_ != nullptr; }
+
   const char* TEST_GetKVChecksum() const { return kv_checksum_; }
 
  private:
+  // Returns a detailed error status by re-processing the footer.
+  // Should only be called when size() == 0 (error marker).
+  Status GetCorruptionStatus() const;
+
   BlockContents contents_;
-  uint32_t restart_offset_;  // Offset in data_ of restart array
+  // Normal state: offset in data_ of restart array.
+  // Error state (size()==0): original data size if footer decode failed,
+  //   otherwise 0. Used by GetCorruptionStatus() to re-decode footer.
+  uint32_t restart_offset_;
   uint32_t num_restarts_;
+  bool is_uniform_{false};
   std::unique_ptr<BlockReadAmpBitmap> read_amp_bitmap_;
   char* kv_checksum_{nullptr};
   uint32_t checksum_size_{0};
@@ -286,6 +306,9 @@ class Block {
   uint32_t block_restart_interval_{0};
   uint8_t protection_bytes_per_key_{0};
   DataBlockHashIndex data_block_hash_index_;
+
+  // Pointer to values section, nullptr if not using separated KV
+  const char* values_section_{nullptr};
 };
 
 // A `BlockIter` iterates over the entries in a `Block`'s data buffer. The
@@ -325,7 +348,7 @@ class BlockIter : public InternalIteratorBase<TValue> {
     assert(!pinned_iters_mgr_ || !pinned_iters_mgr_->PinningEnabled());
 
     data_ = nullptr;
-    current_ = restarts_;
+    current_ = GetKeysEndOffset();
     status_ = s;
 
     // Call cleanup callbacks.
@@ -334,8 +357,10 @@ class BlockIter : public InternalIteratorBase<TValue> {
 
   bool Valid() const override {
     // When status_ is not ok, iter should be invalid.
-    assert(status_.ok() || current_ >= restarts_);
-    return current_ < restarts_;
+    auto key_end = GetKeysEndOffset();
+    assert(status_.ok() || current_ >= key_end);
+    auto valid = current_ < key_end;
+    return valid;
   }
 
   void SeekToFirst() override final {
@@ -426,10 +451,15 @@ class BlockIter : public InternalIteratorBase<TValue> {
   Cache::Handle* cache_handle() { return cache_handle_; }
 
  protected:
-  std::unique_ptr<InternalKeyComparator> icmp_;
+  InternalKeyComparator icmp_;
   const char* data_;       // underlying block contents
   uint32_t num_restarts_;  // Number of uint32_t entries in restart array
 
+  const char* values_section_;
+  // Slice of current entry in the data section. Does not contain value if
+  // values_section_ exists
+  Slice entry_;
+
   // Index of restart block in which current_ or current_-1 falls
   uint32_t restart_index_;
   uint32_t restarts_;  // Offset of restart array (list of fixed32)
@@ -460,6 +490,8 @@ class BlockIter : public InternalIteratorBase<TValue> {
 
   // Per key-value checksum related states
   const char* kv_checksum_;
+  // Index of the next entry to be parsed (used for checksum verification
+  // and to determine if we're at a restart point for separated KV storage)
   int32_t cur_entry_idx_;
   uint32_t block_restart_interval_;
   uint8_t protection_bytes_per_key_;
@@ -502,7 +534,11 @@ class BlockIter : public InternalIteratorBase<TValue> {
     uint32_t count = (num_restarts_ - 1) * block_restart_interval;
     // Add number of keys from the last restart interval
     SeekToRestartPoint(num_restarts_ - 1);
-    while (NextEntryOffset() < restarts_ && status_.ok()) {
+    // For separated KV storage, keys end at values_section_, not at restarts_
+    uint32_t keys_end = values_section_
+                            ? static_cast<uint32_t>(values_section_ - data_)
+                            : restarts_;
+    while (NextEntryOffset() < keys_end && status_.ok()) {
       NextImpl();
       ++count;
     }
@@ -514,7 +550,7 @@ class BlockIter : public InternalIteratorBase<TValue> {
   // Sets raw_key_, value_ to the current parsed key and value.
   // Sets restart_index_ to point to the restart interval that contains
   // the current key.
-  template <typename DecodeEntryFunc>
+  template <typename DecodeEntryFunc, bool StrictCheck = false>
   inline bool ParseNextKey(bool* is_shared);
 
   // protection_bytes_per_key, kv_checksum, and block_restart_interval
@@ -523,22 +559,20 @@ class BlockIter : public InternalIteratorBase<TValue> {
                       uint32_t restarts, uint32_t num_restarts,
                       SequenceNumber global_seqno, bool block_contents_pinned,
                       bool user_defined_timestamp_persisted,
-
                       uint8_t protection_bytes_per_key, const char* kv_checksum,
-                      uint32_t block_restart_interval) {
+                      uint32_t block_restart_interval,
+                      const char* values_section) {
     assert(data_ == nullptr);  // Ensure it is called only once
     assert(num_restarts > 0);  // Ensure the param is valid
-
-    icmp_ = std::make_unique<InternalKeyComparator>(raw_ucmp);
+    assert(raw_ucmp != nullptr);
+    icmp_ = InternalKeyComparator(raw_ucmp);
     data_ = data;
     restarts_ = restarts;
     num_restarts_ = num_restarts;
-    current_ = restarts_;
     restart_index_ = num_restarts_;
+    entry_ = Slice();
     global_seqno_ = global_seqno;
-    if (raw_ucmp != nullptr) {
-      ts_sz_ = raw_ucmp->timestamp_size();
-    }
+    ts_sz_ = raw_ucmp->timestamp_size();
     pad_min_timestamp_ = ts_sz_ > 0 && !user_defined_timestamp_persisted;
     block_contents_pinned_ = block_contents_pinned;
     cache_handle_ = nullptr;
@@ -552,10 +586,13 @@ class BlockIter : public InternalIteratorBase<TValue> {
     assert((protection_bytes_per_key == 0 && kv_checksum == nullptr) ||
            (protection_bytes_per_key > 0 && kv_checksum != nullptr &&
             (block_restart_interval > 0 || num_restarts == 1)));
+
+    values_section_ = values_section;
+    current_ = GetKeysEndOffset();
   }
 
   void CorruptionError(const std::string& error_msg = "bad entry in block") {
-    current_ = restarts_;
+    current_ = GetKeysEndOffset();
     restart_index_ = num_restarts_;
     status_ = Status::Corruption(error_msg);
     raw_key_.Clear();
@@ -571,14 +608,18 @@ class BlockIter : public InternalIteratorBase<TValue> {
     CorruptionError(error_msg);
   }
 
-  void UpdateRawKeyAndMaybePadMinTimestamp(const Slice& key) {
+  void UpdateRawKeyAndMaybePadMinTimestamp(IterKey& raw_key, const Slice& key) {
     if (pad_min_timestamp_) {
-      raw_key_.SetKeyWithPaddedMinTimestamp(key, ts_sz_);
+      raw_key.SetKeyWithPaddedMinTimestamp(key, ts_sz_);
     } else {
-      raw_key_.SetKey(key, false /* copy */);
+      raw_key.SetKey(key, false /* copy */);
     }
   }
 
+  void UpdateRawKeyAndMaybePadMinTimestamp(const Slice& key) {
+    UpdateRawKeyAndMaybePadMinTimestamp(raw_key_, key);
+  }
+
   // Must be called every time a key is found that needs to be returned to user,
   // and may be called when no key is found (as a no-op). Updates `key_`,
   // `key_buf_`, and `key_pinned_` with info about the found key.
@@ -618,18 +659,31 @@ class BlockIter : public InternalIteratorBase<TValue> {
     }
   }
 
-  // Returns the result of `Comparator::Compare()`, where the appropriate
-  // comparator is used for the block contents, the LHS argument is the current
-  // key with global seqno applied, and the RHS argument is `other`.
-  int CompareCurrentKey(const Slice& other) {
+  // Compares two keys using the appropriate comparator for the block contents.
+  // Uses user comparator when the block stores user keys, otherwise uses the
+  // internal key comparator. When global_seqno is not disabled, applies it to
+  // the LHS key for comparison.
+  int CompareKey(const Slice& a, const Slice& b) const {
+    assert(icmp_.user_comparator() != nullptr);
     if (raw_key_.IsUserKey()) {
       assert(global_seqno_ == kDisableGlobalSequenceNumber);
-      return icmp_->user_comparator()->Compare(raw_key_.GetUserKey(), other);
+      return icmp_.user_comparator()->Compare(a, b);
     } else if (global_seqno_ == kDisableGlobalSequenceNumber) {
-      return icmp_->Compare(raw_key_.GetInternalKey(), other);
+      return icmp_.Compare(a, b);
+    }
+    return icmp_.Compare(a, global_seqno_, b, kDisableGlobalSequenceNumber);
+  }
+
+  int CompareKey(const IterKey& a, const Slice& b) const {
+    if (a.IsUserKey()) {
+      return CompareKey(a.GetUserKey(), b);
     }
-    return icmp_->Compare(raw_key_.GetInternalKey(), global_seqno_, other,
-                          kDisableGlobalSequenceNumber);
+    return CompareKey(a.GetInternalKey(), b);
+  }
+
+  // Compares the current key (with global seqno applied) against `other`.
+  int CompareCurrentKey(const Slice& other) const {
+    return CompareKey(raw_key_, other);
   }
 
  private:
@@ -644,28 +698,49 @@ class BlockIter : public InternalIteratorBase<TValue> {
   // Return the offset in data_ just past the end of the current entry.
   inline uint32_t NextEntryOffset() const {
     // NOTE: We don't support blocks bigger than 2GB
-    return static_cast<uint32_t>((value_.data() + value_.size()) - data_);
+    return static_cast<uint32_t>((entry_.data() + entry_.size()) - data_);
+  }
+
+  // Return the offset where the keys section ends.
+  // For separated KV storage, this is the start of the values section.
+  // Otherwise, it's the start of the restart array.
+  inline uint32_t GetKeysEndOffset() const {
+    return values_section_ ? static_cast<uint32_t>(values_section_ - data_)
+                           : restarts_;
   }
 
   uint32_t GetRestartPoint(uint32_t index) const {
     assert(index < num_restarts_);
-    return DecodeFixed32(data_ + restarts_ + index * sizeof(uint32_t));
+    uint32_t offset =
+        DecodeFixed32(data_ + restarts_ + index * sizeof(uint32_t));
+    assert(!values_section_ || offset <= values_section_ - data_);
+    return offset;
   }
 
   void SeekToRestartPoint(uint32_t index) {
     raw_key_.Clear();
     restart_index_ = index;
-    // current_ will be fixed by ParseNextKey();
+    // Set to one before the first entry so ParseNextKey() increments to correct
+    // position
+    cur_entry_idx_ = static_cast<int32_t>(index * block_restart_interval_) - 1;
 
-    // ParseNextKey() starts at the end of value_, so set value_ accordingly
+    // ParseNextKey() starts at the end of entry_, so set value_ accordingly
     uint32_t offset = GetRestartPoint(index);
-    value_ = Slice(data_ + offset, 0);
+    entry_ = Slice(data_ + offset, 0);
   }
 
  protected:
   template <typename DecodeKeyFunc>
-  inline bool BinarySeek(const Slice& target, uint32_t* index,
-                         bool* is_index_key_result);
+  inline bool GetRestartKey(uint32_t index, Slice* key);
+
+  template <typename DecodeKeyFunc>
+  inline bool BinarySeekRestartPointIndex(const Slice& target, uint32_t* index,
+                                          bool* is_index_key_result);
+
+  template <typename DecodeKeyFunc>
+  inline bool InterpolationSeekRestartPointIndex(const Slice& target,
+                                                 uint32_t* index,
+                                                 bool* is_index_key_result);
 
   // Find the first key in restart interval `index` that is >= `target`.
   // If there is no such key, iterator is positioned at the first key in
@@ -692,11 +767,11 @@ class DataBlockIter final : public BlockIter<Slice> {
                   bool user_defined_timestamps_persisted,
                   DataBlockHashIndex* data_block_hash_index,
                   uint8_t protection_bytes_per_key, const char* kv_checksum,
-                  uint32_t block_restart_interval) {
+                  uint32_t block_restart_interval, const char* values_section) {
     InitializeBase(raw_ucmp, data, restarts, num_restarts, global_seqno,
                    block_contents_pinned, user_defined_timestamps_persisted,
                    protection_bytes_per_key, kv_checksum,
-                   block_restart_interval);
+                   block_restart_interval, values_section);
     raw_key_.SetIsUserKey(false);
     read_amp_bitmap_ = read_amp_bitmap;
     last_bitmap_offset_ = current_ + 1;
@@ -753,9 +828,11 @@ class DataBlockIter final : public BlockIter<Slice> {
   // last `current_` value we report to read-amp bitmp
   mutable uint32_t last_bitmap_offset_;
   struct CachedPrevEntry {
-    explicit CachedPrevEntry(uint32_t _offset, const char* _key_ptr,
-                             size_t _key_offset, size_t _key_size, Slice _value)
+    explicit CachedPrevEntry(uint32_t _offset, uint32_t _entry_size,
+                             const char* _key_ptr, size_t _key_offset,
+                             size_t _key_size, Slice _value)
         : offset(_offset),
+          entry_size(_entry_size),
           key_ptr(_key_ptr),
           key_offset(_key_offset),
           key_size(_key_size),
@@ -763,6 +840,8 @@ class DataBlockIter final : public BlockIter<Slice> {
 
     // offset of entry in block
     uint32_t offset;
+    // size of entry (for NextEntryOffset calculation)
+    uint32_t entry_size;
     // Pointer to key data in block (nullptr if key is delta-encoded)
     const char* key_ptr;
     // offset of key in prev_entries_keys_buff_ (0 if key_ptr is not nullptr)
@@ -791,14 +870,15 @@ class MetaBlockIter final : public BlockIter<Slice> {
   MetaBlockIter() : BlockIter() { raw_key_.SetIsUserKey(true); }
   void Initialize(const char* data, uint32_t restarts, uint32_t num_restarts,
                   bool block_contents_pinned, uint8_t protection_bytes_per_key,
-                  const char* kv_checksum, uint32_t block_restart_interval) {
+                  const char* kv_checksum, uint32_t block_restart_interval,
+                  const char* values_section) {
     // Initializes the iterator with a BytewiseComparator and
     // the raw key being a user key.
     InitializeBase(BytewiseComparator(), data, restarts, num_restarts,
                    kDisableGlobalSequenceNumber, block_contents_pinned,
                    /* user_defined_timestamps_persisted */ true,
                    protection_bytes_per_key, kv_checksum,
-                   block_restart_interval);
+                   block_restart_interval, values_section);
     raw_key_.SetIsUserKey(true);
   }
 
@@ -829,22 +909,24 @@ class IndexBlockIter final : public BlockIter<IndexValue> {
   // format.
   // value_is_full, default true, means that no delta encoding is
   // applied to values.
-  void Initialize(const Comparator* raw_ucmp, const char* data,
-                  uint32_t restarts, uint32_t num_restarts,
-                  SequenceNumber global_seqno, BlockPrefixIndex* prefix_index,
-                  bool have_first_key, bool key_includes_seq,
-                  bool value_is_full, bool block_contents_pinned,
-                  bool user_defined_timestamps_persisted,
-                  uint8_t protection_bytes_per_key, const char* kv_checksum,
-                  uint32_t block_restart_interval) {
+  void Initialize(
+      const Comparator* raw_ucmp, const char* data, uint32_t restarts,
+      uint32_t num_restarts, SequenceNumber global_seqno,
+      BlockPrefixIndex* prefix_index, bool have_first_key,
+      bool key_includes_seq, bool value_is_full, bool block_contents_pinned,
+      bool user_defined_timestamps_persisted, uint8_t protection_bytes_per_key,
+      const char* kv_checksum, uint32_t block_restart_interval,
+      const char* values_section,
+      BlockBasedTableOptions::BlockSearchType index_block_search_type) {
     InitializeBase(raw_ucmp, data, restarts, num_restarts,
                    kDisableGlobalSequenceNumber, block_contents_pinned,
                    user_defined_timestamps_persisted, protection_bytes_per_key,
-                   kv_checksum, block_restart_interval);
+                   kv_checksum, block_restart_interval, values_section);
     raw_key_.SetIsUserKey(!key_includes_seq);
     prefix_index_ = prefix_index;
     value_delta_encoded_ = !value_is_full;
     have_first_key_ = have_first_key;
+    index_search_type_ = index_block_search_type;
     if (have_first_key_ && global_seqno != kDisableGlobalSequenceNumber) {
       global_seqno_state_.reset(new GlobalSeqnoState(global_seqno));
     } else {
@@ -894,7 +976,7 @@ class IndexBlockIter final : public BlockIter<IndexValue> {
 
   void SeekForPrevImpl(const Slice&) override {
     assert(false);
-    current_ = restarts_;
+    current_ = GetKeysEndOffset();
     restart_index_ = num_restarts_;
     status_ = Status::InvalidArgument(
         "RocksDB internal error: should never call SeekForPrev() on index "
@@ -939,6 +1021,10 @@ class IndexBlockIter final : public BlockIter<IndexValue> {
   // `pad_min_timestamp_` is true.
   std::string first_internal_key_with_ts_;
 
+  // The search algorithm to use when reading the index block.
+  BlockBasedTableOptions::BlockSearchType index_search_type_ =
+      BlockBasedTableOptions::kBinary;
+
   // Set *prefix_may_exist to false if no key possibly share the same prefix
   // as `target`. If not set, the result position should be the same as total
   // order Seek.
@@ -951,6 +1037,10 @@ class IndexBlockIter final : public BlockIter<IndexValue> {
                             bool* prefix_may_exist);
   inline int CompareBlockKey(uint32_t block_index, const Slice& target);
 
+  template <typename DecodeKeyFunc>
+  bool FindRestartPointForSeek(const Slice& seek_key, uint32_t* index,
+                               bool* skip_linear_scan);
+
   inline bool ParseNextIndexKey();
 
   // When value_delta_encoded_ is enabled it decodes the value which is assumed