@nxtedition/rocksdb 15.4.1 → 16.0.0

package/deps/rocksdb/rocksdb/table/block_based/block_builder.cc

@@ -39,20 +39,64 @@
 
 #include <algorithm>
 #include <cassert>
+#include <cmath>
 
 #include "db/dbformat.h"
+#include "monitoring/statistics_impl.h"
 #include "rocksdb/comparator.h"
+#include "table/block_based/block_util.h"
 #include "table/block_based/data_block_footer.h"
 #include "util/coding.h"
 
 namespace ROCKSDB_NAMESPACE {
 
+namespace {
+
+// Tracks whether restart-point keys are uniformly distributed using Welford's
+// online algorithm to incrementally compute the coefficient of variation (CV)
+// of gaps between consecutive restart keys.
+class UniformDataTracker {
+ public:
+  void AddKey(uint64_t key_value) {
+    if (num_keys_ > 0) {
+      double gap = static_cast<double>(key_value - prev_key_value_);
+      size_t gap_count = num_keys_;
+      double delta = gap - mean_;
+      mean_ += delta / static_cast<double>(gap_count);
+      double delta2 = gap - mean_;
+      m2_ += delta * delta2;
+    }
+    prev_key_value_ = key_value;
+    num_keys_++;
+  }
+
+  // Returns the coefficient of variation (CV) of the key gaps, or -1.0 if
+  // there are not enough data points to compute it.
+  double GetCV() const {
+    size_t gap_count = num_keys_ > 0 ? num_keys_ - 1 : 0;
+    if (gap_count < 2 || mean_ <= 0) {
+      return -1.0;
+    }
+    return std::sqrt(m2_ / static_cast<double>(gap_count)) / mean_;
+  }
+
+ private:
+  uint64_t prev_key_value_ = 0;
+  size_t num_keys_ = 0;
+  double mean_ = 0;
+  double m2_ = 0;
+};
+
+}  // namespace
+
 BlockBuilder::BlockBuilder(
     int block_restart_interval, bool use_delta_encoding,
     bool use_value_delta_encoding,
     BlockBasedTableOptions::DataBlockIndexType index_type,
     double data_block_hash_table_util_ratio, size_t ts_sz,
-    bool persist_user_defined_timestamps, bool is_user_key)
+    bool persist_user_defined_timestamps, bool is_user_key,
+    bool use_separated_kv_storage, Statistics* statistics,
+    double uniform_cv_threshold)
     : block_restart_interval_(block_restart_interval),
       use_delta_encoding_(use_delta_encoding),
       use_value_delta_encoding_(use_value_delta_encoding),
@@ -60,7 +104,10 @@ BlockBuilder::BlockBuilder(
       is_user_key_(is_user_key),
       restarts_(1, 0),  // First restart point is at offset 0
       counter_(0),
-      finished_(false) {
+      finished_(false),
+      uniform_cv_threshold_(uniform_cv_threshold),
+      statistics_(statistics),
+      use_separated_kv_storage_(use_separated_kv_storage) {
   switch (index_type) {
     case BlockBasedTableOptions::kDataBlockBinarySearch:
       break;
@@ -72,20 +119,24 @@ BlockBuilder::BlockBuilder(
       assert(0);
   }
   assert(block_restart_interval_ >= 1);
-  estimate_ = sizeof(uint32_t) + sizeof(uint32_t);
+  estimate_ = sizeof(uint32_t) + sizeof(uint32_t) +
+              (use_separated_kv_storage_ ? sizeof(uint32_t) : 0);
 }
 
 void BlockBuilder::Reset() {
   buffer_.clear();
   restarts_.resize(1);  // First restart point is at offset 0
   assert(restarts_[0] == 0);
-  estimate_ = sizeof(uint32_t) + sizeof(uint32_t);
+  estimate_ = sizeof(uint32_t) + sizeof(uint32_t) +
+              (use_separated_kv_storage_ ? sizeof(uint32_t) : 0);
   counter_ = 0;
   finished_ = false;
   last_key_.clear();
   if (data_block_hash_index_builder_.Valid()) {
     data_block_hash_index_builder_.Reset();
   }
+  values_buffer_.clear();
+
 #ifndef NDEBUG
   add_with_last_key_called_ = false;
 #endif
@@ -116,6 +167,12 @@ size_t BlockBuilder::EstimateSizeAfterKV(const Slice& key,
     estimate += sizeof(uint32_t);  // a new restart entry.
   }
 
+  // For separated KV storage, value_offset varint is written at restart points
+  if (use_separated_kv_storage_ &&
+      (counter_ == 0 || counter_ >= block_restart_interval_)) {
+    estimate += VarintLength(values_buffer_.size());
+  }
+
   estimate += sizeof(int32_t);  // varint for shared prefix length.
   // Note: this is an imprecise estimate as we will have to encoded size, one
   // for shared key and one for non-shared key.
@@ -128,24 +185,34 @@ size_t BlockBuilder::EstimateSizeAfterKV(const Slice& key,
 }
 
 Slice BlockBuilder::Finish() {
+  bool is_uniform = ScanForUniformity();
+
   // Append restart array
+  size_t values_buffer_offset = buffer_.size();
+
+  if (use_separated_kv_storage_) {
+    buffer_.append(values_buffer_);
+  }
+
   for (size_t i = 0; i < restarts_.size(); i++) {
     PutFixed32(&buffer_, restarts_[i]);
   }
 
-  uint32_t num_restarts = static_cast<uint32_t>(restarts_.size());
-  BlockBasedTableOptions::DataBlockIndexType index_type =
-      BlockBasedTableOptions::kDataBlockBinarySearch;
+  DataBlockFooter footer;
+  footer.num_restarts = static_cast<uint32_t>(restarts_.size());
+  footer.index_type = BlockBasedTableOptions::kDataBlockBinarySearch;
+  footer.is_uniform = is_uniform;
   if (data_block_hash_index_builder_.Valid() &&
       CurrentSizeEstimate() <= kMaxBlockSizeSupportedByHashIndex) {
     data_block_hash_index_builder_.Finish(buffer_);
-    index_type = BlockBasedTableOptions::kDataBlockBinaryAndHash;
+    footer.index_type = BlockBasedTableOptions::kDataBlockBinaryAndHash;
   }
 
-  // footer is a packed format of data_block_index_type and num_restarts
-  uint32_t block_footer = PackIndexTypeAndNumRestarts(index_type, num_restarts);
-
-  PutFixed32(&buffer_, block_footer);
+  if (use_separated_kv_storage_) {
+    footer.separated_kv = true;
+    footer.values_section_offset = static_cast<uint32_t>(values_buffer_offset);
+  }
+  footer.EncodeTo(&buffer_);
   finished_ = true;
   return Slice(buffer_);
 }
@@ -221,28 +288,45 @@ inline void BlockBuilder::AddWithLastKeyImpl(const Slice& key,
   }
 
   const size_t non_shared = key_to_persist.size() - shared;
-
+  const size_t previous_value_offset = values_buffer_.size();
   if (use_value_delta_encoding_) {
-    // Add "<shared><non_shared>" to buffer_
-    PutVarint32Varint32(&buffer_, static_cast<uint32_t>(shared),
-                        static_cast<uint32_t>(non_shared));
+    if (use_separated_kv_storage_ && counter_ == 0) {
+      // Add "<shared><non_shared><value_offset>" to buffer_
+      PutVarint32(&buffer_, static_cast<uint32_t>(shared),
+                  static_cast<uint32_t>(non_shared),
+                  static_cast<uint32_t>(values_buffer_.size()));
+    } else {
+      // Add "<shared><non_shared>" to buffer_
+      PutVarint32(&buffer_, static_cast<uint32_t>(shared),
+                  static_cast<uint32_t>(non_shared));
+    }
   } else {
-    // Add "<shared><non_shared><value_size>" to buffer_
-    PutVarint32Varint32Varint32(&buffer_, static_cast<uint32_t>(shared),
-                                static_cast<uint32_t>(non_shared),
-                                static_cast<uint32_t>(value.size()));
+    if (use_separated_kv_storage_ && counter_ == 0) {
+      // Add "<shared><non_shared><value_size><value_offset>" to buffer_
+      PutVarint32(&buffer_, static_cast<uint32_t>(shared),
+                  static_cast<uint32_t>(non_shared),
+                  static_cast<uint32_t>(value.size()),
+                  static_cast<uint32_t>(values_buffer_.size()));
+    } else {
+      // Add "<shared><non_shared><value_size>" to buffer_
+      PutVarint32(&buffer_, static_cast<uint32_t>(shared),
+                  static_cast<uint32_t>(non_shared),
+                  static_cast<uint32_t>(value.size()));
+    }
   }
 
-  // Add string delta to buffer_ followed by value
+  // Add string delta to buffer_
   buffer_.append(key_to_persist.data() + shared, non_shared);
+
+  auto& values_buffer = use_separated_kv_storage_ ? values_buffer_ : buffer_;
   // Use value delta encoding only when the key has shared bytes. This would
   // simplify the decoding, where it can figure which decoding to use simply by
   // looking at the shared bytes size.
   if (shared != 0 && use_value_delta_encoding_) {
     assert(delta_value != nullptr);
-    buffer_.append(delta_value->data(), delta_value->size());
+    values_buffer.append(delta_value->data(), delta_value->size());
   } else {
-    buffer_.append(value.data(), value.size());
+    values_buffer.append(value.data(), value.size());
   }
 
   // TODO(yuzhangyu): make user defined timestamp work with block hash index.
@@ -256,7 +340,8 @@ inline void BlockBuilder::AddWithLastKeyImpl(const Slice& key,
   }
 
   counter_++;
-  estimate_ += buffer_.size() - buffer_size;
+  estimate_ += buffer_.size() - buffer_size + values_buffer_.size() -
+               previous_value_offset;
 }
 
 const Slice BlockBuilder::MaybeStripTimestampFromKey(std::string* key_buf,
@@ -272,4 +357,63 @@ const Slice BlockBuilder::MaybeStripTimestampFromKey(std::string* key_buf,
   }
   return stripped_key;
 }
+
+Slice BlockBuilder::GetRestartKey(uint32_t index, const char* limit) const {
+  assert(index < restarts_.size());
+  const char* p = buffer_.data() + restarts_[index];
+  uint32_t shared;
+  uint32_t non_shared;
+  // When separated KV storage is enabled, restart point entries include an
+  // extra value_offset varint that must be consumed to find the key delta.
+  uint32_t value_offset;
+  uint32_t* value_offset_ptr =
+      use_separated_kv_storage_ ? &value_offset : nullptr;
+  if (use_value_delta_encoding_) {
+    p = DecodeKeyV4()(p, limit, &shared, &non_shared, value_offset_ptr);
+  } else {
+    p = DecodeKey()(p, limit, &shared, &non_shared, value_offset_ptr);
+  }
+  assert(p != nullptr);
+  assert(shared == 0);
+  (void)shared;
+  return Slice(p, non_shared);
+}
+
+bool BlockBuilder::ScanForUniformity() const {
+  if (uniform_cv_threshold_ < 0 || restarts_.size() < 3) {
+    return false;
+  }
+
+  const char* limit = buffer_.data() + buffer_.size();
+
+  Slice first_key = GetRestartKey(0, limit);
+  Slice last_key =
+      GetRestartKey(static_cast<uint32_t>(restarts_.size() - 1), limit);
+
+  // Keys must be long enough for ReadBe64FromKey which strips internal bytes
+  if (!is_user_key_ && (first_key.size() < kNumInternalBytes ||
+                        last_key.size() < kNumInternalBytes)) {
+    return false;
+  }
+
+  size_t prefix_len = first_key.difference_offset(last_key);
+
+  UniformDataTracker tracker;
+  for (size_t i = 0; i < restarts_.size(); i++) {
+    Slice key = GetRestartKey(static_cast<uint32_t>(i), limit);
+    if (!is_user_key_ && key.size() < kNumInternalBytes) {
+      return false;
+    }
+    tracker.AddKey(ReadBe64FromKey(key, is_user_key_, prefix_len));
+  }
+
+  double cv = tracker.GetCV();
+  if (statistics_ != nullptr && cv >= 0) {
+    RecordInHistogram(statistics_, BLOCK_KEY_DISTRIBUTION_CV,
+                      static_cast<uint64_t>(cv * 10000));
+  }
+
+  return cv >= 0 && cv < uniform_cv_threshold_;
+}
+
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/table/block_based/block_builder.h

@@ -18,20 +18,22 @@
 
 namespace ROCKSDB_NAMESPACE {
 
+class Statistics;
+
 class BlockBuilder {
  public:
   BlockBuilder(const BlockBuilder&) = delete;
   void operator=(const BlockBuilder&) = delete;
 
-  explicit BlockBuilder(int block_restart_interval,
-                        bool use_delta_encoding = true,
-                        bool use_value_delta_encoding = false,
-                        BlockBasedTableOptions::DataBlockIndexType index_type =
-                            BlockBasedTableOptions::kDataBlockBinarySearch,
-                        double data_block_hash_table_util_ratio = 0.75,
-                        size_t ts_sz = 0,
-                        bool persist_user_defined_timestamps = true,
-                        bool is_user_key = false);
+  explicit BlockBuilder(
+      int block_restart_interval, bool use_delta_encoding = true,
+      bool use_value_delta_encoding = false,
+      BlockBasedTableOptions::DataBlockIndexType index_type =
+          BlockBasedTableOptions::kDataBlockBinarySearch,
+      double data_block_hash_table_util_ratio = 0.75, size_t ts_sz = 0,
+      bool persist_user_defined_timestamps = true, bool is_user_key = false,
+      bool use_separated_kv_storage = false, Statistics* statistics = nullptr,
+      double uniform_cv_threshold = -1.0);
 
   // Reset the contents as if the BlockBuilder was just constructed.
   void Reset();
@@ -90,6 +92,10 @@ class BlockBuilder {
                                  const Slice* const delta_value,
                                  bool skip_delta_encoding, size_t buffer_size);
 
+  bool ScanForUniformity() const;
+
+  Slice GetRestartKey(uint32_t index, const char* limit) const;
+
   inline const Slice MaybeStripTimestampFromKey(std::string* key_buf,
                                                 const Slice& key);
 
@@ -124,6 +130,16 @@ class BlockBuilder {
   bool finished_;  // Has Finish() been called?
   std::string last_key_;
   DataBlockHashIndexBuilder data_block_hash_index_builder_;
+  const double uniform_cv_threshold_;
+  Statistics* statistics_;
+
+  const bool use_separated_kv_storage_;  // When enabled, keys are stored first,
+                                         // followed by values in a separate
+                                         // section. Value offset is stored as
+                                         // varint only at restart points; for
+                                         // other entries, offset is computed
+                                         // as prev_offset + prev_length.
+  std::string values_buffer_;
 #ifndef NDEBUG
   bool add_with_last_key_called_ = false;
 #endif

package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc

@@ -11,15 +11,17 @@ namespace ROCKSDB_NAMESPACE {
 
 void BlockCreateContext::Create(std::unique_ptr<Block_kData>* parsed_out,
                                 BlockContents&& block) {
-  parsed_out->reset(new Block_kData(
-      std::move(block), table_options->read_amp_bytes_per_bit, statistics));
+  parsed_out->reset(new Block_kData(std::move(block),
+                                    table_options->read_amp_bytes_per_bit,
+                                    statistics, data_block_restart_interval));
   parsed_out->get()->InitializeDataBlockProtectionInfo(protection_bytes_per_key,
                                                        raw_ucmp);
 }
 void BlockCreateContext::Create(std::unique_ptr<Block_kIndex>* parsed_out,
                                 BlockContents&& block) {
   parsed_out->reset(new Block_kIndex(std::move(block),
-                                     /*read_amp_bytes_per_bit*/ 0, statistics));
+                                     /*read_amp_bytes_per_bit*/ 0, statistics,
+                                     index_block_restart_interval));
   parsed_out->get()->InitializeIndexBlockProtectionInfo(
       protection_bytes_per_key, raw_ucmp, index_value_is_full,
       index_has_first_key);
@@ -28,7 +30,8 @@ void BlockCreateContext::Create(
     std::unique_ptr<Block_kFilterPartitionIndex>* parsed_out,
     BlockContents&& block) {
   parsed_out->reset(new Block_kFilterPartitionIndex(
-      std::move(block), /*read_amp_bytes_per_bit*/ 0, statistics));
+      std::move(block), /*read_amp_bytes_per_bit*/ 0, statistics,
+      index_block_restart_interval));
   parsed_out->get()->InitializeIndexBlockProtectionInfo(
       protection_bytes_per_key, raw_ucmp, index_value_is_full,
       index_has_first_key);

package/deps/rocksdb/rocksdb/table/block_based/block_cache.h

@@ -85,7 +85,9 @@ struct BlockCreateContext : public Cache::CreateContext {
                      uint8_t _protection_bytes_per_key,
                      const Comparator* _raw_ucmp,
                      bool _index_value_is_full = false,
-                     bool _index_has_first_key = false)
+                     bool _index_has_first_key = false,
+                     uint32_t _data_block_restart_interval = 0,
+                     uint32_t _index_block_restart_interval = 0)
       : table_options(_table_options),
         ioptions(_ioptions),
         statistics(_statistics),
@@ -93,7 +95,9 @@ struct BlockCreateContext : public Cache::CreateContext {
         raw_ucmp(_raw_ucmp),
         protection_bytes_per_key(_protection_bytes_per_key),
         index_value_is_full(_index_value_is_full),
-        index_has_first_key(_index_has_first_key) {}
+        index_has_first_key(_index_has_first_key),
+        data_block_restart_interval(_data_block_restart_interval),
+        index_block_restart_interval(_index_block_restart_interval) {}
 
   const BlockBasedTableOptions* table_options = nullptr;
   const ImmutableOptions* ioptions = nullptr;
@@ -104,6 +108,9 @@ struct BlockCreateContext : public Cache::CreateContext {
   uint8_t protection_bytes_per_key = 0;
   bool index_value_is_full;
   bool index_has_first_key;
+  // Restart intervals from table properties (0 if not available)
+  uint32_t data_block_restart_interval = 0;
+  uint32_t index_block_restart_interval = 0;
 
   // For TypedCacheInterface
   template <typename TBlocklike>