@nxtedition/rocksdb 8.2.0 → 8.2.2

package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h

@@ -53,7 +53,9 @@ class BlockBasedTableBuilder : public TableBuilder {
   ~BlockBasedTableBuilder();
 
   // Add key,value to the table being constructed.
-  // REQUIRES: key is after any previously added key according to comparator.
+  // REQUIRES: Unless key has type kTypeRangeDeletion, key is after any
+  //           previously added non-kTypeRangeDeletion key according to
+  //           comparator.
   // REQUIRES: Finish(), Abandon() have not been called
   void Add(const Slice& key, const Slice& value) override;
 
@@ -89,6 +91,10 @@ class BlockBasedTableBuilder : public TableBuilder {
   // is enabled.
   uint64_t EstimatedFileSize() const override;
 
+  // Get the size of the "tail" part of a SST file. "Tail" refers to
+  // all blocks after data blocks till the end of the SST file.
+  uint64_t GetTailSize() const override;
+
   bool NeedCompact() const override;
 
   // Get table properties

package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc

@@ -443,11 +443,8 @@ void BlockBasedTableFactory::InitializeOptions() {
     table_options_.block_cache.reset();
   } else if (table_options_.block_cache == nullptr) {
     LRUCacheOptions co;
-    co.capacity = 8 << 20;
-    // It makes little sense to pay overhead for mid-point insertion while the
-    // block size is only 8MB.
-    co.high_pri_pool_ratio = 0.0;
-    co.low_pri_pool_ratio = 0.0;
+    // 32MB, the recommended minimum size for 64 shards, to reduce contention
+    co.capacity = 32 << 20;
     table_options_.block_cache = NewLRUCache(co);
   }
   if (table_options_.block_size_deviation < 0 ||
@@ -570,7 +567,8 @@ Status BlockBasedTableFactory::NewTableReader(
   return BlockBasedTable::Open(
       ro, table_reader_options.ioptions, table_reader_options.env_options,
       table_options_, table_reader_options.internal_comparator, std::move(file),
-      file_size, table_reader, table_reader_cache_res_mgr_,
+      file_size, table_reader_options.block_protection_bytes_per_key,
+      table_reader, table_reader_options.tail_size, table_reader_cache_res_mgr_,
       table_reader_options.prefix_extractor, prefetch_index_and_filter_in_cache,
       table_reader_options.skip_filters, table_reader_options.level,
       table_reader_options.immortal, table_reader_options.largest_seqno,

package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h

@@ -28,6 +28,9 @@ class BlockBasedTableBuilder;
 class RandomAccessFileReader;
 class WritableFileWriter;
 
+// TODO: deprecate this class as it can be replaced with
+// `FileMetaData::tail_size`
+//
 // A class used to track actual bytes written from the tail in the recent SST
 // file opens, and provide a suggestion for following open.
 class TailPrefetchStats {

package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc

@@ -26,10 +26,22 @@ void BlockBasedTableIterator::SeekImpl(const Slice* target,
 
   is_out_of_bound_ = false;
   is_at_first_key_from_index_ = false;
-  if (target && !CheckPrefixMayMatch(*target, IterDirection::kForward)) {
+  seek_stat_state_ = kNone;
+  bool filter_checked = false;
+  if (target &&
+      !CheckPrefixMayMatch(*target, IterDirection::kForward, &filter_checked)) {
     ResetDataIter();
+    RecordTick(table_->GetStatistics(), is_last_level_
+                                            ? LAST_LEVEL_SEEK_FILTERED
+                                            : NON_LAST_LEVEL_SEEK_FILTERED);
     return;
   }
+  if (filter_checked) {
+    seek_stat_state_ = kFilterUsed;
+    RecordTick(table_->GetStatistics(), is_last_level_
+                                            ? LAST_LEVEL_SEEK_FILTER_MATCH
+                                            : NON_LAST_LEVEL_SEEK_FILTER_MATCH);
+  }
 
   bool need_seek_index = true;
   if (block_iter_points_to_real_block_ && block_iter_.Valid()) {
@@ -125,12 +137,23 @@ void BlockBasedTableIterator::SeekImpl(const Slice* target,
 void BlockBasedTableIterator::SeekForPrev(const Slice& target) {
   is_out_of_bound_ = false;
   is_at_first_key_from_index_ = false;
+  seek_stat_state_ = kNone;
+  bool filter_checked = false;
   // For now totally disable prefix seek in auto prefix mode because we don't
   // have logic
-  if (!CheckPrefixMayMatch(target, IterDirection::kBackward)) {
+  if (!CheckPrefixMayMatch(target, IterDirection::kBackward, &filter_checked)) {
     ResetDataIter();
+    RecordTick(table_->GetStatistics(), is_last_level_
+                                            ? LAST_LEVEL_SEEK_FILTERED
+                                            : NON_LAST_LEVEL_SEEK_FILTERED);
     return;
   }
+  if (filter_checked) {
+    seek_stat_state_ = kFilterUsed;
+    RecordTick(table_->GetStatistics(), is_last_level_
+                                            ? LAST_LEVEL_SEEK_FILTER_MATCH
+                                            : NON_LAST_LEVEL_SEEK_FILTER_MATCH);
+  }
 
   SavePrevIndexValue();
 
@@ -185,6 +208,7 @@ void BlockBasedTableIterator::SeekForPrev(const Slice& target) {
 void BlockBasedTableIterator::SeekToLast() {
   is_out_of_bound_ = false;
   is_at_first_key_from_index_ = false;
+  seek_stat_state_ = kNone;
   SavePrevIndexValue();
   index_iter_->SeekToLast();
   if (!index_iter_->Valid()) {
@@ -266,6 +290,14 @@ void BlockBasedTableIterator::InitDataBlock() {
         /*for_compaction=*/is_for_compaction, /*async_read=*/false, s);
     block_iter_points_to_real_block_ = true;
     CheckDataBlockWithinUpperBound();
+    if (!is_for_compaction &&
+        (seek_stat_state_ & kDataBlockReadSinceLastSeek) == 0) {
+      RecordTick(table_->GetStatistics(), is_last_level_
+                                              ? LAST_LEVEL_SEEK_DATA
+                                              : NON_LAST_LEVEL_SEEK_DATA);
+      seek_stat_state_ = static_cast<SeekStatState>(
+          seek_stat_state_ | kDataBlockReadSinceLastSeek | kReportOnUseful);
+    }
   }
 }
 
@@ -320,6 +352,15 @@ void BlockBasedTableIterator::AsyncInitDataBlock(bool is_first_pass) {
   }
   block_iter_points_to_real_block_ = true;
   CheckDataBlockWithinUpperBound();
+
+  if (!is_for_compaction &&
+      (seek_stat_state_ & kDataBlockReadSinceLastSeek) == 0) {
+    RecordTick(table_->GetStatistics(), is_last_level_
+                                            ? LAST_LEVEL_SEEK_DATA
+                                            : NON_LAST_LEVEL_SEEK_DATA);
+    seek_stat_state_ = static_cast<SeekStatState>(
+        seek_stat_state_ | kDataBlockReadSinceLastSeek | kReportOnUseful);
+  }
   async_read_in_progress_ = false;
 }
 

package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h

@@ -41,7 +41,8 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
         block_iter_points_to_real_block_(false),
         check_filter_(check_filter),
         need_upper_bound_check_(need_upper_bound_check),
-        async_read_in_progress_(false) {}
+        async_read_in_progress_(false),
+        is_last_level_(table->IsLastLevel()) {}
 
   ~BlockBasedTableIterator() {}
 
@@ -88,6 +89,18 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
     assert(!is_at_first_key_from_index_);
     assert(Valid());
 
+    if (seek_stat_state_ & kReportOnUseful) {
+      bool filter_used = (seek_stat_state_ & kFilterUsed) != 0;
+      RecordTick(
+          table_->GetStatistics(),
+          filter_used
+              ? (is_last_level_ ? LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH
+                                : NON_LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH)
+              : (is_last_level_ ? LAST_LEVEL_SEEK_DATA_USEFUL_NO_FILTER
+                                : NON_LAST_LEVEL_SEEK_DATA_USEFUL_NO_FILTER));
+      seek_stat_state_ = kDataBlockReadSinceLastSeek;
+    }
+
     return block_iter_.value();
   }
   Status status() const override {
@@ -204,12 +217,24 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
   // bound.
   // If the boundary key hasn't been checked against the upper bound,
   // kUnknown can be used.
-  enum class BlockUpperBound {
+  enum class BlockUpperBound : uint8_t {
     kUpperBoundInCurBlock,
     kUpperBoundBeyondCurBlock,
     kUnknown,
   };
 
+  // State bits for collecting stats on seeks and whether they returned useful
+  // results.
+  enum SeekStatState : uint8_t {
+    kNone = 0,
+    // Most recent seek checked prefix filter (or similar future feature)
+    kFilterUsed = 1 << 0,
+    // Already recorded that a data block was accessed since the last seek.
+    kDataBlockReadSinceLastSeek = 1 << 1,
+    // Have not yet recorded that a value() was accessed.
+    kReportOnUseful = 1 << 2,
+  };
+
   const BlockBasedTable* table_;
   const ReadOptions& read_options_;
   const InternalKeyComparator& icomp_;
@@ -240,6 +265,9 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
 
   bool async_read_in_progress_;
 
+  mutable SeekStatState seek_stat_state_ = SeekStatState::kNone;
+  bool is_last_level_;
+
   // If `target` is null, seek to first.
   void SeekImpl(const Slice* target, bool async_prefetch);
 
@@ -257,16 +285,18 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
   // we need to check and update data_block_within_upper_bound_ accordingly.
   void CheckDataBlockWithinUpperBound();
 
-  bool CheckPrefixMayMatch(const Slice& ikey, IterDirection direction) {
+  bool CheckPrefixMayMatch(const Slice& ikey, IterDirection direction,
+                           bool* filter_checked) {
     if (need_upper_bound_check_ && direction == IterDirection::kBackward) {
       // Upper bound check isn't sufficient for backward direction to
       // guarantee the same result as total order, so disable prefix
       // check.
       return true;
     }
-    if (check_filter_ && !table_->PrefixRangeMayMatch(
-                             ikey, read_options_, prefix_extractor_,
-                             need_upper_bound_check_, &lookup_context_)) {
+    if (check_filter_ &&
+        !table_->PrefixRangeMayMatch(ikey, read_options_, prefix_extractor_,
+                                     need_upper_bound_check_, &lookup_context_,
+                                     filter_checked)) {
       // TODO remember the iterator is invalidated because of prefix
       // match. This can avoid the upper level file iterator to falsely
       // believe the position is the end of the SST file and move to