@nxtedition/rocksdb 8.2.8 → 9.0.1

package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc

@@ -26,15 +26,22 @@ PartitionedFilterBlockBuilder::PartitionedFilterBlockBuilder(
     FilterBitsBuilder* filter_bits_builder, int index_block_restart_interval,
     const bool use_value_delta_encoding,
     PartitionedIndexBuilder* const p_index_builder,
-    const uint32_t partition_size)
+    const uint32_t partition_size, size_t ts_sz,
+    const bool persist_user_defined_timestamps)
     : FullFilterBlockBuilder(_prefix_extractor, whole_key_filtering,
                              filter_bits_builder),
-      index_on_filter_block_builder_(index_block_restart_interval,
-                                     true /*use_delta_encoding*/,
-                                     use_value_delta_encoding),
-      index_on_filter_block_builder_without_seq_(index_block_restart_interval,
-                                                 true /*use_delta_encoding*/,
-                                                 use_value_delta_encoding),
+      index_on_filter_block_builder_(
+          index_block_restart_interval, true /*use_delta_encoding*/,
+          use_value_delta_encoding,
+          BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */,
+          0.75 /* data_block_hash_table_util_ratio */, ts_sz,
+          persist_user_defined_timestamps, false /* is_user_key */),
+      index_on_filter_block_builder_without_seq_(
+          index_block_restart_interval, true /*use_delta_encoding*/,
+          use_value_delta_encoding,
+          BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */,
+          0.75 /* data_block_hash_table_util_ratio */, ts_sz,
+          persist_user_defined_timestamps, true /* is_user_key */),
       p_index_builder_(p_index_builder),
       keys_added_to_partition_(0),
       total_added_in_built_(0) {
@@ -270,7 +277,8 @@ BlockHandle PartitionedFilterBlockReader::GetFilterPartitionHandle(
       table()->get_rep()->get_global_seqno(BlockType::kFilterPartitionIndex),
       &iter, kNullStats, true /* total_order_seek */,
       false /* have_first_key */, index_key_includes_seq(),
-      index_value_is_full());
+      index_value_is_full(), false /* block_contents_pinned */,
+      user_defined_timestamps_persisted());
   iter.Seek(entry);
   if (UNLIKELY(!iter.Valid())) {
     // entry is larger than all the keys. However its prefix might still be
@@ -309,12 +317,12 @@ Status PartitionedFilterBlockReader::GetFilterPartitionBlock(
     read_options.read_tier = kBlockCacheTier;
   }
 
-  const Status s =
-      table()->RetrieveBlock(prefetch_buffer, read_options, fltr_blk_handle,
-                             UncompressionDict::GetEmptyDict(), filter_block,
-                             get_context, lookup_context,
-                             /* for_compaction */ false, /* use_cache */ true,
-                             /* async_read */ false);
+  const Status s = table()->RetrieveBlock(
+      prefetch_buffer, read_options, fltr_blk_handle,
+      UncompressionDict::GetEmptyDict(), filter_block, get_context,
+      lookup_context,
+      /* for_compaction */ false, /* use_cache */ true,
+      /* async_read */ false, /* use_block_cache_for_lookup */ true);
 
   return s;
 }
@@ -470,7 +478,8 @@ Status PartitionedFilterBlockReader::CacheDependencies(
       comparator->user_comparator(),
       rep->get_global_seqno(BlockType::kFilterPartitionIndex), &biter,
       kNullStats, true /* total_order_seek */, false /* have_first_key */,
-      index_key_includes_seq(), index_value_is_full());
+      index_key_includes_seq(), index_value_is_full(),
+      false /* block_contents_pinned */, user_defined_timestamps_persisted());
   // Index partitions are assumed to be consecuitive. Prefetch them all.
   // Read the first block offset
   biter.SeekToFirst();
@@ -488,14 +497,15 @@ Status PartitionedFilterBlockReader::CacheDependencies(
       tail_prefetch_buffer->GetPrefetchOffset() > prefetch_off) {
     rep->CreateFilePrefetchBuffer(
         0, 0, &prefetch_buffer, false /* Implicit autoreadahead */,
-        0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/);
+        0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/,
+        /*upper_bound_offset*/ 0, /*readaheadsize_cb*/ nullptr,
+        /*usage=*/FilePrefetchBufferUsage::kUnknown);
 
     IOOptions opts;
     s = rep->file->PrepareIOOptions(ro, opts);
     if (s.ok()) {
       s = prefetch_buffer->Prefetch(opts, rep->file.get(), prefetch_off,
-                                    static_cast<size_t>(prefetch_len),
-                                    ro.rate_limiter_priority);
+                                    static_cast<size_t>(prefetch_len));
     }
     if (!s.ok()) {
       return s;
@@ -512,7 +522,8 @@ Status PartitionedFilterBlockReader::CacheDependencies(
         prefetch_buffer ? prefetch_buffer.get() : tail_prefetch_buffer, ro,
         handle, UncompressionDict::GetEmptyDict(),
         /* for_compaction */ false, &block, nullptr /* get_context */,
-        &lookup_context, nullptr /* contents */, false);
+        &lookup_context, nullptr /* contents */, false,
+        /* use_block_cache_for_lookup */ true);
     if (!s.ok()) {
       return s;
     }
@@ -551,4 +562,10 @@ bool PartitionedFilterBlockReader::index_value_is_full() const {
   return table()->get_rep()->index_value_is_full;
 }
 
+bool PartitionedFilterBlockReader::user_defined_timestamps_persisted() const {
+  assert(table());
+  assert(table()->get_rep());
+
+  return table()->get_rep()->user_defined_timestamps_persisted;
+}
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.h

@@ -31,7 +31,8 @@ class PartitionedFilterBlockBuilder : public FullFilterBlockBuilder {
       FilterBitsBuilder* filter_bits_builder, int index_block_restart_interval,
       const bool use_value_delta_encoding,
       PartitionedIndexBuilder* const p_index_builder,
-      const uint32_t partition_size);
+      const uint32_t partition_size, size_t ts_sz,
+      const bool persist_user_defined_timestamps);
 
   virtual ~PartitionedFilterBlockBuilder();
 
@@ -172,6 +173,7 @@ class PartitionedFilterBlockReader
   const InternalKeyComparator* internal_comparator() const;
   bool index_key_includes_seq() const;
   bool index_value_is_full() const;
+  bool user_defined_timestamps_persisted() const;
 
  protected:
   // For partition blocks pinned in cache. Can be a subset of blocks

package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc

@@ -57,7 +57,8 @@ class MyPartitionedFilterBlockReader : public PartitionedFilterBlockReader {
 
 class PartitionedFilterBlockTest
     : public testing::Test,
-      virtual public ::testing::WithParamInterface<uint32_t> {
+      virtual public ::testing::WithParamInterface<
+          std::tuple<uint32_t, test::UserDefinedTimestampTestMode>> {
  public:
   Options options_;
   ImmutableOptions ioptions_;
@@ -67,38 +68,64 @@ class PartitionedFilterBlockTest
   std::unique_ptr<BlockBasedTable> table_;
   std::shared_ptr<Cache> cache_;
   int bits_per_key_;
+  size_t ts_sz_;
+  bool user_defined_timestamps_persisted_;
 
-  PartitionedFilterBlockTest()
-      : ioptions_(options_),
-        env_options_(options_),
-        icomp_(options_.comparator),
-        bits_per_key_(10) {
+  PartitionedFilterBlockTest() : bits_per_key_(10) {
+    auto udt_test_mode = std::get<1>(GetParam());
+    if (test::IsUDTEnabled(udt_test_mode)) {
+      options_.comparator = test::BytewiseComparatorWithU64TsWrapper();
+    }
+    ts_sz_ = options_.comparator->timestamp_size();
+    user_defined_timestamps_persisted_ = test::ShouldPersistUDT(udt_test_mode);
+    icomp_ = InternalKeyComparator(options_.comparator);
+    env_options_ = EnvOptions(options_);
+    ioptions_ = ImmutableOptions(options_);
     table_options_.filter_policy.reset(
         NewBloomFilterPolicy(bits_per_key_, false));
-    table_options_.format_version = GetParam();
+    table_options_.format_version = std::get<0>(GetParam());
     table_options_.index_block_restart_interval = 3;
   }
 
   ~PartitionedFilterBlockTest() override {}
 
-  const std::string keys[4] = {"afoo", "bar", "box", "hello"};
-  const std::string missing_keys[2] = {"missing", "other"};
+  static constexpr int kKeyNum = 4;
+  static constexpr int kMissingKeyNum = 2;
+  const std::string keys_without_ts[kKeyNum] = {"afoo", "bar", "box", "hello"};
+  const std::string missing_keys_without_ts[kMissingKeyNum] = {"missing",
+                                                               "other"};
+
+  std::vector<std::string> PrepareKeys(const std::string* orig_keys,
+                                       int number_of_keys) {
+    std::vector<std::string> user_keys;
+    if (ts_sz_ == 0) {
+      user_keys.assign(orig_keys, orig_keys + number_of_keys);
+    } else {
+      for (int i = 0; i < number_of_keys; i++) {
+        std::string key_with_ts;
+        AppendKeyWithMinTimestamp(&key_with_ts, orig_keys[i], ts_sz_);
+        user_keys.push_back(std::move(key_with_ts));
+      }
+    }
+    return user_keys;
+  }
 
   uint64_t MaxIndexSize() {
-    int num_keys = sizeof(keys) / sizeof(*keys);
     uint64_t max_key_size = 0;
-    for (int i = 1; i < num_keys; i++) {
-      max_key_size =
-          std::max(max_key_size, static_cast<uint64_t>(keys[i].size()));
+    for (int i = 0; i < kKeyNum; i++) {
+      // If UDT is enabled, the size of each key would be increased by a
+      // timestamp size.
+      max_key_size = std::max(
+          max_key_size, static_cast<uint64_t>(keys_without_ts[i].size()) +
+                            ts_sz_ * sizeof(static_cast<unsigned char>(0)));
     }
-    uint64_t max_index_size = num_keys * (max_key_size + 8 /*handle*/);
+    uint64_t max_index_size = kKeyNum * (max_key_size + 8 /*handle*/);
     return max_index_size;
   }
 
   uint64_t MaxFilterSize() {
-    int num_keys = sizeof(keys) / sizeof(*keys);
     // General, rough over-approximation
-    return num_keys * bits_per_key_ + (CACHE_LINE_SIZE * 8 + /*metadata*/ 5);
+    return kKeyNum * bits_per_key_ + (CACHE_LINE_SIZE * 8 + /*metadata*/ 5);
   }
 
   uint64_t last_offset = 10;
@@ -112,7 +139,8 @@ class PartitionedFilterBlockTest
   PartitionedIndexBuilder* NewIndexBuilder() {
     const bool kValueDeltaEncoded = true;
     return PartitionedIndexBuilder::CreateIndexBuilder(
-        &icomp_, !kValueDeltaEncoded, table_options_);
+        &icomp_, !kValueDeltaEncoded, table_options_, ts_sz_,
+        user_defined_timestamps_persisted_);
   }
 
   PartitionedFilterBlockBuilder* NewBuilder(
@@ -131,7 +159,8 @@ class PartitionedFilterBlockTest
         BloomFilterPolicy::GetBuilderFromContext(
             FilterBuildingContext(table_options_)),
         table_options_.index_block_restart_interval, !kValueDeltaEncoded,
-        p_index_builder, partition_size);
+        p_index_builder, partition_size, ts_sz_,
+        user_defined_timestamps_persisted_);
   }
 
   PartitionedFilterBlockReader* NewReader(
@@ -152,7 +181,8 @@ class PartitionedFilterBlockTest
     table_.reset(new MockedBlockBasedTable(
         new BlockBasedTable::Rep(ioptions_, env_options_, table_options_,
                                  icomp_, skip_filters, file_size, level,
-                                 immortal_table),
+                                 immortal_table,
+                                 user_defined_timestamps_persisted_),
         pib));
     BlockContents contents(slice);
     CachableEntry<Block> block(
@@ -169,38 +199,41 @@ class PartitionedFilterBlockTest
         NewReader(builder, pib));
     // Querying added keys
     const bool no_io = true;
+    std::vector<std::string> keys = PrepareKeys(keys_without_ts, kKeyNum);
     for (auto key : keys) {
       auto ikey = InternalKey(key, 0, ValueType::kTypeValue);
       const Slice ikey_slice = Slice(*ikey.rep());
-      ASSERT_TRUE(reader->KeyMayMatch(key, !no_io, &ikey_slice,
-                                      /*get_context=*/nullptr,
-                                      /*lookup_context=*/nullptr,
-                                      ReadOptions()));
+      ASSERT_TRUE(reader->KeyMayMatch(
+          StripTimestampFromUserKey(key, ts_sz_), !no_io, &ikey_slice,
+          /*get_context=*/nullptr,
+          /*lookup_context=*/nullptr, ReadOptions()));
     }
     {
       // querying a key twice
       auto ikey = InternalKey(keys[0], 0, ValueType::kTypeValue);
       const Slice ikey_slice = Slice(*ikey.rep());
-      ASSERT_TRUE(reader->KeyMayMatch(keys[0], !no_io, &ikey_slice,
-                                      /*get_context=*/nullptr,
-                                      /*lookup_context=*/nullptr,
-                                      ReadOptions()));
+      ASSERT_TRUE(reader->KeyMayMatch(
+          StripTimestampFromUserKey(keys[0], ts_sz_), !no_io, &ikey_slice,
+          /*get_context=*/nullptr,
+          /*lookup_context=*/nullptr, ReadOptions()));
     }
     // querying missing keys
+    std::vector<std::string> missing_keys =
+        PrepareKeys(missing_keys_without_ts, kMissingKeyNum);
     for (auto key : missing_keys) {
       auto ikey = InternalKey(key, 0, ValueType::kTypeValue);
       const Slice ikey_slice = Slice(*ikey.rep());
       if (empty) {
-        ASSERT_TRUE(reader->KeyMayMatch(key, !no_io, &ikey_slice,
-                                        /*get_context=*/nullptr,
-                                        /*lookup_context=*/nullptr,
-                                        ReadOptions()));
+        ASSERT_TRUE(reader->KeyMayMatch(
+            StripTimestampFromUserKey(key, ts_sz_), !no_io, &ikey_slice,
+            /*get_context=*/nullptr,
+            /*lookup_context=*/nullptr, ReadOptions()));
       } else {
         // assuming a good hash function
-        ASSERT_FALSE(reader->KeyMayMatch(key, !no_io, &ikey_slice,
-                                         /*get_context=*/nullptr,
-                                         /*lookup_context=*/nullptr,
-                                         ReadOptions()));
+        ASSERT_FALSE(reader->KeyMayMatch(
+            StripTimestampFromUserKey(key, ts_sz_), !no_io, &ikey_slice,
+            /*get_context=*/nullptr,
+            /*lookup_context=*/nullptr, ReadOptions()));
       }
     }
   }
@@ -210,17 +243,18 @@ class PartitionedFilterBlockTest
     std::unique_ptr<PartitionedFilterBlockBuilder> builder(
         NewBuilder(pib.get()));
     int i = 0;
-    builder->Add(keys[i]);
+    std::vector<std::string> keys = PrepareKeys(keys_without_ts, kKeyNum);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     CutABlock(pib.get(), keys[i], keys[i + 1]);
     i++;
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     CutABlock(pib.get(), keys[i], keys[i + 1]);
     i++;
-    builder->Add(keys[i]);
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     CutABlock(pib.get(), keys[i], keys[i + 1]);
     i++;
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     CutABlock(pib.get(), keys[i]);
 
     VerifyReader(builder.get(), pib.get());
@@ -231,16 +265,17 @@ class PartitionedFilterBlockTest
     std::unique_ptr<PartitionedIndexBuilder> pib(NewIndexBuilder());
     std::unique_ptr<PartitionedFilterBlockBuilder> builder(
         NewBuilder(pib.get(), prefix_extractor));
+    std::vector<std::string> keys = PrepareKeys(keys_without_ts, kKeyNum);
     int i = 0;
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     i++;
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     CutABlock(pib.get(), keys[i], keys[i + 1]);
     i++;
-    builder->Add(keys[i]);
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     i++;
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     CutABlock(pib.get(), keys[i]);
 
     VerifyReader(builder.get(), pib.get(), prefix_extractor);
@@ -250,15 +285,16 @@ class PartitionedFilterBlockTest
     std::unique_ptr<PartitionedIndexBuilder> pib(NewIndexBuilder());
     std::unique_ptr<PartitionedFilterBlockBuilder> builder(
         NewBuilder(pib.get()));
+    std::vector<std::string> keys = PrepareKeys(keys_without_ts, kKeyNum);
     int i = 0;
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     i++;
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     i++;
-    builder->Add(keys[i]);
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     i++;
-    builder->Add(keys[i]);
+    builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_));
     CutABlock(pib.get(), keys[i]);
 
     VerifyReader(builder.get(), pib.get());
@@ -299,10 +335,12 @@ class PartitionedFilterBlockTest
 };
 
 // Format versions potentially intersting to partitioning
-INSTANTIATE_TEST_CASE_P(FormatVersions, PartitionedFilterBlockTest,
-                        testing::ValuesIn(std::set<uint32_t>{
-                            2, 3, 4, test::kDefaultFormatVersion,
-                            kLatestFormatVersion}));
+INSTANTIATE_TEST_CASE_P(
+    FormatVersions, PartitionedFilterBlockTest,
+    testing::Combine(testing::ValuesIn(std::set<uint32_t>{
+                         2, 3, 4, test::kDefaultFormatVersion,
+                         kLatestFormatVersion}),
+                     testing::ValuesIn(test::GetUDTTestModes())));
 
 TEST_P(PartitionedFilterBlockTest, EmptyBuilder) {
   std::unique_ptr<PartitionedIndexBuilder> pib(NewIndexBuilder());
@@ -337,12 +375,14 @@ TEST_P(PartitionedFilterBlockTest, SamePrefixInMultipleBlocks) {
   std::unique_ptr<PartitionedIndexBuilder> pib(NewIndexBuilder());
   std::unique_ptr<PartitionedFilterBlockBuilder> builder(
       NewBuilder(pib.get(), prefix_extractor.get()));
-  const std::string pkeys[3] = {"p-key10", "p-key20", "p-key30"};
-  builder->Add(pkeys[0]);
+  const std::string pkeys_without_ts[3] = {"p-key10", "p-key20", "p-key30"};
+  std::vector<std::string> pkeys =
+      PrepareKeys(pkeys_without_ts, 3 /* number_of_keys */);
+  builder->Add(StripTimestampFromUserKey(pkeys[0], ts_sz_));
   CutABlock(pib.get(), pkeys[0], pkeys[1]);
-  builder->Add(pkeys[1]);
+  builder->Add(StripTimestampFromUserKey(pkeys[1], ts_sz_));
   CutABlock(pib.get(), pkeys[1], pkeys[2]);
-  builder->Add(pkeys[2]);
+  builder->Add(StripTimestampFromUserKey(pkeys[2], ts_sz_));
   CutABlock(pib.get(), pkeys[2]);
   std::unique_ptr<PartitionedFilterBlockReader> reader(
       NewReader(builder.get(), pib.get()));
@@ -356,7 +396,10 @@ TEST_P(PartitionedFilterBlockTest, SamePrefixInMultipleBlocks) {
                                        ReadOptions()));
   }
   // Non-existent keys but with the same prefix
-  const std::string pnonkeys[4] = {"p-key9", "p-key11", "p-key21", "p-key31"};
+  const std::string pnonkeys_without_ts[4] = {"p-key9", "p-key11", "p-key21",
+                                              "p-key31"};
+  std::vector<std::string> pnonkeys =
+      PrepareKeys(pnonkeys_without_ts, 4 /* number_of_keys */);
   for (auto key : pnonkeys) {
     auto ikey = InternalKey(key, 0, ValueType::kTypeValue);
     const Slice ikey_slice = Slice(*ikey.rep());
@@ -381,23 +424,25 @@ TEST_P(PartitionedFilterBlockTest, PrefixInWrongPartitionBug) {
   // In the bug, searching for prefix "p3" on an index with format version 3,
   // will give the key "p3" and the partition of the keys that are <= p3, i.e.,
   // p2-keys, where the filter for prefix "p3" does not exist.
-  const std::string pkeys[] = {"p1-key1", "p2-key2", "p3-key3", "p4-key3",
-                               "p5-key3"};
-  builder->Add(pkeys[0]);
+  const std::string pkeys_without_ts[] = {"p1-key1", "p2-key2", "p3-key3",
+                                          "p4-key3", "p5-key3"};
+  std::vector<std::string> pkeys =
+      PrepareKeys(pkeys_without_ts, 5 /* number_of_keys */);
+  builder->Add(StripTimestampFromUserKey(pkeys[0], ts_sz_));
   CutABlock(pib.get(), pkeys[0], pkeys[1]);
-  builder->Add(pkeys[1]);
+  builder->Add(StripTimestampFromUserKey(pkeys[1], ts_sz_));
   CutABlock(pib.get(), pkeys[1], pkeys[2]);
-  builder->Add(pkeys[2]);
+  builder->Add(StripTimestampFromUserKey(pkeys[2], ts_sz_));
   CutABlock(pib.get(), pkeys[2], pkeys[3]);
-  builder->Add(pkeys[3]);
+  builder->Add(StripTimestampFromUserKey(pkeys[3], ts_sz_));
   CutABlock(pib.get(), pkeys[3], pkeys[4]);
-  builder->Add(pkeys[4]);
+  builder->Add(StripTimestampFromUserKey(pkeys[4], ts_sz_));
   CutABlock(pib.get(), pkeys[4]);
   std::unique_ptr<PartitionedFilterBlockReader> reader(
       NewReader(builder.get(), pib.get()));
   for (auto key : pkeys) {
     auto prefix = prefix_extractor->Transform(key);
-    auto ikey = InternalKey(prefix, 0, ValueType::kTypeValue);
+    auto ikey = InternalKey(key, 0, ValueType::kTypeValue);
     const Slice ikey_slice = Slice(*ikey.rep());
     ASSERT_TRUE(reader->PrefixMayMatch(prefix,
                                        /*no_io=*/false, &ikey_slice,
@@ -416,7 +461,6 @@ TEST_P(PartitionedFilterBlockTest, OneBlockPerKey) {
 }
 
 TEST_P(PartitionedFilterBlockTest, PartitionCount) {
-  int num_keys = sizeof(keys) / sizeof(*keys);
   table_options_.metadata_block_size =
       std::max(MaxIndexSize(), MaxFilterSize());
   int partitions = TestBlockPerKey();
@@ -424,7 +468,7 @@ TEST_P(PartitionedFilterBlockTest, PartitionCount) {
   // A low number ensures cutting a block after each key
   table_options_.metadata_block_size = 1;
   partitions = TestBlockPerKey();
-  ASSERT_EQ(partitions, num_keys - 1 /* last two keys make one flush */);
+  ASSERT_EQ(partitions, kKeyNum - 1 /* last two keys make one flush */);
 }
 
 }  // namespace ROCKSDB_NAMESPACE
@@ -433,4 +477,4 @@ int main(int argc, char** argv) {
   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
-}
+}

package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_iterator.cc

@@ -91,15 +91,16 @@ void PartitionedIndexIterator::InitPartitionedIndexBlock() {
     //   Enabled from the very first IO when ReadOptions.readahead_size is set.
     block_prefetcher_.PrefetchIfNeeded(
         rep, partitioned_index_handle, read_options_.readahead_size,
-        is_for_compaction, /*no_sequential_checking=*/false,
-        read_options_.rate_limiter_priority);
+        is_for_compaction, /*no_sequential_checking=*/false, read_options_,
+        /*readaheadsize_cb=*/nullptr);
     Status s;
     table_->NewDataBlockIterator<IndexBlockIter>(
         read_options_, partitioned_index_handle, &block_iter_,
         BlockType::kIndex,
         /*get_context=*/nullptr, &lookup_context_,
         block_prefetcher_.prefetch_buffer(),
-        /*for_compaction=*/is_for_compaction, /*async_read=*/false, s);
+        /*for_compaction=*/is_for_compaction, /*async_read=*/false, s,
+        /*use_block_cache_for_lookup=*/true);
     block_iter_points_to_real_block_ = true;
     // We could check upper bound here but it is complicated to reason about
     // upper bound in index iterator. On the other than, in large scans, index

package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc

@@ -75,7 +75,8 @@ InternalIteratorBase<IndexValue>* PartitionIndexReader::NewIterator(
             internal_comparator()->user_comparator(),
             rep->get_global_seqno(BlockType::kIndex), nullptr, kNullStats, true,
             index_has_first_key(), index_key_includes_seq(),
-            index_value_is_full()));
+            index_value_is_full(), false /* block_contents_pinned */,
+            user_defined_timestamps_persisted()));
   } else {
     ReadOptions ro;
     ro.fill_cache = read_options.fill_cache;
@@ -94,7 +95,8 @@ InternalIteratorBase<IndexValue>* PartitionIndexReader::NewIterator(
             internal_comparator()->user_comparator(),
             rep->get_global_seqno(BlockType::kIndex), nullptr, kNullStats, true,
             index_has_first_key(), index_key_includes_seq(),
-            index_value_is_full()));
+            index_value_is_full(), false /* block_contents_pinned */,
+            user_defined_timestamps_persisted()));
 
     it = new PartitionedIndexIterator(
         table(), ro, *internal_comparator(), std::move(index_iter),
@@ -140,7 +142,8 @@ Status PartitionIndexReader::CacheDependencies(
   index_block.GetValue()->NewIndexIterator(
       internal_comparator()->user_comparator(),
       rep->get_global_seqno(BlockType::kIndex), &biter, kNullStats, true,
-      index_has_first_key(), index_key_includes_seq(), index_value_is_full());
+      index_has_first_key(), index_key_includes_seq(), index_value_is_full(),
+      false /* block_contents_pinned */, user_defined_timestamps_persisted());
   // Index partitions are assumed to be consecuitive. Prefetch them all.
   // Read the first block offset
   biter.SeekToFirst();
@@ -166,14 +169,15 @@ Status PartitionIndexReader::CacheDependencies(
       tail_prefetch_buffer->GetPrefetchOffset() > prefetch_off) {
     rep->CreateFilePrefetchBuffer(
         0, 0, &prefetch_buffer, false /*Implicit auto readahead*/,
-        0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/);
+        0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/,
+        /*upper_bound_offset*/ 0, /*readaheadsize_cb*/ nullptr,
+        /*usage=*/FilePrefetchBufferUsage::kUnknown);
     IOOptions opts;
     {
       Status s = rep->file->PrepareIOOptions(ro, opts);
       if (s.ok()) {
         s = prefetch_buffer->Prefetch(opts, rep->file.get(), prefetch_off,
-                                      static_cast<size_t>(prefetch_len),
-                                      ro.rate_limiter_priority);
+                                      static_cast<size_t>(prefetch_len));
       }
       if (!s.ok()) {
         return s;
@@ -197,7 +201,7 @@ Status PartitionIndexReader::CacheDependencies(
         handle, UncompressionDict::GetEmptyDict(),
         /*for_compaction=*/false, &block.As<Block_kIndex>(),
         /*get_context=*/nullptr, &lookup_context, /*contents=*/nullptr,
-        /*async_read=*/false);
+        /*async_read=*/false, /*use_block_cache_for_lookup=*/true);
 
     if (!s.ok()) {
       return s;

package/deps/rocksdb/rocksdb/table/block_based/reader_common.cc

@@ -23,10 +23,14 @@ void ForceReleaseCachedEntry(void* arg, void* h) {
 }
 
 // WART: this is specific to block-based table
-Status VerifyBlockChecksum(ChecksumType type, const char* data,
+Status VerifyBlockChecksum(const Footer& footer, const char* data,
                            size_t block_size, const std::string& file_name,
                            uint64_t offset) {
   PERF_TIMER_GUARD(block_checksum_time);
+
+  assert(footer.GetBlockTrailerSize() == 5);
+  ChecksumType type = footer.checksum_type();
+
   // After block_size bytes is compression type (1 byte), which is part of
   // the checksummed section.
   size_t len = block_size + 1;
@@ -34,6 +38,13 @@ Status VerifyBlockChecksum(ChecksumType type, const char* data,
   uint32_t stored = DecodeFixed32(data + len);
 
   uint32_t computed = ComputeBuiltinChecksum(type, data, len);
+
+  // Unapply context to 'stored' rather than apply to 'computed, for people
+  // who might look for reference crc value in error message
+  uint32_t modifier =
+      ChecksumModifierForContext(footer.base_context_checksum(), offset);
+  stored -= modifier;
+
   if (stored == computed) {
     return Status::OK();
   } else {
@@ -43,8 +54,9 @@ Status VerifyBlockChecksum(ChecksumType type, const char* data,
       computed = crc32c::Unmask(computed);
     }
     return Status::Corruption(
-        "block checksum mismatch: stored = " + std::to_string(stored) +
-        ", computed = " + std::to_string(computed) +
+        "block checksum mismatch: stored" +
+        std::string(modifier ? "(context removed)" : "") + " = " +
+        std::to_string(stored) + ", computed = " + std::to_string(computed) +
         ", type = " + std::to_string(type) + "  in " + file_name + " offset " +
         std::to_string(offset) + " size " + std::to_string(block_size));
   }

package/deps/rocksdb/rocksdb/table/block_based/reader_common.h

@@ -12,6 +12,8 @@
 #include "rocksdb/table.h"
 
 namespace ROCKSDB_NAMESPACE {
+class Footer;
+
 // Release the cached entry and decrement its ref count.
 extern void ForceReleaseCachedEntry(void* arg, void* h);
 
@@ -22,12 +24,13 @@ inline MemoryAllocator* GetMemoryAllocator(
              : nullptr;
 }
 
-// Assumes block has a trailer as in format.h. file_name and offset provided
-// for generating a diagnostic message in returned status.
+// Assumes block has a trailer past `data + block_size` as in format.h.
+// `file_name` provided for generating diagnostic message in returned status.
+// `offset` might be required for proper verification (also used for message).
 //
 // Returns Status::OK() on checksum match, or Status::Corruption() on checksum
 // mismatch.
-extern Status VerifyBlockChecksum(ChecksumType type, const char* data,
+extern Status VerifyBlockChecksum(const Footer& footer, const char* data,
                                   size_t block_size,
                                   const std::string& file_name,
                                   uint64_t offset);

package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc

@@ -63,7 +63,7 @@ Status UncompressionDictReader::ReadUncompressionDictionary(
       UncompressionDict::GetEmptyDict(), uncompression_dict, get_context,
       lookup_context,
       /* for_compaction */ false, use_cache,
-      /* async_read */ false);
+      /* async_read */ false, /* use_block_cache_for_lookup */ true);
 
   if (!s.ok()) {
     ROCKS_LOG_WARN(