@nxtedition/rocksdb 8.2.7 → 9.0.0

package/deps/rocksdb/rocksdb/table/plain/plain_table_reader.cc

@@ -284,9 +284,9 @@ void PlainTableReader::FillBloom(const std::vector<uint32_t>& prefix_hashes) {
 Status PlainTableReader::MmapDataIfNeeded() {
   if (file_info_.is_mmap_mode) {
     // Get mmapped memory.
-    return file_info_.file->Read(
-        IOOptions(), 0, static_cast<size_t>(file_size_), &file_info_.file_data,
-        nullptr, nullptr, Env::IO_TOTAL /* rate_limiter_priority */);
+    return file_info_.file->Read(IOOptions(), 0,
+                                 static_cast<size_t>(file_size_),
+                                 &file_info_.file_data, nullptr, nullptr);
   }
   return Status::OK();
 }

package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc

@@ -109,8 +109,7 @@ Status SstFileDumper::GetTableReader(const std::string& file_path) {
     uint64_t prefetch_off = file_size - prefetch_size;
     IOOptions opts;
     s = prefetch_buffer.Prefetch(opts, file_.get(), prefetch_off,
-                                 static_cast<size_t>(prefetch_size),
-                                 Env::IO_TOTAL /* rate_limiter_priority */);
+                                 static_cast<size_t>(prefetch_size));
 
     s = ReadFooterFromFile(opts, file_.get(), *fs, &prefetch_buffer, file_size,
                            &footer);
@@ -165,10 +164,19 @@ Status SstFileDumper::NewTableReader(
     const ImmutableOptions& /*ioptions*/, const EnvOptions& /*soptions*/,
     const InternalKeyComparator& /*internal_comparator*/, uint64_t file_size,
     std::unique_ptr<TableReader>* /*table_reader*/) {
+  // TODO(yuzhangyu): full support in sst_dump for SST files generated when
+  // `user_defined_timestamps_persisted` is false.
   auto t_opt = TableReaderOptions(
       ioptions_, moptions_.prefix_extractor, soptions_, internal_comparator_,
       0 /* block_protection_bytes_per_key */, false /* skip_filters */,
-      false /* immortal */, true /* force_direct_prefetch */);
+      false /* immortal */, true /* force_direct_prefetch */, -1 /* level */,
+      nullptr /* block_cache_tracer */, 0 /* max_file_size_for_l0_meta_pin */,
+      "" /* cur_db_session_id */, 0 /* cur_file_num */, {} /* unique_id */,
+      0 /* largest_seqno */, 0 /* tail_size */,
+      table_properties_ == nullptr
+          ? true
+          : static_cast<bool>(
+                table_properties_->user_defined_timestamps_persisted));
   // Allow open file with global sequence number for backward compatibility.
   t_opt.largest_seqno = kMaxSequenceNumber;
 
@@ -187,6 +195,7 @@ Status SstFileDumper::NewTableReader(
 }
 
 Status SstFileDumper::VerifyChecksum() {
+  assert(read_options_.verify_checksums);
   // We could pass specific readahead setting into read options if needed.
   return table_reader_->VerifyChecksum(read_options_,
                                        TableReaderCaller::kSSTDumpTool);

package/deps/rocksdb/rocksdb/table/sst_file_writer.cc

@@ -9,6 +9,7 @@
 
 #include "db/db_impl/db_impl.h"
 #include "db/dbformat.h"
+#include "db/wide/wide_column_serialization.h"
 #include "file/writable_file_writer.h"
 #include "rocksdb/file_system.h"
 #include "rocksdb/table.h"
@@ -81,7 +82,8 @@ struct SstFileWriter::Rep {
 
     assert(value_type == kTypeValue || value_type == kTypeMerge ||
            value_type == kTypeDeletion ||
-           value_type == kTypeDeletionWithTimestamp);
+           value_type == kTypeDeletionWithTimestamp ||
+           value_type == kTypeWideColumnEntity);
 
     constexpr SequenceNumber sequence_number = 0;
 
@@ -130,6 +132,24 @@ struct SstFileWriter::Rep {
     return AddImpl(user_key_with_ts, value, value_type);
   }
 
+  Status AddEntity(const Slice& user_key, const WideColumns& columns) {
+    WideColumns sorted_columns(columns);
+    std::sort(sorted_columns.begin(), sorted_columns.end(),
+              [](const WideColumn& lhs, const WideColumn& rhs) {
+                return lhs.name().compare(rhs.name()) < 0;
+              });
+
+    std::string entity;
+    const Status s = WideColumnSerialization::Serialize(sorted_columns, entity);
+    if (!s.ok()) {
+      return s;
+    }
+    if (entity.size() > size_t{std::numeric_limits<uint32_t>::max()}) {
+      return Status::InvalidArgument("wide column entity is too large");
+    }
+    return Add(user_key, entity, kTypeWideColumnEntity);
+  }
+
   Status DeleteRangeImpl(const Slice& begin_key, const Slice& end_key) {
     if (!builder) {
       return Status::InvalidArgument("File is not opened");
@@ -371,6 +391,11 @@ Status SstFileWriter::Put(const Slice& user_key, const Slice& timestamp,
   return rep_->Add(user_key, timestamp, value, ValueType::kTypeValue);
 }
 
+Status SstFileWriter::PutEntity(const Slice& user_key,
+                                const WideColumns& columns) {
+  return rep_->AddEntity(user_key, columns);
+}
+
 Status SstFileWriter::Merge(const Slice& user_key, const Slice& value) {
   return rep_->Add(user_key, value, ValueType::kTypeMerge);
 }

package/deps/rocksdb/rocksdb/table/table_builder.h

@@ -43,7 +43,7 @@ struct TableReaderOptions {
       size_t _max_file_size_for_l0_meta_pin = 0,
       const std::string& _cur_db_session_id = "", uint64_t _cur_file_num = 0,
       UniqueId64x2 _unique_id = {}, SequenceNumber _largest_seqno = 0,
-      uint64_t _tail_size = 0)
+      uint64_t _tail_size = 0, bool _user_defined_timestamps_persisted = true)
       : ioptions(_ioptions),
         prefix_extractor(_prefix_extractor),
         env_options(_env_options),
@@ -59,7 +59,8 @@ struct TableReaderOptions {
         cur_file_num(_cur_file_num),
         unique_id(_unique_id),
         block_protection_bytes_per_key(_block_protection_bytes_per_key),
-        tail_size(_tail_size) {}
+        tail_size(_tail_size),
+        user_defined_timestamps_persisted(_user_defined_timestamps_persisted) {}
 
   const ImmutableOptions& ioptions;
   const std::shared_ptr<const SliceTransform>& prefix_extractor;
@@ -93,6 +94,9 @@ struct TableReaderOptions {
   uint8_t block_protection_bytes_per_key;
 
   uint64_t tail_size;
+
+  // Whether the key in the table contains user-defined timestamps.
+  bool user_defined_timestamps_persisted;
 };
 
 struct TableBuilderOptions {

package/deps/rocksdb/rocksdb/table/table_properties.cc

@@ -109,6 +109,10 @@ std::string TableProperties::ToString(const std::string& prop_delim,
   AppendProperty(result, "comparator name",
                  comparator_name.empty() ? std::string("N/A") : comparator_name,
                  prop_delim, kv_delim);
+  AppendProperty(result, "user defined timestamps persisted",
+                 user_defined_timestamps_persisted ? std::string("true")
+                                                   : std::string("false"),
+                 prop_delim, kv_delim);
 
   AppendProperty(
       result, "merge operator name",
@@ -305,6 +309,8 @@ const std::string TablePropertiesNames::kSequenceNumberTimeMapping =
     "rocksdb.seqno.time.map";
 const std::string TablePropertiesNames::kTailStartOffset =
     "rocksdb.tail.start.offset";
+const std::string TablePropertiesNames::kUserDefinedTimestampsPersisted =
+    "rocksdb.user.defined.timestamps.persisted";
 
 #ifndef NDEBUG
 // WARNING: TEST_SetRandomTableProperties assumes the following layout of

package/deps/rocksdb/rocksdb/table/table_test.cc

@@ -1330,7 +1330,7 @@ class FileChecksumTestHelper {
     uint64_t offset = 0;
     Status s;
     s = file_reader_->Read(IOOptions(), offset, 2048, &result, scratch.get(),
-                           nullptr, Env::IO_TOTAL /* rate_limiter_priority */);
+                           nullptr);
     if (!s.ok()) {
       return s;
     }
@@ -1338,8 +1338,7 @@ class FileChecksumTestHelper {
       file_checksum_generator->Update(scratch.get(), result.size());
       offset += static_cast<uint64_t>(result.size());
       s = file_reader_->Read(IOOptions(), offset, 2048, &result, scratch.get(),
-                             nullptr,
-                             Env::IO_TOTAL /* rate_limiter_priority */);
+                             nullptr);
       if (!s.ok()) {
         return s;
       }
@@ -4472,11 +4471,12 @@ TEST(TableTest, FooterTests) {
   BlockHandle index(data_size + 5, index_size);
   BlockHandle meta_index(data_size + index_size + 2 * 5, metaindex_size);
   uint64_t footer_offset = data_size + metaindex_size + index_size + 3 * 5;
+  uint32_t base_context_checksum = 123456789;
   {
     // legacy block based
     FooterBuilder footer;
-    footer.Build(kBlockBasedTableMagicNumber, /* format_version */ 0,
-                 footer_offset, kCRC32c, meta_index, index);
+    ASSERT_OK(footer.Build(kBlockBasedTableMagicNumber, /* format_version */ 0,
+                           footer_offset, kCRC32c, meta_index, index));
     Footer decoded_footer;
     ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset));
     ASSERT_EQ(decoded_footer.table_magic_number(), kBlockBasedTableMagicNumber);
@@ -4486,6 +4486,7 @@ TEST(TableTest, FooterTests) {
     ASSERT_EQ(decoded_footer.index_handle().offset(), index.offset());
     ASSERT_EQ(decoded_footer.index_handle().size(), index.size());
     ASSERT_EQ(decoded_footer.format_version(), 0U);
+    ASSERT_EQ(decoded_footer.base_context_checksum(), 0U);
     ASSERT_EQ(decoded_footer.GetBlockTrailerSize(), 5U);
     // Ensure serialized with legacy magic
     ASSERT_EQ(
@@ -4495,9 +4496,11 @@ TEST(TableTest, FooterTests) {
   // block based, various checksums, various versions
   for (auto t : GetSupportedChecksums()) {
     for (uint32_t fv = 1; IsSupportedFormatVersion(fv); ++fv) {
+      uint32_t maybe_bcc =
+          FormatVersionUsesContextChecksum(fv) ? base_context_checksum : 0U;
       FooterBuilder footer;
-      footer.Build(kBlockBasedTableMagicNumber, fv, footer_offset, t,
-                   meta_index, index);
+      ASSERT_OK(footer.Build(kBlockBasedTableMagicNumber, fv, footer_offset, t,
+                             meta_index, index, maybe_bcc));
       Footer decoded_footer;
       ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset));
       ASSERT_EQ(decoded_footer.table_magic_number(),
@@ -4506,18 +4509,44 @@ TEST(TableTest, FooterTests) {
       ASSERT_EQ(decoded_footer.metaindex_handle().offset(),
                 meta_index.offset());
       ASSERT_EQ(decoded_footer.metaindex_handle().size(), meta_index.size());
-      ASSERT_EQ(decoded_footer.index_handle().offset(), index.offset());
-      ASSERT_EQ(decoded_footer.index_handle().size(), index.size());
+      if (FormatVersionUsesIndexHandleInFooter(fv)) {
+        ASSERT_EQ(decoded_footer.index_handle().offset(), index.offset());
+        ASSERT_EQ(decoded_footer.index_handle().size(), index.size());
+      }
       ASSERT_EQ(decoded_footer.format_version(), fv);
       ASSERT_EQ(decoded_footer.GetBlockTrailerSize(), 5U);
+
+      if (FormatVersionUsesContextChecksum(fv)) {
+        ASSERT_EQ(decoded_footer.base_context_checksum(),
+                  base_context_checksum);
+
+        // Bad offset should fail footer checksum
+        decoded_footer = Footer();
+        ASSERT_NOK(
+            decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset - 1));
+      } else {
+        ASSERT_EQ(decoded_footer.base_context_checksum(), 0U);
+      }
+
+      // Too big metaindex size should also fail encoding only in new footer
+      uint64_t big_metaindex_size = 0x100000007U;
+      uint64_t big_footer_offset =
+          data_size + big_metaindex_size + index_size + 3 * 5;
+      BlockHandle big_metaindex =
+          BlockHandle(data_size + index_size + 2 * 5, big_metaindex_size);
+      ASSERT_NE(footer
+                    .Build(kBlockBasedTableMagicNumber, fv, big_footer_offset,
+                           t, big_metaindex, index, maybe_bcc)
+                    .ok(),
+                FormatVersionUsesContextChecksum(fv));
     }
   }
 
   {
     // legacy plain table
     FooterBuilder footer;
-    footer.Build(kPlainTableMagicNumber, /* format_version */ 0, footer_offset,
-                 kNoChecksum, meta_index);
+    ASSERT_OK(footer.Build(kPlainTableMagicNumber, /* format_version */ 0,
+                           footer_offset, kNoChecksum, meta_index));
     Footer decoded_footer;
     ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset));
     ASSERT_EQ(decoded_footer.table_magic_number(), kPlainTableMagicNumber);
@@ -4536,8 +4565,8 @@ TEST(TableTest, FooterTests) {
   {
     // xxhash plain table (not currently used)
     FooterBuilder footer;
-    footer.Build(kPlainTableMagicNumber, /* format_version */ 1, footer_offset,
-                 kxxHash, meta_index);
+    ASSERT_OK(footer.Build(kPlainTableMagicNumber, /* format_version */ 1,
+                           footer_offset, kxxHash, meta_index));
     Footer decoded_footer;
     ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset));
     ASSERT_EQ(decoded_footer.table_magic_number(), kPlainTableMagicNumber);
@@ -5211,9 +5240,13 @@ TEST_P(BlockBasedTableTest, PropertiesMetaBlockLast) {
     }
   }
   ASSERT_EQ(kPropertiesBlockName, key_at_max_offset);
-  // index handle is stored in footer rather than metaindex block, so need
-  // separate logic to verify it comes before properties block.
-  ASSERT_GT(max_offset, footer.index_handle().offset());
+  if (FormatVersionUsesIndexHandleInFooter(footer.format_version())) {
+    // If index handle is stored in footer rather than metaindex block,
+    // need separate logic to verify it comes before properties block.
+    ASSERT_GT(max_offset, footer.index_handle().offset());
+  } else {
+    ASSERT_TRUE(footer.index_handle().IsNull());
+  }
   c.ResetTableReader();
 }
 
@@ -5362,16 +5395,13 @@ TEST_F(BBTTailPrefetchTest, FilePrefetchBufferMinOffset) {
   IOOptions opts;
   buffer.TryReadFromCache(opts, nullptr /* reader */, 500 /* offset */,
                           10 /* n */, nullptr /* result */,
-                          nullptr /* status */,
-                          Env::IO_TOTAL /* rate_limiter_priority */);
+                          nullptr /* status */);
   buffer.TryReadFromCache(opts, nullptr /* reader */, 480 /* offset */,
                           10 /* n */, nullptr /* result */,
-                          nullptr /* status */,
-                          Env::IO_TOTAL /* rate_limiter_priority */);
+                          nullptr /* status */);
   buffer.TryReadFromCache(opts, nullptr /* reader */, 490 /* offset */,
                           10 /* n */, nullptr /* result */,
-                          nullptr /* status */,
-                          Env::IO_TOTAL /* rate_limiter_priority */);
+                          nullptr /* status */);
   ASSERT_EQ(480, buffer.min_offset_read());
 }
 

package/deps/rocksdb/rocksdb/test_util/secondary_cache_test_util.h

@@ -42,12 +42,19 @@ class WithCacheType : public TestCreateContext {
   };
 
   static constexpr auto kLRU = "lru";
-  static constexpr auto kHyperClock = "hyper_clock";
+  static constexpr auto kFixedHyperClock = "fixed_hyper_clock";
+  static constexpr auto kAutoHyperClock = "auto_hyper_clock";
 
   // For options other than capacity
   size_t estimated_value_size_ = 1;
 
-  virtual const std::string& Type() = 0;
+  virtual const std::string& Type() const = 0;
+
+  static bool IsHyperClock(const std::string& type) {
+    return type == kFixedHyperClock || type == kAutoHyperClock;
+  }
+
+  bool IsHyperClock() const { return IsHyperClock(Type()); }
 
   std::shared_ptr<Cache> NewCache(
       size_t capacity,
@@ -62,8 +69,11 @@ class WithCacheType : public TestCreateContext {
       }
       return lru_opts.MakeSharedCache();
     }
-    if (type == kHyperClock) {
-      HyperClockCacheOptions hc_opts{capacity, estimated_value_size_};
+    if (IsHyperClock(type)) {
+      HyperClockCacheOptions hc_opts{
+          capacity, type == kFixedHyperClock ? estimated_value_size_ : 0};
+      hc_opts.min_avg_entry_charge =
+          std::max(size_t{1}, estimated_value_size_ / 2);
       hc_opts.hash_seed = 0;  // deterministic tests
       if (modify_opts_fn) {
         modify_opts_fn(hc_opts);
@@ -105,14 +115,16 @@ class WithCacheType : public TestCreateContext {
 
 class WithCacheTypeParam : public WithCacheType,
                            public testing::WithParamInterface<std::string> {
-  const std::string& Type() override { return GetParam(); }
+  const std::string& Type() const override { return GetParam(); }
 };
 
 constexpr auto kLRU = WithCacheType::kLRU;
-constexpr auto kHyperClock = WithCacheType::kHyperClock;
+constexpr auto kFixedHyperClock = WithCacheType::kFixedHyperClock;
+constexpr auto kAutoHyperClock = WithCacheType::kAutoHyperClock;
 
 inline auto GetTestingCacheTypes() {
-  return testing::Values(std::string(kLRU), std::string(kHyperClock));
+  return testing::Values(std::string(kLRU), std::string(kFixedHyperClock),
+                         std::string(kAutoHyperClock));
 }
 
 }  // namespace secondary_cache_test_util

package/deps/rocksdb/rocksdb/test_util/sync_point.h

@@ -85,7 +85,9 @@ class SyncPoint {
   };
 
   // call once at the beginning of a test to setup the dependency between
-  // sync points
+  // sync points. Specifically, execution will not be allowed to proceed past
+  // each successor until execution has reached the corresponding predecessor,
+  // in any thread.
   void LoadDependency(const std::vector<SyncPointPair>& dependencies);
 
   // call once at the beginning of a test to setup the dependency between

package/deps/rocksdb/rocksdb/test_util/testutil.cc

@@ -39,7 +39,10 @@ namespace test {
 
 const uint32_t kDefaultFormatVersion = BlockBasedTableOptions().format_version;
 const std::set<uint32_t> kFooterFormatVersionsToTest{
+    // Non-legacy, before big footer changes
     5U,
+    // After big footer changes
+    6U,
     // In case any interesting future changes
     kDefaultFormatVersion,
     kLatestFormatVersion,
@@ -72,6 +75,22 @@ std::string RandomKey(Random* rnd, int len, RandomKeyType type) {
   return result;
 }
 
+const std::vector<UserDefinedTimestampTestMode>& GetUDTTestModes() {
+  static std::vector<UserDefinedTimestampTestMode> udt_test_modes = {
+      UserDefinedTimestampTestMode::kStripUserDefinedTimestamp,
+      UserDefinedTimestampTestMode::kNormal,
+      UserDefinedTimestampTestMode::kNone};
+  return udt_test_modes;
+}
+
+bool IsUDTEnabled(const UserDefinedTimestampTestMode& test_mode) {
+  return test_mode != UserDefinedTimestampTestMode::kNone;
+}
+
+bool ShouldPersistUDT(const UserDefinedTimestampTestMode& test_mode) {
+  return test_mode != UserDefinedTimestampTestMode::kStripUserDefinedTimestamp;
+}
+
 extern Slice CompressibleString(Random* rnd, double compressed_fraction,
                                 int len, std::string* dst) {
   int raw = static_cast<int>(len * compressed_fraction);
@@ -134,6 +153,16 @@ const Comparator* BytewiseComparatorWithU64TsWrapper() {
   return user_comparator;
 }
 
+const Comparator* ReverseBytewiseComparatorWithU64TsWrapper() {
+  ConfigOptions config_options;
+  const Comparator* user_comparator = nullptr;
+  Status s = Comparator::CreateFromString(
+      config_options, "rocksdb.ReverseBytewiseComparator.u64ts",
+      &user_comparator);
+  s.PermitUncheckedError();
+  return user_comparator;
+}
+
 void CorruptKeyType(InternalKey* ikey) {
   std::string keystr = ikey->Encode().ToString();
   keystr[keystr.size() - 8] = kTypeLogData;

package/deps/rocksdb/rocksdb/test_util/testutil.h

@@ -52,6 +52,22 @@ enum RandomKeyType : char { RANDOM, LARGEST, SMALLEST, MIDDLE };
 extern std::string RandomKey(Random* rnd, int len,
                              RandomKeyType type = RandomKeyType::RANDOM);
 
+enum class UserDefinedTimestampTestMode {
+  // Test does not enable user-defined timestamp feature.
+  kNone,
+  // Test enables user-defined timestamp feature. Write/read with min timestamps
+  kNormal,
+  // Test enables user-defined timestamp feature. Write/read with min timestamps
+  // Set `persist_user_defined_timestamps` to false.
+  kStripUserDefinedTimestamp,
+};
+
+extern const std::vector<UserDefinedTimestampTestMode>& GetUDTTestModes();
+
+extern bool IsUDTEnabled(const UserDefinedTimestampTestMode& test_mode);
+
+extern bool ShouldPersistUDT(const UserDefinedTimestampTestMode& test_mode);
+
 // Store in *dst a string of length "len" that will compress to
 // "N*compressed_fraction" bytes and return a Slice that references
 // the generated data.
@@ -116,6 +132,9 @@ extern const Comparator* Uint64Comparator();
 // A wrapper api for getting the ComparatorWithU64Ts<BytewiseComparator>
 extern const Comparator* BytewiseComparatorWithU64TsWrapper();
 
+// A wrapper api for getting the ComparatorWithU64Ts<ReverseBytewiseComparator>
+extern const Comparator* ReverseBytewiseComparatorWithU64TsWrapper();
+
 class StringSink : public FSWritableFile {
  public:
   std::string contents_;

package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc

@@ -602,6 +602,12 @@ DEFINE_uint32(
     "compress_format_version == 2 -- decompressed size is included"
     " in the block header in varint32 format.");
 
+DEFINE_bool(use_tiered_volatile_cache, false,
+            "If use_compressed_secondary_cache is true and "
+            "use_tiered_volatile_cache is true, then allocate a tiered cache "
+            "that distributes cache reservations proportionally over both "
+            "the caches.");
+
 DEFINE_int64(simcache_size, -1,
              "Number of bytes to use as a simcache of "
              "uncompressed data. Nagative value disables simcache.");
@@ -2828,7 +2834,7 @@ class Benchmark {
       std::string input_str(len, 'y');
       std::string compressed;
       CompressionOptions opts;
-      CompressionContext context(FLAGS_compression_type_e);
+      CompressionContext context(FLAGS_compression_type_e, opts);
       CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
                            FLAGS_compression_type_e,
                            FLAGS_sample_for_compression);
@@ -3009,19 +3015,54 @@ class Benchmark {
   }
 
   static std::shared_ptr<Cache> NewCache(int64_t capacity) {
+    CompressedSecondaryCacheOptions secondary_cache_opts;
+    bool use_tiered_cache = false;
     if (capacity <= 0) {
       return nullptr;
     }
+    if (FLAGS_use_compressed_secondary_cache) {
+      secondary_cache_opts.capacity = FLAGS_compressed_secondary_cache_size;
+      secondary_cache_opts.num_shard_bits =
+          FLAGS_compressed_secondary_cache_numshardbits;
+      secondary_cache_opts.high_pri_pool_ratio =
+          FLAGS_compressed_secondary_cache_high_pri_pool_ratio;
+      secondary_cache_opts.low_pri_pool_ratio =
+          FLAGS_compressed_secondary_cache_low_pri_pool_ratio;
+      secondary_cache_opts.compression_type =
+          FLAGS_compressed_secondary_cache_compression_type_e;
+      secondary_cache_opts.compress_format_version =
+          FLAGS_compressed_secondary_cache_compress_format_version;
+      if (FLAGS_use_tiered_volatile_cache) {
+        use_tiered_cache = true;
+      }
+    }
     if (FLAGS_cache_type == "clock_cache") {
       fprintf(stderr, "Old clock cache implementation has been removed.\n");
       exit(1);
-    } else if (FLAGS_cache_type == "hyper_clock_cache") {
-      HyperClockCacheOptions hcco{
-          static_cast<size_t>(capacity),
-          static_cast<size_t>(FLAGS_block_size) /*estimated_entry_charge*/,
-          FLAGS_cache_numshardbits};
-      hcco.hash_seed = GetCacheHashSeed();
-      return hcco.MakeSharedCache();
+    } else if (EndsWith(FLAGS_cache_type, "hyper_clock_cache")) {
+      size_t estimated_entry_charge;
+      if (FLAGS_cache_type == "fixed_hyper_clock_cache" ||
+          FLAGS_cache_type == "hyper_clock_cache") {
+        estimated_entry_charge = FLAGS_block_size;
+      } else if (FLAGS_cache_type == "auto_hyper_clock_cache") {
+        estimated_entry_charge = 0;
+      } else {
+        fprintf(stderr, "Cache type not supported.");
+        exit(1);
+      }
+      HyperClockCacheOptions opts(FLAGS_cache_size, estimated_entry_charge,
+                                  FLAGS_cache_numshardbits);
+      opts.hash_seed = GetCacheHashSeed();
+      if (use_tiered_cache) {
+        TieredVolatileCacheOptions tiered_opts;
+        opts.capacity += secondary_cache_opts.capacity;
+        tiered_opts.cache_type = PrimaryCacheType::kCacheTypeHCC;
+        tiered_opts.cache_opts = &opts;
+        tiered_opts.comp_cache_opts = secondary_cache_opts;
+        return NewTieredVolatileCache(tiered_opts);
+      } else {
+        return opts.MakeSharedCache();
+      }
     } else if (FLAGS_cache_type == "lru_cache") {
       LRUCacheOptions opts(
           static_cast<size_t>(capacity), FLAGS_cache_numshardbits,
@@ -3040,26 +3081,21 @@ class Benchmark {
           exit(1);
         }
         opts.secondary_cache = secondary_cache;
-      }
-
-      if (FLAGS_use_compressed_secondary_cache) {
-        CompressedSecondaryCacheOptions secondary_cache_opts;
-        secondary_cache_opts.capacity = FLAGS_compressed_secondary_cache_size;
-        secondary_cache_opts.num_shard_bits =
-            FLAGS_compressed_secondary_cache_numshardbits;
-        secondary_cache_opts.high_pri_pool_ratio =
-            FLAGS_compressed_secondary_cache_high_pri_pool_ratio;
-        secondary_cache_opts.low_pri_pool_ratio =
-            FLAGS_compressed_secondary_cache_low_pri_pool_ratio;
-        secondary_cache_opts.compression_type =
-            FLAGS_compressed_secondary_cache_compression_type_e;
-        secondary_cache_opts.compress_format_version =
-            FLAGS_compressed_secondary_cache_compress_format_version;
+      } else if (FLAGS_use_compressed_secondary_cache && !use_tiered_cache) {
         opts.secondary_cache =
             NewCompressedSecondaryCache(secondary_cache_opts);
       }
 
-      return opts.MakeSharedCache();
+      if (use_tiered_cache) {
+        TieredVolatileCacheOptions tiered_opts;
+        opts.capacity += secondary_cache_opts.capacity;
+        tiered_opts.cache_type = PrimaryCacheType::kCacheTypeLRU;
+        tiered_opts.cache_opts = &opts;
+        tiered_opts.comp_cache_opts = secondary_cache_opts;
+        return NewTieredVolatileCache(tiered_opts);
+      } else {
+        return opts.MakeSharedCache();
+      }
     } else {
       fprintf(stderr, "Cache type not supported.");
       exit(1);
@@ -3966,7 +4002,8 @@ class Benchmark {
     bool ok = true;
     std::string compressed;
     CompressionOptions opts;
-    CompressionContext context(FLAGS_compression_type_e);
+    opts.level = FLAGS_compression_level;
+    CompressionContext context(FLAGS_compression_type_e, opts);
     CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
                          FLAGS_compression_type_e,
                          FLAGS_sample_for_compression);
@@ -3995,8 +4032,10 @@ class Benchmark {
     Slice input = gen.Generate(FLAGS_block_size);
     std::string compressed;
 
-    CompressionContext compression_ctx(FLAGS_compression_type_e);
     CompressionOptions compression_opts;
+    compression_opts.level = FLAGS_compression_level;
+    CompressionContext compression_ctx(FLAGS_compression_type_e,
+                                       compression_opts);
     CompressionInfo compression_info(
         compression_opts, compression_ctx, CompressionDict::GetEmptyDict(),
         FLAGS_compression_type_e, FLAGS_sample_for_compression);

package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc

@@ -1310,7 +1310,8 @@ void DBLoaderCommand::DoCommand() {
 namespace {
 
 void DumpManifestFile(Options options, std::string file, bool verbose, bool hex,
-                      bool json) {
+                      bool json,
+                      const std::vector<ColumnFamilyDescriptor>& cf_descs) {
   EnvOptions sopt;
   std::string dbname("dummy");
   std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
@@ -1326,7 +1327,7 @@ void DumpManifestFile(Options options, std::string file, bool verbose, bool hex,
   VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc,
                       /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
                       /*db_id*/ "", /*db_session_id*/ "");
-  Status s = versions.DumpManifest(options, file, verbose, hex, json);
+  Status s = versions.DumpManifest(options, file, verbose, hex, json, cf_descs);
   if (!s.ok()) {
     fprintf(stderr, "Error in processing file %s %s\n", file.c_str(),
             s.ToString().c_str());
@@ -1438,7 +1439,8 @@ void ManifestDumpCommand::DoCommand() {
     fprintf(stdout, "Processing Manifest file %s\n", manifestfile.c_str());
   }
 
-  DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_);
+  DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_,
+                   column_families_);
 
   if (verbose_) {
     fprintf(stdout, "Processing Manifest file %s done\n", manifestfile.c_str());
@@ -2046,7 +2048,7 @@ void DBDumperCommand::DoCommand() {
         break;
       case kDescriptorFile:
         DumpManifestFile(options_, path_, /* verbose_ */ false, is_key_hex_,
-                         /*  json_ */ false);
+                         /*  json_ */ false, column_families_);
         break;
       case kBlobFile:
         DumpBlobFile(path_, is_key_hex_, is_value_hex_,
@@ -3706,7 +3708,8 @@ void DBFileDumperCommand::DoCommand() {
   manifest_filepath = NormalizePath(manifest_filepath);
 
   std::cout << manifest_filepath << std::endl;
-  DumpManifestFile(options_, manifest_filepath, false, false, false);
+  DumpManifestFile(options_, manifest_filepath, false, false, false,
+                   column_families_);
   std::cout << std::endl;
 
   std::vector<ColumnFamilyMetaData> column_families;

package/deps/rocksdb/rocksdb/tools/ldb_tool.cc

@@ -93,6 +93,7 @@ void LDBCommandRunner::PrintHelp(const LDBOptions& ldb_options,
   BatchPutCommand::Help(ret);
   ScanCommand::Help(ret);
   DeleteCommand::Help(ret);
+  SingleDeleteCommand::Help(ret);
   DeleteRangeCommand::Help(ret);
   DBQuerierCommand::Help(ret);
   ApproxSizeCommand::Help(ret);