@nxtedition/rocksdb 8.2.7 → 9.0.0

package/deps/rocksdb/rocksdb/db/convenience.cc

@@ -7,6 +7,7 @@
 
 #include "rocksdb/convenience.h"
 
+#include "db/convenience_impl.h"
 #include "db/db_impl/db_impl.h"
 #include "util/cast_util.h"
 
@@ -39,9 +40,25 @@ Status VerifySstFileChecksum(const Options& options,
 }
 Status VerifySstFileChecksum(const Options& options,
                              const EnvOptions& env_options,
-                             const ReadOptions& read_options,
+                             const ReadOptions& _read_options,
                              const std::string& file_path,
                              const SequenceNumber& largest_seqno) {
+  if (_read_options.io_activity != Env::IOActivity::kUnknown) {
+    return Status::InvalidArgument(
+        "Can only call VerifySstFileChecksum with `ReadOptions::io_activity` "
+        "is "
+        "`Env::IOActivity::kUnknown`");
+  }
+  ReadOptions read_options(_read_options);
+  return VerifySstFileChecksumInternal(options, env_options, read_options,
+                                       file_path, largest_seqno);
+}
+
+Status VerifySstFileChecksumInternal(const Options& options,
+                                     const EnvOptions& env_options,
+                                     const ReadOptions& read_options,
+                                     const std::string& file_path,
+                                     const SequenceNumber& largest_seqno) {
   std::unique_ptr<FSRandomAccessFile> file;
   uint64_t file_size;
   InternalKeyComparator internal_comparator(options.comparator);
@@ -68,8 +85,8 @@ Status VerifySstFileChecksum(const Options& options,
       !kImmortal, false /* force_direct_prefetch */, -1 /* level */);
   reader_options.largest_seqno = largest_seqno;
   s = ioptions.table_factory->NewTableReader(
-      reader_options, std::move(file_reader), file_size, &table_reader,
-      false /* prefetch_index_and_filter_in_cache */);
+      read_options, reader_options, std::move(file_reader), file_size,
+      &table_reader, false /* prefetch_index_and_filter_in_cache */);
   if (!s.ok()) {
     return s;
   }

package/deps/rocksdb/rocksdb/db/convenience_impl.h

@@ -0,0 +1,15 @@
+// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+#include "rocksdb/db.h"
+
+namespace ROCKSDB_NAMESPACE {
+Status VerifySstFileChecksumInternal(const Options& options,
+                                     const EnvOptions& env_options,
+                                     const ReadOptions& read_options,
+                                     const std::string& file_path,
+                                     const SequenceNumber& largest_seqno = 0);
+}  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/db/corruption_test.cc

@@ -450,6 +450,7 @@ TEST_F(CorruptionTest, TableFile) {
 
 TEST_F(CorruptionTest, VerifyChecksumReadahead) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   SpecialEnv senv(base_env_);
   options.env = &senv;
   // Disable block cache as we are going to check checksum for
@@ -503,6 +504,7 @@ TEST_F(CorruptionTest, VerifyChecksumReadahead) {
 
 TEST_F(CorruptionTest, TableFileIndexData) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   // very big, we'll trigger flushes manually
   options.write_buffer_size = 100 * 1024 * 1024;
   Reopen(&options);
@@ -659,6 +661,7 @@ TEST_F(CorruptionTest, CorruptedDescriptor) {
 
 TEST_F(CorruptionTest, CompactionInputError) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   Reopen(&options);
   Build(10);
@@ -680,6 +683,7 @@ TEST_F(CorruptionTest, CompactionInputError) {
 
 TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   options.paranoid_checks = true;
   options.write_buffer_size = 131072;
@@ -777,6 +781,7 @@ TEST_F(CorruptionTest, RangeDeletionCorrupted) {
 TEST_F(CorruptionTest, FileSystemStateCorrupted) {
   for (int iter = 0; iter < 2; ++iter) {
     Options options;
+    options.level_compaction_dynamic_level_bytes = false;
     options.env = env_.get();
     options.paranoid_checks = true;
     options.create_if_missing = true;
@@ -816,6 +821,7 @@ static const auto& corruption_modes = {
 
 TEST_F(CorruptionTest, ParanoidFileChecksOnFlush) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   options.check_flush_compaction_key_order = false;
   options.paranoid_file_checks = true;
@@ -844,6 +850,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnFlush) {
 
 TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   options.paranoid_file_checks = true;
   options.create_if_missing = true;
@@ -877,6 +884,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) {
 
 TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeFirst) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   options.check_flush_compaction_key_order = false;
   options.paranoid_file_checks = true;
@@ -913,6 +921,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeFirst) {
 
 TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRange) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   options.check_flush_compaction_key_order = false;
   options.paranoid_file_checks = true;
@@ -952,6 +961,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRange) {
 
 TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeLast) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   options.check_flush_compaction_key_order = false;
   options.paranoid_file_checks = true;
@@ -988,6 +998,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeLast) {
 
 TEST_F(CorruptionTest, LogCorruptionErrorsInCompactionIterator) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   options.create_if_missing = true;
   options.allow_data_in_errors = true;
@@ -1017,6 +1028,7 @@ TEST_F(CorruptionTest, LogCorruptionErrorsInCompactionIterator) {
 
 TEST_F(CorruptionTest, CompactionKeyOrderCheck) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   options.paranoid_file_checks = false;
   options.create_if_missing = true;
@@ -1044,6 +1056,7 @@ TEST_F(CorruptionTest, CompactionKeyOrderCheck) {
 
 TEST_F(CorruptionTest, FlushKeyOrderCheck) {
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   options.paranoid_file_checks = false;
   options.create_if_missing = true;
@@ -1097,6 +1110,7 @@ TEST_F(CorruptionTest, DisableKeyOrderCheck) {
 TEST_F(CorruptionTest, VerifyWholeTableChecksum) {
   CloseDb();
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.env = env_.get();
   ASSERT_OK(DestroyDB(dbname_, options));
   options.create_if_missing = true;
@@ -1182,6 +1196,7 @@ INSTANTIATE_TEST_CASE_P(CorruptionTest, CrashDuringRecoveryWithCorruptionTest,
 TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecovery) {
   CloseDb();
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.track_and_verify_wals_in_manifest =
       track_and_verify_wals_in_manifest_;
   options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
@@ -1354,6 +1369,7 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecovery) {
 TEST_P(CrashDuringRecoveryWithCorruptionTest, TxnDbCrashDuringRecovery) {
   CloseDb();
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
   options.track_and_verify_wals_in_manifest =
       track_and_verify_wals_in_manifest_;
@@ -1551,6 +1567,7 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, TxnDbCrashDuringRecovery) {
 TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecoveryWithFlush) {
   CloseDb();
   Options options;
+  options.level_compaction_dynamic_level_bytes = false;
   options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
   options.avoid_flush_during_recovery = false;
   options.env = env_.get();

package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc

@@ -39,6 +39,7 @@ class CuckooTableDBTest : public testing::Test {
 
   Options CurrentOptions() {
     Options options;
+    options.level_compaction_dynamic_level_bytes = false;
     options.table_factory.reset(NewCuckooTableFactory());
     options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true));
     options.allow_mmap_reads = true;

package/deps/rocksdb/rocksdb/db/db_basic_test.cc

@@ -1204,9 +1204,23 @@ TEST_F(DBBasicTest, DBClose) {
   delete db;
   ASSERT_EQ(env->GetCloseCount(), 2);
 
+  // close by WaitForCompact() with close_db option
+  options.create_if_missing = false;
+  s = DB::Open(options, dbname, &db);
+  ASSERT_OK(s);
+  ASSERT_TRUE(db != nullptr);
+  WaitForCompactOptions wait_for_compact_options = WaitForCompactOptions();
+  wait_for_compact_options.close_db = true;
+  s = db->WaitForCompact(wait_for_compact_options);
+  ASSERT_EQ(env->GetCloseCount(), 3);
+  // see TestLogger::CloseHelper()
+  ASSERT_EQ(s, Status::IOError());
+
+  delete db;
+  ASSERT_EQ(env->GetCloseCount(), 3);
+
   // Provide our own logger and ensure DB::Close() does not close it
   options.info_log.reset(new TestEnv::TestLogger(env));
-  options.create_if_missing = false;
   s = DB::Open(options, dbname, &db);
   ASSERT_OK(s);
   ASSERT_TRUE(db != nullptr);
@@ -1214,9 +1228,9 @@ TEST_F(DBBasicTest, DBClose) {
   s = db->Close();
   ASSERT_EQ(s, Status::OK());
   delete db;
-  ASSERT_EQ(env->GetCloseCount(), 2);
-  options.info_log.reset();
   ASSERT_EQ(env->GetCloseCount(), 3);
+  options.info_log.reset();
+  ASSERT_EQ(env->GetCloseCount(), 4);
 }
 
 TEST_F(DBBasicTest, DBCloseAllDirectoryFDs) {

package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc

@@ -741,10 +741,15 @@ TEST_F(DBBlockCacheTest, AddRedundantStats) {
   int iterations_tested = 0;
   for (std::shared_ptr<Cache> base_cache :
        {NewLRUCache(capacity, num_shard_bits),
+        // FixedHyperClockCache
         HyperClockCacheOptions(
             capacity,
             BlockBasedTableOptions().block_size /*estimated_value_size*/,
             num_shard_bits)
+            .MakeSharedCache(),
+        // AutoHyperClockCache
+        HyperClockCacheOptions(capacity, 0 /*estimated_value_size*/,
+                               num_shard_bits)
             .MakeSharedCache()}) {
     if (!base_cache) {
       // Skip clock cache when not supported

package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc

@@ -654,7 +654,7 @@ TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
     for (i = 0; i < maxKey; i++) {
       ASSERT_OK(Put(Key(i), Key(i)));
     }
-    Flush();
+    ASSERT_OK(Flush());
   };
   auto GetFn = [&]() {
     int i;
@@ -792,7 +792,7 @@ TEST_F(DBBloomFilterTest, BloomFilterRate) {
     }
     // Add a large key to make the file contain wide range
     ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
-    Flush(1);
+    ASSERT_OK(Flush(1));
 
     // Check if they can be found
     for (int i = 0; i < maxKey; i++) {
@@ -1696,7 +1696,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
     table_options.format_version = 5;
     options.table_factory.reset(NewBlockBasedTableFactory(table_options));
 
-    TryReopen(options);
+    ASSERT_OK(TryReopen(options));
     CreateAndReopenWithCF({fifo ? "abe" : "bob"}, options);
 
     const int maxKey = 10000;
@@ -1705,7 +1705,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
     }
     // Add a large key to make the file contain wide range
     ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
-    Flush(1);
+    ASSERT_OK(Flush(1));
     EXPECT_EQ(policy->DumpTestReport(),
               fifo ? "cf=abe,s=kCompactionStyleFIFO,n=7,l=0,b=0,r=kFlush\n"
                    : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n");
@@ -1713,7 +1713,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
     for (int i = maxKey / 2; i < maxKey; i++) {
       ASSERT_OK(Put(1, Key(i), Key(i)));
     }
-    Flush(1);
+    ASSERT_OK(Flush(1));
     EXPECT_EQ(policy->DumpTestReport(),
               fifo ? "cf=abe,s=kCompactionStyleFIFO,n=7,l=0,b=0,r=kFlush\n"
                    : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n");
@@ -2261,7 +2261,7 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTest) {
   ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
   ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count);
 
-  Flush();
+  ASSERT_OK(Flush());
 
   // sanity checks
   ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
@@ -2311,7 +2311,7 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
   ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count);
   ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
 
-  Flush();
+  ASSERT_OK(Flush());
 
   iter.reset(dbfull()->NewIterator(ReadOptions()));
 
@@ -2379,7 +2379,7 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
     snprintf(buf, sizeof(buf), "%02d______:end", i + 1);
     keystr = std::string(buf);
     ASSERT_OK(dbtest->Put(keystr, keystr));
-    dbtest->Flush();
+    ASSERT_OK(dbtest->Flush());
   }
 
   // GROUP 2
@@ -2390,7 +2390,7 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
     snprintf(buf, sizeof(buf), "%02d______:end", small_range_sstfiles + i + 1);
     keystr = std::string(buf);
     ASSERT_OK(dbtest->Put(keystr, keystr));
-    dbtest->Flush();
+    ASSERT_OK(dbtest->Flush());
   }
 }
 }  // anonymous namespace
@@ -2614,7 +2614,7 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
   compact_options.target_level = 7;
   ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr));
 
-  ASSERT_EQ(trivial_move, 1);
+  ASSERT_GE(trivial_move, 1);
   ASSERT_EQ(non_trivial_move, 0);
 
   prev_cache_filter_hits = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
@@ -2853,7 +2853,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) {
     ASSERT_OK(Put("foo", "bar"));
     ASSERT_OK(Put("foq1", "bar1"));
     ASSERT_OK(Put("fpa", "0"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
     std::unique_ptr<Iterator> iter_old(db_->NewIterator(read_options));
     ASSERT_EQ(CountIter(iter_old, "foo"), 4);
     EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0);
@@ -2981,7 +2981,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterNewColumnFamily) {
     ASSERT_OK(Put(2, "foo5", "bar5"));
     ASSERT_OK(Put(2, "foq6", "bar6"));
     ASSERT_OK(Put(2, "fpq7", "bar7"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
     {
       std::unique_ptr<Iterator> iter(
           db_->NewIterator(read_options, handles_[2]));
@@ -3031,17 +3031,17 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) {
     ASSERT_OK(Put("foo", "bar"));
     ASSERT_OK(Put("foo1", "bar1"));
     ASSERT_OK(Put("fpa", "0"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
     ASSERT_OK(Put("foo3", "bar3"));
     ASSERT_OK(Put("foo4", "bar4"));
     ASSERT_OK(Put("foo5", "bar5"));
     ASSERT_OK(Put("fpb", "1"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
     ASSERT_OK(Put("foo6", "bar6"));
     ASSERT_OK(Put("foo7", "bar7"));
     ASSERT_OK(Put("foo8", "bar8"));
     ASSERT_OK(Put("fpc", "2"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
 
     ReadOptions read_options;
     read_options.prefix_same_as_start = true;