@nxtedition/rocksdb 8.2.0 → 8.2.2

package/deps/rocksdb/rocksdb/db/db_statistics_test.cc

@@ -20,76 +20,115 @@ class DBStatisticsTest : public DBTestBase {
 };
 
 TEST_F(DBStatisticsTest, CompressionStatsTest) {
-  CompressionType type;
-
-  if (Snappy_Supported()) {
-    type = kSnappyCompression;
-    fprintf(stderr, "using snappy\n");
-  } else if (Zlib_Supported()) {
-    type = kZlibCompression;
-    fprintf(stderr, "using zlib\n");
-  } else if (BZip2_Supported()) {
-    type = kBZip2Compression;
-    fprintf(stderr, "using bzip2\n");
-  } else if (LZ4_Supported()) {
-    type = kLZ4Compression;
-    fprintf(stderr, "using lz4\n");
-  } else if (XPRESS_Supported()) {
-    type = kXpressCompression;
-    fprintf(stderr, "using xpress\n");
-  } else if (ZSTD_Supported()) {
-    type = kZSTD;
-    fprintf(stderr, "using ZSTD\n");
-  } else {
-    fprintf(stderr, "skipping test, compression disabled\n");
-    return;
-  }
+  for (CompressionType type : GetSupportedCompressions()) {
+    if (type == kNoCompression) {
+      continue;
+    }
+    if (type == kBZip2Compression) {
+      // Weird behavior in this test
+      continue;
+    }
+    SCOPED_TRACE("Compression type: " + std::to_string(type));
 
-  Options options = CurrentOptions();
-  options.compression = type;
-  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
-  options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
-  DestroyAndReopen(options);
+    Options options = CurrentOptions();
+    options.compression = type;
+    options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+    options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
+    BlockBasedTableOptions bbto;
+    bbto.enable_index_compression = false;
+    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+    DestroyAndReopen(options);
 
-  int kNumKeysWritten = 100000;
+    auto PopStat = [&](Tickers t) -> uint64_t {
+      return options.statistics->getAndResetTickerCount(t);
+    };
 
-  // Check that compressions occur and are counted when compression is turned on
-  Random rnd(301);
-  for (int i = 0; i < kNumKeysWritten; ++i) {
-    // compressible string
-    ASSERT_OK(Put(Key(i), rnd.RandomString(128) + std::string(128, 'a')));
-  }
-  ASSERT_OK(Flush());
-  ASSERT_GT(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED), 0);
+    int kNumKeysWritten = 100;
+    double compress_to = 0.5;
+    // About three KVs per block
+    int len = static_cast<int>(BlockBasedTableOptions().block_size / 3);
+    int uncomp_est = kNumKeysWritten * (len + 20);
 
-  for (int i = 0; i < kNumKeysWritten; ++i) {
-    auto r = Get(Key(i));
-  }
-  ASSERT_GT(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED), 0);
+    Random rnd(301);
+    std::string buf;
 
-  options.compression = kNoCompression;
-  DestroyAndReopen(options);
-  uint64_t currentCompressions =
-      options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
-  uint64_t currentDecompressions =
-      options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED);
-
-  // Check that compressions do not occur when turned off
-  for (int i = 0; i < kNumKeysWritten; ++i) {
-    // compressible string
-    ASSERT_OK(Put(Key(i), rnd.RandomString(128) + std::string(128, 'a')));
-  }
-  ASSERT_OK(Flush());
-  ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED) -
-                currentCompressions,
-            0);
+    // Check that compressions occur and are counted when compression is turned
+    // on
+    for (int i = 0; i < kNumKeysWritten; ++i) {
+      ASSERT_OK(
+          Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
+    }
+    ASSERT_OK(Flush());
+    EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED));
+    EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10);
+    EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO),
+                 uncomp_est / 10);
+
+    EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
+    EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
+    EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
+
+    // And decompressions
+    for (int i = 0; i < kNumKeysWritten; ++i) {
+      auto r = Get(Key(i));
+    }
+    EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED));
+    EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10);
+    EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM),
+                 uncomp_est / 10);
+
+    EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
+    EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
+    EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
+    EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
+
+    // Check when compression is rejected.
+    DestroyAndReopen(options);
 
-  for (int i = 0; i < kNumKeysWritten; ++i) {
-    auto r = Get(Key(i));
+    for (int i = 0; i < kNumKeysWritten; ++i) {
+      ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len)));
+    }
+    ASSERT_OK(Flush());
+    for (int i = 0; i < kNumKeysWritten; ++i) {
+      auto r = Get(Key(i));
+    }
+    EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
+    EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED),
+                 uncomp_est / 10);
+
+    EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
+    EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
+    EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
+    EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
+    EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
+    EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
+    EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
+    EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
+
+    // Check when compression is disabled.
+    options.compression = kNoCompression;
+    DestroyAndReopen(options);
+
+    for (int i = 0; i < kNumKeysWritten; ++i) {
+      ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len)));
+    }
+    ASSERT_OK(Flush());
+    for (int i = 0; i < kNumKeysWritten; ++i) {
+      auto r = Get(Key(i));
+    }
+    EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
+    EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED),
+                 uncomp_est / 10);
+
+    EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
+    EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
+    EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
+    EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
+    EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
+    EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
+    EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
+    EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
   }
-  ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED) -
-                currentDecompressions,
-            0);
 }
 
 TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) {
@@ -203,6 +242,46 @@ TEST_F(DBStatisticsTest, VerifyChecksumReadStat) {
   }
 }
 
+TEST_F(DBStatisticsTest, BlockChecksumStats) {
+  Options options = CurrentOptions();
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+  Reopen(options);
+
+  // Scenario 0: only WAL data. Not verified so require ticker to be zero.
+  ASSERT_OK(Put("foo", "value"));
+  ASSERT_OK(db_->VerifyChecksum());
+  ASSERT_EQ(0,
+            options.statistics->getTickerCount(BLOCK_CHECKSUM_COMPUTE_COUNT));
+  ASSERT_EQ(0,
+            options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT));
+
+  // Scenario 1: Flushed table verified in `VerifyChecksum()`. This opens a
+  // `TableReader` to verify each of the four blocks (meta-index, table
+  // properties, index, and data block).
+  ASSERT_OK(Flush());
+  ASSERT_OK(options.statistics->Reset());
+  ASSERT_OK(db_->VerifyChecksum());
+  ASSERT_EQ(4,
+            options.statistics->getTickerCount(BLOCK_CHECKSUM_COMPUTE_COUNT));
+  ASSERT_EQ(0,
+            options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT));
+
+  // Scenario 2: Corrupted table verified in `VerifyChecksum()`. The corruption
+  // is in the fourth and final verified block, i.e., the data block.
+  std::unordered_map<std::string, uint64_t> table_files;
+  ASSERT_OK(GetAllDataFiles(kTableFile, &table_files));
+  ASSERT_EQ(1, table_files.size());
+  std::string table_name = table_files.begin()->first;
+  // Assumes the data block starts at offset zero.
+  ASSERT_OK(test::CorruptFile(options.env, table_name, 0 /* offset */,
+                              3 /* bytes_to_corrupt */));
+  ASSERT_OK(options.statistics->Reset());
+  ASSERT_NOK(db_->VerifyChecksum());
+  ASSERT_EQ(4,
+            options.statistics->getTickerCount(BLOCK_CHECKSUM_COMPUTE_COUNT));
+  ASSERT_EQ(1,
+            options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT));
+}
 
 }  // namespace ROCKSDB_NAMESPACE
 

package/deps/rocksdb/rocksdb/db/db_test.cc

@@ -66,7 +66,7 @@
 #include "util/compression.h"
 #include "util/mutexlock.h"
 #include "util/random.h"
-#include "util/rate_limiter.h"
+#include "util/rate_limiter_impl.h"
 #include "util/string_util.h"
 #include "utilities/merge_operators.h"
 
@@ -353,7 +353,7 @@ TEST_F(DBTest, MixedSlowdownOptionsInQueue) {
           for (int i = 0; i < 2; ++i) {
             threads.emplace_back(write_no_slowdown_func);
           }
-          // Sleep for 2s to allow the threads to insert themselves into the
+          // Sleep for 3s to allow the threads to insert themselves into the
           // write queue
           env_->SleepForMicroseconds(3000000ULL);
         }
@@ -424,7 +424,7 @@ TEST_F(DBTest, MixedSlowdownOptionsStop) {
           for (int i = 0; i < 2; ++i) {
             threads.emplace_back(write_no_slowdown_func);
           }
-          // Sleep for 2s to allow the threads to insert themselves into the
+          // Sleep for 3s to allow the threads to insert themselves into the
           // write queue
           env_->SleepForMicroseconds(3000000ULL);
         }
@@ -583,7 +583,7 @@ TEST_F(DBTest, LevelReopenWithFIFO) {
         TryReopenWithColumnFamilies({"default", "pikachu"}, fifo_options));
     // For FIFO to pick a compaction
     ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]));
-    ASSERT_OK(dbfull()->TEST_WaitForCompact(false));
+    ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
     for (int g = 0; g < kKeyCount; ++g) {
       std::string get_key = std::string(1, char('a' + g));
       int status_index = i / kKeyCount;
@@ -3107,6 +3107,13 @@ class ModelDB : public DB {
     return Status::NotSupported("Not implemented.");
   }
 
+  using DB::ClipColumnFamily;
+  virtual Status ClipColumnFamily(ColumnFamilyHandle* /*column_family*/,
+                                  const Slice& /*begin*/,
+                                  const Slice& /*end*/) override {
+    return Status::NotSupported("Not implemented.");
+  }
+
   using DB::GetPropertiesOfAllTables;
   Status GetPropertiesOfAllTables(
       ColumnFamilyHandle* /*column_family*/,
@@ -3460,6 +3467,7 @@ static bool CompareIterators(int step, DB* model, DB* db,
       ok = false;
     }
   }
+  (void)count;
   delete miter;
   delete dbiter;
   return ok;
@@ -4936,7 +4944,7 @@ TEST_P(DBTestWithParam, PreShutdownMultipleCompaction) {
   ASSERT_GE(operation_count[ThreadStatus::OP_COMPACTION], 1);
   CancelAllBackgroundWork(db_);
   TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:VerifyPreshutdown");
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
   // Record the number of compactions at a time.
   for (int i = 0; i < ThreadStatus::NUM_OP_TYPES; ++i) {
     operation_count[i] = 0;
@@ -5023,7 +5031,7 @@ TEST_P(DBTestWithParam, PreShutdownCompactionMiddle) {
   CancelAllBackgroundWork(db_);
   TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:Preshutdown");
   TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:VerifyPreshutdown");
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
   // Record the number of compactions at a time.
   for (int i = 0; i < ThreadStatus::NUM_OP_TYPES; ++i) {
     operation_count[i] = 0;
@@ -6886,7 +6894,20 @@ TEST_F(DBTest, CreateColumnFamilyShouldFailOnIncompatibleOptions) {
 TEST_F(DBTest, RowCache) {
   Options options = CurrentOptions();
   options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
-  options.row_cache = NewLRUCache(8192);
+  LRUCacheOptions cache_options;
+  cache_options.capacity = 8192;
+  options.row_cache = cache_options.MakeSharedGeneralCache();
+  // BEGIN check that Cache classes as aliases of each other.
+  // Currently, GeneralCache and BlockCache are aliases for Cache.
+  // This is expected to change (carefully, intentionally)
+  std::shared_ptr<GeneralCache> general_cache = options.row_cache;
+  std::shared_ptr<Cache> cache = general_cache;
+  std::shared_ptr<BlockCache> block_cache = general_cache;
+  general_cache = cache;
+  block_cache = cache;
+  general_cache = block_cache;
+  cache = block_cache;
+  // END check that Cache classes as aliases of each other.
   DestroyAndReopen(options);
 
   ASSERT_OK(Put("foo", "bar"));