@nxtedition/rocksdb 15.4.1 → 16.0.0

package/deps/rocksdb/rocksdb/db/db_options_test.cc

@@ -1689,6 +1689,46 @@ TEST_F(DBOptionsTest, SetOptionsNoManifestWrite) {
   ASSERT_EQ(Get("x"), "x");
 }
 
+TEST_F(DBOptionsTest, SetOptionsMultipleColumnFamilies) {
+  Options options;
+  options.create_if_missing = true;
+  options.env = CurrentOptions().env;
+  options.disable_auto_compactions = true;
+  Reopen(options);
+
+  // Create two additional column families
+  CreateColumnFamilies({"cf1", "cf2"}, options);
+  ReopenWithColumnFamilies({"default", "cf1", "cf2"}, options);
+
+  // Verify initial state - auto compaction should be disabled
+  ASSERT_TRUE(dbfull()->GetOptions(handles_[0]).disable_auto_compactions);
+  ASSERT_TRUE(dbfull()->GetOptions(handles_[1]).disable_auto_compactions);
+  ASSERT_TRUE(dbfull()->GetOptions(handles_[2]).disable_auto_compactions);
+
+  // Set options on multiple column families at once
+  ASSERT_OK(dbfull()->SetOptions({handles_[1], handles_[2]},
+                                 {{"disable_auto_compactions", "false"}}));
+
+  ASSERT_TRUE(
+      dbfull()->GetOptions(handles_[0]).disable_auto_compactions);  // unchanged
+  ASSERT_FALSE(
+      dbfull()->GetOptions(handles_[1]).disable_auto_compactions);  // changed
+  ASSERT_FALSE(
+      dbfull()->GetOptions(handles_[2]).disable_auto_compactions);  // changed
+
+  std::unordered_map<ColumnFamilyHandle*,
+                     std::unordered_map<std::string, std::string>>
+      options_map;
+  options_map[handles_[0]] = {{"disable_auto_compactions", "false"}};
+  options_map[handles_[1]] = {{"disable_auto_compactions", "true"}};
+  options_map[handles_[2]] = {{"disable_auto_compactions", "true"}};
+  ASSERT_OK(dbfull()->SetOptions(options_map));
+
+  ASSERT_FALSE(dbfull()->GetOptions(handles_[0]).disable_auto_compactions);
+  ASSERT_TRUE(dbfull()->GetOptions(handles_[1]).disable_auto_compactions);
+  ASSERT_TRUE(dbfull()->GetOptions(handles_[2]).disable_auto_compactions);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/db/db_properties_test.cc

@@ -384,7 +384,7 @@ TEST_F(DBPropertiesTest, AggregatedTableProperties) {
 
     // Hold open a snapshot to prevent range tombstones from being compacted
     // away.
-    ManagedSnapshot snapshot(db_);
+    ManagedSnapshot snapshot(db_.get());
 
     Random rnd(5632);
     for (int table = 1; table <= kTableCount; ++table) {
@@ -582,7 +582,7 @@ TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) {
   DestroyAndReopen(options);
 
   // Hold open a snapshot to prevent range tombstones from being compacted away.
-  ManagedSnapshot snapshot(db_);
+  ManagedSnapshot snapshot(db_.get());
 
   std::string level_tp_strings[kMaxLevel];
   std::string tp_string;
@@ -1519,16 +1519,14 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
 
 // Excluded from RocksDB lite tests due to `GetPropertiesOfAllTables()` usage.
 TEST_F(DBPropertiesTest, BlockAddForCompressionSampling) {
-  // Sampled compression requires at least one of the following four types.
-  if (!Snappy_Supported() && !Zlib_Supported() && !LZ4_Supported() &&
-      !ZSTD_Supported()) {
-    return;
-  }
-
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   options.table_properties_collector_factories.emplace_back(
       std::make_shared<BlockCountingTablePropertiesCollectorFactory>());
+  options.compression = kNoCompression;
+
+  bool fast_sampling_supported = Snappy_Supported() || LZ4_Supported();
+  bool slow_sampling_supported = ZSTD_Supported() || Zlib_Supported();
 
   for (bool sample_for_compression : {false, true}) {
     // For simplicity/determinism, sample 100% when enabled, or 0% when disabled
@@ -1542,10 +1540,11 @@ TEST_F(DBPropertiesTest, BlockAddForCompressionSampling) {
     // L1_0 ["a", "b"]
     //
     // L0_0 was created by flush. L1_0 was created by compaction. Each file
-    // contains one data block.
+    // contains one data block with enough data to be compressible.
     for (int i = 0; i < 3; ++i) {
-      ASSERT_OK(Put("a", "val"));
-      ASSERT_OK(Put("b", "val"));
+      for (int j = 0; j < 50; ++j) {
+        ASSERT_OK(Put(std::to_string(j), "thisismyvalue"));
+      }
       ASSERT_OK(Flush());
       if (i == 1) {
         ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
@@ -1558,13 +1557,33 @@ TEST_F(DBPropertiesTest, BlockAddForCompressionSampling) {
     ASSERT_OK(db_->GetPropertiesOfAllTables(&file_to_props));
     ASSERT_EQ(2, file_to_props.size());
     for (const auto& file_and_props : file_to_props) {
-      auto& user_props = file_and_props.second->user_collected_properties;
+      auto& props = *file_and_props.second;
+      auto& user_props = props.user_collected_properties;
       ASSERT_TRUE(user_props.find(BlockCountingTablePropertiesCollector::
                                       kNumSampledBlocksPropertyName) !=
                   user_props.end());
       ASSERT_EQ(user_props.at(BlockCountingTablePropertiesCollector::
                                   kNumSampledBlocksPropertyName),
                 std::to_string(sample_for_compression ? 1 : 0));
+      if (sample_for_compression) {
+        EXPECT_GT(props.fast_compression_estimated_data_size, 0);
+        EXPECT_GT(props.slow_compression_estimated_data_size, 0);
+        if (fast_sampling_supported) {
+          EXPECT_LT(props.fast_compression_estimated_data_size,
+                    props.data_size);
+          if (slow_sampling_supported) {
+            EXPECT_LT(props.slow_compression_estimated_data_size,
+                      props.fast_compression_estimated_data_size);
+          }
+        }
+        if (slow_sampling_supported) {
+          EXPECT_LT(props.slow_compression_estimated_data_size,
+                    props.data_size);
+        }
+      } else {
+        EXPECT_EQ(props.fast_compression_estimated_data_size, 0);
+        EXPECT_EQ(props.slow_compression_estimated_data_size, 0);
+      }
     }
   }
 }
@@ -1845,7 +1864,7 @@ TEST_F(DBPropertiesTest, MinObsoleteSstNumberToKeep) {
   options.listeners.push_back(listener);
   options.level0_file_num_compaction_trigger = kNumL0Files;
   DestroyAndReopen(options);
-  listener->SetDB(db_);
+  listener->SetDB(db_.get());
 
   for (int i = 0; i < kNumL0Files; ++i) {
     // Make sure they overlap in keyspace to prevent trivial move

package/deps/rocksdb/rocksdb/db/db_range_del_test.cc

@@ -2047,7 +2047,7 @@ TEST_F(DBRangeDelTest, IteratorReseek) {
   // Immutable memtable
   ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(1),
                              Key(2)));
-  ASSERT_OK(static_cast_with_check<DBImpl>(db_)->TEST_SwitchMemtable());
+  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
   std::string value;
   ASSERT_TRUE(dbfull()->GetProperty(db_->DefaultColumnFamily(),
                                     "rocksdb.num-immutable-mem-table", &value));

package/deps/rocksdb/rocksdb/db/db_readonly_with_timestamp_test.cc

@@ -237,7 +237,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
          it->Next(), ++count, ++key) {
       CheckIterUserEntry(it.get(), Key1(key), kTypeValue,
                          "value" + std::to_string(i), write_timestamps[i]);
-      get_value_and_check(db_, read_opts, it->key(), it->value(),
+      get_value_and_check(db_.get(), read_opts, it->key(), it->value(),
                           write_timestamps[i]);
     }
     ASSERT_OK(it->status());
@@ -250,7 +250,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
          it->Prev(), ++count, --key) {
       CheckIterUserEntry(it.get(), Key1(key), kTypeValue,
                          "value" + std::to_string(i), write_timestamps[i]);
-      get_value_and_check(db_, read_opts, it->key(), it->value(),
+      get_value_and_check(db_.get(), read_opts, it->key(), it->value(),
                           write_timestamps[i]);
     }
     ASSERT_OK(it->status());
@@ -272,7 +272,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
            it->Valid(); it->Next(), ++key, ++count) {
         CheckIterUserEntry(it.get(), Key1(key), kTypeValue,
                            "value" + std::to_string(i), write_timestamps[i]);
-        get_value_and_check(db_, read_opts, it->key(), it->value(),
+        get_value_and_check(db_.get(), read_opts, it->key(), it->value(),
                             write_timestamps[i]);
       }
       ASSERT_OK(it->status());
@@ -282,7 +282,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
            it->Valid(); it->Prev(), --key, ++count) {
         CheckIterUserEntry(it.get(), Key1(key - 1), kTypeValue,
                            "value" + std::to_string(i), write_timestamps[i]);
-        get_value_and_check(db_, read_opts, it->key(), it->value(),
+        get_value_and_check(db_.get(), read_opts, it->key(), it->value(),
                             write_timestamps[i]);
       }
       ASSERT_OK(it->status());

package/deps/rocksdb/rocksdb/db/db_secondary_test.cc

@@ -56,12 +56,11 @@ class DBSecondaryTestBase : public DBBasicTestWithTimestampBase {
       ASSERT_OK(db_secondary_->DestroyColumnFamilyHandle(h));
     }
     handles_secondary_.clear();
-    delete db_secondary_;
-    db_secondary_ = nullptr;
+    db_secondary_.reset();
   }
 
   DBImplSecondary* db_secondary_full() {
-    return static_cast<DBImplSecondary*>(db_secondary_);
+    return static_cast<DBImplSecondary*>(db_secondary_.get());
   }
 
   void CheckFileTypeCounts(const std::string& dir, int expected_log,
@@ -69,7 +68,7 @@ class DBSecondaryTestBase : public DBBasicTestWithTimestampBase {
 
   std::string secondary_path_;
   std::vector<ColumnFamilyHandle*> handles_secondary_;
-  DB* db_secondary_;
+  std::unique_ptr<DB> db_secondary_;
 };
 
 void DBSecondaryTestBase::OpenSecondary(const Options& options) {
@@ -152,8 +151,8 @@ TEST_F(DBSecondaryTest, NonExistingDb) {
   options.env = env_;
   options.max_open_files = -1;
   const std::string dbname = "/doesnt/exist";
-  Status s =
-      DB::OpenAsSecondary(options, dbname, secondary_path_, &db_secondary_);
+  std::unique_ptr<DB> dbptr;
+  Status s = DB::OpenAsSecondary(options, dbname, secondary_path_, &dbptr);
   ASSERT_TRUE(s.IsIOError());
 }
 
@@ -182,7 +181,7 @@ TEST_F(DBSecondaryTest, ReopenAsSecondary) {
 
   ReadOptions ropts;
   ropts.verify_checksums = true;
-  auto db1 = static_cast<DBImplSecondary*>(db_);
+  auto db1 = static_cast<DBImplSecondary*>(db_.get());
   ASSERT_NE(nullptr, db1);
   Iterator* iter = db1->NewIterator(ropts);
   ASSERT_NE(nullptr, iter);
@@ -834,7 +833,7 @@ TEST_F(DBSecondaryTest, OpenWithSubsetOfColumnFamilies) {
   options1.max_open_files = -1;
   OpenSecondary(options1);
   ASSERT_EQ(0, handles_secondary_.size());
-  ASSERT_NE(nullptr, db_secondary_);
+  ASSERT_NE(nullptr, db_secondary_.get());
 
   ASSERT_OK(Put(0 /*cf*/, "foo", "foo_value"));
   ASSERT_OK(Put(1 /*cf*/, "foo", "foo_value"));
@@ -1152,7 +1151,7 @@ TEST_F(DBSecondaryTest, DISABLED_SwitchWAL) {
   for (int k = 0; k != 16; ++k) {
     ASSERT_OK(Put("key" + std::to_string(k), "value" + std::to_string(k)));
     ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
-    verify_db(dbfull(), db_secondary_);
+    verify_db(dbfull(), db_secondary_.get());
   }
 }
 
@@ -1221,7 +1220,7 @@ TEST_F(DBSecondaryTest, DISABLED_SwitchWALMultiColumnFamilies) {
     TEST_SYNC_POINT(
         "DBSecondaryTest::SwitchWALMultipleColumnFamilies:BeforeCatchUp");
     ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
-    verify_db(dbfull(), handles_, db_secondary_, handles_secondary_);
+    verify_db(dbfull(), handles_, db_secondary_.get(), handles_secondary_);
     SyncPoint::GetInstance()->ClearTrace();
   }
 }
@@ -1357,7 +1356,7 @@ TEST_F(DBSecondaryTest, OpenWithTransactionDB) {
   TransactionDBOptions txn_db_opts;
   ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, &txn_db));
   ASSERT_NE(txn_db, nullptr);
-  db_ = txn_db;
+  db_.reset(txn_db);
 
   std::vector<std::string> cfs = {"new_CF"};
   CreateColumnFamilies(cfs, options);
@@ -1561,7 +1560,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
          it->Next(), ++count, ++key) {
       CheckIterUserEntry(it.get(), Key1(key), kTypeValue,
                          "value" + std::to_string(i), write_timestamps[i]);
-      get_value_and_check(db_, read_opts, it->key(), it->value(),
+      get_value_and_check(db_.get(), read_opts, it->key(), it->value(),
                           write_timestamps[i]);
     }
     ASSERT_OK(it->status());
@@ -1574,7 +1573,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
          it->Prev(), ++count, --key) {
       CheckIterUserEntry(it.get(), Key1(key), kTypeValue,
                          "value" + std::to_string(i), write_timestamps[i]);
-      get_value_and_check(db_, read_opts, it->key(), it->value(),
+      get_value_and_check(db_.get(), read_opts, it->key(), it->value(),
                           write_timestamps[i]);
     }
     ASSERT_OK(it->status());
@@ -1596,7 +1595,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
            it->Valid(); it->Next(), ++key, ++count) {
         CheckIterUserEntry(it.get(), Key1(key), kTypeValue,
                            "value" + std::to_string(i), write_timestamps[i]);
-        get_value_and_check(db_, read_opts, it->key(), it->value(),
+        get_value_and_check(db_.get(), read_opts, it->key(), it->value(),
                             write_timestamps[i]);
       }
       ASSERT_OK(it->status());
@@ -1606,7 +1605,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
            it->Valid(); it->Prev(), --key, ++count) {
         CheckIterUserEntry(it.get(), Key1(key - 1), kTypeValue,
                            "value" + std::to_string(i), write_timestamps[i]);
-        get_value_and_check(db_, read_opts, it->key(), it->value(),
+        get_value_and_check(db_.get(), read_opts, it->key(), it->value(),
                             write_timestamps[i]);
       }
       ASSERT_OK(it->status());
@@ -1828,6 +1827,60 @@ TEST_F(DBSecondaryTestWithTimestamp, Iterators) {
   Close();
 }
 
+TEST_F(DBSecondaryTest, GetLiveFilesOnSecondary) {
+  Options options;
+  options.env = env_;
+  options.level0_file_num_compaction_trigger = 4;
+  Reopen(options);
+
+  // Write some data and flush to create SST files on the primary.
+  for (int i = 0; i < 3; ++i) {
+    ASSERT_OK(Put("key" + std::to_string(i), "value" + std::to_string(i)));
+    ASSERT_OK(Flush());
+  }
+
+  // Open secondary and verify GetLiveFiles works.
+  Options options1;
+  options1.env = env_;
+  options1.max_open_files = -1;
+  OpenSecondary(options1);
+
+  std::vector<std::string> live_files;
+  uint64_t manifest_size = 0;
+  ASSERT_OK(db_secondary_->GetLiveFiles(live_files, &manifest_size));
+  ASSERT_GT(live_files.size(), 0);
+  ASSERT_GT(manifest_size, 0);
+
+  // Should contain SST files, CURRENT, MANIFEST, and OPTIONS.
+  bool has_sst = false;
+  bool has_current = false;
+  bool has_manifest = false;
+  for (const auto& f : live_files) {
+    if (f.find(".sst") != std::string::npos) {
+      has_sst = true;
+    } else if (f.find("CURRENT") != std::string::npos) {
+      has_current = true;
+    } else if (f.find("MANIFEST") != std::string::npos) {
+      has_manifest = true;
+    }
+  }
+  ASSERT_TRUE(has_sst);
+  ASSERT_TRUE(has_current);
+  ASSERT_TRUE(has_manifest);
+
+  // Write more data on primary, catch up, and verify the file list updates.
+  ASSERT_OK(Put("key3", "value3"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
+
+  std::vector<std::string> live_files_after;
+  uint64_t manifest_size_after = 0;
+  ASSERT_OK(
+      db_secondary_->GetLiveFiles(live_files_after, &manifest_size_after));
+  ASSERT_GT(live_files_after.size(), live_files.size());
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/db/db_sst_test.cc

@@ -1535,7 +1535,7 @@ TEST_F(DBSSTTest, OpenDBWithInfiniteMaxOpenFiles) {
 
     for (const auto& level : files) {
       for (const auto& file : level) {
-        ASSERT_TRUE(file.table_reader_handle != nullptr);
+        ASSERT_TRUE(file.fd.pinned_reader.Get() != nullptr);
       }
     }
 

package/deps/rocksdb/rocksdb/db/db_statistics_test.cc

@@ -321,7 +321,7 @@ TEST_F(DBStatisticsTest, BytesWrittenStats) {
     options.enable_pipelined_write = enable_pipelined_write;
     ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, &txn_db));
     ASSERT_NE(txn_db, nullptr);
-    db_ = txn_db->GetBaseDB();
+    db_.reset(txn_db);
 
     WriteOptions wopts;
     TransactionOptions txn_opts;
@@ -351,8 +351,7 @@ TEST_F(DBStatisticsTest, BytesWrittenStats) {
                   WriteBatchInternal::kHeader);
 
     // Cleanup
-    db_ = nullptr;
-    delete txn_db;
+    db_.reset();
   }
 }
 

package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc

@@ -69,14 +69,6 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
 
   // Create 4 tables
   for (int table = 0; table < 4; ++table) {
-    // Use old meta name for table properties for one file
-    if (table == 3) {
-      SyncPoint::GetInstance()->SetCallBack(
-          "BlockBasedTableBuilder::WritePropertiesBlock:Meta", [&](void* meta) {
-            *static_cast<const std::string**>(meta) = &kPropertiesBlockOldName;
-          });
-      SyncPoint::GetInstance()->EnableProcessing();
-    }
     // Build file
     for (int i = 0; i < 10 + table; ++i) {
       ASSERT_OK(
@@ -84,7 +76,6 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
     }
     ASSERT_OK(db_->Flush(FlushOptions()));
   }
-  SyncPoint::GetInstance()->DisableProcessing();
   std::string original_session_id;
   ASSERT_OK(db_->GetDbSessionId(original_session_id));
 
@@ -99,7 +90,7 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
   // Clear out auto-opened files
   dbfull()->TEST_table_cache()->EraseUnRefEntries();
   ASSERT_EQ(dbfull()->TEST_table_cache()->GetUsage(), 0U);
-  VerifyTableProperties(db_, 10 + 11 + 12 + 13);
+  VerifyTableProperties(db_.get(), 10 + 11 + 12 + 13);
 
   // 2. Put two tables to table cache and
   Reopen(options);
@@ -112,7 +103,7 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
     Get(std::to_string(i * 100 + 0));
   }
 
-  VerifyTableProperties(db_, 10 + 11 + 12 + 13);
+  VerifyTableProperties(db_.get(), 10 + 11 + 12 + 13);
 
   // 3. Put all tables to table cache
   Reopen(options);
@@ -120,7 +111,7 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
   for (int i = 0; i < 4; ++i) {
     Get(std::to_string(i * 100 + 0));
   }
-  VerifyTableProperties(db_, 10 + 11 + 12 + 13);
+  VerifyTableProperties(db_.get(), 10 + 11 + 12 + 13);
 
   // 4. Try to read CORRUPT properties (a) directly from file, and (b)
   // through reader on Get
@@ -169,10 +160,7 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
   SyncPoint::GetInstance()->DisableProcessing();
 }
 
-TEST_F(DBTablePropertiesTest, InvalidIgnored) {
-  // RocksDB versions 2.5 - 2.7 generate some properties that Block considers
-  // invalid in some way. This approximates that.
-
+TEST_F(DBTablePropertiesTest, InvalidReportedAsCorruption) {
   // Inject properties block data that Block considers invalid
   SyncPoint::GetInstance()->SetCallBack(
       "BlockBasedTableBuilder::WritePropertiesBlock:BlockData",
@@ -189,13 +177,10 @@ TEST_F(DBTablePropertiesTest, InvalidIgnored) {
   for (int i = 0; i < 10; ++i) {
     ASSERT_OK(db_->Put(WriteOptions(), std::to_string(i), "val"));
   }
-  ASSERT_OK(db_->Flush(FlushOptions()));
+  // Corrupted properties block should be detected and reported as corruption
+  ASSERT_TRUE(db_->Flush(FlushOptions()).IsCorruption());
 
   SyncPoint::GetInstance()->DisableProcessing();
-
-  // Not crashing is good enough
-  TablePropertiesCollection props;
-  ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
 }
 
 TEST_F(DBTablePropertiesTest, CreateOnDeletionCollectorFactory) {