@nxtedition/rocksdb 15.4.1 → 16.0.0

package/deps/rocksdb/rocksdb/db/db_flush_test.cc

@@ -17,6 +17,7 @@
 #include "port/port.h"
 #include "port/stack_trace.h"
 #include "rocksdb/utilities/transaction_db.h"
+#include "table/block_based/block_based_table_builder.h"
 #include "test_util/sync_point.h"
 #include "test_util/testutil.h"
 #include "util/cast_util.h"
@@ -33,6 +34,17 @@ static std::string NEW_VALUE = "NewValue";
 class DBFlushTest : public DBTestBase {
  public:
   DBFlushTest() : DBTestBase("db_flush_test", /*env_do_fsync=*/true) {}
+
+  // Wait for all background flush callbacks to complete before Close().
+  // Close() calls db_.reset() which nulls the unique_ptr before the destructor
+  // runs, so a concurrent OnFlushCompleted reading test_->db_ would race.
+  // WaitForCompact() waits for bg_flush_scheduled_==0 (which is decremented
+  // after listener callbacks return) without setting shutting_down_ (so
+  // callbacks are not skipped) and without requiring NumNotFlushed()==0 (which
+  // would hang when mempurge leaves a memtable in imm()).
+  Status WaitForFlushCallbacks() {
+    return db_->WaitForCompact(WaitForCompactOptions());
+  }
 };
 
 class DBFlushDirectIOTest : public DBFlushTest,
@@ -101,7 +113,7 @@ TEST_F(DBFlushTest, SyncFail) {
   TEST_SYNC_POINT("DBFlushTest::SyncFail:2");
   fault_injection_env->SetFilesystemActive(true);
   // Now the background job will do the flush; wait for it.
-  // Returns the IO error happend during flush.
+  // Returns the IO error happened during flush.
   ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable());
   ASSERT_EQ("", FilesPerLevel());  // flush failed.
   Destroy(options);
@@ -518,11 +530,11 @@ TEST_F(DBFlushTest, StatisticsGarbageInsertAndDeletes) {
   // Note : one set of delete for KEY1, KEY2, KEY3 is written to
   // SSTable to propagate the delete operations to K-V pairs
   // that could have been inserted into the database during past Flush
-  // opeartions.
+  // operations.
   EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH -=
       KEY1.size() + KEY2.size() + KEY3.size() + 3 * sizeof(uint64_t);
 
-  // Additional useful paylaod.
+  // Additional useful payload.
   ASSERT_OK(Delete(KEY4));
   ASSERT_OK(Delete(KEY5));
   ASSERT_OK(Delete(KEY6));
@@ -614,7 +626,7 @@ TEST_F(DBFlushTest, StatisticsGarbageRangeDeletes) {
 
   // Note : one set of deleteRange for (KEY1, KEY2) and (KEY2, KEY3) is written
   // to SSTable to propagate the deleteRange operations to K-V pairs that could
-  // have been inserted into the database during past Flush opeartions.
+  // have been inserted into the database during past Flush operations.
   EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH -=
       (KEY1.size() + KEY2.size() + sizeof(uint64_t)) +
       (KEY2.size() + KEY3.size() + sizeof(uint64_t));
@@ -709,7 +721,7 @@ class TestFlushListener : public EventListener {
     // that assumption does not hold (see the test case MultiDBMultiListeners
     // below).
     ASSERT_TRUE(test_);
-    if (db == test_->db_) {
+    if (db == test_->db_.get()) {
       std::vector<std::vector<FileMetaData>> files_by_level;
       test_->dbfull()->TEST_GetFilesMetaData(db->DefaultColumnFamily(),
                                              &files_by_level);
@@ -842,7 +854,7 @@ TEST_F(DBFlushTest, FixFlushReasonRaceFromConcurrentFlushes) {
     ASSERT_OK(Put(1, Key(idx), std::string(1, 'v')));
   }
 
-  // To coerce a manual flush happenning in the middle of GetLiveFiles's flush,
+  // To coerce a manual flush happening in the middle of GetLiveFiles's flush,
   // we need to pause background flush thread and enable it later.
   std::shared_ptr<test::SleepingBackgroundTask> sleeping_task =
       std::make_shared<test::SleepingBackgroundTask>();
@@ -851,7 +863,7 @@ TEST_F(DBFlushTest, FixFlushReasonRaceFromConcurrentFlushes) {
                  sleeping_task.get(), Env::Priority::HIGH);
   sleeping_task->WaitUntilSleeping();
 
-  // Coerce a manual flush happenning in the middle of GetLiveFiles's flush
+  // Coerce a manual flush happening in the middle of GetLiveFiles's flush
   bool get_live_files_paused_at_sync_point = false;
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
       "DBImpl::AtomicFlushMemTables:AfterScheduleFlush", [&](void* /* arg */) {
@@ -1055,6 +1067,7 @@ TEST_F(DBFlushTest, MemPurgeBasic) {
   ASSERT_EQ(Get(RNDKEY2), p_rv2);
   ASSERT_EQ(Get(RNDKEY3), p_rv3);
 
+  ASSERT_OK(WaitForFlushCallbacks());
   Close();
 }
 
@@ -1169,6 +1182,7 @@ TEST_F(DBFlushTest, MemPurgeBasicToggle) {
   // We expect no mempurge at all.
   EXPECT_EQ(mempurge_count.exchange(0), ZERO);
 
+  ASSERT_OK(WaitForFlushCallbacks());
   Close();
 }
 // End of MemPurgeBasicToggle, which is not
@@ -1425,10 +1439,11 @@ TEST_F(DBFlushTest, MemPurgeDeleteAndDeleteRange) {
     delete iter;
   }
 
+  ASSERT_OK(WaitForFlushCallbacks());
   Close();
 }
 
-// Create a Compaction Fitler that will be invoked
+// Create a Compaction Filter that will be invoked
 // at flush time and will update the value of a KV pair
 // if the key string is "lower" than the filter_key_ string.
 class ConditionalUpdateFilter : public CompactionFilter {
@@ -1879,6 +1894,85 @@ TEST_F(DBFlushTest, MemPurgeCorrectLogNumberAndSSTFileCreation) {
   Close();
 }
 
+// Reproduction for MemPurge memtable ID ordering bug.
+// When MemPurge runs, it releases db_mutex_. During that window, new
+// memtables can be switched to the immutable list with higher IDs. When
+// MemPurge re-acquires the mutex and adds its output memtable using the
+// stale ID from mems_.back(), the ordering assertion fires.
+TEST_F(DBFlushTest, MemPurgeIdOrdering) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.compression = kNoCompression;
+  options.inplace_update_support = false;
+  options.allow_concurrent_memtable_write = true;
+  options.write_buffer_size = 1 << 20;  // 1MB
+  // Allow enough immutable memtables so writes don't stall while the
+  // flush thread is paused in the sync point.
+  options.max_write_buffer_number = 8;
+  // Always attempt MemPurge on flush.
+  options.experimental_mempurge_threshold = 1.0;
+  ASSERT_OK(TryReopen(options));
+
+  // Coordinate via LoadDependency:
+  //   1. Flush thread hits BeforeReacquireMutex -> foreground unblocks
+  //   2. Foreground writes data, triggers memtable switch
+  //   3. Foreground hits SwitchDone -> flush thread unblocks at
+  //   AfterWaitForTest
+  //   4. Flush thread re-acquires mutex, tries AddMemTable with stale ID
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"FlushJob::MemPurge:BeforeReacquireMutex",
+        "DBFlushTest::MemPurgeIdOrdering:StartWriting"},
+       {"DBFlushTest::MemPurgeIdOrdering:SwitchDone",
+        "FlushJob::MemPurge:AfterWaitForTest"}});
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  const int kValueSize = 10240;  // 10KB values like MemPurgeBasic
+  const int kNumKeys = 30;
+  Random rnd(301);
+
+  // Fill the memtable with overwrites of the same keys so MemPurge
+  // can compact them down (high garbage ratio). Each round is ~300KB,
+  // need ~3 rounds to approach the 1MB write_buffer_size.
+  for (int round = 0; round < 3; round++) {
+    for (int i = 0; i < kNumKeys; i++) {
+      ASSERT_OK(Put("key" + std::to_string(i), rnd.RandomString(kValueSize)));
+    }
+  }
+
+  // One more round should trigger a flush with MemPurge in the background.
+  for (int i = 0; i < kNumKeys; i++) {
+    ASSERT_OK(Put("key" + std::to_string(i), rnd.RandomString(kValueSize)));
+  }
+
+  // Block until MemPurge reaches the sync point (db_mutex_ released).
+  TEST_SYNC_POINT("DBFlushTest::MemPurgeIdOrdering:StartWriting");
+
+  // Now MemPurge is paused with db_mutex_ released. Write enough data
+  // to fill another memtable and trigger a switch. This creates an
+  // immutable memtable with a higher ID than the one being mempurged.
+  for (int i = 0; i < kNumKeys * 4; i++) {
+    ASSERT_OK(Put("newkey" + std::to_string(i), rnd.RandomString(kValueSize)));
+  }
+
+  // Let MemPurge continue -- it will re-acquire the mutex and try to
+  // add the output memtable with the stale ID.
+  TEST_SYNC_POINT("DBFlushTest::MemPurgeIdOrdering:SwitchDone");
+
+  // Allow background work to finish.
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+
+  // If the bug is present, the assertion in AddMemTable fires before
+  // we reach here.
+  for (int i = 0; i < kNumKeys; i++) {
+    ASSERT_NE(Get("key" + std::to_string(i)), "NOT_FOUND");
+  }
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  Close();
+}
+
 TEST_P(DBFlushDirectIOTest, DirectIO) {
   Options options;
   options.create_if_missing = true;
@@ -2533,7 +2627,7 @@ TEST_F(DBFlushTest, TombstoneVisibleInSnapshot) {
 
   ASSERT_OK(db_->Put(WriteOptions(), "foo", "value0"));
 
-  ManagedSnapshot snapshot_guard(db_);
+  ManagedSnapshot snapshot_guard(db_.get());
 
   ColumnFamilyHandle* default_cf = db_->DefaultColumnFamily();
   ASSERT_OK(db_->Flush(FlushOptions(), default_cf));
@@ -2574,7 +2668,7 @@ TEST_P(DBAtomicFlushTest, ManualFlushUnder2PC) {
   txn_db_opts.write_policy = TxnDBWritePolicy::WRITE_COMMITTED;
   ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, &txn_db));
   ASSERT_NE(txn_db, nullptr);
-  db_ = txn_db;
+  db_.reset(txn_db);
 
   // Create two more columns other than default CF.
   std::vector<std::string> cfs = {"puppy", "kitty"};
@@ -2638,9 +2732,8 @@ TEST_P(DBAtomicFlushTest, ManualFlushUnder2PC) {
   // it means atomic flush didn't write the min_log_number_to_keep to MANIFEST.
   cfs.push_back(kDefaultColumnFamilyName);
   ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
-  DBImpl* db_impl = static_cast<DBImpl*>(db_);
-  ASSERT_TRUE(db_impl->allow_2pc());
-  ASSERT_NE(db_impl->MinLogNumberToKeep(), 0);
+  ASSERT_TRUE(dbfull()->allow_2pc());
+  ASSERT_NE(dbfull()->MinLogNumberToKeep(), 0);
 }
 
 TEST_P(DBAtomicFlushTest, ManualAtomicFlush) {
@@ -3707,6 +3800,55 @@ INSTANTIATE_TEST_CASE_P(
                      // larger than the max allowed padding size
                      testing::Values(4, kLowSpaceOverheadRatio)));
 
+// Test that when the table builder's io_status becomes bad during flush
+// (simulating write fault injection), BuildTable properly propagates the
+// builder's IO error instead of producing a misleading Corruption from the
+// num_entries mismatch check.
+TEST_F(DBFlushTest, BuilderWriteFaultPropagationDuringFlush) {
+  Options options = CurrentOptions();
+  options.flush_verify_memtable_count = true;
+  options.table_factory.reset(
+      NewBlockBasedTableFactory(BlockBasedTableOptions()));
+
+  DestroyAndReopen(options);
+
+  for (int i = 0; i < 100; i++) {
+    ASSERT_OK(Put("key" + std::to_string(i),
+                  std::string(100, 'v') + std::to_string(i)));
+  }
+
+  // Skip all Add() calls to simulate entries not being committed (builder
+  // stays empty), as happens when fault injection causes early returns.
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlockBasedTableBuilder::Add::skip",
+      [&](void* skip) { *(bool*)skip = true; });
+
+  // Inject an IOError into the builder's status before the empty check.
+  // This simulates the scenario where write fault injection puts the builder
+  // in an error state, causing all Add() calls to return early (ok() is false)
+  // and leaving the builder empty.
+  SyncPoint::GetInstance()->SetCallBack(
+      "BuildTable:BeforeCheckEmpty", [&](void* arg) {
+        auto* builder = static_cast<BlockBasedTableBuilder*>(
+            static_cast<TableBuilder*>(arg));
+        builder->TEST_InjectIOError();
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  Status s = Flush();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  // With the fix, the builder's IOError is propagated instead of the
+  // misleading Corruption from the key count mismatch check.
+  ASSERT_NOK(s);
+  ASSERT_TRUE(s.IsIOError())
+      << "Expected IOError from builder error propagation, got: "
+      << s.ToString();
+
+  Close();
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc

@@ -91,8 +91,21 @@ Status CompactedDBImpl::Get(const ReadOptions& _read_options,
           /*b_has_ts=*/false) < 0) {
     return Status::NotFound();
   }
-  Status s = f.fd.table_reader->Get(read_options, lkey.internal_key(),
-                                    &get_context, nullptr);
+  TableReader* t = nullptr;
+  TableCache::TypedHandle* handle = nullptr;
+  Status s = cfd_->table_cache()->FindTable(
+      read_options, cfd_->table_cache()->file_options(),
+      cfd_->internal_comparator(), *f.file_metadata, &handle,
+      version_->GetMutableCFOptions(), &t, false /* no_io */,
+      nullptr /* file_read_hist */, false /* skip_filters */, files_level_,
+      true /* prefetch_index_and_filter_in_cache */,
+      0 /* max_file_size_for_l0_meta_pin */, f.file_metadata->temperature,
+      true /* pin_table_handle */);
+  if (s.ok()) {
+    assert(handle == nullptr);
+    // Use TableReader directly to avoid extra table_cache->Get() overheads
+    s = t->Get(read_options, lkey.internal_key(), &get_context, nullptr);
+  }
   if (!s.ok() && !s.IsNotFound()) {
     return s;
   }
@@ -152,7 +165,6 @@ void CompactedDBImpl::MultiGet(const ReadOptions& _read_options,
   }
 
   GetWithTimestampReadCallback read_cb(kMaxSequenceNumber);
-  autovector<TableReader*, 16> reader_list;
   for (size_t i = 0; i < num_keys; ++i) {
     const Slice& key = keys[i];
     LookupKey lkey(key, kMaxSequenceNumber, read_options.timestamp);
@@ -162,38 +174,40 @@ void CompactedDBImpl::MultiGet(const ReadOptions& _read_options,
             ExtractUserKeyAndStripTimestamp(f.smallest_key,
                                             user_comparator_->timestamp_size()),
             /*b_has_ts=*/false) < 0) {
-      reader_list.push_back(nullptr);
-    } else {
-      f.fd.table_reader->Prepare(lkey.internal_key());
-      reader_list.push_back(f.fd.table_reader);
+      statuses[i] = Status::NotFound();
+      continue;
     }
-  }
-  for (size_t i = 0; i < num_keys; ++i) {
-    statuses[i] = Status::NotFound();
-  }
-  int idx = 0;
-  for (auto* r : reader_list) {
-    if (r != nullptr) {
-      PinnableSlice& pinnable_val = values[idx];
-      LookupKey lkey(keys[idx], kMaxSequenceNumber, read_options.timestamp);
-      std::string* timestamp = timestamps ? &timestamps[idx] : nullptr;
-      GetContext get_context(
-          user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound,
-          lkey.user_key(), &pinnable_val, /*columns=*/nullptr,
-          user_comparator_->timestamp_size() > 0 ? timestamp : nullptr, nullptr,
-          nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb);
-      Status status =
-          r->Get(read_options, lkey.internal_key(), &get_context, nullptr);
-      assert(static_cast<size_t>(idx) < num_keys);
-      if (!status.ok() && !status.IsNotFound()) {
-        statuses[idx] = status;
+    PinnableSlice& pinnable_val = values[i];
+    std::string* timestamp = timestamps ? &timestamps[i] : nullptr;
+    GetContext get_context(
+        user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound,
+        lkey.user_key(), &pinnable_val, /*columns=*/nullptr,
+        user_comparator_->timestamp_size() > 0 ? timestamp : nullptr, nullptr,
+        nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb);
+    TableReader* t = nullptr;
+    TableCache::TypedHandle* handle = nullptr;
+    Status status = cfd_->table_cache()->FindTable(
+        read_options, cfd_->table_cache()->file_options(),
+        cfd_->internal_comparator(), *f.file_metadata, &handle,
+        version_->GetMutableCFOptions(), &t, false /* no_io */,
+        nullptr /* file_read_hist */, false /* skip_filters */, files_level_,
+        true /* prefetch_index_and_filter_in_cache */,
+        0 /* max_file_size_for_l0_meta_pin */, f.file_metadata->temperature,
+        true /* pin_table_handle */);
+    if (status.ok()) {
+      assert(handle == nullptr);
+      // Use TableReader directly to avoid extra table_cache->Get() overheads
+      status = t->Get(read_options, lkey.internal_key(), &get_context, nullptr);
+    }
+    if (!status.ok() && !status.IsNotFound()) {
+      statuses[i] = status;
+    } else {
+      if (get_context.State() == GetContext::kFound) {
+        statuses[i] = Status::OK();
       } else {
-        if (get_context.State() == GetContext::kFound) {
-          statuses[idx] = Status::OK();
-        }
+        statuses[i] = Status::NotFound();
       }
     }
-    ++idx;
   }
 }
 
@@ -230,6 +244,7 @@ Status CompactedDBImpl::Init(const Options& options) {
       return Status::NotSupported("Both L0 and other level contain files");
     }
     files_ = l0;
+    files_level_ = 0;
     return Status::OK();
   }
 
@@ -242,6 +257,7 @@ Status CompactedDBImpl::Init(const Options& options) {
   int level = vstorage->num_non_empty_levels() - 1;
   if (vstorage->LevelFilesBrief(level).num_files > 0) {
     files_ = vstorage->LevelFilesBrief(level);
+    files_level_ = level;
     return Status::OK();
   }
   return Status::NotSupported("no file exists");
@@ -261,6 +277,13 @@ Status CompactedDBImpl::Open(const Options& options, const std::string& dbname,
   std::unique_ptr<CompactedDBImpl> db(new CompactedDBImpl(db_options, dbname));
   Status s = db->Init(options);
   if (s.ok()) {
+    {
+      InstrumentedMutexLock l(&db->mutex_);
+      db->opened_successfully_ = true;
+      if (db->immutable_db_options_.open_files_async) {
+        db->ScheduleAsyncFileOpening();
+      }
+    }
     s = db->StartPeriodicTaskScheduler();
   }
   if (s.ok()) {

package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h

@@ -143,5 +143,6 @@ class CompactedDBImpl : public DBImpl {
   Version* version_;
   const Comparator* user_comparator_;
   LevelFilesBrief files_;
+  int files_level_{0};
 };
 }  // namespace ROCKSDB_NAMESPACE