@nxtedition/rocksdb 13.5.7 → 13.5.9

package/deps/rocksdb/rocksdb/db/deletefile_test.cc

@@ -135,57 +135,6 @@ class DeleteFileTest : public DBTestBase {
   }
 };
 
-TEST_F(DeleteFileTest, AddKeysAndQueryLevels) {
-  Options options = CurrentOptions();
-  SetOptions(&options);
-  Destroy(options);
-  options.create_if_missing = true;
-  Reopen(options);
-
-  CreateTwoLevels();
-  std::vector<LiveFileMetaData> metadata;
-  db_->GetLiveFilesMetaData(&metadata);
-
-  std::string level1file;
-  int level1keycount = 0;
-  std::string level2file;
-  int level2keycount = 0;
-  int level1index = 0;
-  int level2index = 1;
-
-  ASSERT_EQ((int)metadata.size(), 2);
-  if (metadata[0].level == 2) {
-    level1index = 1;
-    level2index = 0;
-  }
-
-  level1file = metadata[level1index].name;
-  int startkey = atoi(metadata[level1index].smallestkey.c_str());
-  int endkey = atoi(metadata[level1index].largestkey.c_str());
-  level1keycount = (endkey - startkey + 1);
-  level2file = metadata[level2index].name;
-  startkey = atoi(metadata[level2index].smallestkey.c_str());
-  endkey = atoi(metadata[level2index].largestkey.c_str());
-  level2keycount = (endkey - startkey + 1);
-
-  // COntrolled setup. Levels 1 and 2 should both have 50K files.
-  // This is a little fragile as it depends on the current
-  // compaction heuristics.
-  ASSERT_EQ(level1keycount, 50000);
-  ASSERT_EQ(level2keycount, 50000);
-
-  Status status = db_->DeleteFile("0.sst");
-  ASSERT_TRUE(status.IsInvalidArgument());
-
-  // intermediate level files cannot be deleted.
-  status = db_->DeleteFile(level1file);
-  ASSERT_TRUE(status.IsInvalidArgument());
-
-  // Lowest level file deletion should succeed.
-  status = db_->DeleteFile(level2file);
-  ASSERT_OK(status);
-}
-
 TEST_F(DeleteFileTest, PurgeObsoleteFilesTest) {
   Options options = CurrentOptions();
   SetOptions(&options);
@@ -496,145 +445,6 @@ TEST_F(DeleteFileTest, BackgroundPurgeTestMultipleJobs) {
   CheckFileTypeCounts(dbname_, 0, 1, 1);
 }
 
-TEST_F(DeleteFileTest, DeleteFileWithIterator) {
-  Options options = CurrentOptions();
-  SetOptions(&options);
-  Destroy(options);
-  options.create_if_missing = true;
-  Reopen(options);
-
-  CreateTwoLevels();
-  ReadOptions read_options;
-  Iterator* it = db_->NewIterator(read_options);
-  ASSERT_OK(it->status());
-  std::vector<LiveFileMetaData> metadata;
-  db_->GetLiveFilesMetaData(&metadata);
-
-  std::string level2file;
-
-  ASSERT_EQ(metadata.size(), static_cast<size_t>(2));
-  if (metadata[0].level == 1) {
-    level2file = metadata[1].name;
-  } else {
-    level2file = metadata[0].name;
-  }
-
-  Status status = db_->DeleteFile(level2file);
-  fprintf(stdout, "Deletion status %s: %s\n", level2file.c_str(),
-          status.ToString().c_str());
-  ASSERT_OK(status);
-  it->SeekToFirst();
-  int numKeysIterated = 0;
-  while (it->Valid()) {
-    numKeysIterated++;
-    it->Next();
-  }
-  ASSERT_EQ(numKeysIterated, 50000);
-  delete it;
-}
-
-TEST_F(DeleteFileTest, DeleteLogFiles) {
-  Options options = CurrentOptions();
-  SetOptions(&options);
-  Destroy(options);
-  options.create_if_missing = true;
-  Reopen(options);
-
-  AddKeys(10, 0);
-  VectorLogPtr logfiles;
-  ASSERT_OK(db_->GetSortedWalFiles(logfiles));
-  ASSERT_GT(logfiles.size(), 0UL);
-  // Take the last log file which is expected to be alive and try to delete it
-  // Should not succeed because live logs are not allowed to be deleted
-  std::unique_ptr<LogFile> alive_log = std::move(logfiles.back());
-  ASSERT_EQ(alive_log->Type(), kAliveLogFile);
-  ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName()));
-  fprintf(stdout, "Deleting alive log file %s\n",
-          alive_log->PathName().c_str());
-  ASSERT_NOK(db_->DeleteFile(alive_log->PathName()));
-  ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName()));
-  logfiles.clear();
-
-  // Call Flush to bring about a new working log file and add more keys
-  // Call Flush again to flush out memtable and move alive log to archived log
-  // and try to delete the archived log file
-  FlushOptions fopts;
-  ASSERT_OK(db_->Flush(fopts));
-  AddKeys(10, 0);
-  ASSERT_OK(db_->Flush(fopts));
-  ASSERT_OK(db_->GetSortedWalFiles(logfiles));
-  ASSERT_GT(logfiles.size(), 0UL);
-  std::unique_ptr<LogFile> archived_log = std::move(logfiles.front());
-  ASSERT_EQ(archived_log->Type(), kArchivedLogFile);
-  ASSERT_OK(env_->FileExists(wal_dir_ + "/" + archived_log->PathName()));
-  fprintf(stdout, "Deleting archived log file %s\n",
-          archived_log->PathName().c_str());
-  ASSERT_OK(db_->DeleteFile(archived_log->PathName()));
-  ASSERT_TRUE(
-      env_->FileExists(wal_dir_ + "/" + archived_log->PathName()).IsNotFound());
-}
-
-TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) {
-  Options options = CurrentOptions();
-  SetOptions(&options);
-  Destroy(options);
-  options.create_if_missing = true;
-  Reopen(options);
-  CreateAndReopenWithCF({"new_cf"}, options);
-
-  Random rnd(5);
-  for (int i = 0; i < 1000; ++i) {
-    ASSERT_OK(db_->Put(WriteOptions(), handles_[1], test::RandomKey(&rnd, 10),
-                       test::RandomKey(&rnd, 10)));
-  }
-  ASSERT_OK(db_->Flush(FlushOptions(), handles_[1]));
-  for (int i = 0; i < 1000; ++i) {
-    ASSERT_OK(db_->Put(WriteOptions(), handles_[1], test::RandomKey(&rnd, 10),
-                       test::RandomKey(&rnd, 10)));
-  }
-  ASSERT_OK(db_->Flush(FlushOptions(), handles_[1]));
-
-  std::vector<LiveFileMetaData> metadata;
-  db_->GetLiveFilesMetaData(&metadata);
-  ASSERT_EQ(2U, metadata.size());
-  ASSERT_EQ("new_cf", metadata[0].column_family_name);
-  ASSERT_EQ("new_cf", metadata[1].column_family_name);
-  auto old_file = metadata[0].smallest_seqno < metadata[1].smallest_seqno
-                      ? metadata[0].name
-                      : metadata[1].name;
-  auto new_file = metadata[0].smallest_seqno > metadata[1].smallest_seqno
-                      ? metadata[0].name
-                      : metadata[1].name;
-  ASSERT_TRUE(db_->DeleteFile(new_file).IsInvalidArgument());
-  ASSERT_OK(db_->DeleteFile(old_file));
-
-  {
-    std::unique_ptr<Iterator> itr(db_->NewIterator(ReadOptions(), handles_[1]));
-    ASSERT_OK(itr->status());
-    int count = 0;
-    for (itr->SeekToFirst(); itr->Valid(); itr->Next()) {
-      ASSERT_OK(itr->status());
-      ++count;
-    }
-    ASSERT_OK(itr->status());
-    ASSERT_EQ(count, 1000);
-  }
-
-  Close();
-  ReopenWithColumnFamilies({kDefaultColumnFamilyName, "new_cf"}, options);
-
-  {
-    std::unique_ptr<Iterator> itr(db_->NewIterator(ReadOptions(), handles_[1]));
-    int count = 0;
-    for (itr->SeekToFirst(); itr->Valid(); itr->Next()) {
-      ASSERT_OK(itr->status());
-      ++count;
-    }
-    ASSERT_OK(itr->status());
-    ASSERT_EQ(count, 1000);
-  }
-}
-
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/db/error_handler.cc

@@ -227,7 +227,7 @@ std::map<std::tuple<BackgroundErrorReason, bool>, Status::Severity>
          Status::Severity::kFatalError},
 };
 
-void ErrorHandler::CancelErrorRecovery() {
+void ErrorHandler::CancelErrorRecoveryForShutDown() {
   db_mutex_->AssertHeld();
 
   // We'll release the lock before calling sfm, so make sure no new
@@ -275,9 +275,6 @@ void ErrorHandler::HandleKnownErrors(const Status& bg_err,
     return;
   }
 
-  ROCKS_LOG_INFO(db_options_.info_log,
-                 "ErrorHandler: Set regular background error\n");
-
   bool paranoid = db_options_.paranoid_checks;
   Status::Severity sev = Status::Severity::kFatalError;
   Status new_bg_err;
@@ -335,12 +332,21 @@ void ErrorHandler::HandleKnownErrors(const Status& bg_err,
     if (!s.ok() && (s.severity() > bg_error_.severity())) {
       bg_error_ = s;
     } else {
+      ROCKS_LOG_INFO(db_options_.info_log,
+                     "ErrorHandler: Hit less severe background error\n");
+
       // This error is less severe than previously encountered error. Don't
       // take any further action
       return;
     }
   }
 
+  bool stop = bg_error_.severity() >= Status::Severity::kHardError;
+  ROCKS_LOG_INFO(
+      db_options_.info_log,
+      "ErrorHandler: Set regular background error, auto_recovery=%d, stop=%d\n",
+      int{auto_recovery}, int{stop});
+
   recover_context_ = context;
   if (auto_recovery) {
     recovery_in_prog_ = true;
@@ -351,7 +357,7 @@ void ErrorHandler::HandleKnownErrors(const Status& bg_err,
       RecoverFromNoSpace();
     }
   }
-  if (bg_error_.severity() >= Status::Severity::kHardError) {
+  if (stop) {
     is_db_stopped_.store(true, std::memory_order_release);
   }
 }
@@ -573,6 +579,8 @@ Status ErrorHandler::ClearBGError() {
 
   // Signal that recovery succeeded
   if (recovery_error_.ok()) {
+    // If this assertion fails, it means likely bg error is not set after a
+    // file is quarantined during MANIFEST write.
     assert(files_to_quarantine_.empty());
     Status old_bg_error = bg_error_;
     // old_bg_error is only for notifying listeners, so may not be checked
@@ -585,8 +593,15 @@ Status ErrorHandler::ClearBGError() {
     recovery_error_.PermitUncheckedError();
     recovery_in_prog_ = false;
     soft_error_no_bg_work_ = false;
-    EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, old_bg_error,
-                                           bg_error_, db_mutex_);
+    if (!db_->shutdown_initiated_) {
+      // NotifyOnErrorRecoveryEnd() may release and re-acquire db_mutex_.
+      // Prevent DB from being closed while we notify listeners. DB close will
+      // wait until allow_db_shutdown_ = true, see ReadyForShutdown().
+      allow_db_shutdown_ = false;
+      EventHelpers::NotifyOnErrorRecoveryEnd(
+          db_options_.listeners, old_bg_error, bg_error_, db_mutex_);
+      allow_db_shutdown_ = true;
+    }
   }
   return recovery_error_;
 }

package/deps/rocksdb/rocksdb/db/error_handler.h

@@ -44,6 +44,7 @@ class ErrorHandler {
         auto_recovery_(false),
         recovery_in_prog_(false),
         soft_error_no_bg_work_(false),
+        allow_db_shutdown_(true),
         is_db_stopped_(false),
         bg_error_stats_(db_options.statistics) {
     // Clear the checked flag for uninitialized errors
@@ -63,6 +64,12 @@ class ErrorHandler {
 
   Status GetRecoveryError() const { return recovery_error_; }
 
+  // REQUIREs: db mutex held
+  //
+  // Returns non-OK status if encountered error during recovery.
+  // Returns OK if bg error is successfully cleared. May releases and
+  // re-acquire db mutex to notify listeners. However, DB close (if initiated)
+  // will be blocked until db mutex is released after return.
   Status ClearBGError();
 
   bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); }
@@ -79,8 +86,14 @@ class ErrorHandler {
 
   bool IsRecoveryInProgress() { return recovery_in_prog_; }
 
+  // REQUIRES: db mutex held
+  bool ReadyForShutdown() {
+    db_mutex_->AssertHeld();
+    return !recovery_in_prog_ && allow_db_shutdown_;
+  }
+
   Status RecoverFromBGError(bool is_manual = false);
-  void CancelErrorRecovery();
+  void CancelErrorRecoveryForShutDown();
 
   void EndAutoRecovery();
 
@@ -121,6 +134,8 @@ class ErrorHandler {
   // A flag to indicate that for the soft error, we should not allow any
   // background work except the work is from recovery.
   bool soft_error_no_bg_work_;
+  // Used in ClearBGError() to prevent DB from being closed.
+  bool allow_db_shutdown_;
 
   // Used to store the context for recover, such as flush reason.
   DBRecoverContext recover_context_;

package/deps/rocksdb/rocksdb/db/event_helpers.cc

@@ -77,13 +77,17 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished(
     TableFileCreationReason reason, const Status& s,
     const std::string& file_checksum,
     const std::string& file_checksum_func_name) {
-  if (s.ok() && event_logger) {
+  if (!event_logger && listeners.empty()) {
+    s.PermitUncheckedError();
+    return;
+  }
+
+  if (event_logger) {
     JSONWriter jwriter;
     AppendCurrentTime(&jwriter);
     jwriter << "cf_name" << cf_name << "job" << job_id << "event"
-            << "table_file_creation"
-            << "file_number" << fd.GetNumber() << "file_size"
-            << fd.GetFileSize() << "file_checksum"
+            << "table_file_creation" << "file_number" << fd.GetNumber()
+            << "file_size" << fd.GetFileSize() << "file_checksum"
             << Slice(file_checksum).ToString(true) << "file_checksum_func_name"
             << file_checksum_func_name << "smallest_seqno" << fd.smallest_seqno
             << "largest_seqno" << fd.largest_seqno;
@@ -166,6 +170,8 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished(
       jwriter << "oldest_blob_file_number" << oldest_blob_file_number;
     }
 
+    jwriter << "status" << s.ToString();
+
     jwriter.EndObject();
 
     event_logger->Log(jwriter);
@@ -196,19 +202,22 @@ void EventHelpers::LogAndNotifyTableFileDeletion(
     const std::string& file_path, const Status& status,
     const std::string& dbname,
     const std::vector<std::shared_ptr<EventListener>>& listeners) {
-  JSONWriter jwriter;
-  AppendCurrentTime(&jwriter);
-
-  jwriter << "job" << job_id << "event"
-          << "table_file_deletion"
-          << "file_number" << file_number;
-  if (!status.ok()) {
-    jwriter << "status" << status.ToString();
+  if (!event_logger && listeners.empty()) {
+    status.PermitUncheckedError();
+    return;
   }
 
-  jwriter.EndObject();
+  if (event_logger) {
+    JSONWriter jwriter;
+    AppendCurrentTime(&jwriter);
+
+    jwriter << "job" << job_id << "event" << "table_file_deletion"
+            << "file_number" << file_number << "status" << status.ToString();
+
+    jwriter.EndObject();
 
-  event_logger->Log(jwriter);
+    event_logger->Log(jwriter);
+  }
 
   if (listeners.empty()) {
     return;
@@ -276,15 +285,20 @@ void EventHelpers::LogAndNotifyBlobFileCreationFinished(
     const std::string& file_checksum,
     const std::string& file_checksum_func_name, uint64_t total_blob_count,
     uint64_t total_blob_bytes) {
-  if (s.ok() && event_logger) {
+  if (!event_logger && listeners.empty()) {
+    s.PermitUncheckedError();
+    return;
+  }
+
+  if (event_logger) {
     JSONWriter jwriter;
     AppendCurrentTime(&jwriter);
     jwriter << "cf_name" << cf_name << "job" << job_id << "event"
-            << "blob_file_creation"
-            << "file_number" << file_number << "total_blob_count"
-            << total_blob_count << "total_blob_bytes" << total_blob_bytes
-            << "file_checksum" << file_checksum << "file_checksum_func_name"
-            << file_checksum_func_name << "status" << s.ToString();
+            << "blob_file_creation" << "file_number" << file_number
+            << "total_blob_count" << total_blob_count << "total_blob_bytes"
+            << total_blob_bytes << "file_checksum" << file_checksum
+            << "file_checksum_func_name" << file_checksum_func_name << "status"
+            << s.ToString();
 
     jwriter.EndObject();
     event_logger->Log(jwriter);
@@ -307,16 +321,17 @@ void EventHelpers::LogAndNotifyBlobFileDeletion(
     const std::vector<std::shared_ptr<EventListener>>& listeners, int job_id,
     uint64_t file_number, const std::string& file_path, const Status& status,
     const std::string& dbname) {
+  if (!event_logger && listeners.empty()) {
+    status.PermitUncheckedError();
+    return;
+  }
+
   if (event_logger) {
     JSONWriter jwriter;
     AppendCurrentTime(&jwriter);
 
-    jwriter << "job" << job_id << "event"
-            << "blob_file_deletion"
-            << "file_number" << file_number;
-    if (!status.ok()) {
-      jwriter << "status" << status.ToString();
-    }
+    jwriter << "job" << job_id << "event" << "blob_file_deletion"
+            << "file_number" << file_number << "status" << status.ToString();
 
     jwriter.EndObject();
     event_logger->Log(jwriter);

package/deps/rocksdb/rocksdb/db/experimental.cc

@@ -37,7 +37,6 @@ Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
   return db->PromoteL0(column_family, target_level);
 }
 
-
 Status SuggestCompactRange(DB* db, const Slice* begin, const Slice* end) {
   return SuggestCompactRange(db, db->DefaultColumnFamily(), begin, end);
 }
@@ -58,7 +57,8 @@ Status GetFileChecksumsFromCurrentManifest(FileSystem* fs,
   }
   assert(checksum_list);
 
-  const ReadOptions read_options(Env::IOActivity::kReadManifest);
+  const ReadOptions read_options(
+      Env::IOActivity::kGetFileChecksumsFromCurrentManifest);
   checksum_list->reset();
 
   std::unique_ptr<SequentialFileReader> file_reader;
@@ -75,7 +75,8 @@ Status GetFileChecksumsFromCurrentManifest(FileSystem* fs,
 
   struct LogReporter : public log::Reader::Reporter {
     Status* status_ptr;
-    void Corruption(size_t /*bytes*/, const Status& st) override {
+    void Corruption(size_t /*bytes*/, const Status& st,
+                    uint64_t /*log_number*/ = kMaxSequenceNumber) override {
       if (status_ptr->ok()) {
         *status_ptr = st;
       }