@nxtedition/rocksdb 5.2.36 → 5.2.39

package/deps/liburing/liburing/test/unlink.c

@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: run various nop tests
+ *
+ */
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+
+#include "liburing.h"
+
+static int test_unlink(struct io_uring *ring, const char *old)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	if (!sqe) {
+		fprintf(stderr, "get sqe failed\n");
+		goto err;
+	}
+	io_uring_prep_unlinkat(sqe, AT_FDCWD, old, 0);
+	
+	ret = io_uring_submit(ring);
+	if (ret <= 0) {
+		fprintf(stderr, "sqe submit failed: %d\n", ret);
+		goto err;
+	}
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret < 0) {
+		fprintf(stderr, "wait completion %d\n", ret);
+		goto err;
+	}
+	ret = cqe->res;
+	io_uring_cqe_seen(ring, cqe);
+	return ret;
+err:
+	return 1;
+}
+
+static int stat_file(const char *buf)
+{
+	struct stat sb;
+
+	if (!stat(buf, &sb))
+		return 0;
+
+	return errno;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	char buf[32] = "./XXXXXX";
+	int ret;
+
+	if (argc > 1)
+		return 0;
+
+	ret = io_uring_queue_init(1, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "ring setup failed: %d\n", ret);
+		return 1;
+	}
+
+	ret = mkstemp(buf);
+	if (ret < 0) {
+		perror("mkstemp");
+		return 1;
+	}
+	close(ret);
+
+	if (stat_file(buf) != 0) {
+		perror("stat");
+		return 1;
+	}
+
+	ret = test_unlink(&ring, buf);
+	if (ret < 0) {
+		if (ret == -EBADF || ret == -EINVAL) {
+			fprintf(stdout, "Unlink not supported, skipping\n");
+			unlink(buf);
+			return 0;
+		}
+		fprintf(stderr, "rename: %s\n", strerror(-ret));
+		goto err;
+	} else if (ret)
+		goto err;
+
+	ret = stat_file(buf);
+	if (ret != ENOENT) {
+		fprintf(stderr, "stat got %s\n", strerror(ret));
+		return 1;
+	}
+
+	ret = test_unlink(&ring, "/3/2/3/1/z/y");
+	if (ret != -ENOENT) {
+		fprintf(stderr, "invalid unlink got %s\n", strerror(-ret));
+		return 1;
+	}
+
+	return 0;
+err:
+	unlink(buf);
+	return 1;
+}

package/deps/liburing/liburing/test/wakeup-hang.c

@@ -0,0 +1,162 @@
+/* SPDX-License-Identifier: MIT */
+#include <sys/eventfd.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+#include <liburing.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <sys/time.h>
+
+struct thread_data {
+	struct io_uring *ring;
+	int write_fd;
+};
+
+static void error_exit(char *message)
+{
+	perror(message);
+	exit(1);
+}
+
+static void *listener_thread(void *data)
+{
+	struct thread_data *td = data;
+	struct io_uring_cqe *cqe;
+	int ret;
+
+        ret = io_uring_wait_cqe(td->ring, &cqe);
+        if (ret < 0) {
+        	fprintf(stderr, "Error waiting for completion: %s\n",
+                	strerror(-ret));
+		goto err;
+        }
+	if (cqe->res < 0) {
+		fprintf(stderr, "Error in async operation: %s\n", strerror(-cqe->res));
+		goto err;
+        }
+	io_uring_cqe_seen(td->ring, cqe);
+	return NULL;
+err:
+	return (void *) 1;
+}
+
+static void *wakeup_io_uring(void *data)
+{
+	struct thread_data *td = data;
+	int res;
+
+	res = eventfd_write(td->write_fd, (eventfd_t) 1L);
+	if (res < 0) {
+		perror("eventfd_write");
+		return (void *) 1;
+	}
+	return NULL;
+}
+
+static int test_pipes(void)
+{
+	struct io_uring_sqe *sqe;
+	struct thread_data td;
+	struct io_uring ring;
+	pthread_t t1, t2;
+	int ret, fds[2];
+	void *pret;
+
+	if (pipe(fds) < 0)
+		error_exit("eventfd");
+
+	ret = io_uring_queue_init(8, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "Unable to setup io_uring: %s\n", strerror(-ret));
+		return 1;
+	}
+
+	td.write_fd = fds[1];
+	td.ring = &ring;
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_poll_add(sqe, fds[0], POLLIN);
+	sqe->user_data = 2;
+	ret = io_uring_submit(&ring);
+	if (ret != 1) {
+		fprintf(stderr, "ring_submit=%d\n", ret);
+		return 1;
+	}
+
+	pthread_create(&t1, NULL, listener_thread, &td);
+
+	sleep(1);
+
+	pthread_create(&t2, NULL, wakeup_io_uring, &td);
+	pthread_join(t1, &pret);
+
+	io_uring_queue_exit(&ring);
+	return pret != NULL;
+}
+
+static int test_eventfd(void)
+{
+	struct io_uring_sqe *sqe;
+	struct thread_data td;
+	struct io_uring ring;
+	pthread_t t1, t2;
+	int efd, ret;
+	void *pret;
+
+	efd = eventfd(0, 0);
+	if (efd < 0)
+		error_exit("eventfd");
+
+	ret = io_uring_queue_init(8, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "Unable to setup io_uring: %s\n", strerror(-ret));
+		return 1;
+	}
+
+	td.write_fd = efd;
+	td.ring = &ring;
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_poll_add(sqe, efd, POLLIN);
+	sqe->user_data = 2;
+	ret = io_uring_submit(&ring);
+	if (ret != 1) {
+		fprintf(stderr, "ring_submit=%d\n", ret);
+		return 1;
+	}
+
+	pthread_create(&t1, NULL, listener_thread, &td);
+
+	sleep(1);
+
+	pthread_create(&t2, NULL, wakeup_io_uring, &td);
+	pthread_join(t1, &pret);
+
+	io_uring_queue_exit(&ring);
+	return pret != NULL;
+}
+
+int main(int argc, char *argv[])
+{
+	int ret;
+
+	if (argc > 1)
+		return 0;
+
+	ret = test_pipes();
+	if (ret) {
+		fprintf(stderr, "test_pipe failed\n");
+		return ret;
+	}
+
+	ret = test_eventfd();
+	if (ret) {
+		fprintf(stderr, "test_eventfd failed\n");
+		return ret;
+	}
+
+	return 0;
+}

package/deps/liburing/liburing.gyp

@@ -0,0 +1,20 @@
+{
+  "variables": { "openssl_fips": "0" },
+  "targets":
+    [
+      {
+        "target_name": "liburing",
+        "type": "static_library",
+        "include_dirs": ["linux", "liburing/src/include"],
+        "direct_dependent_settings":
+          { "include_dirs": ["linux", "liburing/src/include"] },
+        "sources":
+          [
+            "liburing/src/queue.c",
+            "liburing/src/register.c",
+            "liburing/src/setup.c",
+            "liburing/src/syscall.c",
+          ],
+      },
+    ],
+}

package/deps/rocksdb/rocksdb/db/corruption_test.cc

@@ -337,6 +337,68 @@ TEST_F(CorruptionTest, Recovery) {
   Check(36, 36);
 }
 
+TEST_F(CorruptionTest, PostPITRCorruptionWALsRetained) {
+  // Repro for bug where WALs following the point-in-time recovery were not
+  // retained leading to the next recovery failing.
+  CloseDb();
+
+  options_.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
+
+  const std::string test_cf_name = "test_cf";
+  std::vector<ColumnFamilyDescriptor> cf_descs;
+  cf_descs.emplace_back(kDefaultColumnFamilyName, ColumnFamilyOptions());
+  cf_descs.emplace_back(test_cf_name, ColumnFamilyOptions());
+
+  uint64_t log_num;
+  {
+    options_.create_missing_column_families = true;
+    std::vector<ColumnFamilyHandle*> cfhs;
+    ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_));
+
+    ASSERT_OK(db_->Put(WriteOptions(), cfhs[0], "k", "v"));
+    ASSERT_OK(db_->Put(WriteOptions(), cfhs[1], "k", "v"));
+    ASSERT_OK(db_->Put(WriteOptions(), cfhs[0], "k2", "v2"));
+    std::vector<uint64_t> file_nums;
+    GetSortedWalFiles(file_nums);
+    log_num = file_nums.back();
+    for (auto* cfh : cfhs) {
+      delete cfh;
+    }
+    CloseDb();
+  }
+
+  CorruptFileWithTruncation(FileType::kWalFile, log_num,
+                            /*bytes_to_truncate=*/1);
+
+  {
+    // Recover "k" -> "v" for both CFs. "k2" -> "v2" is lost due to truncation.
+    options_.avoid_flush_during_recovery = true;
+    std::vector<ColumnFamilyHandle*> cfhs;
+    ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_));
+    // Flush one but not both CFs and write some data so there's a seqno gap
+    // between the PITR corruption and the next DB session's first WAL.
+    ASSERT_OK(db_->Put(WriteOptions(), cfhs[1], "k2", "v2"));
+    ASSERT_OK(db_->Flush(FlushOptions(), cfhs[1]));
+
+    for (auto* cfh : cfhs) {
+      delete cfh;
+    }
+    CloseDb();
+  }
+
+  // With the bug, this DB open would remove the WALs following the PITR
+  // corruption. Then, the next recovery would fail.
+  for (int i = 0; i < 2; ++i) {
+    std::vector<ColumnFamilyHandle*> cfhs;
+    ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_));
+
+    for (auto* cfh : cfhs) {
+      delete cfh;
+    }
+    CloseDb();
+  }
+}
+
 TEST_F(CorruptionTest, RecoverWriteError) {
   env_->writable_file_error_ = true;
   Status s = TryReopen();

package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h

@@ -1240,43 +1240,6 @@ class DBImpl : public DB {
 
   std::atomic<bool> shutting_down_;
 
-  // RecoveryContext struct stores the context about version edits along
-  // with corresponding column_family_data and column_family_options.
-  class RecoveryContext {
-   public:
-    ~RecoveryContext() {
-      for (auto& edit_list : edit_lists_) {
-        for (auto* edit : edit_list) {
-          delete edit;
-        }
-        edit_list.clear();
-      }
-      cfds_.clear();
-      mutable_cf_opts_.clear();
-      edit_lists_.clear();
-      files_to_delete_.clear();
-    }
-
-    void UpdateVersionEdits(ColumnFamilyData* cfd, const VersionEdit& edit) {
-      if (map_.find(cfd->GetID()) == map_.end()) {
-        uint32_t size = static_cast<uint32_t>(map_.size());
-        map_.emplace(cfd->GetID(), size);
-        cfds_.emplace_back(cfd);
-        mutable_cf_opts_.emplace_back(cfd->GetLatestMutableCFOptions());
-        edit_lists_.emplace_back(autovector<VersionEdit*>());
-      }
-      uint32_t i = map_[cfd->GetID()];
-      edit_lists_[i].emplace_back(new VersionEdit(edit));
-    }
-
-    std::unordered_map<uint32_t, uint32_t> map_;  // cf_id to index;
-    autovector<ColumnFamilyData*> cfds_;
-    autovector<const MutableCFOptions*> mutable_cf_opts_;
-    autovector<autovector<VersionEdit*>> edit_lists_;
-    // files_to_delete_ contains sst files
-    std::set<std::string> files_to_delete_;
-  };
-
   // Except in DB::Open(), WriteOptionsFile can only be called when:
   // Persist options to options file.
   // If need_mutex_lock = false, the method will lock DB mutex.
@@ -1393,19 +1356,16 @@ class DBImpl : public DB {
   // be made to the descriptor are added to *edit.
   // recovered_seq is set to less than kMaxSequenceNumber if the log's tail is
   // skipped.
-  // recovery_ctx stores the context about version edits and all those
-  // edits are persisted to new Manifest after successfully syncing the new WAL.
   virtual Status Recover(
       const std::vector<ColumnFamilyDescriptor>& column_families,
       bool read_only = false, bool error_if_wal_file_exists = false,
       bool error_if_data_exists_in_wals = false,
-      uint64_t* recovered_seq = nullptr,
-      RecoveryContext* recovery_ctx = nullptr);
+      uint64_t* recovered_seq = nullptr);
 
   virtual bool OwnTablesAndLogs() const { return true; }
 
   // Set DB identity file, and write DB ID to manifest if necessary.
-  Status SetDBId(bool read_only, RecoveryContext* recovery_ctx);
+  Status SetDBId(bool read_only);
 
   // REQUIRES: db mutex held when calling this function, but the db mutex can
   // be released and re-acquired. Db mutex will be held when the function
@@ -1414,15 +1374,12 @@ class DBImpl : public DB {
   // not referenced in the MANIFEST (e.g.
   // 1. It's best effort recovery;
   // 2. The VersionEdits referencing the SST files are appended to
-  // RecoveryContext, DB crashes when syncing the MANIFEST, the VersionEdits are
+  // MANIFEST, DB crashes when syncing the MANIFEST, the VersionEdits are
   // still not synced to MANIFEST during recovery.)
-  // It stores the SST files to be deleted in RecoveryContext. In the
+  // We delete these SST files. In the
   // meantime, we find out the largest file number present in the paths, and
   // bump up the version set's next_file_number_ to be 1 + largest_file_number.
-  // recovery_ctx stores the context about version edits and files to be
-  // deleted. All those edits are persisted to new Manifest after successfully
-  // syncing the new WAL.
-  Status DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx);
+  Status DeleteUnreferencedSstFiles();
 
   // SetDbSessionId() should be called in the constuctor DBImpl()
   // to ensure that db_session_id_ gets updated every time the DB is opened
@@ -1432,11 +1389,6 @@ class DBImpl : public DB {
   Status FailIfTsSizesMismatch(const ColumnFamilyHandle* column_family,
                                const Slice& ts) const;
 
-  // recovery_ctx stores the context about version edits and
-  // LogAndApplyForRecovery persist all those edits to new Manifest after
-  // successfully syncing new WAL.
-  Status LogAndApplyForRecovery(const RecoveryContext& recovery_ctx);
-
  private:
   friend class DB;
   friend class ErrorHandler;
@@ -1691,10 +1643,9 @@ class DBImpl : public DB {
 
   // REQUIRES: log_numbers are sorted in ascending order
   // corrupted_log_found is set to true if we recover from a corrupted log file.
-  Status RecoverLogFiles(std::vector<uint64_t>& log_numbers,
+  Status RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
                          SequenceNumber* next_sequence, bool read_only,
-                         bool* corrupted_log_found,
-                         RecoveryContext* recovery_ctx);
+                         bool* corrupted_log_found);
 
   // The following two methods are used to flush a memtable to
   // storage. The first one is used at database RecoveryTime (when the
@@ -1704,12 +1655,6 @@ class DBImpl : public DB {
   Status WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
                                      MemTable* mem, VersionEdit* edit);
 
-  // Move all the WAL files starting from corrupted WAL found to
-  // max_wal_number to avoid column family inconsistency error on recovery. It
-  // also removes the deleted file from the vector wal_numbers.
-  void MoveCorruptedWalFiles(std::vector<uint64_t>& wal_numbers,
-                             uint64_t corrupted_wal_number);
-
   // Get the size of a log file and, if truncate is true, truncate the
   // log file to its actual size, thereby freeing preallocated space.
   // Return success even if truncate fails

package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc

@@ -863,7 +863,7 @@ uint64_t PrecomputeMinLogNumberToKeep2PC(
   return min_log_number_to_keep;
 }
 
-Status DBImpl::SetDBId(bool read_only, RecoveryContext* recovery_ctx) {
+Status DBImpl::SetDBId(bool read_only) {
   Status s;
   // Happens when immutable_db_options_.write_dbid_to_manifest is set to true
   // the very first time.
@@ -890,14 +890,14 @@ Status DBImpl::SetDBId(bool read_only, RecoveryContext* recovery_ctx) {
     }
     s = GetDbIdentityFromIdentityFile(&db_id_);
     if (immutable_db_options_.write_dbid_to_manifest && s.ok()) {
-      assert(!read_only);
-      assert(recovery_ctx != nullptr);
-      assert(versions_->GetColumnFamilySet() != nullptr);
       VersionEdit edit;
       edit.SetDBId(db_id_);
+      Options options;
+      MutableCFOptions mutable_cf_options(options);
       versions_->db_id_ = db_id_;
-      recovery_ctx->UpdateVersionEdits(
-          versions_->GetColumnFamilySet()->GetDefault(), edit);
+      s = versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(),
+                                 mutable_cf_options, &edit, &mutex_, nullptr,
+                                 /* new_descriptor_log */ false);
     }
   } else if (!read_only) {
     s = SetIdentityFile(env_, dbname_, db_id_);
@@ -905,7 +905,7 @@ Status DBImpl::SetDBId(bool read_only, RecoveryContext* recovery_ctx) {
   return s;
 }
 
-Status DBImpl::DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx) {
+Status DBImpl::DeleteUnreferencedSstFiles() {
   mutex_.AssertHeld();
   std::vector<std::string> paths;
   paths.push_back(NormalizePath(dbname_ + std::string(1, kFilePathSeparator)));
@@ -925,6 +925,7 @@ Status DBImpl::DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx) {
 
   uint64_t next_file_number = versions_->current_next_file_number();
   uint64_t largest_file_number = next_file_number;
+  std::set<std::string> files_to_delete;
   Status s;
   for (const auto& path : paths) {
     std::vector<std::string> files;
@@ -942,9 +943,8 @@ Status DBImpl::DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx) {
       const std::string normalized_fpath = path + fname;
       largest_file_number = std::max(largest_file_number, number);
       if (type == kTableFile && number >= next_file_number &&
-          recovery_ctx->files_to_delete_.find(normalized_fpath) ==
-              recovery_ctx->files_to_delete_.end()) {
-        recovery_ctx->files_to_delete_.insert(normalized_fpath);
+          files_to_delete.find(normalized_fpath) == files_to_delete.end()) {
+        files_to_delete.insert(normalized_fpath);
       }
     }
   }
@@ -961,7 +961,21 @@ Status DBImpl::DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx) {
   assert(versions_->GetColumnFamilySet());
   ColumnFamilyData* default_cfd = versions_->GetColumnFamilySet()->GetDefault();
   assert(default_cfd);
-  recovery_ctx->UpdateVersionEdits(default_cfd, edit);
+  s = versions_->LogAndApply(
+      default_cfd, *default_cfd->GetLatestMutableCFOptions(), &edit, &mutex_,
+      directories_.GetDbDir(), /*new_descriptor_log*/ false);
+  if (!s.ok()) {
+    return s;
+  }
+
+  mutex_.Unlock();
+  for (const auto& fname : files_to_delete) {
+    s = env_->DeleteFile(fname);
+    if (!s.ok()) {
+      break;
+    }
+  }
+  mutex_.Lock();
   return s;
 }