@nxtedition/rocksdb 8.0.4 → 8.1.0

package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc

@@ -16,9 +16,11 @@
 #include <atomic>
 #include <cstddef>
 #include <cstdint>
+#include <exception>
 #include <limits>
 #include <memory>
 #include <random>
+#include <stdexcept>
 #include <string>
 #include <utility>
 
@@ -754,7 +756,7 @@ class BackupEngineTest : public testing::Test {
   void CloseBackupEngine() { backup_engine_.reset(nullptr); }
 
   // cross-cutting test of GetBackupInfo
-  void AssertBackupInfoConsistency() {
+  void AssertBackupInfoConsistency(bool allow_excluded = false) {
     std::vector<BackupInfo> backup_info;
     backup_engine_->GetBackupInfo(&backup_info, /*with file details*/ true);
     std::map<std::string, uint64_t> file_sizes;
@@ -774,6 +776,9 @@ class BackupEngineTest : public testing::Test {
         sum_for_backup += file.size;
       }
       ASSERT_EQ(backup.size, sum_for_backup);
+      if (!allow_excluded) {
+        ASSERT_EQ(backup.excluded_files.size(), 0);
+      }
     }
 
     std::vector<BackupID> corrupt_backup_ids;
@@ -3094,10 +3099,25 @@ TEST_F(BackupEngineTest, OpenBackupAsReadOnlyDB) {
 
 TEST_F(BackupEngineTest, ProgressCallbackDuringBackup) {
   DestroyDBWithoutCheck(dbname_, options_);
-  // Too big for this small DB
-  engine_options_->callback_trigger_interval_size = 100000;
+
   OpenDBAndBackupEngine(true);
   FillDB(db_.get(), 0, 100);
+
+  // First test exception handling
+  // Easily small enough for this small DB
+  engine_options_->callback_trigger_interval_size = 1000;
+  OpenBackupEngine();
+  ASSERT_TRUE(
+      backup_engine_->CreateNewBackup(db_.get(), true, []() { throw 42; })
+          .IsAborted());
+  ASSERT_TRUE(backup_engine_
+                  ->CreateNewBackup(db_.get(), true,
+                                    []() { throw std::out_of_range("blah"); })
+                  .IsAborted());
+
+  // Too big for this small DB
+  engine_options_->callback_trigger_interval_size = 100000;
+  OpenBackupEngine();
   bool is_callback_invoked = false;
   ASSERT_OK(backup_engine_->CreateNewBackup(
       db_.get(), true,
@@ -4180,7 +4200,7 @@ TEST_F(BackupEngineTest, FileTemperatures) {
         &info, /*include_file_details*/ true));
     ASSERT_GT(info.file_details.size(), 2);
     for (auto& e : info.file_details) {
-      ASSERT_EQ(expected_temps[e.file_number], e.temperature);
+      EXPECT_EQ(expected_temps[e.file_number], e.temperature);
     }
 
     // Restore backup to another virtual (tiered) dir
@@ -4202,6 +4222,177 @@ TEST_F(BackupEngineTest, FileTemperatures) {
   }
 }
 
+TEST_F(BackupEngineTest, ExcludeFiles) {
+  // Required for excluding files
+  engine_options_->schema_version = 2;
+
+  // Need a sufficent set of file numbers
+  options_.level0_file_num_compaction_trigger = 100;
+
+  OpenDBAndBackupEngine(true, false, kShareWithChecksum);
+  // Need a sufficent set of file numbers
+  const int keys_iteration = 5000;
+  FillDB(db_.get(), 0, keys_iteration / 3);
+  FillDB(db_.get(), keys_iteration / 3, keys_iteration * 2 / 3);
+  FillDB(db_.get(), keys_iteration * 2 / 3, keys_iteration);
+  CloseAndReopenDB();
+
+  BackupEngine* alt_backup_engine;
+  BackupEngineOptions alt_engine_options{*engine_options_};
+  // Use an alternate Env to test that support
+  std::string backup_alt_chroot = test::PerThreadDBPath("db_alt_backups");
+  EXPECT_OK(Env::Default()->CreateDirIfMissing(backup_alt_chroot));
+  alt_engine_options.backup_dir = "/altbk";
+  std::shared_ptr<FileSystem> alt_fs{
+      NewChrootFileSystem(FileSystem::Default(), backup_alt_chroot)};
+  std::unique_ptr<Env> alt_env{new CompositeEnvWrapper(Env::Default(), alt_fs)};
+  alt_engine_options.backup_env = alt_env.get();
+
+  ASSERT_OK(BackupEngine::Open(test_db_env_.get(), alt_engine_options,
+                               &alt_backup_engine));
+
+  // Ensure each backup is same set of files
+  db_.reset();
+  DB* db = nullptr;
+  ASSERT_OK(DB::OpenForReadOnly(options_, dbname_, &db));
+
+  // A callback that throws should cleanly fail the backup creation.
+  // Do this early to ensure later operations still work.
+  CreateBackupOptions cbo;
+  cbo.exclude_files_callback = [](MaybeExcludeBackupFile* /*files_begin*/,
+                                  MaybeExcludeBackupFile* /*files_end*/) {
+    throw 42;
+  };
+  ASSERT_TRUE(backup_engine_->CreateNewBackup(cbo, db).IsAborted());
+  cbo.exclude_files_callback = [](MaybeExcludeBackupFile* /*files_begin*/,
+                                  MaybeExcludeBackupFile* /*files_end*/) {
+    throw std::out_of_range("blah");
+  };
+  ASSERT_TRUE(backup_engine_->CreateNewBackup(cbo, db).IsAborted());
+
+  // Include files only in given bucket, based on modulus and remainder
+  constexpr int modulus = 4;
+  int remainder = 0;
+
+  cbo.exclude_files_callback = [&remainder](MaybeExcludeBackupFile* files_begin,
+                                            MaybeExcludeBackupFile* files_end) {
+    for (auto* f = files_begin; f != files_end; ++f) {
+      std::string s = StringSplit(f->info.relative_file, '/').back();
+      s = s.substr(0, s.find("_"));
+      int64_t num = std::strtoll(s.c_str(), nullptr, /*base*/ 10);
+      // Exclude if not a match
+      f->exclude_decision = (num % modulus) != remainder;
+    }
+  };
+
+  BackupID first_id{};
+  BackupID last_alt_id{};
+  remainder = 0;
+  ASSERT_OK(backup_engine_->CreateNewBackup(cbo, db, &first_id));
+  AssertBackupInfoConsistency(/*allow excluded*/ true);
+  remainder = 1;
+  ASSERT_OK(alt_backup_engine->CreateNewBackup(cbo, db));
+  AssertBackupInfoConsistency(/*allow excluded*/ true);
+  remainder = 2;
+  ASSERT_OK(backup_engine_->CreateNewBackup(cbo, db));
+  AssertBackupInfoConsistency(/*allow excluded*/ true);
+  remainder = 3;
+  ASSERT_OK(alt_backup_engine->CreateNewBackup(cbo, db, &last_alt_id));
+  AssertBackupInfoConsistency(/*allow excluded*/ true);
+
+  // Close DB
+  ASSERT_OK(db->Close());
+  delete db;
+  db = nullptr;
+
+  for (auto be_pair :
+       {std::make_pair(backup_engine_.get(), alt_backup_engine),
+        std::make_pair(alt_backup_engine, backup_engine_.get())}) {
+    DestroyDB(dbname_, options_);
+    RestoreOptions ro;
+    // Fails without alternate dir
+    ASSERT_TRUE(be_pair.first->RestoreDBFromLatestBackup(dbname_, dbname_, ro)
+                    .IsInvalidArgument());
+
+    DestroyDB(dbname_, options_);
+    // Works with alternate dir
+    ro.alternate_dirs.push_front(be_pair.second);
+    ASSERT_OK(be_pair.first->RestoreDBFromLatestBackup(dbname_, dbname_, ro));
+
+    // Check DB contents
+    db = OpenDB();
+    AssertExists(db, 0, keys_iteration);
+    delete db;
+  }
+
+  // Should still work after close and re-open
+  CloseBackupEngine();
+  OpenBackupEngine();
+
+  for (auto be_pair :
+       {std::make_pair(backup_engine_.get(), alt_backup_engine),
+        std::make_pair(alt_backup_engine, backup_engine_.get())}) {
+    DestroyDB(dbname_, options_);
+    RestoreOptions ro;
+    ro.alternate_dirs.push_front(be_pair.second);
+    ASSERT_OK(be_pair.first->RestoreDBFromLatestBackup(dbname_, dbname_, ro));
+  }
+
+  // Deletion semantics are tricky when within a single backup dir one backup
+  // includes a file and the other backup excluded the file. The excluded one
+  // does not have a persistent record of metadata like file checksum, etc.
+  // Although it would be possible to amend the backup with the excluded file,
+  // that is not currently supported (unless you open the backup as read-only
+  // DB and take another backup of it). The "excluded" reference to the file
+  // is like a weak reference: it doesn't prevent the file from being deleted
+  // if all the backups with "included" references to it are deleted.
+  CloseBackupEngine();
+  OpenBackupEngine();
+
+  AssertBackupInfoConsistency(/*allow excluded*/ true);
+
+  ASSERT_OK(backup_engine_->DeleteBackup(first_id));
+  ASSERT_OK(alt_backup_engine->DeleteBackup(last_alt_id));
+
+  // Includes check for any leaked backup files
+  AssertBackupInfoConsistency(/*allow excluded*/ true);
+
+  // Excluded file(s) deleted, unable to restore
+  for (auto be_pair :
+       {std::make_pair(backup_engine_.get(), alt_backup_engine),
+        std::make_pair(alt_backup_engine, backup_engine_.get())}) {
+    RestoreOptions ro;
+    ro.alternate_dirs.push_front(be_pair.second);
+    ASSERT_TRUE(be_pair.first->RestoreDBFromLatestBackup(dbname_, dbname_, ro)
+                    .IsInvalidArgument());
+  }
+
+  // Close & Re-open (no crash, etc.)
+  CloseBackupEngine();
+  OpenBackupEngine();
+
+  AssertBackupInfoConsistency(/*allow excluded*/ true);
+
+  // Excluded file(s) deleted, unable to restore
+  for (auto be_pair :
+       {std::make_pair(backup_engine_.get(), alt_backup_engine),
+        std::make_pair(alt_backup_engine, backup_engine_.get())}) {
+    RestoreOptions ro;
+    ro.alternate_dirs.push_front(be_pair.second);
+    ASSERT_TRUE(be_pair.first->RestoreDBFromLatestBackup(dbname_, dbname_, ro)
+                    .IsInvalidArgument());
+  }
+
+  // Ensure files are not leaked after removing everything.
+  ASSERT_OK(backup_engine_->DeleteBackup(first_id + 1));
+  ASSERT_OK(alt_backup_engine->DeleteBackup(last_alt_id - 1));
+
+  // Includes check for leaked backups files
+  AssertBackupInfoConsistency(/*allow excluded*/ false);
+
+  delete alt_backup_engine;
+}
+
 }  // namespace
 
 }  // namespace ROCKSDB_NAMESPACE

package/index.js

@@ -168,6 +168,23 @@ class RocksLevel extends AbstractLevel {
     return callback[kPromise]
   }
 
+	_getMergeOperands(key, options, callback) {
+    callback = fromCallback(callback, kPromise)
+
+    try {
+      this[kRef]()
+      binding.db_get_merge_operands(this[kContext], key, options ?? EMPTY, (err, val) => {
+        callback(err, val)
+        this[kUnref]()
+      })
+    } catch (err) {
+      process.nextTick(callback, err)
+      this[kUnref]()
+    }
+
+    return callback[kPromise]
+	}
+
   _del (key, options, callback) {
     callback = fromCallback(callback, kPromise)
 

package/max_rev_operator.h

@@ -0,0 +1,100 @@
+#pragma once
+
+#include <rocksdb/slice.h>
+#include <rocksdb/merge_operator.h>
+
+int compareRev(const rocksdb::Slice& a, const rocksdb::Slice& b) {
+  auto indexA = 0UL;
+  auto indexB = 0UL;
+  const auto endA = a.size();
+  const auto endB = b.size();
+
+  // Compare the revision number
+  auto result = 0;
+  const auto end = std::min(endA, endB);
+  while (indexA < end && indexB < end) {
+    const auto ac = a[indexA++];
+    const auto bc = b[indexB++];
+
+    if (ac == '-') {
+      if (bc == '-') {
+        break;
+      }
+      return -1;
+    } else if (bc == '-') {
+      return 1;
+    }
+
+    if (!result) {
+      result = ac == bc ? 0 : ac < bc ? -1 : 1;
+    }
+  }
+
+  if (result) {
+    return result;
+  }
+
+  // Compare the rest
+  while (indexA < end && indexB < end) {
+    const auto ac = a[indexA++];
+    const auto bc = b[indexB++];
+    if (ac != bc) {
+      return ac < bc ? -1 : 1;
+    }
+  }
+
+  return endA - endB;
+}
+
+class MaxRevOperator : public rocksdb::MergeOperator {
+ public:
+  bool FullMergeV2(const MergeOperationInput& merge_in,
+                   MergeOperationOutput* merge_out) const override {
+    rocksdb::Slice& max = merge_out->existing_operand;
+    if (merge_in.existing_value) {
+      max = rocksdb::Slice(merge_in.existing_value->data(),
+                  merge_in.existing_value->size());
+    } else if (max.data() == nullptr) {
+      max = rocksdb::Slice();
+    }
+
+    for (const auto& op : merge_in.operand_list) {
+      if (compareRev(max, op) < 0) {
+        max = op;
+      }
+    }
+
+    return true;
+  }
+
+  bool PartialMerge(const rocksdb::Slice& /*key*/, const rocksdb::Slice& left_operand,
+                    const rocksdb::Slice& right_operand, std::string* new_value,
+                    rocksdb::Logger* /*logger*/) const override {
+    if (compareRev(left_operand, right_operand) >= 0) {
+      new_value->assign(left_operand.data(), left_operand.size());
+    } else {
+      new_value->assign(right_operand.data(), right_operand.size());
+    }
+    return true;
+  }
+
+  bool PartialMergeMulti(const rocksdb::Slice& /*key*/,
+                         const std::deque<rocksdb::Slice>& operand_list,
+                         std::string* new_value,
+                         rocksdb::Logger* /*logger*/) const override {
+    rocksdb::Slice max;
+    for (const auto& operand : operand_list) {
+      if (compareRev(max, operand) < 0) {
+        max = operand;
+      }
+    }
+
+    new_value->assign(max.data(), max.size());
+    return true;
+  }
+
+  static const char* kClassName() { return "MaxRevOperator"; }
+  static const char* kNickName() { return "maxRev"; }
+  const char* Name() const override { return kClassName(); }
+  const char* NickName() const override { return kNickName(); }
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "8.0.4",
+  "version": "8.1.0",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",