@nxtedition/rocksdb 8.1.2 → 8.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/deps/rocksdb/rocksdb/CMakeLists.txt +0 -9
  2. package/deps/rocksdb/rocksdb/Makefile +1 -13
  3. package/deps/rocksdb/rocksdb/TARGETS +0 -4
  4. package/deps/rocksdb/rocksdb/db/blob/blob_counting_iterator.h +0 -11
  5. package/deps/rocksdb/rocksdb/db/builder.cc +4 -13
  6. package/deps/rocksdb/rocksdb/db/builder.h +1 -2
  7. package/deps/rocksdb/rocksdb/db/c.cc +0 -15
  8. package/deps/rocksdb/rocksdb/db/c_test.c +0 -3
  9. package/deps/rocksdb/rocksdb/db/column_family.cc +5 -11
  10. package/deps/rocksdb/rocksdb/db/column_family.h +0 -20
  11. package/deps/rocksdb/rocksdb/db/compaction/clipping_iterator.h +0 -5
  12. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +37 -31
  13. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +0 -5
  14. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +7 -24
  15. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +1 -17
  16. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +1 -4
  17. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +2 -4
  18. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +6 -9
  19. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +26 -104
  20. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +0 -5
  21. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +18 -11
  22. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +16 -17
  23. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +6 -19
  24. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +5 -5
  25. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +22 -22
  26. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +5 -5
  27. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +52 -81
  28. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +1 -5
  29. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +5 -5
  30. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +2 -8
  31. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +0 -8
  32. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +138 -266
  33. package/deps/rocksdb/rocksdb/db/corruption_test.cc +1 -86
  34. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +3 -2
  35. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +123 -897
  36. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +20 -31
  37. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +9 -5
  38. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +28 -121
  39. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +1 -1
  40. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +2 -3
  41. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +0 -3
  42. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +4 -8
  43. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +0 -10
  44. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +4 -325
  45. package/deps/rocksdb/rocksdb/db/db_test.cc +0 -3
  46. package/deps/rocksdb/rocksdb/db/db_test2.cc +8 -233
  47. package/deps/rocksdb/rocksdb/db/db_test_util.h +0 -3
  48. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +0 -129
  49. package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +0 -21
  50. package/deps/rocksdb/rocksdb/db/dbformat.cc +0 -25
  51. package/deps/rocksdb/rocksdb/db/dbformat.h +0 -2
  52. package/deps/rocksdb/rocksdb/db/experimental.cc +2 -3
  53. package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +0 -3
  54. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +13 -92
  55. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +1 -38
  56. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +110 -14
  57. package/deps/rocksdb/rocksdb/db/flush_job.cc +4 -6
  58. package/deps/rocksdb/rocksdb/db/history_trimming_iterator.h +0 -4
  59. package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +53 -56
  60. package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +4 -3
  61. package/deps/rocksdb/rocksdb/db/memtable.cc +1 -1
  62. package/deps/rocksdb/rocksdb/db/merge_helper.cc +0 -4
  63. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +10 -10
  64. package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter_test.cc +1 -1
  65. package/deps/rocksdb/rocksdb/db/repair.cc +22 -65
  66. package/deps/rocksdb/rocksdb/db/repair_test.cc +0 -54
  67. package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +26 -26
  68. package/deps/rocksdb/rocksdb/db/table_properties_collector.h +1 -3
  69. package/deps/rocksdb/rocksdb/db/version_builder.cc +43 -90
  70. package/deps/rocksdb/rocksdb/db/version_builder.h +0 -20
  71. package/deps/rocksdb/rocksdb/db/version_builder_test.cc +93 -218
  72. package/deps/rocksdb/rocksdb/db/version_edit.cc +1 -27
  73. package/deps/rocksdb/rocksdb/db/version_edit.h +9 -33
  74. package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +6 -13
  75. package/deps/rocksdb/rocksdb/db/version_edit_handler.h +6 -17
  76. package/deps/rocksdb/rocksdb/db/version_edit_test.cc +17 -19
  77. package/deps/rocksdb/rocksdb/db/version_set.cc +26 -166
  78. package/deps/rocksdb/rocksdb/db/version_set.h +2 -32
  79. package/deps/rocksdb/rocksdb/db/version_set_test.cc +31 -65
  80. package/deps/rocksdb/rocksdb/db/write_thread.cc +2 -5
  81. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +0 -1
  82. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +32 -31
  83. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.h +1 -2
  84. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +6 -8
  85. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +0 -4
  86. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +4 -11
  87. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +15 -16
  88. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +1 -13
  89. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +75 -0
  90. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +0 -8
  91. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +0 -6
  92. package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +1 -7
  93. package/deps/rocksdb/rocksdb/include/rocksdb/metadata.h +3 -9
  94. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +1 -2
  95. package/deps/rocksdb/rocksdb/include/rocksdb/status.h +0 -3
  96. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h +9 -69
  97. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
  98. package/deps/rocksdb/rocksdb/memory/arena.cc +87 -23
  99. package/deps/rocksdb/rocksdb/memory/arena.h +31 -25
  100. package/deps/rocksdb/rocksdb/memory/arena_test.cc +0 -90
  101. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +26 -26
  102. package/deps/rocksdb/rocksdb/src.mk +0 -2
  103. package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +2 -3
  104. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +2 -3
  105. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +2 -6
  106. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +1 -1
  107. package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +3 -3
  108. package/deps/rocksdb/rocksdb/table/format.cc +20 -24
  109. package/deps/rocksdb/rocksdb/table/format.h +2 -5
  110. package/deps/rocksdb/rocksdb/table/merging_iterator.cc +105 -54
  111. package/deps/rocksdb/rocksdb/table/merging_iterator.h +0 -80
  112. package/deps/rocksdb/rocksdb/table/meta_blocks.cc +2 -2
  113. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +1 -1
  114. package/deps/rocksdb/rocksdb/table/table_test.cc +6 -7
  115. package/deps/rocksdb/rocksdb/test_util/testutil.h +0 -10
  116. package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.h +2 -2
  117. package/deps/rocksdb/rocksdb/util/bloom_test.cc +1 -1
  118. package/deps/rocksdb/rocksdb/util/status.cc +0 -7
  119. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +74 -250
  120. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +4 -199
  121. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +0 -1
  122. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +0 -39
  123. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +0 -9
  124. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +0 -59
  125. package/deps/rocksdb/rocksdb.gyp +0 -3
  126. package/index.js +2 -2
  127. package/package.json +1 -1
  128. package/prebuilds/darwin-arm64/node.napi.node +0 -0
  129. package/prebuilds/linux-x64/node.napi.node +0 -0
  130. package/deps/rocksdb/rocksdb/port/mmap.cc +0 -98
  131. package/deps/rocksdb/rocksdb/port/mmap.h +0 -70
  132. package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +0 -142
  133. package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +0 -241
@@ -13,13 +13,11 @@
13
13
  #include <atomic>
14
14
  #include <cinttypes>
15
15
  #include <cstdlib>
16
- #include <exception>
17
16
  #include <functional>
18
17
  #include <future>
19
18
  #include <limits>
20
19
  #include <map>
21
20
  #include <mutex>
22
- #include <optional>
23
21
  #include <sstream>
24
22
  #include <string>
25
23
  #include <thread>
@@ -157,10 +155,16 @@ class BackupEngineImpl {
157
155
 
158
156
  void GetCorruptedBackups(std::vector<BackupID>* corrupt_backup_ids) const;
159
157
 
160
- IOStatus RestoreDBFromBackup(
161
- const RestoreOptions& options, BackupID backup_id,
162
- const std::string& db_dir, const std::string& wal_dir,
163
- const std::list<const BackupEngineImpl*>& locked_restore_from_dirs) const;
158
+ IOStatus RestoreDBFromBackup(const RestoreOptions& options,
159
+ BackupID backup_id, const std::string& db_dir,
160
+ const std::string& wal_dir) const;
161
+
162
+ IOStatus RestoreDBFromLatestBackup(const RestoreOptions& options,
163
+ const std::string& db_dir,
164
+ const std::string& wal_dir) const {
165
+ // Note: don't read latest_valid_backup_id_ outside of lock
166
+ return RestoreDBFromBackup(options, kLatestBackupIDMarker, db_dir, wal_dir);
167
+ }
164
168
 
165
169
  IOStatus VerifyBackup(BackupID backup_id,
166
170
  bool verify_with_checksum = false) const;
@@ -216,7 +220,6 @@ class BackupEngineImpl {
216
220
  FileInfo& operator=(const FileInfo&) = delete;
217
221
 
218
222
  int refs;
219
- // Relative path from backup dir
220
223
  const std::string filename;
221
224
  const uint64_t size;
222
225
  // crc32c checksum as hex. empty == unknown / unavailable
@@ -230,7 +233,7 @@ class BackupEngineImpl {
230
233
  const std::string db_session_id;
231
234
  Temperature temp;
232
235
 
233
- std::string GetDbFileName() const {
236
+ std::string GetDbFileName() {
234
237
  std::string rv;
235
238
  // extract the filename part
236
239
  size_t slash = filename.find_last_of('/');
@@ -412,10 +415,6 @@ class BackupEngineImpl {
412
415
 
413
416
  IOStatus AddFile(std::shared_ptr<FileInfo> file_info);
414
417
 
415
- void AddExcludedFile(const std::string& relative_file) {
416
- excluded_files_.emplace_back(relative_file);
417
- }
418
-
419
418
  IOStatus Delete(bool delete_meta = true);
420
419
 
421
420
  bool Empty() const { return files_.empty(); }
@@ -432,10 +431,6 @@ class BackupEngineImpl {
432
431
  return files_;
433
432
  }
434
433
 
435
- const std::vector<BackupExcludedFileInfo>& GetExcludedFiles() const {
436
- return excluded_files_;
437
- }
438
-
439
434
  // @param abs_path_to_size Pre-fetched file sizes (bytes).
440
435
  IOStatus LoadFromFile(
441
436
  const std::string& backup_dir,
@@ -493,7 +488,6 @@ class BackupEngineImpl {
493
488
  std::string const meta_tmp_filename_;
494
489
  // files with relative paths (without "/" prefix!!)
495
490
  std::vector<std::shared_ptr<FileInfo>> files_;
496
- std::vector<BackupExcludedFileInfo> excluded_files_;
497
491
  std::unordered_map<std::string, std::shared_ptr<FileInfo>>* file_infos_;
498
492
  Env* env_;
499
493
  mutable std::shared_ptr<Env> env_for_open_;
@@ -687,19 +681,17 @@ class BackupEngineImpl {
687
681
  return *this;
688
682
  }
689
683
 
690
- CopyOrCreateWorkItem(std::string _src_path, std::string _dst_path,
691
- const Temperature _src_temperature,
692
- const Temperature _dst_temperature,
693
- std::string _contents, Env* _src_env, Env* _dst_env,
694
- EnvOptions _src_env_options, bool _sync,
695
- RateLimiter* _rate_limiter, uint64_t _size_limit,
696
- Statistics* _stats,
697
- std::function<void()> _progress_callback = {},
698
- const std::string& _src_checksum_func_name =
699
- kUnknownFileChecksumFuncName,
700
- const std::string& _src_checksum_hex = "",
701
- const std::string& _db_id = "",
702
- const std::string& _db_session_id = "")
684
+ CopyOrCreateWorkItem(
685
+ std::string _src_path, std::string _dst_path,
686
+ const Temperature _src_temperature, const Temperature _dst_temperature,
687
+ std::string _contents, Env* _src_env, Env* _dst_env,
688
+ EnvOptions _src_env_options, bool _sync, RateLimiter* _rate_limiter,
689
+ uint64_t _size_limit, Statistics* _stats,
690
+ std::function<void()> _progress_callback = []() {},
691
+ const std::string& _src_checksum_func_name =
692
+ kUnknownFileChecksumFuncName,
693
+ const std::string& _src_checksum_hex = "",
694
+ const std::string& _db_id = "", const std::string& _db_session_id = "")
703
695
  : src_path(std::move(_src_path)),
704
696
  dst_path(std::move(_dst_path)),
705
697
  src_temperature(_src_temperature),
@@ -766,9 +758,6 @@ class BackupEngineImpl {
766
758
  dst_relative(std::move(_dst_relative)) {}
767
759
  };
768
760
 
769
- using BackupWorkItemPair =
770
- std::pair<CopyOrCreateWorkItem, BackupAfterCopyOrCreateWorkItem>;
771
-
772
761
  struct RestoreAfterCopyOrCreateWorkItem {
773
762
  std::future<CopyOrCreateResult> result;
774
763
  std::string from_file;
@@ -818,14 +807,13 @@ class BackupEngineImpl {
818
807
  // @param contents If non-empty, the file will be created with these contents.
819
808
  IOStatus AddBackupFileWorkItem(
820
809
  std::unordered_set<std::string>& live_dst_paths,
821
- std::deque<BackupAfterCopyOrCreateWorkItem>& backup_items_to_finish,
822
- std::deque<BackupWorkItemPair>* excludable_items, BackupID backup_id,
823
- bool shared, const std::string& src_dir,
810
+ std::vector<BackupAfterCopyOrCreateWorkItem>& backup_items_to_finish,
811
+ BackupID backup_id, bool shared, const std::string& src_dir,
824
812
  const std::string& fname, // starts with "/"
825
813
  const EnvOptions& src_env_options, RateLimiter* rate_limiter,
826
814
  FileType file_type, uint64_t size_bytes, Statistics* stats,
827
815
  uint64_t size_limit = 0, bool shared_checksum = false,
828
- std::function<void()> progress_callback = {},
816
+ std::function<void()> progress_callback = []() {},
829
817
  const std::string& contents = std::string(),
830
818
  const std::string& src_checksum_func_name = kUnknownFileChecksumFuncName,
831
819
  const std::string& src_checksum_str = kUnknownFileChecksum,
@@ -935,37 +923,8 @@ class BackupEngineImplThreadSafe : public BackupEngine,
935
923
  IOStatus RestoreDBFromBackup(const RestoreOptions& options,
936
924
  BackupID backup_id, const std::string& db_dir,
937
925
  const std::string& wal_dir) const override {
938
- // TSAN reports a lock inversion (potential deadlock) if we acquire read
939
- // locks in different orders. Assuming the implementation of RWMutex
940
- // allows simultaneous read locks, there should be no deadlock, because
941
- // there is no write lock involved here. Nevertheless, to appease TSAN and
942
- // in case of degraded RWMutex implementation, we lock the BackupEngines
943
- // including this one and those in options.alternate_dirs in a consistent
944
- // order.
945
- // However, locked_restore_from_dirs is kept in "search" order.
946
- std::list<const BackupEngineImpl*> locked_restore_from_dirs;
947
- std::vector<port::RWMutex*> mutexes;
948
-
949
- // Add `this`
950
- locked_restore_from_dirs.emplace_back(&impl_);
951
- mutexes.push_back(&mutex_);
952
-
953
- // Add alternates
954
- for (BackupEngineReadOnlyBase* be : options.alternate_dirs) {
955
- BackupEngineImplThreadSafe* bets =
956
- static_cast_with_check<BackupEngineImplThreadSafe>(
957
- be->AsBackupEngine());
958
- locked_restore_from_dirs.emplace_back(&bets->impl_);
959
- mutexes.push_back(&bets->mutex_);
960
- }
961
-
962
- // Acquire read locks in pointer order
963
- std::sort(mutexes.begin(), mutexes.end());
964
- std::vector<ReadLock> locks(mutexes.begin(), mutexes.end());
965
-
966
- // Impl
967
- return impl_.RestoreDBFromBackup(options, backup_id, db_dir, wal_dir,
968
- locked_restore_from_dirs);
926
+ ReadLock lock(&mutex_);
927
+ return impl_.RestoreDBFromBackup(options, backup_id, db_dir, wal_dir);
969
928
  }
970
929
 
971
930
  using BackupEngine::RestoreDBFromLatestBackup;
@@ -982,8 +941,6 @@ class BackupEngineImplThreadSafe : public BackupEngine,
982
941
  return impl_.VerifyBackup(backup_id, verify_with_checksum);
983
942
  }
984
943
 
985
- BackupEngine* AsBackupEngine() override { return this; }
986
-
987
944
  // Not public API but needed
988
945
  IOStatus Initialize() {
989
946
  // No locking needed
@@ -1009,7 +966,6 @@ class BackupEngineImplThreadSafe : public BackupEngine,
1009
966
  mutable port::RWMutex mutex_;
1010
967
  BackupEngineImpl impl_;
1011
968
  };
1012
-
1013
969
  } // namespace
1014
970
 
1015
971
  IOStatus BackupEngine::Open(const BackupEngineOptions& options, Env* env,
@@ -1344,12 +1300,6 @@ IOStatus BackupEngineImpl::CreateNewBackupWithMetadata(
1344
1300
  return IOStatus::InvalidArgument("App metadata too large");
1345
1301
  }
1346
1302
 
1347
- bool maybe_exclude_items = bool{options.exclude_files_callback};
1348
- if (maybe_exclude_items && options_.schema_version < 2) {
1349
- return IOStatus::InvalidArgument(
1350
- "exclude_files_callback requires schema_version >= 2");
1351
- }
1352
-
1353
1303
  if (options.decrease_background_thread_cpu_priority) {
1354
1304
  if (options.background_thread_cpu_priority < threads_cpu_priority_) {
1355
1305
  threads_cpu_priority_.store(options.background_thread_cpu_priority);
@@ -1414,8 +1364,7 @@ IOStatus BackupEngineImpl::CreateNewBackupWithMetadata(
1414
1364
  // live file.
1415
1365
  std::unordered_set<std::string> live_dst_paths;
1416
1366
 
1417
- std::deque<BackupWorkItemPair> excludable_items;
1418
- std::deque<BackupAfterCopyOrCreateWorkItem> backup_items_to_finish;
1367
+ std::vector<BackupAfterCopyOrCreateWorkItem> backup_items_to_finish;
1419
1368
  // Add a CopyOrCreateWorkItem to the channel for each live file
1420
1369
  Status disabled = db->DisableFileDeletions();
1421
1370
  DBOptions db_options = db->GetDBOptions();
@@ -1487,8 +1436,7 @@ IOStatus BackupEngineImpl::CreateNewBackupWithMetadata(
1487
1436
  break;
1488
1437
  }
1489
1438
  io_st = AddBackupFileWorkItem(
1490
- live_dst_paths, backup_items_to_finish,
1491
- maybe_exclude_items ? &excludable_items : nullptr, new_backup_id,
1439
+ live_dst_paths, backup_items_to_finish, new_backup_id,
1492
1440
  options_.share_table_files &&
1493
1441
  (type == kTableFile || type == kBlobFile),
1494
1442
  src_dirname, fname, src_env_options, rate_limiter, type,
@@ -1503,8 +1451,7 @@ IOStatus BackupEngineImpl::CreateNewBackupWithMetadata(
1503
1451
  FileType type) {
1504
1452
  Log(options_.info_log, "add file for backup %s", fname.c_str());
1505
1453
  return AddBackupFileWorkItem(
1506
- live_dst_paths, backup_items_to_finish,
1507
- maybe_exclude_items ? &excludable_items : nullptr, new_backup_id,
1454
+ live_dst_paths, backup_items_to_finish, new_backup_id,
1508
1455
  false /* shared */, "" /* src_dir */, fname,
1509
1456
  EnvOptions() /* src_env_options */, rate_limiter, type,
1510
1457
  contents.size(), db_options.statistics.get(), 0 /* size_limit */,
@@ -1517,46 +1464,7 @@ IOStatus BackupEngineImpl::CreateNewBackupWithMetadata(
1517
1464
  new_backup->SetSequenceNumber(sequence_number);
1518
1465
  }
1519
1466
  }
1520
- ROCKS_LOG_INFO(options_.info_log, "add files for backup done.");
1521
- if (io_s.ok() && maybe_exclude_items) {
1522
- assert(options.exclude_files_callback);
1523
- size_t count = excludable_items.size();
1524
- std::vector<MaybeExcludeBackupFile> maybe_exclude_files;
1525
- maybe_exclude_files.reserve(count);
1526
- for (auto& e : excludable_items) {
1527
- maybe_exclude_files.emplace_back(
1528
- BackupExcludedFileInfo(e.second.dst_relative));
1529
- }
1530
- if (count > 0) {
1531
- try {
1532
- options.exclude_files_callback(
1533
- &maybe_exclude_files.front(),
1534
- /*end pointer*/ &maybe_exclude_files.back() + 1);
1535
- } catch (const std::exception& exn) {
1536
- io_s = IOStatus::Aborted("Exception in exclude_files_callback: " +
1537
- std::string(exn.what()));
1538
- } catch (...) {
1539
- io_s = IOStatus::Aborted("Unknown exception in exclude_files_callback");
1540
- }
1541
- }
1542
- if (io_s.ok()) {
1543
- for (size_t i = 0; i < count; ++i) {
1544
- auto& e = excludable_items[i];
1545
- if (maybe_exclude_files[i].exclude_decision) {
1546
- new_backup.get()->AddExcludedFile(e.second.dst_relative);
1547
- } else {
1548
- files_to_copy_or_create_.write(std::move(e.first));
1549
- backup_items_to_finish.push_back(std::move(e.second));
1550
- }
1551
- }
1552
- }
1553
- excludable_items.clear();
1554
- } else {
1555
- assert(!options.exclude_files_callback);
1556
- assert(excludable_items.empty());
1557
- }
1558
- ROCKS_LOG_INFO(options_.info_log,
1559
- "dispatch files for backup done, wait for finish.");
1467
+ ROCKS_LOG_INFO(options_.info_log, "add files for backup done, wait finish.");
1560
1468
  IOStatus item_io_status;
1561
1469
  for (auto& item : backup_items_to_finish) {
1562
1470
  item.result.wait();
@@ -1727,11 +1635,6 @@ IOStatus BackupEngineImpl::DeleteBackupNoGC(BackupID backup_id) {
1727
1635
  return io_s;
1728
1636
  }
1729
1637
  backups_.erase(backup);
1730
- if (backups_.empty()) {
1731
- latest_valid_backup_id_ = 0;
1732
- } else {
1733
- latest_valid_backup_id_ = backups_.rbegin()->first;
1734
- }
1735
1638
  } else {
1736
1639
  auto corrupt = corrupt_backups_.find(backup_id);
1737
1640
  if (corrupt == corrupt_backups_.end()) {
@@ -1791,7 +1694,7 @@ void BackupEngineImpl::SetBackupInfoFromBackupMeta(
1791
1694
  auto& file_details = backup_info->file_details;
1792
1695
  file_details.reserve(meta.GetFiles().size());
1793
1696
  for (auto& file_ptr : meta.GetFiles()) {
1794
- BackupFileInfo& finfo = file_details.emplace_back();
1697
+ BackupFileInfo& finfo = *file_details.emplace(file_details.end());
1795
1698
  finfo.relative_filename = file_ptr->filename;
1796
1699
  finfo.size = file_ptr->size;
1797
1700
  finfo.directory = dir;
@@ -1803,10 +1706,7 @@ void BackupEngineImpl::SetBackupInfoFromBackupMeta(
1803
1706
  finfo.file_type = type;
1804
1707
  }
1805
1708
  // TODO: temperature, file_checksum, file_checksum_func_name
1806
- // finfo.temperature = file_ptr->temp;
1807
1709
  }
1808
- backup_info->excluded_files = meta.GetExcludedFiles();
1809
-
1810
1710
  backup_info->name_for_open = GetAbsolutePath(GetPrivateFileRel(id));
1811
1711
  backup_info->name_for_open.pop_back(); // remove trailing '/'
1812
1712
  backup_info->env_for_open = meta.GetEnvForOpen();
@@ -1864,8 +1764,7 @@ void BackupEngineImpl::GetCorruptedBackups(
1864
1764
 
1865
1765
  IOStatus BackupEngineImpl::RestoreDBFromBackup(
1866
1766
  const RestoreOptions& options, BackupID backup_id,
1867
- const std::string& db_dir, const std::string& wal_dir,
1868
- const std::list<const BackupEngineImpl*>& locked_restore_from_dirs) const {
1767
+ const std::string& db_dir, const std::string& wal_dir) const {
1869
1768
  assert(initialized_);
1870
1769
  if (backup_id == kLatestBackupIDMarker) {
1871
1770
  // Note: Read latest_valid_backup_id_ inside of lock
@@ -1925,37 +1824,6 @@ IOStatus BackupEngineImpl::RestoreDBFromBackup(
1925
1824
  DeleteChildren(db_dir);
1926
1825
  }
1927
1826
 
1928
- // Files to restore, and from where (taking into account excluded files)
1929
- std::vector<std::pair<const BackupEngineImpl*, const FileInfo*>>
1930
- restore_file_infos;
1931
- restore_file_infos.reserve(backup->GetFiles().size() +
1932
- backup->GetExcludedFiles().size());
1933
-
1934
- for (const auto& ef : backup->GetExcludedFiles()) {
1935
- const std::string& file = ef.relative_file;
1936
-
1937
- bool found = false;
1938
- for (auto be : locked_restore_from_dirs) {
1939
- auto it = be->backuped_file_infos_.find(file);
1940
- if (it != backuped_file_infos_.end()) {
1941
- restore_file_infos.emplace_back(be, &*it->second);
1942
- found = true;
1943
- break;
1944
- }
1945
- }
1946
- if (!found) {
1947
- return IOStatus::InvalidArgument(
1948
- "Excluded file " + file + " not found in other backups nor in " +
1949
- std::to_string(locked_restore_from_dirs.size() - 1) +
1950
- " alternate backup directories");
1951
- }
1952
- }
1953
-
1954
- // Non-excluded files
1955
- for (const auto& file_info_shared : backup->GetFiles()) {
1956
- restore_file_infos.emplace_back(this, &*file_info_shared);
1957
- }
1958
-
1959
1827
  IOStatus io_s;
1960
1828
  std::vector<RestoreAfterCopyOrCreateWorkItem> restore_items_to_finish;
1961
1829
  std::string temporary_current_file;
@@ -1963,13 +1831,8 @@ IOStatus BackupEngineImpl::RestoreDBFromBackup(
1963
1831
  std::unique_ptr<FSDirectory> db_dir_for_fsync;
1964
1832
  std::unique_ptr<FSDirectory> wal_dir_for_fsync;
1965
1833
 
1966
- for (const auto& engine_and_file_info : restore_file_infos) {
1967
- const FileInfo* file_info = engine_and_file_info.second;
1834
+ for (const auto& file_info : backup->GetFiles()) {
1968
1835
  const std::string& file = file_info->filename;
1969
- std::string absolute_file =
1970
- engine_and_file_info.first->GetAbsolutePath(file);
1971
- Env* src_env = engine_and_file_info.first->backup_env_;
1972
-
1973
1836
  // 1. get DB filename
1974
1837
  std::string dst = file_info->GetDbFileName();
1975
1838
 
@@ -2013,8 +1876,8 @@ IOStatus BackupEngineImpl::RestoreDBFromBackup(
2013
1876
  ROCKS_LOG_INFO(options_.info_log, "Restoring %s to %s\n", file.c_str(),
2014
1877
  dst.c_str());
2015
1878
  CopyOrCreateWorkItem copy_or_create_work_item(
2016
- absolute_file, dst, Temperature::kUnknown /* src_temp */,
2017
- file_info->temp, "" /* contents */, src_env, db_env_,
1879
+ GetAbsolutePath(file), dst, Temperature::kUnknown /* src_temp */,
1880
+ file_info->temp, "" /* contents */, backup_env_, db_env_,
2018
1881
  EnvOptions() /* src_env_options */, options_.sync,
2019
1882
  options_.restore_rate_limiter.get(), file_info->size,
2020
1883
  nullptr /* stats */);
@@ -2254,19 +2117,8 @@ IOStatus BackupEngineImpl::CopyOrCreateFile(
2254
2117
  while (*bytes_toward_next_callback >=
2255
2118
  options_.callback_trigger_interval_size) {
2256
2119
  *bytes_toward_next_callback -= options_.callback_trigger_interval_size;
2257
- if (progress_callback) {
2258
- std::lock_guard<std::mutex> lock(byte_report_mutex_);
2259
- try {
2260
- progress_callback();
2261
- } catch (const std::exception& exn) {
2262
- io_s = IOStatus::Aborted("Exception in progress_callback: " +
2263
- std::string(exn.what()));
2264
- break;
2265
- } catch (...) {
2266
- io_s = IOStatus::Aborted("Unknown exception in progress_callback");
2267
- break;
2268
- }
2269
- }
2120
+ std::lock_guard<std::mutex> lock(byte_report_mutex_);
2121
+ progress_callback();
2270
2122
  }
2271
2123
  } while (io_s.ok() && contents.empty() && data.size() > 0 && size_limit > 0);
2272
2124
 
@@ -2287,12 +2139,11 @@ IOStatus BackupEngineImpl::CopyOrCreateFile(
2287
2139
  // fname will always start with "/"
2288
2140
  IOStatus BackupEngineImpl::AddBackupFileWorkItem(
2289
2141
  std::unordered_set<std::string>& live_dst_paths,
2290
- std::deque<BackupAfterCopyOrCreateWorkItem>& backup_items_to_finish,
2291
- std::deque<BackupWorkItemPair>* excludable_items, BackupID backup_id,
2292
- bool shared, const std::string& src_dir, const std::string& fname,
2293
- const EnvOptions& src_env_options, RateLimiter* rate_limiter,
2294
- FileType file_type, uint64_t size_bytes, Statistics* stats,
2295
- uint64_t size_limit, bool shared_checksum,
2142
+ std::vector<BackupAfterCopyOrCreateWorkItem>& backup_items_to_finish,
2143
+ BackupID backup_id, bool shared, const std::string& src_dir,
2144
+ const std::string& fname, const EnvOptions& src_env_options,
2145
+ RateLimiter* rate_limiter, FileType file_type, uint64_t size_bytes,
2146
+ Statistics* stats, uint64_t size_limit, bool shared_checksum,
2296
2147
  std::function<void()> progress_callback, const std::string& contents,
2297
2148
  const std::string& src_checksum_func_name,
2298
2149
  const std::string& src_checksum_str, const Temperature src_temperature) {
@@ -2486,6 +2337,8 @@ IOStatus BackupEngineImpl::AddBackupFileWorkItem(
2486
2337
 
2487
2338
  // Step 3: Add work item
2488
2339
  if (!contents.empty() || need_to_copy) {
2340
+ ROCKS_LOG_INFO(options_.info_log, "Copying %s to %s", fname.c_str(),
2341
+ copy_dest_path->c_str());
2489
2342
  CopyOrCreateWorkItem copy_or_create_work_item(
2490
2343
  src_dir.empty() ? "" : src_path, *copy_dest_path, src_temperature,
2491
2344
  Temperature::kUnknown /*dst_temp*/, contents, db_env_, backup_env_,
@@ -2495,21 +2348,8 @@ IOStatus BackupEngineImpl::AddBackupFileWorkItem(
2495
2348
  BackupAfterCopyOrCreateWorkItem after_copy_or_create_work_item(
2496
2349
  copy_or_create_work_item.result.get_future(), shared, need_to_copy,
2497
2350
  backup_env_, temp_dest_path, final_dest_path, dst_relative);
2498
- if (excludable_items != nullptr && shared && shared_checksum &&
2499
- need_to_copy) {
2500
- ROCKS_LOG_INFO(options_.info_log, "Copying (if not excluded) %s to %s",
2501
- fname.c_str(), copy_dest_path->c_str());
2502
- excludable_items->emplace_back(std::move(copy_or_create_work_item),
2503
- std::move(after_copy_or_create_work_item));
2504
- } else {
2505
- // For files known not excluded, can start copying even before finishing
2506
- // the checkpoint
2507
- ROCKS_LOG_INFO(options_.info_log, "Copying %s to %s", fname.c_str(),
2508
- copy_dest_path->c_str());
2509
- files_to_copy_or_create_.write(std::move(copy_or_create_work_item));
2510
- backup_items_to_finish.push_back(
2511
- std::move(after_copy_or_create_work_item));
2512
- }
2351
+ files_to_copy_or_create_.write(std::move(copy_or_create_work_item));
2352
+ backup_items_to_finish.push_back(std::move(after_copy_or_create_work_item));
2513
2353
  } else {
2514
2354
  std::promise<CopyOrCreateResult> promise_result;
2515
2355
  BackupAfterCopyOrCreateWorkItem after_copy_or_create_work_item(
@@ -2900,7 +2740,6 @@ const std::string kAppMetaDataFieldName{"metadata"};
2900
2740
  const std::string kFileCrc32cFieldName{"crc32"};
2901
2741
  const std::string kFileSizeFieldName{"size"};
2902
2742
  const std::string kTemperatureFieldName{"temp"};
2903
- const std::string kExcludedFieldName{"ni::excluded"};
2904
2743
 
2905
2744
  // Marks a (future) field that should cause failure if not recognized.
2906
2745
  // Other fields are assumed to be ignorable. For example, in the future
@@ -2921,7 +2760,8 @@ const std::string kNonIgnorableFieldPrefix{"ni::"};
2921
2760
  // ...
2922
2761
  //----------------------------------------------------------
2923
2762
  //
2924
- // For schema version 2.x:
2763
+ // For schema version 2.x (not in public APIs, but
2764
+ // forward-compatibility started):
2925
2765
  //----------------------------------------------------------
2926
2766
  // schema_version <ver>
2927
2767
  // <timestamp>
@@ -3064,6 +2904,20 @@ IOStatus BackupEngineImpl::BackupMeta::LoadFromFile(
3064
2904
 
3065
2905
  const std::string& filename = components[0];
3066
2906
 
2907
+ uint64_t actual_size;
2908
+ const std::shared_ptr<FileInfo> file_info = GetFile(filename);
2909
+ if (file_info) {
2910
+ actual_size = file_info->size;
2911
+ } else {
2912
+ std::string abs_path = backup_dir + "/" + filename;
2913
+ auto e = abs_path_to_size.find(abs_path);
2914
+ if (e == abs_path_to_size.end()) {
2915
+ return IOStatus::Corruption(
2916
+ "Pathname in meta file not found on disk: " + abs_path);
2917
+ }
2918
+ actual_size = e->second;
2919
+ }
2920
+
3067
2921
  if (schema_major_version >= 2) {
3068
2922
  if (components.size() % 2 != 1) {
3069
2923
  return IOStatus::Corruption(
@@ -3085,10 +2939,8 @@ IOStatus BackupEngineImpl::BackupMeta::LoadFromFile(
3085
2939
  }
3086
2940
  }
3087
2941
 
3088
- std::optional<uint64_t> expected_size{};
3089
2942
  std::string checksum_hex;
3090
2943
  Temperature temp = Temperature::kUnknown;
3091
- bool excluded = false;
3092
2944
  for (unsigned i = 1; i < components.size(); i += 2) {
3093
2945
  const std::string& field_name = components[i];
3094
2946
  const std::string& field_data = components[i + 1];
@@ -3102,7 +2954,14 @@ IOStatus BackupEngineImpl::BackupMeta::LoadFromFile(
3102
2954
  }
3103
2955
  checksum_hex = ChecksumInt32ToHex(checksum_value);
3104
2956
  } else if (field_name == kFileSizeFieldName) {
3105
- expected_size = std::strtoull(field_data.c_str(), nullptr, /*base*/ 10);
2957
+ uint64_t ex_size =
2958
+ std::strtoull(field_data.c_str(), nullptr, /*base*/ 10);
2959
+ if (ex_size != actual_size) {
2960
+ return IOStatus::Corruption(
2961
+ "For file " + filename + " expected size " +
2962
+ std::to_string(ex_size) + " but found size" +
2963
+ std::to_string(actual_size));
2964
+ }
3106
2965
  } else if (field_name == kTemperatureFieldName) {
3107
2966
  auto iter = temperature_string_map.find(field_data);
3108
2967
  if (iter != temperature_string_map.end()) {
@@ -3113,15 +2972,6 @@ IOStatus BackupEngineImpl::BackupMeta::LoadFromFile(
3113
2972
  // be safe.
3114
2973
  temp = Temperature::kUnknown;
3115
2974
  }
3116
- } else if (field_name == kExcludedFieldName) {
3117
- if (field_data == "true") {
3118
- excluded = true;
3119
- } else if (field_data == "false") {
3120
- excluded = false;
3121
- } else {
3122
- return IOStatus::NotSupported("Unrecognized value \"" + field_data +
3123
- "\" for field " + field_name);
3124
- }
3125
2975
  } else if (StartsWith(field_name, kNonIgnorableFieldPrefix)) {
3126
2976
  return IOStatus::NotSupported("Unrecognized non-ignorable file field " +
3127
2977
  field_name + " (from future version?)");
@@ -3134,29 +2984,8 @@ IOStatus BackupEngineImpl::BackupMeta::LoadFromFile(
3134
2984
  }
3135
2985
  }
3136
2986
 
3137
- if (excluded) {
3138
- excluded_files_.emplace_back(filename);
3139
- } else {
3140
- // Verify file exists, with expected size
3141
- std::string abs_path = backup_dir + "/" + filename;
3142
- auto e = abs_path_to_size.find(abs_path);
3143
- if (e == abs_path_to_size.end()) {
3144
- return IOStatus::Corruption(
3145
- "Pathname in meta file not found on disk: " + abs_path);
3146
- }
3147
- uint64_t actual_size = e->second;
3148
- if (expected_size.has_value() && *expected_size != actual_size) {
3149
- return IOStatus::Corruption("For file " + filename + " expected size " +
3150
- std::to_string(*expected_size) +
3151
- " but found size" +
3152
- std::to_string(actual_size));
3153
- }
3154
-
3155
- // NOTE: FileInfo will be coalesced for sharing later (AddFile below)
3156
- files.emplace_back(
3157
- std::make_shared<FileInfo>(filename, actual_size, checksum_hex,
3158
- /*id*/ "", /*sid*/ "", temp));
3159
- }
2987
+ files.emplace_back(new FileInfo(filename, actual_size, checksum_hex,
2988
+ /*id*/ "", /*sid*/ "", temp));
3160
2989
  }
3161
2990
 
3162
2991
  if (footer_present) {
@@ -3212,7 +3041,7 @@ IOStatus BackupEngineImpl::BackupMeta::LoadFromFile(
3212
3041
  const std::vector<std::string> minor_version_strings{
3213
3042
  "", // invalid major version 0
3214
3043
  "", // implicit major version 1
3215
- "2.1",
3044
+ "2.0",
3216
3045
  };
3217
3046
 
3218
3047
  IOStatus BackupEngineImpl::BackupMeta::StoreToFile(
@@ -3290,11 +3119,6 @@ IOStatus BackupEngineImpl::BackupMeta::StoreToFile(
3290
3119
  buf << "\n";
3291
3120
  }
3292
3121
 
3293
- for (const auto& file : excluded_files_) {
3294
- assert(schema_version >= 2);
3295
- buf << file.relative_file << " " << kExcludedFieldName << " true\n";
3296
- }
3297
-
3298
3122
  if (schema_test_options && !schema_test_options->footer_fields.empty()) {
3299
3123
  buf << kFooterMarker << "\n";
3300
3124
  for (auto& e : schema_test_options->footer_fields) {