@nxtedition/rocksdb 8.1.2 → 8.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/deps/rocksdb/rocksdb/CMakeLists.txt +0 -9
  2. package/deps/rocksdb/rocksdb/Makefile +1 -13
  3. package/deps/rocksdb/rocksdb/TARGETS +0 -4
  4. package/deps/rocksdb/rocksdb/db/blob/blob_counting_iterator.h +0 -11
  5. package/deps/rocksdb/rocksdb/db/builder.cc +4 -13
  6. package/deps/rocksdb/rocksdb/db/builder.h +1 -2
  7. package/deps/rocksdb/rocksdb/db/c.cc +0 -15
  8. package/deps/rocksdb/rocksdb/db/c_test.c +0 -3
  9. package/deps/rocksdb/rocksdb/db/column_family.cc +5 -11
  10. package/deps/rocksdb/rocksdb/db/column_family.h +0 -20
  11. package/deps/rocksdb/rocksdb/db/compaction/clipping_iterator.h +0 -5
  12. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +37 -31
  13. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +0 -5
  14. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +7 -24
  15. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +1 -17
  16. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +1 -4
  17. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +2 -4
  18. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +6 -9
  19. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +26 -104
  20. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +0 -5
  21. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +18 -11
  22. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +16 -17
  23. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +6 -19
  24. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +5 -5
  25. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +22 -22
  26. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +5 -5
  27. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +52 -81
  28. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +1 -5
  29. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +5 -5
  30. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +2 -8
  31. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +0 -8
  32. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +138 -266
  33. package/deps/rocksdb/rocksdb/db/corruption_test.cc +1 -86
  34. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +3 -2
  35. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +123 -897
  36. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +20 -31
  37. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +9 -5
  38. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +28 -121
  39. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +1 -1
  40. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +2 -3
  41. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +0 -3
  42. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +4 -8
  43. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +0 -10
  44. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +4 -325
  45. package/deps/rocksdb/rocksdb/db/db_test.cc +0 -3
  46. package/deps/rocksdb/rocksdb/db/db_test2.cc +8 -233
  47. package/deps/rocksdb/rocksdb/db/db_test_util.h +0 -3
  48. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +0 -129
  49. package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +0 -21
  50. package/deps/rocksdb/rocksdb/db/dbformat.cc +0 -25
  51. package/deps/rocksdb/rocksdb/db/dbformat.h +0 -2
  52. package/deps/rocksdb/rocksdb/db/experimental.cc +2 -3
  53. package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +0 -3
  54. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +13 -92
  55. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +1 -38
  56. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +110 -14
  57. package/deps/rocksdb/rocksdb/db/flush_job.cc +4 -6
  58. package/deps/rocksdb/rocksdb/db/history_trimming_iterator.h +0 -4
  59. package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +53 -56
  60. package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +4 -3
  61. package/deps/rocksdb/rocksdb/db/memtable.cc +1 -1
  62. package/deps/rocksdb/rocksdb/db/merge_helper.cc +0 -4
  63. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +10 -10
  64. package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter_test.cc +1 -1
  65. package/deps/rocksdb/rocksdb/db/repair.cc +22 -65
  66. package/deps/rocksdb/rocksdb/db/repair_test.cc +0 -54
  67. package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +26 -26
  68. package/deps/rocksdb/rocksdb/db/table_properties_collector.h +1 -3
  69. package/deps/rocksdb/rocksdb/db/version_builder.cc +43 -90
  70. package/deps/rocksdb/rocksdb/db/version_builder.h +0 -20
  71. package/deps/rocksdb/rocksdb/db/version_builder_test.cc +93 -218
  72. package/deps/rocksdb/rocksdb/db/version_edit.cc +1 -27
  73. package/deps/rocksdb/rocksdb/db/version_edit.h +9 -33
  74. package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +6 -13
  75. package/deps/rocksdb/rocksdb/db/version_edit_handler.h +6 -17
  76. package/deps/rocksdb/rocksdb/db/version_edit_test.cc +17 -19
  77. package/deps/rocksdb/rocksdb/db/version_set.cc +26 -166
  78. package/deps/rocksdb/rocksdb/db/version_set.h +2 -32
  79. package/deps/rocksdb/rocksdb/db/version_set_test.cc +31 -65
  80. package/deps/rocksdb/rocksdb/db/write_thread.cc +2 -5
  81. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +0 -1
  82. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +32 -31
  83. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.h +1 -2
  84. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +6 -8
  85. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +0 -4
  86. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +4 -11
  87. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +15 -16
  88. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +1 -13
  89. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +75 -0
  90. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +0 -8
  91. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +0 -6
  92. package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +1 -7
  93. package/deps/rocksdb/rocksdb/include/rocksdb/metadata.h +3 -9
  94. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +1 -2
  95. package/deps/rocksdb/rocksdb/include/rocksdb/status.h +0 -3
  96. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h +9 -69
  97. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
  98. package/deps/rocksdb/rocksdb/memory/arena.cc +87 -23
  99. package/deps/rocksdb/rocksdb/memory/arena.h +31 -25
  100. package/deps/rocksdb/rocksdb/memory/arena_test.cc +0 -90
  101. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +26 -26
  102. package/deps/rocksdb/rocksdb/src.mk +0 -2
  103. package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +2 -3
  104. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +2 -3
  105. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +2 -6
  106. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +1 -1
  107. package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +3 -3
  108. package/deps/rocksdb/rocksdb/table/format.cc +20 -24
  109. package/deps/rocksdb/rocksdb/table/format.h +2 -5
  110. package/deps/rocksdb/rocksdb/table/merging_iterator.cc +105 -54
  111. package/deps/rocksdb/rocksdb/table/merging_iterator.h +0 -80
  112. package/deps/rocksdb/rocksdb/table/meta_blocks.cc +2 -2
  113. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +1 -1
  114. package/deps/rocksdb/rocksdb/table/table_test.cc +6 -7
  115. package/deps/rocksdb/rocksdb/test_util/testutil.h +0 -10
  116. package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.h +2 -2
  117. package/deps/rocksdb/rocksdb/util/bloom_test.cc +1 -1
  118. package/deps/rocksdb/rocksdb/util/status.cc +0 -7
  119. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +74 -250
  120. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +4 -199
  121. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +0 -1
  122. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +0 -39
  123. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +0 -9
  124. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +0 -59
  125. package/deps/rocksdb/rocksdb.gyp +0 -3
  126. package/index.js +2 -2
  127. package/package.json +1 -1
  128. package/prebuilds/darwin-arm64/node.napi.node +0 -0
  129. package/prebuilds/linux-x64/node.napi.node +0 -0
  130. package/deps/rocksdb/rocksdb/port/mmap.cc +0 -98
  131. package/deps/rocksdb/rocksdb/port/mmap.h +0 -70
  132. package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +0 -142
  133. package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +0 -241
@@ -9,10 +9,8 @@
9
9
 
10
10
  #include <tuple>
11
11
 
12
- #include "compaction/compaction_picker_universal.h"
13
12
  #include "db/blob/blob_index.h"
14
13
  #include "db/db_test_util.h"
15
- #include "db/dbformat.h"
16
14
  #include "env/mock_env.h"
17
15
  #include "port/port.h"
18
16
  #include "port/stack_trace.h"
@@ -1026,70 +1024,6 @@ TEST_F(DBCompactionTest, CompactionSstPartitioner) {
1026
1024
  ASSERT_EQ("B", Get("bbbb1"));
1027
1025
  }
1028
1026
 
1029
- TEST_F(DBCompactionTest, CompactionSstPartitionWithManualCompaction) {
1030
- Options options = CurrentOptions();
1031
- options.compaction_style = kCompactionStyleLevel;
1032
- options.level0_file_num_compaction_trigger = 3;
1033
-
1034
- DestroyAndReopen(options);
1035
-
1036
- // create first file and flush to l0
1037
- ASSERT_OK(Put("000015", "A"));
1038
- ASSERT_OK(Put("000025", "B"));
1039
- ASSERT_OK(Flush());
1040
- ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
1041
-
1042
- // create second file and flush to l0
1043
- ASSERT_OK(Put("000015", "A2"));
1044
- ASSERT_OK(Put("000025", "B2"));
1045
- ASSERT_OK(Flush());
1046
- ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
1047
-
1048
- // CONTROL 1: compact without partitioner
1049
- CompactRangeOptions compact_options;
1050
- compact_options.bottommost_level_compaction =
1051
- BottommostLevelCompaction::kForceOptimized;
1052
- ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
1053
-
1054
- // Check (compacted but no partitioning yet)
1055
- std::vector<LiveFileMetaData> files;
1056
- dbfull()->GetLiveFilesMetaData(&files);
1057
- ASSERT_EQ(1, files.size());
1058
-
1059
- // Install partitioner
1060
- std::shared_ptr<SstPartitionerFactory> factory(
1061
- NewSstPartitionerFixedPrefixFactory(5));
1062
- options.sst_partitioner_factory = factory;
1063
- Reopen(options);
1064
-
1065
- // CONTROL 2: request compaction on range with no partition boundary and no
1066
- // overlap with actual entries
1067
- Slice from("000017");
1068
- Slice to("000019");
1069
- ASSERT_OK(dbfull()->CompactRange(compact_options, &from, &to));
1070
-
1071
- // Check (no partitioning yet)
1072
- files.clear();
1073
- dbfull()->GetLiveFilesMetaData(&files);
1074
- ASSERT_EQ(1, files.size());
1075
- ASSERT_EQ("A2", Get("000015"));
1076
- ASSERT_EQ("B2", Get("000025"));
1077
-
1078
- // TEST: request compaction overlapping with partition boundary but no
1079
- // actual entries
1080
- // NOTE: `to` is INCLUSIVE
1081
- from = Slice("000019");
1082
- to = Slice("000020");
1083
- ASSERT_OK(dbfull()->CompactRange(compact_options, &from, &to));
1084
-
1085
- // Check (must be partitioned)
1086
- files.clear();
1087
- dbfull()->GetLiveFilesMetaData(&files);
1088
- ASSERT_EQ(2, files.size());
1089
- ASSERT_EQ("A2", Get("000015"));
1090
- ASSERT_EQ("B2", Get("000025"));
1091
- }
1092
-
1093
1027
  TEST_F(DBCompactionTest, CompactionSstPartitionerNonTrivial) {
1094
1028
  Options options = CurrentOptions();
1095
1029
  options.compaction_style = kCompactionStyleLevel;
@@ -6245,231 +6179,6 @@ TEST_P(DBCompactionTestWithParam, FixFileIngestionCompactionDeadlock) {
6245
6179
  Close();
6246
6180
  }
6247
6181
 
6248
- class DBCompactionTestWithOngoingFileIngestionParam
6249
- : public DBCompactionTest,
6250
- public testing::WithParamInterface<std::string> {
6251
- public:
6252
- DBCompactionTestWithOngoingFileIngestionParam() : DBCompactionTest() {
6253
- compaction_path_to_test_ = GetParam();
6254
- }
6255
- void SetupOptions() {
6256
- options_ = CurrentOptions();
6257
- options_.create_if_missing = true;
6258
-
6259
- if (compaction_path_to_test_ == "RefitLevelCompactRange") {
6260
- options_.num_levels = 7;
6261
- } else {
6262
- options_.num_levels = 3;
6263
- }
6264
- options_.compaction_style = CompactionStyle::kCompactionStyleLevel;
6265
- if (compaction_path_to_test_ == "AutoCompaction") {
6266
- options_.disable_auto_compactions = false;
6267
- options_.level0_file_num_compaction_trigger = 1;
6268
- } else {
6269
- options_.disable_auto_compactions = true;
6270
- }
6271
- }
6272
-
6273
- void PauseCompactionThread() {
6274
- sleeping_task_.reset(new test::SleepingBackgroundTask());
6275
- env_->SetBackgroundThreads(1, Env::LOW);
6276
- env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
6277
- sleeping_task_.get(), Env::Priority::LOW);
6278
- sleeping_task_->WaitUntilSleeping();
6279
- }
6280
-
6281
- void ResumeCompactionThread() {
6282
- if (sleeping_task_) {
6283
- sleeping_task_->WakeUp();
6284
- sleeping_task_->WaitUntilDone();
6285
- }
6286
- }
6287
-
6288
- void SetupFilesToForceFutureFilesIngestedToCertainLevel() {
6289
- SstFileWriter sst_file_writer(EnvOptions(), options_);
6290
- std::string dummy = dbname_ + "/dummy.sst";
6291
- ASSERT_OK(sst_file_writer.Open(dummy));
6292
- ASSERT_OK(sst_file_writer.Put("k2", "dummy"));
6293
- ASSERT_OK(sst_file_writer.Finish());
6294
- ASSERT_OK(db_->IngestExternalFile({dummy}, IngestExternalFileOptions()));
6295
- // L2 is made to contain a file overlapped with files to be ingested in
6296
- // later steps on key "k2". This will force future files ingested to L1 or
6297
- // above.
6298
- ASSERT_EQ("0,0,1", FilesPerLevel(0));
6299
- }
6300
-
6301
- void SetupSyncPoints() {
6302
- if (compaction_path_to_test_ == "AutoCompaction") {
6303
- SyncPoint::GetInstance()->SetCallBack(
6304
- "ExternalSstFileIngestionJob::Run", [&](void*) {
6305
- SyncPoint::GetInstance()->LoadDependency(
6306
- {{"DBImpl::BackgroundCompaction():AfterPickCompaction",
6307
- "VersionSet::LogAndApply:WriteManifest"}});
6308
- });
6309
- } else if (compaction_path_to_test_ == "NonRefitLevelCompactRange") {
6310
- SyncPoint::GetInstance()->SetCallBack(
6311
- "ExternalSstFileIngestionJob::Run", [&](void*) {
6312
- SyncPoint::GetInstance()->LoadDependency(
6313
- {{"ColumnFamilyData::CompactRange:Return",
6314
- "VersionSet::LogAndApply:WriteManifest"}});
6315
- });
6316
- } else if (compaction_path_to_test_ == "RefitLevelCompactRange") {
6317
- SyncPoint::GetInstance()->SetCallBack(
6318
- "ExternalSstFileIngestionJob::Run", [&](void*) {
6319
- SyncPoint::GetInstance()->LoadDependency(
6320
- {{"DBImpl::CompactRange:PostRefitLevel",
6321
- "VersionSet::LogAndApply:WriteManifest"}});
6322
- });
6323
- } else if (compaction_path_to_test_ == "CompactFiles") {
6324
- SyncPoint::GetInstance()->SetCallBack(
6325
- "ExternalSstFileIngestionJob::Run", [&](void*) {
6326
- SyncPoint::GetInstance()->LoadDependency(
6327
- {{"DBImpl::CompactFilesImpl::PostSanitizeCompactionInputFiles",
6328
- "VersionSet::LogAndApply:WriteManifest"}});
6329
- });
6330
- } else {
6331
- assert(false);
6332
- }
6333
- SyncPoint::GetInstance()->LoadDependency(
6334
- {{"ExternalSstFileIngestionJob::Run", "PreCompaction"}});
6335
- SyncPoint::GetInstance()->EnableProcessing();
6336
- }
6337
-
6338
- void RunCompactionOverlappedWithFileIngestion() {
6339
- if (compaction_path_to_test_ == "AutoCompaction") {
6340
- TEST_SYNC_POINT("PreCompaction");
6341
- ResumeCompactionThread();
6342
- // Without proper range conflict check,
6343
- // this would have been `Status::Corruption` about overlapping ranges
6344
- Status s = dbfull()->TEST_WaitForCompact();
6345
- EXPECT_OK(s);
6346
- } else if (compaction_path_to_test_ == "NonRefitLevelCompactRange") {
6347
- CompactRangeOptions cro;
6348
- cro.change_level = false;
6349
- std::string start_key = "k1";
6350
- Slice start(start_key);
6351
- std::string end_key = "k4";
6352
- Slice end(end_key);
6353
- TEST_SYNC_POINT("PreCompaction");
6354
- // Without proper range conflict check,
6355
- // this would have been `Status::Corruption` about overlapping ranges
6356
- Status s = dbfull()->CompactRange(cro, &start, &end);
6357
- EXPECT_OK(s);
6358
- } else if (compaction_path_to_test_ == "RefitLevelCompactRange") {
6359
- CompactRangeOptions cro;
6360
- cro.change_level = true;
6361
- cro.target_level = 5;
6362
- std::string start_key = "k1";
6363
- Slice start(start_key);
6364
- std::string end_key = "k4";
6365
- Slice end(end_key);
6366
- TEST_SYNC_POINT("PreCompaction");
6367
- Status s = dbfull()->CompactRange(cro, &start, &end);
6368
- // Without proper range conflict check,
6369
- // this would have been `Status::Corruption` about overlapping ranges
6370
- // To see this, remove the fix AND replace
6371
- // `DBImpl::CompactRange:PostRefitLevel` in sync point dependency with
6372
- // `DBImpl::ReFitLevel:PostRegisterCompaction`
6373
- EXPECT_TRUE(s.IsNotSupported());
6374
- EXPECT_TRUE(s.ToString().find("some ongoing compaction's output") !=
6375
- std::string::npos);
6376
- } else if (compaction_path_to_test_ == "CompactFiles") {
6377
- ColumnFamilyMetaData cf_meta_data;
6378
- db_->GetColumnFamilyMetaData(&cf_meta_data);
6379
- ASSERT_EQ(cf_meta_data.levels[0].files.size(), 1);
6380
- std::vector<std::string> input_files;
6381
- for (const auto& file : cf_meta_data.levels[0].files) {
6382
- input_files.push_back(file.name);
6383
- }
6384
- TEST_SYNC_POINT("PreCompaction");
6385
- Status s = db_->CompactFiles(CompactionOptions(), input_files, 1);
6386
- // Without proper range conflict check,
6387
- // this would have been `Status::Corruption` about overlapping ranges
6388
- EXPECT_TRUE(s.IsAborted());
6389
- EXPECT_TRUE(
6390
- s.ToString().find(
6391
- "A running compaction is writing to the same output level") !=
6392
- std::string::npos);
6393
- } else {
6394
- assert(false);
6395
- }
6396
- }
6397
-
6398
- void DisableSyncPoints() {
6399
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
6400
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
6401
- }
6402
-
6403
- protected:
6404
- std::string compaction_path_to_test_;
6405
- Options options_;
6406
- std::shared_ptr<test::SleepingBackgroundTask> sleeping_task_;
6407
- };
6408
-
6409
- INSTANTIATE_TEST_CASE_P(DBCompactionTestWithOngoingFileIngestionParam,
6410
- DBCompactionTestWithOngoingFileIngestionParam,
6411
- ::testing::Values("AutoCompaction",
6412
- "NonRefitLevelCompactRange",
6413
- "RefitLevelCompactRange",
6414
- "CompactFiles"));
6415
-
6416
- TEST_P(DBCompactionTestWithOngoingFileIngestionParam, RangeConflictCheck) {
6417
- SetupOptions();
6418
- DestroyAndReopen(options_);
6419
-
6420
- if (compaction_path_to_test_ == "AutoCompaction") {
6421
- PauseCompactionThread();
6422
- }
6423
-
6424
- if (compaction_path_to_test_ != "RefitLevelCompactRange") {
6425
- SetupFilesToForceFutureFilesIngestedToCertainLevel();
6426
- }
6427
-
6428
- // Create s1
6429
- ASSERT_OK(Put("k1", "v"));
6430
- ASSERT_OK(Put("k4", "v"));
6431
- ASSERT_OK(Flush());
6432
- if (compaction_path_to_test_ == "RefitLevelCompactRange") {
6433
- MoveFilesToLevel(6 /* level */);
6434
- ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel(0));
6435
- } else {
6436
- ASSERT_EQ("1,0,1", FilesPerLevel(0));
6437
- }
6438
-
6439
- // To coerce following sequence of events
6440
- // Timeline Thread 1 (Ingest s2) Thread 2 (Compact s1)
6441
- // t0 | Decide to output to Lk
6442
- // t1 | Release lock in LogAndApply()
6443
- // t2 | Acquire lock
6444
- // t3 | Decides to compact to Lk
6445
- // | Expected to fail due to range
6446
- // | conflict check with file
6447
- // | ingestion
6448
- // t4 | Release lock in LogAndApply()
6449
- // t5 | Acquire lock again and finish
6450
- // t6 | Acquire lock again and finish
6451
- SetupSyncPoints();
6452
-
6453
- // Ingest s2
6454
- port::Thread thread1([&] {
6455
- SstFileWriter sst_file_writer(EnvOptions(), options_);
6456
- std::string s2 = dbname_ + "/ingested_s2.sst";
6457
- ASSERT_OK(sst_file_writer.Open(s2));
6458
- ASSERT_OK(sst_file_writer.Put("k2", "v2"));
6459
- ASSERT_OK(sst_file_writer.Put("k3", "v2"));
6460
- ASSERT_OK(sst_file_writer.Finish());
6461
- ASSERT_OK(db_->IngestExternalFile({s2}, IngestExternalFileOptions()));
6462
- });
6463
-
6464
- // Compact s1. Without proper range conflict check,
6465
- // this will encounter overlapping file corruption.
6466
- port::Thread thread2([&] { RunCompactionOverlappedWithFileIngestion(); });
6467
-
6468
- thread1.join();
6469
- thread2.join();
6470
- DisableSyncPoints();
6471
- }
6472
-
6473
6182
  TEST_F(DBCompactionTest, ConsistencyFailTest) {
6474
6183
  Options options = CurrentOptions();
6475
6184
  options.force_consistency_checks = true;
@@ -6562,655 +6271,172 @@ void IngestOneKeyValue(DBImpl* db, const std::string& key,
6562
6271
  ASSERT_OK(db->IngestExternalFile({info.file_path}, ingest_opt));
6563
6272
  }
6564
6273
 
6565
- class DBCompactionTestL0FilesMisorderCorruption : public DBCompactionTest {
6566
- public:
6567
- DBCompactionTestL0FilesMisorderCorruption() : DBCompactionTest() {}
6568
- void SetupOptions(const CompactionStyle compaciton_style,
6569
- const std::string& compaction_path_to_test = "") {
6570
- options_ = CurrentOptions();
6571
- options_.create_if_missing = true;
6572
- options_.compression = kNoCompression;
6573
-
6574
- options_.force_consistency_checks = true;
6575
- options_.compaction_style = compaciton_style;
6576
-
6577
- if (compaciton_style == CompactionStyle::kCompactionStyleLevel) {
6578
- options_.num_levels = 7;
6579
- // Level compaction's PickIntraL0Compaction() impl detail requires
6580
- // `options.level0_file_num_compaction_trigger` to be
6581
- // at least 2 files less than the actual number of level 0 files
6582
- // (i.e, 7 by design in this test)
6583
- options_.level0_file_num_compaction_trigger = 5;
6584
- options_.max_background_compactions = 2;
6585
- options_.write_buffer_size = 2 << 20;
6586
- options_.max_write_buffer_number = 6;
6587
- } else if (compaciton_style == CompactionStyle::kCompactionStyleUniversal) {
6588
- // TODO: expand test coverage to num_lvels > 1 for universal compacion,
6589
- // which requires careful unit test design to compact to level 0 despite
6590
- // num_levels > 1
6591
- options_.num_levels = 1;
6592
- options_.level0_file_num_compaction_trigger = 5;
6593
-
6594
- CompactionOptionsUniversal universal_options;
6595
- if (compaction_path_to_test == "PickCompactionToReduceSizeAmp") {
6596
- universal_options.max_size_amplification_percent = 50;
6597
- } else if (compaction_path_to_test ==
6598
- "PickCompactionToReduceSortedRuns") {
6599
- universal_options.max_size_amplification_percent = 400;
6600
- } else if (compaction_path_to_test == "PickDeleteTriggeredCompaction") {
6601
- universal_options.max_size_amplification_percent = 400;
6602
- universal_options.min_merge_width = 6;
6603
- }
6604
- options_.compaction_options_universal = universal_options;
6605
- } else if (compaciton_style == CompactionStyle::kCompactionStyleFIFO) {
6606
- options_.max_open_files = -1;
6607
- options_.num_levels = 1;
6608
- options_.level0_file_num_compaction_trigger = 3;
6609
-
6610
- CompactionOptionsFIFO fifo_options;
6611
- if (compaction_path_to_test == "FindIntraL0Compaction" ||
6612
- compaction_path_to_test == "CompactRange") {
6613
- fifo_options.allow_compaction = true;
6614
- fifo_options.age_for_warm = 0;
6615
- } else if (compaction_path_to_test == "CompactFile") {
6616
- fifo_options.allow_compaction = false;
6617
- fifo_options.age_for_warm = 0;
6618
- }
6619
- options_.compaction_options_fifo = fifo_options;
6620
- }
6621
-
6622
- if (compaction_path_to_test == "CompactFile" ||
6623
- compaction_path_to_test == "CompactRange") {
6624
- options_.disable_auto_compactions = true;
6625
- } else {
6626
- options_.disable_auto_compactions = false;
6627
- }
6628
- }
6629
-
6630
- void Destroy(const Options& options) {
6631
- if (snapshot_) {
6632
- assert(db_);
6633
- db_->ReleaseSnapshot(snapshot_);
6634
- snapshot_ = nullptr;
6635
- }
6636
- DBTestBase::Destroy(options);
6637
- }
6638
-
6639
- void Reopen(const Options& options) {
6640
- DBTestBase::Reopen(options);
6641
- if (options.compaction_style != CompactionStyle::kCompactionStyleLevel) {
6642
- // To force assigning the global seqno to ingested file
6643
- // for our test purpose.
6644
- assert(snapshot_ == nullptr);
6645
- snapshot_ = db_->GetSnapshot();
6646
- }
6647
- }
6648
-
6649
- void DestroyAndReopen(Options& options) {
6650
- Destroy(options);
6651
- Reopen(options);
6652
- }
6653
-
6654
- void PauseCompactionThread() {
6655
- sleeping_task_.reset(new test::SleepingBackgroundTask());
6656
- env_->SetBackgroundThreads(1, Env::LOW);
6657
- env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
6658
- sleeping_task_.get(), Env::Priority::LOW);
6659
- sleeping_task_->WaitUntilSleeping();
6660
- }
6661
-
6662
- void ResumeCompactionThread() {
6663
- if (sleeping_task_) {
6664
- sleeping_task_->WakeUp();
6665
- sleeping_task_->WaitUntilDone();
6666
- }
6667
- }
6274
+ TEST_P(DBCompactionTestWithParam,
6275
+ FlushAfterIntraL0CompactionCheckConsistencyFail) {
6276
+ Options options = CurrentOptions();
6277
+ options.force_consistency_checks = true;
6278
+ options.compression = kNoCompression;
6279
+ options.level0_file_num_compaction_trigger = 5;
6280
+ options.max_background_compactions = 2;
6281
+ options.max_subcompactions = max_subcompactions_;
6282
+ DestroyAndReopen(options);
6668
6283
 
6669
- void AddFilesMarkedForPeriodicCompaction(const size_t num_files) {
6670
- assert(options_.compaction_style ==
6671
- CompactionStyle::kCompactionStyleUniversal);
6672
- VersionSet* const versions = dbfull()->GetVersionSet();
6673
- assert(versions);
6674
- ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault();
6675
- assert(cfd);
6676
- Version* const current = cfd->current();
6677
- assert(current);
6284
+ const size_t kValueSize = 1 << 20;
6285
+ Random rnd(301);
6286
+ std::atomic<int> pick_intra_l0_count(0);
6287
+ std::string value(rnd.RandomString(kValueSize));
6678
6288
 
6679
- VersionStorageInfo* const storage_info = current->storage_info();
6680
- assert(storage_info);
6289
+ // The L0->L1 must be picked before we begin ingesting files to trigger
6290
+ // intra-L0 compaction, and must not finish until after an intra-L0
6291
+ // compaction has been picked.
6292
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
6293
+ {{"LevelCompactionPicker::PickCompaction:Return",
6294
+ "DBCompactionTestWithParam::"
6295
+ "FlushAfterIntraL0CompactionCheckConsistencyFail:L0ToL1Ready"},
6296
+ {"LevelCompactionPicker::PickCompactionBySize:0",
6297
+ "CompactionJob::Run():Start"}});
6298
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
6299
+ "FindIntraL0Compaction",
6300
+ [&](void* /*arg*/) { pick_intra_l0_count.fetch_add(1); });
6681
6301
 
6682
- const std::vector<FileMetaData*> level0_files = storage_info->LevelFiles(0);
6683
- assert(level0_files.size() == num_files);
6302
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
6684
6303
 
6685
- for (FileMetaData* f : level0_files) {
6686
- storage_info->TEST_AddFileMarkedForPeriodicCompaction(0, f);
6687
- }
6304
+ // prevents trivial move
6305
+ for (int i = 0; i < 10; ++i) {
6306
+ ASSERT_OK(Put(Key(i), "")); // prevents trivial move
6688
6307
  }
6308
+ ASSERT_OK(Flush());
6309
+ Compact("", Key(99));
6310
+ ASSERT_EQ(0, NumTableFilesAtLevel(0));
6689
6311
 
6690
- void AddFilesMarkedForCompaction(const size_t num_files) {
6691
- assert(options_.compaction_style ==
6692
- CompactionStyle::kCompactionStyleUniversal);
6693
- VersionSet* const versions = dbfull()->GetVersionSet();
6694
- assert(versions);
6695
- ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault();
6696
- assert(cfd);
6697
- Version* const current = cfd->current();
6698
- assert(current);
6699
-
6700
- VersionStorageInfo* const storage_info = current->storage_info();
6701
- assert(storage_info);
6702
-
6703
- const std::vector<FileMetaData*> level0_files = storage_info->LevelFiles(0);
6704
- assert(level0_files.size() == num_files);
6705
-
6706
- for (FileMetaData* f : level0_files) {
6707
- storage_info->TEST_AddFileMarkedForCompaction(0, f);
6708
- }
6312
+ // Flush 5 L0 sst.
6313
+ for (int i = 0; i < 5; ++i) {
6314
+ ASSERT_OK(Put(Key(i + 1), value));
6315
+ ASSERT_OK(Flush());
6709
6316
  }
6317
+ ASSERT_EQ(5, NumTableFilesAtLevel(0));
6710
6318
 
6711
- void SetupSyncPoints(const std::string& compaction_path_to_test) {
6712
- compaction_path_sync_point_called_.store(false);
6713
- if (compaction_path_to_test == "FindIntraL0Compaction" &&
6714
- options_.compaction_style == CompactionStyle::kCompactionStyleLevel) {
6715
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
6716
- "PostPickFileToCompact", [&](void* arg) {
6717
- bool* picked_file_to_compact = (bool*)arg;
6718
- // To trigger intra-L0 compaction specifically,
6719
- // we mock PickFileToCompact()'s result to be false
6720
- *picked_file_to_compact = false;
6721
- });
6722
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
6723
- "FindIntraL0Compaction", [&](void* /*arg*/) {
6724
- compaction_path_sync_point_called_.store(true);
6725
- });
6726
-
6727
- } else if (compaction_path_to_test == "PickPeriodicCompaction") {
6728
- assert(options_.compaction_style ==
6729
- CompactionStyle::kCompactionStyleUniversal);
6730
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
6731
- "PostPickPeriodicCompaction", [&](void* compaction_arg) {
6732
- Compaction* compaction = (Compaction*)compaction_arg;
6733
- if (compaction != nullptr) {
6734
- compaction_path_sync_point_called_.store(true);
6735
- }
6736
- });
6737
- } else if (compaction_path_to_test == "PickCompactionToReduceSizeAmp") {
6738
- assert(options_.compaction_style ==
6739
- CompactionStyle::kCompactionStyleUniversal);
6740
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
6741
- "PickCompactionToReduceSizeAmpReturnNonnullptr", [&](void* /*arg*/) {
6742
- compaction_path_sync_point_called_.store(true);
6743
- });
6744
- } else if (compaction_path_to_test == "PickCompactionToReduceSortedRuns") {
6745
- assert(options_.compaction_style ==
6746
- CompactionStyle::kCompactionStyleUniversal);
6747
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
6748
- "PickCompactionToReduceSortedRunsReturnNonnullptr",
6749
- [&](void* /*arg*/) {
6750
- compaction_path_sync_point_called_.store(true);
6751
- });
6752
- } else if (compaction_path_to_test == "PickDeleteTriggeredCompaction") {
6753
- assert(options_.compaction_style ==
6754
- CompactionStyle::kCompactionStyleUniversal);
6755
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
6756
- "PickDeleteTriggeredCompactionReturnNonnullptr", [&](void* /*arg*/) {
6757
- compaction_path_sync_point_called_.store(true);
6758
- });
6759
- } else if ((compaction_path_to_test == "FindIntraL0Compaction" ||
6760
- compaction_path_to_test == "CompactRange") &&
6761
- options_.compaction_style ==
6762
- CompactionStyle::kCompactionStyleFIFO) {
6763
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
6764
- "FindIntraL0Compaction", [&](void* /*arg*/) {
6765
- compaction_path_sync_point_called_.store(true);
6766
- });
6767
- }
6768
-
6769
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
6770
- }
6319
+ // Put one key, to make smallest log sequence number in this memtable is less
6320
+ // than sst which would be ingested in next step.
6321
+ ASSERT_OK(Put(Key(0), "a"));
6771
6322
 
6772
- bool SyncPointsCalled() { return compaction_path_sync_point_called_.load(); }
6323
+ ASSERT_EQ(5, NumTableFilesAtLevel(0));
6324
+ TEST_SYNC_POINT(
6325
+ "DBCompactionTestWithParam::"
6326
+ "FlushAfterIntraL0CompactionCheckConsistencyFail:L0ToL1Ready");
6773
6327
 
6774
- void DisableSyncPoints() {
6775
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
6776
- ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
6328
+ // Ingest 5 L0 sst. And this files would trigger PickIntraL0Compaction.
6329
+ for (int i = 5; i < 10; i++) {
6330
+ ASSERT_EQ(i, NumTableFilesAtLevel(0));
6331
+ IngestOneKeyValue(dbfull(), Key(i), value, options);
6777
6332
  }
6778
6333
 
6779
- // Return the largest seqno of the latest L0 file based on file number
6780
- SequenceNumber GetLatestL0FileLargestSeqnoHelper() {
6781
- VersionSet* const versions = dbfull()->GetVersionSet();
6782
- assert(versions);
6783
- ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault();
6784
- assert(cfd);
6785
- Version* const current = cfd->current();
6786
- assert(current);
6787
- VersionStorageInfo* const storage_info = current->storage_info();
6788
- assert(storage_info);
6789
- const std::vector<FileMetaData*> level0_files = storage_info->LevelFiles(0);
6790
- assert(level0_files.size() >= 1);
6791
-
6792
- uint64_t latest_file_num = 0;
6793
- uint64_t latest_file_largest_seqno = 0;
6794
- for (FileMetaData* f : level0_files) {
6795
- if (f->fd.GetNumber() > latest_file_num) {
6796
- latest_file_num = f->fd.GetNumber();
6797
- latest_file_largest_seqno = f->fd.largest_seqno;
6798
- }
6799
- }
6334
+ // Put one key, to make biggest log sequence number in this memtable is bigger
6335
+ // than sst which would be ingested in next step.
6336
+ ASSERT_OK(Put(Key(2), "b"));
6337
+ ASSERT_OK(dbfull()->TEST_WaitForCompact());
6338
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
6339
+ std::vector<std::vector<FileMetaData>> level_to_files;
6340
+ dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
6341
+ &level_to_files);
6342
+ ASSERT_GT(level_to_files[0].size(), 0);
6343
+ ASSERT_GT(pick_intra_l0_count.load(), 0);
6800
6344
 
6801
- return latest_file_largest_seqno;
6802
- }
6345
+ ASSERT_OK(Flush());
6346
+ }
6803
6347
 
6804
- protected:
6805
- Options options_;
6348
+ TEST_P(DBCompactionTestWithParam,
6349
+ IntraL0CompactionAfterFlushCheckConsistencyFail) {
6350
+ Options options = CurrentOptions();
6351
+ options.force_consistency_checks = true;
6352
+ options.compression = kNoCompression;
6353
+ options.level0_file_num_compaction_trigger = 5;
6354
+ options.max_background_compactions = 2;
6355
+ options.max_subcompactions = max_subcompactions_;
6356
+ options.write_buffer_size = 2 << 20;
6357
+ options.max_write_buffer_number = 6;
6358
+ DestroyAndReopen(options);
6806
6359
 
6807
- private:
6808
- const Snapshot* snapshot_ = nullptr;
6809
- std::atomic<bool> compaction_path_sync_point_called_;
6810
- std::shared_ptr<test::SleepingBackgroundTask> sleeping_task_;
6811
- };
6360
+ const size_t kValueSize = 1 << 20;
6361
+ Random rnd(301);
6362
+ std::string value(rnd.RandomString(kValueSize));
6363
+ std::string value2(rnd.RandomString(kValueSize));
6364
+ std::string bigvalue = value + value;
6812
6365
 
6813
- TEST_F(DBCompactionTestL0FilesMisorderCorruption,
6814
- FlushAfterIntraL0LevelCompactionWithIngestedFile) {
6815
- SetupOptions(CompactionStyle::kCompactionStyleLevel, "");
6816
- DestroyAndReopen(options_);
6817
- // Prevents trivial move
6366
+ // prevents trivial move
6818
6367
  for (int i = 0; i < 10; ++i) {
6819
- ASSERT_OK(Put(Key(i), "")); // Prevents trivial move
6368
+ ASSERT_OK(Put(Key(i), "")); // prevents trivial move
6820
6369
  }
6821
6370
  ASSERT_OK(Flush());
6822
6371
  Compact("", Key(99));
6823
6372
  ASSERT_EQ(0, NumTableFilesAtLevel(0));
6824
6373
 
6825
- // To get accurate NumTableFilesAtLevel(0) when the number reaches
6826
- // options_.level0_file_num_compaction_trigger
6827
- PauseCompactionThread();
6828
-
6829
- // To create below LSM tree
6830
- // (key:value@n indicates key-value pair has seqno "n", L0 is sorted):
6831
- //
6832
- // memtable: m1[ 5:new@12 .. 1:new@8, 0:new@7]
6833
- // L0: s6[6:new@13], s5[5:old@6] ... s1[1:old@2],s0[0:old@1]
6834
- //
6835
- // (1) Make 6 L0 sst (i.e, s0 - s5)
6374
+ std::atomic<int> pick_intra_l0_count(0);
6375
+ // The L0->L1 must be picked before we begin ingesting files to trigger
6376
+ // intra-L0 compaction, and must not finish until after an intra-L0
6377
+ // compaction has been picked.
6378
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
6379
+ {{"LevelCompactionPicker::PickCompaction:Return",
6380
+ "DBCompactionTestWithParam::"
6381
+ "IntraL0CompactionAfterFlushCheckConsistencyFail:L0ToL1Ready"},
6382
+ {"LevelCompactionPicker::PickCompactionBySize:0",
6383
+ "CompactionJob::Run():Start"}});
6384
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
6385
+ "FindIntraL0Compaction",
6386
+ [&](void* /*arg*/) { pick_intra_l0_count.fetch_add(1); });
6387
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
6388
+ // Make 6 L0 sst.
6836
6389
  for (int i = 0; i < 6; ++i) {
6837
6390
  if (i % 2 == 0) {
6838
- IngestOneKeyValue(dbfull(), Key(i), "old", options_);
6391
+ IngestOneKeyValue(dbfull(), Key(i), value, options);
6839
6392
  } else {
6840
- ASSERT_OK(Put(Key(i), "old"));
6393
+ ASSERT_OK(Put(Key(i), value));
6841
6394
  ASSERT_OK(Flush());
6842
6395
  }
6843
6396
  }
6397
+
6844
6398
  ASSERT_EQ(6, NumTableFilesAtLevel(0));
6845
6399
 
6846
- // (2) Create m1
6400
+ // Stop run flush job
6401
+ env_->SetBackgroundThreads(1, Env::HIGH);
6402
+ test::SleepingBackgroundTask sleeping_tasks;
6403
+ env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_tasks,
6404
+ Env::Priority::HIGH);
6405
+ sleeping_tasks.WaitUntilSleeping();
6406
+
6407
+ // Put many keys to make memtable request to flush
6847
6408
  for (int i = 0; i < 6; ++i) {
6848
- ASSERT_OK(Put(Key(i), "new"));
6409
+ ASSERT_OK(Put(Key(i), bigvalue));
6849
6410
  }
6850
- ASSERT_EQ(6, NumTableFilesAtLevel(0));
6851
6411
 
6852
- // (3) Ingest file (i.e, s6) to trigger IntraL0Compaction()
6853
- for (int i = 6; i < 7; ++i) {
6412
+ ASSERT_EQ(6, NumTableFilesAtLevel(0));
6413
+ TEST_SYNC_POINT(
6414
+ "DBCompactionTestWithParam::"
6415
+ "IntraL0CompactionAfterFlushCheckConsistencyFail:L0ToL1Ready");
6416
+ // ingest file to trigger IntraL0Compaction
6417
+ for (int i = 6; i < 10; ++i) {
6854
6418
  ASSERT_EQ(i, NumTableFilesAtLevel(0));
6855
- IngestOneKeyValue(dbfull(), Key(i), "new", options_);
6419
+ IngestOneKeyValue(dbfull(), Key(i), value2, options);
6856
6420
  }
6857
6421
 
6858
- SetupSyncPoints("FindIntraL0Compaction");
6859
- ResumeCompactionThread();
6860
-
6422
+ // Wake up flush job
6423
+ sleeping_tasks.WakeUp();
6424
+ sleeping_tasks.WaitUntilDone();
6861
6425
  ASSERT_OK(dbfull()->TEST_WaitForCompact());
6426
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
6862
6427
 
6863
- ASSERT_TRUE(SyncPointsCalled());
6864
- DisableSyncPoints();
6865
-
6866
- // After compaction, we have LSM tree:
6867
- //
6868
- // memtable: m1[ 5:new@12 .. 1:new@8, 0:new@7]
6869
- // L0: s7[6:new@13, 5:old@6 .. 0:old@1]
6870
- ASSERT_EQ(1, NumTableFilesAtLevel(0));
6871
- SequenceNumber compact_output_file_largest_seqno =
6872
- GetLatestL0FileLargestSeqnoHelper();
6873
-
6874
- ASSERT_OK(Flush());
6875
- // After flush, we have LSM tree:
6876
- //
6877
- // L0: s8[5:new@12 .. 0:new@7],s7[6:new@13, 5:old@5 .. 0:old@1]
6878
- ASSERT_EQ(2, NumTableFilesAtLevel(0));
6879
- SequenceNumber flushed_file_largest_seqno =
6880
- GetLatestL0FileLargestSeqnoHelper();
6881
-
6882
- // To verify there isn't any file misorder leading to returning a old value
6883
- // of Key(0) - Key(5) , which is caused by flushed table s8 has a
6884
- // smaller largest seqno than the compaction output file s7's largest seqno
6885
- // while the flushed table has the newer version of the values than the
6886
- // compaction output file's.
6887
- ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno);
6428
+ uint64_t error_count = 0;
6429
+ db_->GetIntProperty("rocksdb.background-errors", &error_count);
6430
+ ASSERT_EQ(error_count, 0);
6431
+ ASSERT_GT(pick_intra_l0_count.load(), 0);
6888
6432
  for (int i = 0; i < 6; ++i) {
6889
- ASSERT_EQ("new", Get(Key(i)));
6433
+ ASSERT_EQ(bigvalue, Get(Key(i)));
6890
6434
  }
6891
- for (int i = 6; i < 7; ++i) {
6892
- ASSERT_EQ("new", Get(Key(i)));
6435
+ for (int i = 6; i < 10; ++i) {
6436
+ ASSERT_EQ(value2, Get(Key(i)));
6893
6437
  }
6894
6438
  }
6895
6439
 
6896
- TEST_F(DBCompactionTestL0FilesMisorderCorruption,
6897
- FlushAfterIntraL0UniversalCompactionWithIngestedFile) {
6898
- for (const std::string compaction_path_to_test :
6899
- {"PickPeriodicCompaction", "PickCompactionToReduceSizeAmp",
6900
- "PickCompactionToReduceSortedRuns", "PickDeleteTriggeredCompaction"}) {
6901
- SetupOptions(CompactionStyle::kCompactionStyleUniversal,
6902
- compaction_path_to_test);
6903
- DestroyAndReopen(options_);
6904
-
6905
- // To get accurate NumTableFilesAtLevel(0) when the number reaches
6906
- // options_.level0_file_num_compaction_trigger
6907
- PauseCompactionThread();
6908
-
6909
- // To create below LSM tree
6910
- // (key:value@n indicates key-value pair has seqno "n", L0 is sorted):
6911
- //
6912
- // memtable: m1 [ k2:new@8, k1:new@7]
6913
- // L0: s4[k9:dummy@10], s3[k8:dummy@9],
6914
- // s2[k7:old@6, k6:old@5].. s0[k3:old@2, k1:old@1]
6915
- //
6916
- // (1) Create 3 existing SST file (i.e, s0 - s2)
6917
- ASSERT_OK(Put("k1", "old"));
6918
- ASSERT_OK(Put("k3", "old"));
6919
- ASSERT_OK(Flush());
6920
- ASSERT_EQ(1, NumTableFilesAtLevel(0));
6921
- ASSERT_OK(Put("k4", "old"));
6922
- ASSERT_OK(Put("k5", "old"));
6923
- ASSERT_OK(Flush());
6924
- ASSERT_EQ(2, NumTableFilesAtLevel(0));
6925
- ASSERT_OK(Put("k6", "old"));
6926
- ASSERT_OK(Put("k7", "old"));
6927
- ASSERT_OK(Flush());
6928
- ASSERT_EQ(3, NumTableFilesAtLevel(0));
6929
-
6930
- // (2) Create m1. Noted that it contains a overlaped key with s0
6931
- ASSERT_OK(Put("k1", "new")); // overlapped key
6932
- ASSERT_OK(Put("k2", "new"));
6933
-
6934
- // (3) Ingest two SST files s3, s4
6935
- IngestOneKeyValue(dbfull(), "k8", "dummy", options_);
6936
- IngestOneKeyValue(dbfull(), "k9", "dummy", options_);
6937
- // Up to now, L0 contains s0 - s4
6938
- ASSERT_EQ(5, NumTableFilesAtLevel(0));
6939
-
6940
- if (compaction_path_to_test == "PickPeriodicCompaction") {
6941
- AddFilesMarkedForPeriodicCompaction(5);
6942
- } else if (compaction_path_to_test == "PickDeleteTriggeredCompaction") {
6943
- AddFilesMarkedForCompaction(5);
6944
- }
6945
-
6946
- SetupSyncPoints(compaction_path_to_test);
6947
- ResumeCompactionThread();
6948
-
6949
- ASSERT_OK(dbfull()->TEST_WaitForCompact());
6950
-
6951
- ASSERT_TRUE(SyncPointsCalled())
6952
- << "failed for compaction path to test: " << compaction_path_to_test;
6953
- DisableSyncPoints();
6954
-
6955
- // After compaction, we have LSM tree:
6956
- //
6957
- // memtable: m1[ k2:new@8, k1:new@7]
6958
- // L0: s5[k9:dummy@10, k8@dummy@9, k7:old@6 .. k3:old@2, k1:old@1]
6959
- ASSERT_EQ(1, NumTableFilesAtLevel(0))
6960
- << "failed for compaction path to test: " << compaction_path_to_test;
6961
- SequenceNumber compact_output_file_largest_seqno =
6962
- GetLatestL0FileLargestSeqnoHelper();
6963
-
6964
- ASSERT_OK(Flush()) << "failed for compaction path to test: "
6965
- << compaction_path_to_test;
6966
- // After flush, we have LSM tree:
6967
- //
6968
- // L0: s6[k2:new@8, k1:new@7],
6969
- // s5[k9:dummy@10, k8@dummy@9, k7:old@6 .. k3:old@2, k1:old@1]
6970
- ASSERT_EQ(2, NumTableFilesAtLevel(0))
6971
- << "failed for compaction path to test: " << compaction_path_to_test;
6972
- SequenceNumber flushed_file_largest_seqno =
6973
- GetLatestL0FileLargestSeqnoHelper();
6974
-
6975
- // To verify there isn't any file misorder leading to returning a old
6976
- // value of "k1" , which is caused by flushed table s6 has a
6977
- // smaller largest seqno than the compaction output file s5's largest seqno
6978
- // while the flushed table has the newer version of the value
6979
- // than the compaction output file's.
6980
- ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno)
6981
- << "failed for compaction path to test: " << compaction_path_to_test;
6982
- EXPECT_EQ(Get("k1"), "new")
6983
- << "failed for compaction path to test: " << compaction_path_to_test;
6984
- }
6985
-
6986
- Destroy(options_);
6987
- }
6988
-
6989
- TEST_F(DBCompactionTestL0FilesMisorderCorruption,
6990
- FlushAfterIntraL0FIFOCompactionWithIngestedFile) {
6991
- for (const std::string compaction_path_to_test : {"FindIntraL0Compaction"}) {
6992
- SetupOptions(CompactionStyle::kCompactionStyleFIFO,
6993
- compaction_path_to_test);
6994
- DestroyAndReopen(options_);
6995
-
6996
- // To create below LSM tree
6997
- // (key:value@n indicates key-value pair has seqno "n", L0 is sorted):
6998
- //
6999
- // memtable: m1 [ k2:new@4, k1:new@3]
7000
- // L0: s2[k5:dummy@6], s1[k4:dummy@5], s0[k3:old@2, k1:old@1]
7001
- //
7002
- // (1) Create an existing SST file s0
7003
- ASSERT_OK(Put("k1", "old"));
7004
- ASSERT_OK(Put("k3", "old"));
7005
- ASSERT_OK(Flush());
7006
- ASSERT_EQ(1, NumTableFilesAtLevel(0));
7007
-
7008
- // (2) Create memtable m1. Noted that it contains a overlaped key with s0
7009
- ASSERT_OK(Put("k1", "new")); // overlapped key
7010
- ASSERT_OK(Put("k2", "new"));
7011
-
7012
- // To get accurate NumTableFilesAtLevel(0) when the number reaches
7013
- // options_.level0_file_num_compaction_trigger
7014
- PauseCompactionThread();
7015
-
7016
- // (3) Ingest two SST files s1, s2
7017
- IngestOneKeyValue(dbfull(), "k4", "dummy", options_);
7018
- IngestOneKeyValue(dbfull(), "k5", "dummy", options_);
7019
- // Up to now, L0 contains s0, s1, s2
7020
- ASSERT_EQ(3, NumTableFilesAtLevel(0));
7021
-
7022
- SetupSyncPoints(compaction_path_to_test);
7023
- ResumeCompactionThread();
7024
-
7025
- ASSERT_OK(dbfull()->TEST_WaitForCompact());
7026
-
7027
- ASSERT_TRUE(SyncPointsCalled())
7028
- << "failed for compaction path to test: " << compaction_path_to_test;
7029
- DisableSyncPoints();
7030
- // After compaction, we have LSM tree:
7031
- //
7032
- // memtable: m1 [ k2:new@4, k1:new@3]
7033
- // L0: s3[k5:dummy@6, k4:dummy@5, k3:old@2, k1:old@1]
7034
- ASSERT_EQ(1, NumTableFilesAtLevel(0))
7035
- << "failed for compaction path to test: " << compaction_path_to_test;
7036
- SequenceNumber compact_output_file_largest_seqno =
7037
- GetLatestL0FileLargestSeqnoHelper();
7038
-
7039
- ASSERT_OK(Flush()) << "failed for compaction path to test: "
7040
- << compaction_path_to_test;
7041
- // After flush, we have LSM tree:
7042
- //
7043
- // L0: s4[k2:new@4, k1:new@3], s3[k5:dummy@6, k4:dummy@5, k3:old@2,
7044
- // k1:old@1]
7045
- ASSERT_EQ(2, NumTableFilesAtLevel(0))
7046
- << "failed for compaction path to test: " << compaction_path_to_test;
7047
- SequenceNumber flushed_file_largest_seqno =
7048
- GetLatestL0FileLargestSeqnoHelper();
7049
-
7050
- // To verify there isn't any file misorder leading to returning a old
7051
- // value of "k1" , which is caused by flushed table s4 has a
7052
- // smaller largest seqno than the compaction output file s3's largest seqno
7053
- // while the flushed table has the newer version of the value
7054
- // than the compaction output file's.
7055
- ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno)
7056
- << "failed for compaction path to test: " << compaction_path_to_test;
7057
- EXPECT_EQ(Get("k1"), "new")
7058
- << "failed for compaction path to test: " << compaction_path_to_test;
7059
- }
7060
-
7061
- Destroy(options_);
7062
- }
7063
-
7064
- class DBCompactionTestL0FilesMisorderCorruptionWithParam
7065
- : public DBCompactionTestL0FilesMisorderCorruption,
7066
- public testing::WithParamInterface<CompactionStyle> {
7067
- public:
7068
- DBCompactionTestL0FilesMisorderCorruptionWithParam()
7069
- : DBCompactionTestL0FilesMisorderCorruption() {}
7070
- };
7071
-
7072
- // TODO: add `CompactionStyle::kCompactionStyleLevel` to testing parameter,
7073
- // which requires careful unit test
7074
- // design for ingesting file to L0 and CompactRange()/CompactFile() to L0
7075
- INSTANTIATE_TEST_CASE_P(
7076
- DBCompactionTestL0FilesMisorderCorruptionWithParam,
7077
- DBCompactionTestL0FilesMisorderCorruptionWithParam,
7078
- ::testing::Values(CompactionStyle::kCompactionStyleUniversal,
7079
- CompactionStyle::kCompactionStyleFIFO));
7080
-
7081
- TEST_P(DBCompactionTestL0FilesMisorderCorruptionWithParam,
7082
- FlushAfterIntraL0CompactFileWithIngestedFile) {
7083
- SetupOptions(GetParam(), "CompactFile");
7084
- DestroyAndReopen(options_);
7085
-
7086
- // To create below LSM tree
7087
- // (key:value@n indicates key-value pair has seqno "n", L0 is sorted):
7088
- //
7089
- // memtable: m1 [ k2:new@4, k1:new@3]
7090
- // L0: s2[k5:dummy@6], s1[k4:dummy@5], s0[k3:old@2, k1:old@1]
7091
- //
7092
- // (1) Create an existing SST file s0
7093
- ASSERT_OK(Put("k1", "old"));
7094
- ASSERT_OK(Put("k3", "old"));
7095
- ASSERT_OK(Flush());
7096
- ASSERT_EQ(1, NumTableFilesAtLevel(0));
7097
-
7098
- // (2) Create memtable m1. Noted that it contains a overlaped key with s0
7099
- ASSERT_OK(Put("k1", "new")); // overlapped key
7100
- ASSERT_OK(Put("k2", "new"));
7101
-
7102
- // (3) Ingest two SST files s1, s2
7103
- IngestOneKeyValue(dbfull(), "k4", "dummy", options_);
7104
- IngestOneKeyValue(dbfull(), "k5", "dummy", options_);
7105
- // Up to now, L0 contains s0, s1, s2
7106
- ASSERT_EQ(3, NumTableFilesAtLevel(0));
7107
-
7108
- ColumnFamilyMetaData cf_meta_data;
7109
- db_->GetColumnFamilyMetaData(&cf_meta_data);
7110
- ASSERT_EQ(cf_meta_data.levels[0].files.size(), 3);
7111
- std::vector<std::string> input_files;
7112
- for (const auto& file : cf_meta_data.levels[0].files) {
7113
- input_files.push_back(file.name);
7114
- }
7115
- ASSERT_EQ(input_files.size(), 3);
7116
-
7117
- Status s = db_->CompactFiles(CompactionOptions(), input_files, 0);
7118
- // After compaction, we have LSM tree:
7119
- //
7120
- // memtable: m1 [ k2:new@4, k1:new@3]
7121
- // L0: s3[k5:dummy@6, k4:dummy@5, k3:old@2, k1:old@1]
7122
- ASSERT_OK(s);
7123
- ASSERT_EQ(1, NumTableFilesAtLevel(0));
7124
- SequenceNumber compact_output_file_largest_seqno =
7125
- GetLatestL0FileLargestSeqnoHelper();
7126
-
7127
- ASSERT_OK(Flush());
7128
- // After flush, we have LSM tree:
7129
- //
7130
- // L0: s4[k2:new@4, k1:new@3], s3[k5:dummy@6, k4:dummy@5, k3:old@2,
7131
- // k1:old@1]
7132
- ASSERT_EQ(2, NumTableFilesAtLevel(0));
7133
- SequenceNumber flushed_file_largest_seqno =
7134
- GetLatestL0FileLargestSeqnoHelper();
7135
-
7136
- // To verify there isn't any file misorder leading to returning a old value
7137
- // of "1" , which is caused by flushed table s4 has a smaller
7138
- // largest seqno than the compaction output file s3's largest seqno while the
7139
- // flushed table has the newer version of the value than the
7140
- // compaction output file's.
7141
- ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno);
7142
- EXPECT_EQ(Get("k1"), "new");
7143
-
7144
- Destroy(options_);
7145
- }
7146
-
7147
- TEST_P(DBCompactionTestL0FilesMisorderCorruptionWithParam,
7148
- FlushAfterIntraL0CompactRangeWithIngestedFile) {
7149
- SetupOptions(GetParam(), "CompactRange");
7150
- DestroyAndReopen(options_);
7151
-
7152
- // To create below LSM tree
7153
- // (key:value@n indicates key-value pair has seqno "n", L0 is sorted):
7154
- //
7155
- // memtable: m1 [ k2:new@4, k1:new@3]
7156
- // L0: s2[k5:dummy@6], s1[k4:dummy@5], s0[k3:old@2, k1:old@1]
7157
- //
7158
- // (1) Create an existing SST file s0
7159
- ASSERT_OK(Put("k1", "old"));
7160
- ASSERT_OK(Put("k3", "old"));
7161
- ASSERT_OK(Flush());
7162
- ASSERT_EQ(1, NumTableFilesAtLevel(0));
7163
-
7164
- // (2) Create memtable m1. Noted that it contains a overlaped key with s0
7165
- ASSERT_OK(Put("k1", "new")); // overlapped key
7166
- ASSERT_OK(Put("k2", "new"));
7167
-
7168
- // (3) Ingest two SST files s1, s2
7169
- IngestOneKeyValue(dbfull(), "k4", "dummy", options_);
7170
- IngestOneKeyValue(dbfull(), "k5", "dummy", options_);
7171
- // Up to now, L0 contains s0, s1, s2
7172
- ASSERT_EQ(3, NumTableFilesAtLevel(0));
7173
-
7174
- if (options_.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
7175
- SetupSyncPoints("CompactRange");
7176
- }
7177
- // `start` and `end` is carefully chosen so that compact range:
7178
- // (1) doesn't overlap with memtable therefore the memtable won't be flushed
7179
- // (2) should target at compacting s0 with s1 and s2
7180
- Slice start("k3"), end("k5");
7181
- ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end));
7182
- // After compaction, we have LSM tree:
7183
- //
7184
- // memtable: m1 [ k2:new@4, k1:new@3]
7185
- // L0: s3[k5:dummy@6, k4:dummy@5, k3:old@2, k1:old@1]
7186
- if (options_.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
7187
- ASSERT_TRUE(SyncPointsCalled());
7188
- DisableSyncPoints();
7189
- }
7190
- ASSERT_EQ(1, NumTableFilesAtLevel(0));
7191
- SequenceNumber compact_output_file_largest_seqno =
7192
- GetLatestL0FileLargestSeqnoHelper();
7193
-
7194
- ASSERT_OK(Flush());
7195
- // After flush, we have LSM tree:
7196
- //
7197
- // L0: s4[k2:new@4, k1:new@3], s3[k5:dummy@6, k4:dummy@5, k3:old@2,
7198
- // k1:old@1]
7199
- ASSERT_EQ(2, NumTableFilesAtLevel(0));
7200
- SequenceNumber flushed_file_largest_seqno =
7201
- GetLatestL0FileLargestSeqnoHelper();
7202
-
7203
- // To verify there isn't any file misorder leading to returning a old value
7204
- // of "k1" , which is caused by flushed table s4 has a smaller
7205
- // largest seqno than the compaction output file s3's largest seqno while the
7206
- // flushed table has the newer version of the value than the
7207
- // compaction output file's.
7208
- ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno);
7209
- EXPECT_EQ(Get("k1"), "new");
7210
-
7211
- Destroy(options_);
7212
- }
7213
-
7214
6440
  TEST_P(DBCompactionTestWithBottommostParam, SequenceKeysManualCompaction) {
7215
6441
  constexpr int kSstNum = 10;
7216
6442
  Options options = CurrentOptions();