@nxtedition/rocksdb 8.1.2 → 8.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deps/rocksdb/rocksdb/CMakeLists.txt +0 -9
- package/deps/rocksdb/rocksdb/Makefile +1 -13
- package/deps/rocksdb/rocksdb/TARGETS +0 -4
- package/deps/rocksdb/rocksdb/db/blob/blob_counting_iterator.h +0 -11
- package/deps/rocksdb/rocksdb/db/builder.cc +4 -13
- package/deps/rocksdb/rocksdb/db/builder.h +1 -2
- package/deps/rocksdb/rocksdb/db/c.cc +0 -15
- package/deps/rocksdb/rocksdb/db/c_test.c +0 -3
- package/deps/rocksdb/rocksdb/db/column_family.cc +5 -11
- package/deps/rocksdb/rocksdb/db/column_family.h +0 -20
- package/deps/rocksdb/rocksdb/db/compaction/clipping_iterator.h +0 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +37 -31
- package/deps/rocksdb/rocksdb/db/compaction/compaction.h +0 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +7 -24
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +1 -17
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +1 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +2 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +6 -9
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +26 -104
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +0 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +18 -11
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +16 -17
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +6 -19
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +5 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +22 -22
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +5 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +52 -81
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +1 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +5 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +2 -8
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +0 -8
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +138 -266
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +1 -86
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +3 -2
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +123 -897
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +20 -31
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +9 -5
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +28 -121
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +2 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +0 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +4 -8
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +0 -10
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +4 -325
- package/deps/rocksdb/rocksdb/db/db_test.cc +0 -3
- package/deps/rocksdb/rocksdb/db/db_test2.cc +8 -233
- package/deps/rocksdb/rocksdb/db/db_test_util.h +0 -3
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +0 -129
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +0 -21
- package/deps/rocksdb/rocksdb/db/dbformat.cc +0 -25
- package/deps/rocksdb/rocksdb/db/dbformat.h +0 -2
- package/deps/rocksdb/rocksdb/db/experimental.cc +2 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +0 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +13 -92
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +1 -38
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +110 -14
- package/deps/rocksdb/rocksdb/db/flush_job.cc +4 -6
- package/deps/rocksdb/rocksdb/db/history_trimming_iterator.h +0 -4
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +53 -56
- package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +4 -3
- package/deps/rocksdb/rocksdb/db/memtable.cc +1 -1
- package/deps/rocksdb/rocksdb/db/merge_helper.cc +0 -4
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +10 -10
- package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/repair.cc +22 -65
- package/deps/rocksdb/rocksdb/db/repair_test.cc +0 -54
- package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +26 -26
- package/deps/rocksdb/rocksdb/db/table_properties_collector.h +1 -3
- package/deps/rocksdb/rocksdb/db/version_builder.cc +43 -90
- package/deps/rocksdb/rocksdb/db/version_builder.h +0 -20
- package/deps/rocksdb/rocksdb/db/version_builder_test.cc +93 -218
- package/deps/rocksdb/rocksdb/db/version_edit.cc +1 -27
- package/deps/rocksdb/rocksdb/db/version_edit.h +9 -33
- package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +6 -13
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +6 -17
- package/deps/rocksdb/rocksdb/db/version_edit_test.cc +17 -19
- package/deps/rocksdb/rocksdb/db/version_set.cc +26 -166
- package/deps/rocksdb/rocksdb/db/version_set.h +2 -32
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +31 -65
- package/deps/rocksdb/rocksdb/db/write_thread.cc +2 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +0 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +32 -31
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.h +1 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +6 -8
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +0 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +4 -11
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +15 -16
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +1 -13
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +75 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +0 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +0 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +1 -7
- package/deps/rocksdb/rocksdb/include/rocksdb/metadata.h +3 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +1 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/status.h +0 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h +9 -69
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/memory/arena.cc +87 -23
- package/deps/rocksdb/rocksdb/memory/arena.h +31 -25
- package/deps/rocksdb/rocksdb/memory/arena_test.cc +0 -90
- package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +26 -26
- package/deps/rocksdb/rocksdb/src.mk +0 -2
- package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +2 -3
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +2 -3
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +2 -6
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +3 -3
- package/deps/rocksdb/rocksdb/table/format.cc +20 -24
- package/deps/rocksdb/rocksdb/table/format.h +2 -5
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +105 -54
- package/deps/rocksdb/rocksdb/table/merging_iterator.h +0 -80
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +2 -2
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +1 -1
- package/deps/rocksdb/rocksdb/table/table_test.cc +6 -7
- package/deps/rocksdb/rocksdb/test_util/testutil.h +0 -10
- package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.h +2 -2
- package/deps/rocksdb/rocksdb/util/bloom_test.cc +1 -1
- package/deps/rocksdb/rocksdb/util/status.cc +0 -7
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +74 -250
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +4 -199
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +0 -39
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +0 -9
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +0 -59
- package/deps/rocksdb/rocksdb.gyp +0 -3
- package/index.js +2 -2
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
- package/deps/rocksdb/rocksdb/port/mmap.cc +0 -98
- package/deps/rocksdb/rocksdb/port/mmap.h +0 -70
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +0 -142
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +0 -241
|
@@ -9,10 +9,8 @@
|
|
|
9
9
|
|
|
10
10
|
#include <tuple>
|
|
11
11
|
|
|
12
|
-
#include "compaction/compaction_picker_universal.h"
|
|
13
12
|
#include "db/blob/blob_index.h"
|
|
14
13
|
#include "db/db_test_util.h"
|
|
15
|
-
#include "db/dbformat.h"
|
|
16
14
|
#include "env/mock_env.h"
|
|
17
15
|
#include "port/port.h"
|
|
18
16
|
#include "port/stack_trace.h"
|
|
@@ -1026,70 +1024,6 @@ TEST_F(DBCompactionTest, CompactionSstPartitioner) {
|
|
|
1026
1024
|
ASSERT_EQ("B", Get("bbbb1"));
|
|
1027
1025
|
}
|
|
1028
1026
|
|
|
1029
|
-
TEST_F(DBCompactionTest, CompactionSstPartitionWithManualCompaction) {
|
|
1030
|
-
Options options = CurrentOptions();
|
|
1031
|
-
options.compaction_style = kCompactionStyleLevel;
|
|
1032
|
-
options.level0_file_num_compaction_trigger = 3;
|
|
1033
|
-
|
|
1034
|
-
DestroyAndReopen(options);
|
|
1035
|
-
|
|
1036
|
-
// create first file and flush to l0
|
|
1037
|
-
ASSERT_OK(Put("000015", "A"));
|
|
1038
|
-
ASSERT_OK(Put("000025", "B"));
|
|
1039
|
-
ASSERT_OK(Flush());
|
|
1040
|
-
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
1041
|
-
|
|
1042
|
-
// create second file and flush to l0
|
|
1043
|
-
ASSERT_OK(Put("000015", "A2"));
|
|
1044
|
-
ASSERT_OK(Put("000025", "B2"));
|
|
1045
|
-
ASSERT_OK(Flush());
|
|
1046
|
-
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
1047
|
-
|
|
1048
|
-
// CONTROL 1: compact without partitioner
|
|
1049
|
-
CompactRangeOptions compact_options;
|
|
1050
|
-
compact_options.bottommost_level_compaction =
|
|
1051
|
-
BottommostLevelCompaction::kForceOptimized;
|
|
1052
|
-
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
1053
|
-
|
|
1054
|
-
// Check (compacted but no partitioning yet)
|
|
1055
|
-
std::vector<LiveFileMetaData> files;
|
|
1056
|
-
dbfull()->GetLiveFilesMetaData(&files);
|
|
1057
|
-
ASSERT_EQ(1, files.size());
|
|
1058
|
-
|
|
1059
|
-
// Install partitioner
|
|
1060
|
-
std::shared_ptr<SstPartitionerFactory> factory(
|
|
1061
|
-
NewSstPartitionerFixedPrefixFactory(5));
|
|
1062
|
-
options.sst_partitioner_factory = factory;
|
|
1063
|
-
Reopen(options);
|
|
1064
|
-
|
|
1065
|
-
// CONTROL 2: request compaction on range with no partition boundary and no
|
|
1066
|
-
// overlap with actual entries
|
|
1067
|
-
Slice from("000017");
|
|
1068
|
-
Slice to("000019");
|
|
1069
|
-
ASSERT_OK(dbfull()->CompactRange(compact_options, &from, &to));
|
|
1070
|
-
|
|
1071
|
-
// Check (no partitioning yet)
|
|
1072
|
-
files.clear();
|
|
1073
|
-
dbfull()->GetLiveFilesMetaData(&files);
|
|
1074
|
-
ASSERT_EQ(1, files.size());
|
|
1075
|
-
ASSERT_EQ("A2", Get("000015"));
|
|
1076
|
-
ASSERT_EQ("B2", Get("000025"));
|
|
1077
|
-
|
|
1078
|
-
// TEST: request compaction overlapping with partition boundary but no
|
|
1079
|
-
// actual entries
|
|
1080
|
-
// NOTE: `to` is INCLUSIVE
|
|
1081
|
-
from = Slice("000019");
|
|
1082
|
-
to = Slice("000020");
|
|
1083
|
-
ASSERT_OK(dbfull()->CompactRange(compact_options, &from, &to));
|
|
1084
|
-
|
|
1085
|
-
// Check (must be partitioned)
|
|
1086
|
-
files.clear();
|
|
1087
|
-
dbfull()->GetLiveFilesMetaData(&files);
|
|
1088
|
-
ASSERT_EQ(2, files.size());
|
|
1089
|
-
ASSERT_EQ("A2", Get("000015"));
|
|
1090
|
-
ASSERT_EQ("B2", Get("000025"));
|
|
1091
|
-
}
|
|
1092
|
-
|
|
1093
1027
|
TEST_F(DBCompactionTest, CompactionSstPartitionerNonTrivial) {
|
|
1094
1028
|
Options options = CurrentOptions();
|
|
1095
1029
|
options.compaction_style = kCompactionStyleLevel;
|
|
@@ -6245,231 +6179,6 @@ TEST_P(DBCompactionTestWithParam, FixFileIngestionCompactionDeadlock) {
|
|
|
6245
6179
|
Close();
|
|
6246
6180
|
}
|
|
6247
6181
|
|
|
6248
|
-
class DBCompactionTestWithOngoingFileIngestionParam
|
|
6249
|
-
: public DBCompactionTest,
|
|
6250
|
-
public testing::WithParamInterface<std::string> {
|
|
6251
|
-
public:
|
|
6252
|
-
DBCompactionTestWithOngoingFileIngestionParam() : DBCompactionTest() {
|
|
6253
|
-
compaction_path_to_test_ = GetParam();
|
|
6254
|
-
}
|
|
6255
|
-
void SetupOptions() {
|
|
6256
|
-
options_ = CurrentOptions();
|
|
6257
|
-
options_.create_if_missing = true;
|
|
6258
|
-
|
|
6259
|
-
if (compaction_path_to_test_ == "RefitLevelCompactRange") {
|
|
6260
|
-
options_.num_levels = 7;
|
|
6261
|
-
} else {
|
|
6262
|
-
options_.num_levels = 3;
|
|
6263
|
-
}
|
|
6264
|
-
options_.compaction_style = CompactionStyle::kCompactionStyleLevel;
|
|
6265
|
-
if (compaction_path_to_test_ == "AutoCompaction") {
|
|
6266
|
-
options_.disable_auto_compactions = false;
|
|
6267
|
-
options_.level0_file_num_compaction_trigger = 1;
|
|
6268
|
-
} else {
|
|
6269
|
-
options_.disable_auto_compactions = true;
|
|
6270
|
-
}
|
|
6271
|
-
}
|
|
6272
|
-
|
|
6273
|
-
void PauseCompactionThread() {
|
|
6274
|
-
sleeping_task_.reset(new test::SleepingBackgroundTask());
|
|
6275
|
-
env_->SetBackgroundThreads(1, Env::LOW);
|
|
6276
|
-
env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
|
|
6277
|
-
sleeping_task_.get(), Env::Priority::LOW);
|
|
6278
|
-
sleeping_task_->WaitUntilSleeping();
|
|
6279
|
-
}
|
|
6280
|
-
|
|
6281
|
-
void ResumeCompactionThread() {
|
|
6282
|
-
if (sleeping_task_) {
|
|
6283
|
-
sleeping_task_->WakeUp();
|
|
6284
|
-
sleeping_task_->WaitUntilDone();
|
|
6285
|
-
}
|
|
6286
|
-
}
|
|
6287
|
-
|
|
6288
|
-
void SetupFilesToForceFutureFilesIngestedToCertainLevel() {
|
|
6289
|
-
SstFileWriter sst_file_writer(EnvOptions(), options_);
|
|
6290
|
-
std::string dummy = dbname_ + "/dummy.sst";
|
|
6291
|
-
ASSERT_OK(sst_file_writer.Open(dummy));
|
|
6292
|
-
ASSERT_OK(sst_file_writer.Put("k2", "dummy"));
|
|
6293
|
-
ASSERT_OK(sst_file_writer.Finish());
|
|
6294
|
-
ASSERT_OK(db_->IngestExternalFile({dummy}, IngestExternalFileOptions()));
|
|
6295
|
-
// L2 is made to contain a file overlapped with files to be ingested in
|
|
6296
|
-
// later steps on key "k2". This will force future files ingested to L1 or
|
|
6297
|
-
// above.
|
|
6298
|
-
ASSERT_EQ("0,0,1", FilesPerLevel(0));
|
|
6299
|
-
}
|
|
6300
|
-
|
|
6301
|
-
void SetupSyncPoints() {
|
|
6302
|
-
if (compaction_path_to_test_ == "AutoCompaction") {
|
|
6303
|
-
SyncPoint::GetInstance()->SetCallBack(
|
|
6304
|
-
"ExternalSstFileIngestionJob::Run", [&](void*) {
|
|
6305
|
-
SyncPoint::GetInstance()->LoadDependency(
|
|
6306
|
-
{{"DBImpl::BackgroundCompaction():AfterPickCompaction",
|
|
6307
|
-
"VersionSet::LogAndApply:WriteManifest"}});
|
|
6308
|
-
});
|
|
6309
|
-
} else if (compaction_path_to_test_ == "NonRefitLevelCompactRange") {
|
|
6310
|
-
SyncPoint::GetInstance()->SetCallBack(
|
|
6311
|
-
"ExternalSstFileIngestionJob::Run", [&](void*) {
|
|
6312
|
-
SyncPoint::GetInstance()->LoadDependency(
|
|
6313
|
-
{{"ColumnFamilyData::CompactRange:Return",
|
|
6314
|
-
"VersionSet::LogAndApply:WriteManifest"}});
|
|
6315
|
-
});
|
|
6316
|
-
} else if (compaction_path_to_test_ == "RefitLevelCompactRange") {
|
|
6317
|
-
SyncPoint::GetInstance()->SetCallBack(
|
|
6318
|
-
"ExternalSstFileIngestionJob::Run", [&](void*) {
|
|
6319
|
-
SyncPoint::GetInstance()->LoadDependency(
|
|
6320
|
-
{{"DBImpl::CompactRange:PostRefitLevel",
|
|
6321
|
-
"VersionSet::LogAndApply:WriteManifest"}});
|
|
6322
|
-
});
|
|
6323
|
-
} else if (compaction_path_to_test_ == "CompactFiles") {
|
|
6324
|
-
SyncPoint::GetInstance()->SetCallBack(
|
|
6325
|
-
"ExternalSstFileIngestionJob::Run", [&](void*) {
|
|
6326
|
-
SyncPoint::GetInstance()->LoadDependency(
|
|
6327
|
-
{{"DBImpl::CompactFilesImpl::PostSanitizeCompactionInputFiles",
|
|
6328
|
-
"VersionSet::LogAndApply:WriteManifest"}});
|
|
6329
|
-
});
|
|
6330
|
-
} else {
|
|
6331
|
-
assert(false);
|
|
6332
|
-
}
|
|
6333
|
-
SyncPoint::GetInstance()->LoadDependency(
|
|
6334
|
-
{{"ExternalSstFileIngestionJob::Run", "PreCompaction"}});
|
|
6335
|
-
SyncPoint::GetInstance()->EnableProcessing();
|
|
6336
|
-
}
|
|
6337
|
-
|
|
6338
|
-
void RunCompactionOverlappedWithFileIngestion() {
|
|
6339
|
-
if (compaction_path_to_test_ == "AutoCompaction") {
|
|
6340
|
-
TEST_SYNC_POINT("PreCompaction");
|
|
6341
|
-
ResumeCompactionThread();
|
|
6342
|
-
// Without proper range conflict check,
|
|
6343
|
-
// this would have been `Status::Corruption` about overlapping ranges
|
|
6344
|
-
Status s = dbfull()->TEST_WaitForCompact();
|
|
6345
|
-
EXPECT_OK(s);
|
|
6346
|
-
} else if (compaction_path_to_test_ == "NonRefitLevelCompactRange") {
|
|
6347
|
-
CompactRangeOptions cro;
|
|
6348
|
-
cro.change_level = false;
|
|
6349
|
-
std::string start_key = "k1";
|
|
6350
|
-
Slice start(start_key);
|
|
6351
|
-
std::string end_key = "k4";
|
|
6352
|
-
Slice end(end_key);
|
|
6353
|
-
TEST_SYNC_POINT("PreCompaction");
|
|
6354
|
-
// Without proper range conflict check,
|
|
6355
|
-
// this would have been `Status::Corruption` about overlapping ranges
|
|
6356
|
-
Status s = dbfull()->CompactRange(cro, &start, &end);
|
|
6357
|
-
EXPECT_OK(s);
|
|
6358
|
-
} else if (compaction_path_to_test_ == "RefitLevelCompactRange") {
|
|
6359
|
-
CompactRangeOptions cro;
|
|
6360
|
-
cro.change_level = true;
|
|
6361
|
-
cro.target_level = 5;
|
|
6362
|
-
std::string start_key = "k1";
|
|
6363
|
-
Slice start(start_key);
|
|
6364
|
-
std::string end_key = "k4";
|
|
6365
|
-
Slice end(end_key);
|
|
6366
|
-
TEST_SYNC_POINT("PreCompaction");
|
|
6367
|
-
Status s = dbfull()->CompactRange(cro, &start, &end);
|
|
6368
|
-
// Without proper range conflict check,
|
|
6369
|
-
// this would have been `Status::Corruption` about overlapping ranges
|
|
6370
|
-
// To see this, remove the fix AND replace
|
|
6371
|
-
// `DBImpl::CompactRange:PostRefitLevel` in sync point dependency with
|
|
6372
|
-
// `DBImpl::ReFitLevel:PostRegisterCompaction`
|
|
6373
|
-
EXPECT_TRUE(s.IsNotSupported());
|
|
6374
|
-
EXPECT_TRUE(s.ToString().find("some ongoing compaction's output") !=
|
|
6375
|
-
std::string::npos);
|
|
6376
|
-
} else if (compaction_path_to_test_ == "CompactFiles") {
|
|
6377
|
-
ColumnFamilyMetaData cf_meta_data;
|
|
6378
|
-
db_->GetColumnFamilyMetaData(&cf_meta_data);
|
|
6379
|
-
ASSERT_EQ(cf_meta_data.levels[0].files.size(), 1);
|
|
6380
|
-
std::vector<std::string> input_files;
|
|
6381
|
-
for (const auto& file : cf_meta_data.levels[0].files) {
|
|
6382
|
-
input_files.push_back(file.name);
|
|
6383
|
-
}
|
|
6384
|
-
TEST_SYNC_POINT("PreCompaction");
|
|
6385
|
-
Status s = db_->CompactFiles(CompactionOptions(), input_files, 1);
|
|
6386
|
-
// Without proper range conflict check,
|
|
6387
|
-
// this would have been `Status::Corruption` about overlapping ranges
|
|
6388
|
-
EXPECT_TRUE(s.IsAborted());
|
|
6389
|
-
EXPECT_TRUE(
|
|
6390
|
-
s.ToString().find(
|
|
6391
|
-
"A running compaction is writing to the same output level") !=
|
|
6392
|
-
std::string::npos);
|
|
6393
|
-
} else {
|
|
6394
|
-
assert(false);
|
|
6395
|
-
}
|
|
6396
|
-
}
|
|
6397
|
-
|
|
6398
|
-
void DisableSyncPoints() {
|
|
6399
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
6400
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
|
6401
|
-
}
|
|
6402
|
-
|
|
6403
|
-
protected:
|
|
6404
|
-
std::string compaction_path_to_test_;
|
|
6405
|
-
Options options_;
|
|
6406
|
-
std::shared_ptr<test::SleepingBackgroundTask> sleeping_task_;
|
|
6407
|
-
};
|
|
6408
|
-
|
|
6409
|
-
INSTANTIATE_TEST_CASE_P(DBCompactionTestWithOngoingFileIngestionParam,
|
|
6410
|
-
DBCompactionTestWithOngoingFileIngestionParam,
|
|
6411
|
-
::testing::Values("AutoCompaction",
|
|
6412
|
-
"NonRefitLevelCompactRange",
|
|
6413
|
-
"RefitLevelCompactRange",
|
|
6414
|
-
"CompactFiles"));
|
|
6415
|
-
|
|
6416
|
-
TEST_P(DBCompactionTestWithOngoingFileIngestionParam, RangeConflictCheck) {
|
|
6417
|
-
SetupOptions();
|
|
6418
|
-
DestroyAndReopen(options_);
|
|
6419
|
-
|
|
6420
|
-
if (compaction_path_to_test_ == "AutoCompaction") {
|
|
6421
|
-
PauseCompactionThread();
|
|
6422
|
-
}
|
|
6423
|
-
|
|
6424
|
-
if (compaction_path_to_test_ != "RefitLevelCompactRange") {
|
|
6425
|
-
SetupFilesToForceFutureFilesIngestedToCertainLevel();
|
|
6426
|
-
}
|
|
6427
|
-
|
|
6428
|
-
// Create s1
|
|
6429
|
-
ASSERT_OK(Put("k1", "v"));
|
|
6430
|
-
ASSERT_OK(Put("k4", "v"));
|
|
6431
|
-
ASSERT_OK(Flush());
|
|
6432
|
-
if (compaction_path_to_test_ == "RefitLevelCompactRange") {
|
|
6433
|
-
MoveFilesToLevel(6 /* level */);
|
|
6434
|
-
ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel(0));
|
|
6435
|
-
} else {
|
|
6436
|
-
ASSERT_EQ("1,0,1", FilesPerLevel(0));
|
|
6437
|
-
}
|
|
6438
|
-
|
|
6439
|
-
// To coerce following sequence of events
|
|
6440
|
-
// Timeline Thread 1 (Ingest s2) Thread 2 (Compact s1)
|
|
6441
|
-
// t0 | Decide to output to Lk
|
|
6442
|
-
// t1 | Release lock in LogAndApply()
|
|
6443
|
-
// t2 | Acquire lock
|
|
6444
|
-
// t3 | Decides to compact to Lk
|
|
6445
|
-
// | Expected to fail due to range
|
|
6446
|
-
// | conflict check with file
|
|
6447
|
-
// | ingestion
|
|
6448
|
-
// t4 | Release lock in LogAndApply()
|
|
6449
|
-
// t5 | Acquire lock again and finish
|
|
6450
|
-
// t6 | Acquire lock again and finish
|
|
6451
|
-
SetupSyncPoints();
|
|
6452
|
-
|
|
6453
|
-
// Ingest s2
|
|
6454
|
-
port::Thread thread1([&] {
|
|
6455
|
-
SstFileWriter sst_file_writer(EnvOptions(), options_);
|
|
6456
|
-
std::string s2 = dbname_ + "/ingested_s2.sst";
|
|
6457
|
-
ASSERT_OK(sst_file_writer.Open(s2));
|
|
6458
|
-
ASSERT_OK(sst_file_writer.Put("k2", "v2"));
|
|
6459
|
-
ASSERT_OK(sst_file_writer.Put("k3", "v2"));
|
|
6460
|
-
ASSERT_OK(sst_file_writer.Finish());
|
|
6461
|
-
ASSERT_OK(db_->IngestExternalFile({s2}, IngestExternalFileOptions()));
|
|
6462
|
-
});
|
|
6463
|
-
|
|
6464
|
-
// Compact s1. Without proper range conflict check,
|
|
6465
|
-
// this will encounter overlapping file corruption.
|
|
6466
|
-
port::Thread thread2([&] { RunCompactionOverlappedWithFileIngestion(); });
|
|
6467
|
-
|
|
6468
|
-
thread1.join();
|
|
6469
|
-
thread2.join();
|
|
6470
|
-
DisableSyncPoints();
|
|
6471
|
-
}
|
|
6472
|
-
|
|
6473
6182
|
TEST_F(DBCompactionTest, ConsistencyFailTest) {
|
|
6474
6183
|
Options options = CurrentOptions();
|
|
6475
6184
|
options.force_consistency_checks = true;
|
|
@@ -6562,655 +6271,172 @@ void IngestOneKeyValue(DBImpl* db, const std::string& key,
|
|
|
6562
6271
|
ASSERT_OK(db->IngestExternalFile({info.file_path}, ingest_opt));
|
|
6563
6272
|
}
|
|
6564
6273
|
|
|
6565
|
-
|
|
6566
|
-
|
|
6567
|
-
|
|
6568
|
-
|
|
6569
|
-
|
|
6570
|
-
|
|
6571
|
-
|
|
6572
|
-
|
|
6573
|
-
|
|
6574
|
-
options_.force_consistency_checks = true;
|
|
6575
|
-
options_.compaction_style = compaciton_style;
|
|
6576
|
-
|
|
6577
|
-
if (compaciton_style == CompactionStyle::kCompactionStyleLevel) {
|
|
6578
|
-
options_.num_levels = 7;
|
|
6579
|
-
// Level compaction's PickIntraL0Compaction() impl detail requires
|
|
6580
|
-
// `options.level0_file_num_compaction_trigger` to be
|
|
6581
|
-
// at least 2 files less than the actual number of level 0 files
|
|
6582
|
-
// (i.e, 7 by design in this test)
|
|
6583
|
-
options_.level0_file_num_compaction_trigger = 5;
|
|
6584
|
-
options_.max_background_compactions = 2;
|
|
6585
|
-
options_.write_buffer_size = 2 << 20;
|
|
6586
|
-
options_.max_write_buffer_number = 6;
|
|
6587
|
-
} else if (compaciton_style == CompactionStyle::kCompactionStyleUniversal) {
|
|
6588
|
-
// TODO: expand test coverage to num_lvels > 1 for universal compacion,
|
|
6589
|
-
// which requires careful unit test design to compact to level 0 despite
|
|
6590
|
-
// num_levels > 1
|
|
6591
|
-
options_.num_levels = 1;
|
|
6592
|
-
options_.level0_file_num_compaction_trigger = 5;
|
|
6593
|
-
|
|
6594
|
-
CompactionOptionsUniversal universal_options;
|
|
6595
|
-
if (compaction_path_to_test == "PickCompactionToReduceSizeAmp") {
|
|
6596
|
-
universal_options.max_size_amplification_percent = 50;
|
|
6597
|
-
} else if (compaction_path_to_test ==
|
|
6598
|
-
"PickCompactionToReduceSortedRuns") {
|
|
6599
|
-
universal_options.max_size_amplification_percent = 400;
|
|
6600
|
-
} else if (compaction_path_to_test == "PickDeleteTriggeredCompaction") {
|
|
6601
|
-
universal_options.max_size_amplification_percent = 400;
|
|
6602
|
-
universal_options.min_merge_width = 6;
|
|
6603
|
-
}
|
|
6604
|
-
options_.compaction_options_universal = universal_options;
|
|
6605
|
-
} else if (compaciton_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
6606
|
-
options_.max_open_files = -1;
|
|
6607
|
-
options_.num_levels = 1;
|
|
6608
|
-
options_.level0_file_num_compaction_trigger = 3;
|
|
6609
|
-
|
|
6610
|
-
CompactionOptionsFIFO fifo_options;
|
|
6611
|
-
if (compaction_path_to_test == "FindIntraL0Compaction" ||
|
|
6612
|
-
compaction_path_to_test == "CompactRange") {
|
|
6613
|
-
fifo_options.allow_compaction = true;
|
|
6614
|
-
fifo_options.age_for_warm = 0;
|
|
6615
|
-
} else if (compaction_path_to_test == "CompactFile") {
|
|
6616
|
-
fifo_options.allow_compaction = false;
|
|
6617
|
-
fifo_options.age_for_warm = 0;
|
|
6618
|
-
}
|
|
6619
|
-
options_.compaction_options_fifo = fifo_options;
|
|
6620
|
-
}
|
|
6621
|
-
|
|
6622
|
-
if (compaction_path_to_test == "CompactFile" ||
|
|
6623
|
-
compaction_path_to_test == "CompactRange") {
|
|
6624
|
-
options_.disable_auto_compactions = true;
|
|
6625
|
-
} else {
|
|
6626
|
-
options_.disable_auto_compactions = false;
|
|
6627
|
-
}
|
|
6628
|
-
}
|
|
6629
|
-
|
|
6630
|
-
void Destroy(const Options& options) {
|
|
6631
|
-
if (snapshot_) {
|
|
6632
|
-
assert(db_);
|
|
6633
|
-
db_->ReleaseSnapshot(snapshot_);
|
|
6634
|
-
snapshot_ = nullptr;
|
|
6635
|
-
}
|
|
6636
|
-
DBTestBase::Destroy(options);
|
|
6637
|
-
}
|
|
6638
|
-
|
|
6639
|
-
void Reopen(const Options& options) {
|
|
6640
|
-
DBTestBase::Reopen(options);
|
|
6641
|
-
if (options.compaction_style != CompactionStyle::kCompactionStyleLevel) {
|
|
6642
|
-
// To force assigning the global seqno to ingested file
|
|
6643
|
-
// for our test purpose.
|
|
6644
|
-
assert(snapshot_ == nullptr);
|
|
6645
|
-
snapshot_ = db_->GetSnapshot();
|
|
6646
|
-
}
|
|
6647
|
-
}
|
|
6648
|
-
|
|
6649
|
-
void DestroyAndReopen(Options& options) {
|
|
6650
|
-
Destroy(options);
|
|
6651
|
-
Reopen(options);
|
|
6652
|
-
}
|
|
6653
|
-
|
|
6654
|
-
void PauseCompactionThread() {
|
|
6655
|
-
sleeping_task_.reset(new test::SleepingBackgroundTask());
|
|
6656
|
-
env_->SetBackgroundThreads(1, Env::LOW);
|
|
6657
|
-
env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
|
|
6658
|
-
sleeping_task_.get(), Env::Priority::LOW);
|
|
6659
|
-
sleeping_task_->WaitUntilSleeping();
|
|
6660
|
-
}
|
|
6661
|
-
|
|
6662
|
-
void ResumeCompactionThread() {
|
|
6663
|
-
if (sleeping_task_) {
|
|
6664
|
-
sleeping_task_->WakeUp();
|
|
6665
|
-
sleeping_task_->WaitUntilDone();
|
|
6666
|
-
}
|
|
6667
|
-
}
|
|
6274
|
+
TEST_P(DBCompactionTestWithParam,
|
|
6275
|
+
FlushAfterIntraL0CompactionCheckConsistencyFail) {
|
|
6276
|
+
Options options = CurrentOptions();
|
|
6277
|
+
options.force_consistency_checks = true;
|
|
6278
|
+
options.compression = kNoCompression;
|
|
6279
|
+
options.level0_file_num_compaction_trigger = 5;
|
|
6280
|
+
options.max_background_compactions = 2;
|
|
6281
|
+
options.max_subcompactions = max_subcompactions_;
|
|
6282
|
+
DestroyAndReopen(options);
|
|
6668
6283
|
|
|
6669
|
-
|
|
6670
|
-
|
|
6671
|
-
|
|
6672
|
-
|
|
6673
|
-
assert(versions);
|
|
6674
|
-
ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault();
|
|
6675
|
-
assert(cfd);
|
|
6676
|
-
Version* const current = cfd->current();
|
|
6677
|
-
assert(current);
|
|
6284
|
+
const size_t kValueSize = 1 << 20;
|
|
6285
|
+
Random rnd(301);
|
|
6286
|
+
std::atomic<int> pick_intra_l0_count(0);
|
|
6287
|
+
std::string value(rnd.RandomString(kValueSize));
|
|
6678
6288
|
|
|
6679
|
-
|
|
6680
|
-
|
|
6289
|
+
// The L0->L1 must be picked before we begin ingesting files to trigger
|
|
6290
|
+
// intra-L0 compaction, and must not finish until after an intra-L0
|
|
6291
|
+
// compaction has been picked.
|
|
6292
|
+
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
|
|
6293
|
+
{{"LevelCompactionPicker::PickCompaction:Return",
|
|
6294
|
+
"DBCompactionTestWithParam::"
|
|
6295
|
+
"FlushAfterIntraL0CompactionCheckConsistencyFail:L0ToL1Ready"},
|
|
6296
|
+
{"LevelCompactionPicker::PickCompactionBySize:0",
|
|
6297
|
+
"CompactionJob::Run():Start"}});
|
|
6298
|
+
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
6299
|
+
"FindIntraL0Compaction",
|
|
6300
|
+
[&](void* /*arg*/) { pick_intra_l0_count.fetch_add(1); });
|
|
6681
6301
|
|
|
6682
|
-
|
|
6683
|
-
assert(level0_files.size() == num_files);
|
|
6302
|
+
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
|
6684
6303
|
|
|
6685
|
-
|
|
6686
|
-
|
|
6687
|
-
|
|
6304
|
+
// prevents trivial move
|
|
6305
|
+
for (int i = 0; i < 10; ++i) {
|
|
6306
|
+
ASSERT_OK(Put(Key(i), "")); // prevents trivial move
|
|
6688
6307
|
}
|
|
6308
|
+
ASSERT_OK(Flush());
|
|
6309
|
+
Compact("", Key(99));
|
|
6310
|
+
ASSERT_EQ(0, NumTableFilesAtLevel(0));
|
|
6689
6311
|
|
|
6690
|
-
|
|
6691
|
-
|
|
6692
|
-
|
|
6693
|
-
|
|
6694
|
-
assert(versions);
|
|
6695
|
-
ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault();
|
|
6696
|
-
assert(cfd);
|
|
6697
|
-
Version* const current = cfd->current();
|
|
6698
|
-
assert(current);
|
|
6699
|
-
|
|
6700
|
-
VersionStorageInfo* const storage_info = current->storage_info();
|
|
6701
|
-
assert(storage_info);
|
|
6702
|
-
|
|
6703
|
-
const std::vector<FileMetaData*> level0_files = storage_info->LevelFiles(0);
|
|
6704
|
-
assert(level0_files.size() == num_files);
|
|
6705
|
-
|
|
6706
|
-
for (FileMetaData* f : level0_files) {
|
|
6707
|
-
storage_info->TEST_AddFileMarkedForCompaction(0, f);
|
|
6708
|
-
}
|
|
6312
|
+
// Flush 5 L0 sst.
|
|
6313
|
+
for (int i = 0; i < 5; ++i) {
|
|
6314
|
+
ASSERT_OK(Put(Key(i + 1), value));
|
|
6315
|
+
ASSERT_OK(Flush());
|
|
6709
6316
|
}
|
|
6317
|
+
ASSERT_EQ(5, NumTableFilesAtLevel(0));
|
|
6710
6318
|
|
|
6711
|
-
|
|
6712
|
-
|
|
6713
|
-
|
|
6714
|
-
options_.compaction_style == CompactionStyle::kCompactionStyleLevel) {
|
|
6715
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
6716
|
-
"PostPickFileToCompact", [&](void* arg) {
|
|
6717
|
-
bool* picked_file_to_compact = (bool*)arg;
|
|
6718
|
-
// To trigger intra-L0 compaction specifically,
|
|
6719
|
-
// we mock PickFileToCompact()'s result to be false
|
|
6720
|
-
*picked_file_to_compact = false;
|
|
6721
|
-
});
|
|
6722
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
6723
|
-
"FindIntraL0Compaction", [&](void* /*arg*/) {
|
|
6724
|
-
compaction_path_sync_point_called_.store(true);
|
|
6725
|
-
});
|
|
6726
|
-
|
|
6727
|
-
} else if (compaction_path_to_test == "PickPeriodicCompaction") {
|
|
6728
|
-
assert(options_.compaction_style ==
|
|
6729
|
-
CompactionStyle::kCompactionStyleUniversal);
|
|
6730
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
6731
|
-
"PostPickPeriodicCompaction", [&](void* compaction_arg) {
|
|
6732
|
-
Compaction* compaction = (Compaction*)compaction_arg;
|
|
6733
|
-
if (compaction != nullptr) {
|
|
6734
|
-
compaction_path_sync_point_called_.store(true);
|
|
6735
|
-
}
|
|
6736
|
-
});
|
|
6737
|
-
} else if (compaction_path_to_test == "PickCompactionToReduceSizeAmp") {
|
|
6738
|
-
assert(options_.compaction_style ==
|
|
6739
|
-
CompactionStyle::kCompactionStyleUniversal);
|
|
6740
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
6741
|
-
"PickCompactionToReduceSizeAmpReturnNonnullptr", [&](void* /*arg*/) {
|
|
6742
|
-
compaction_path_sync_point_called_.store(true);
|
|
6743
|
-
});
|
|
6744
|
-
} else if (compaction_path_to_test == "PickCompactionToReduceSortedRuns") {
|
|
6745
|
-
assert(options_.compaction_style ==
|
|
6746
|
-
CompactionStyle::kCompactionStyleUniversal);
|
|
6747
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
6748
|
-
"PickCompactionToReduceSortedRunsReturnNonnullptr",
|
|
6749
|
-
[&](void* /*arg*/) {
|
|
6750
|
-
compaction_path_sync_point_called_.store(true);
|
|
6751
|
-
});
|
|
6752
|
-
} else if (compaction_path_to_test == "PickDeleteTriggeredCompaction") {
|
|
6753
|
-
assert(options_.compaction_style ==
|
|
6754
|
-
CompactionStyle::kCompactionStyleUniversal);
|
|
6755
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
6756
|
-
"PickDeleteTriggeredCompactionReturnNonnullptr", [&](void* /*arg*/) {
|
|
6757
|
-
compaction_path_sync_point_called_.store(true);
|
|
6758
|
-
});
|
|
6759
|
-
} else if ((compaction_path_to_test == "FindIntraL0Compaction" ||
|
|
6760
|
-
compaction_path_to_test == "CompactRange") &&
|
|
6761
|
-
options_.compaction_style ==
|
|
6762
|
-
CompactionStyle::kCompactionStyleFIFO) {
|
|
6763
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
6764
|
-
"FindIntraL0Compaction", [&](void* /*arg*/) {
|
|
6765
|
-
compaction_path_sync_point_called_.store(true);
|
|
6766
|
-
});
|
|
6767
|
-
}
|
|
6768
|
-
|
|
6769
|
-
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
|
6770
|
-
}
|
|
6319
|
+
// Put one key, to make smallest log sequence number in this memtable is less
|
|
6320
|
+
// than sst which would be ingested in next step.
|
|
6321
|
+
ASSERT_OK(Put(Key(0), "a"));
|
|
6771
6322
|
|
|
6772
|
-
|
|
6323
|
+
ASSERT_EQ(5, NumTableFilesAtLevel(0));
|
|
6324
|
+
TEST_SYNC_POINT(
|
|
6325
|
+
"DBCompactionTestWithParam::"
|
|
6326
|
+
"FlushAfterIntraL0CompactionCheckConsistencyFail:L0ToL1Ready");
|
|
6773
6327
|
|
|
6774
|
-
|
|
6775
|
-
|
|
6776
|
-
|
|
6328
|
+
// Ingest 5 L0 sst. And this files would trigger PickIntraL0Compaction.
|
|
6329
|
+
for (int i = 5; i < 10; i++) {
|
|
6330
|
+
ASSERT_EQ(i, NumTableFilesAtLevel(0));
|
|
6331
|
+
IngestOneKeyValue(dbfull(), Key(i), value, options);
|
|
6777
6332
|
}
|
|
6778
6333
|
|
|
6779
|
-
//
|
|
6780
|
-
|
|
6781
|
-
|
|
6782
|
-
|
|
6783
|
-
|
|
6784
|
-
|
|
6785
|
-
|
|
6786
|
-
|
|
6787
|
-
|
|
6788
|
-
|
|
6789
|
-
const std::vector<FileMetaData*> level0_files = storage_info->LevelFiles(0);
|
|
6790
|
-
assert(level0_files.size() >= 1);
|
|
6791
|
-
|
|
6792
|
-
uint64_t latest_file_num = 0;
|
|
6793
|
-
uint64_t latest_file_largest_seqno = 0;
|
|
6794
|
-
for (FileMetaData* f : level0_files) {
|
|
6795
|
-
if (f->fd.GetNumber() > latest_file_num) {
|
|
6796
|
-
latest_file_num = f->fd.GetNumber();
|
|
6797
|
-
latest_file_largest_seqno = f->fd.largest_seqno;
|
|
6798
|
-
}
|
|
6799
|
-
}
|
|
6334
|
+
// Put one key, to make biggest log sequence number in this memtable is bigger
|
|
6335
|
+
// than sst which would be ingested in next step.
|
|
6336
|
+
ASSERT_OK(Put(Key(2), "b"));
|
|
6337
|
+
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
6338
|
+
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
|
6339
|
+
std::vector<std::vector<FileMetaData>> level_to_files;
|
|
6340
|
+
dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
|
|
6341
|
+
&level_to_files);
|
|
6342
|
+
ASSERT_GT(level_to_files[0].size(), 0);
|
|
6343
|
+
ASSERT_GT(pick_intra_l0_count.load(), 0);
|
|
6800
6344
|
|
|
6801
|
-
|
|
6802
|
-
|
|
6345
|
+
ASSERT_OK(Flush());
|
|
6346
|
+
}
|
|
6803
6347
|
|
|
6804
|
-
|
|
6805
|
-
|
|
6348
|
+
TEST_P(DBCompactionTestWithParam,
|
|
6349
|
+
IntraL0CompactionAfterFlushCheckConsistencyFail) {
|
|
6350
|
+
Options options = CurrentOptions();
|
|
6351
|
+
options.force_consistency_checks = true;
|
|
6352
|
+
options.compression = kNoCompression;
|
|
6353
|
+
options.level0_file_num_compaction_trigger = 5;
|
|
6354
|
+
options.max_background_compactions = 2;
|
|
6355
|
+
options.max_subcompactions = max_subcompactions_;
|
|
6356
|
+
options.write_buffer_size = 2 << 20;
|
|
6357
|
+
options.max_write_buffer_number = 6;
|
|
6358
|
+
DestroyAndReopen(options);
|
|
6806
6359
|
|
|
6807
|
-
|
|
6808
|
-
|
|
6809
|
-
std::
|
|
6810
|
-
std::
|
|
6811
|
-
|
|
6360
|
+
const size_t kValueSize = 1 << 20;
|
|
6361
|
+
Random rnd(301);
|
|
6362
|
+
std::string value(rnd.RandomString(kValueSize));
|
|
6363
|
+
std::string value2(rnd.RandomString(kValueSize));
|
|
6364
|
+
std::string bigvalue = value + value;
|
|
6812
6365
|
|
|
6813
|
-
|
|
6814
|
-
FlushAfterIntraL0LevelCompactionWithIngestedFile) {
|
|
6815
|
-
SetupOptions(CompactionStyle::kCompactionStyleLevel, "");
|
|
6816
|
-
DestroyAndReopen(options_);
|
|
6817
|
-
// Prevents trivial move
|
|
6366
|
+
// prevents trivial move
|
|
6818
6367
|
for (int i = 0; i < 10; ++i) {
|
|
6819
|
-
ASSERT_OK(Put(Key(i), "")); //
|
|
6368
|
+
ASSERT_OK(Put(Key(i), "")); // prevents trivial move
|
|
6820
6369
|
}
|
|
6821
6370
|
ASSERT_OK(Flush());
|
|
6822
6371
|
Compact("", Key(99));
|
|
6823
6372
|
ASSERT_EQ(0, NumTableFilesAtLevel(0));
|
|
6824
6373
|
|
|
6825
|
-
|
|
6826
|
-
//
|
|
6827
|
-
|
|
6828
|
-
|
|
6829
|
-
|
|
6830
|
-
|
|
6831
|
-
|
|
6832
|
-
|
|
6833
|
-
|
|
6834
|
-
|
|
6835
|
-
|
|
6374
|
+
std::atomic<int> pick_intra_l0_count(0);
|
|
6375
|
+
// The L0->L1 must be picked before we begin ingesting files to trigger
|
|
6376
|
+
// intra-L0 compaction, and must not finish until after an intra-L0
|
|
6377
|
+
// compaction has been picked.
|
|
6378
|
+
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
|
|
6379
|
+
{{"LevelCompactionPicker::PickCompaction:Return",
|
|
6380
|
+
"DBCompactionTestWithParam::"
|
|
6381
|
+
"IntraL0CompactionAfterFlushCheckConsistencyFail:L0ToL1Ready"},
|
|
6382
|
+
{"LevelCompactionPicker::PickCompactionBySize:0",
|
|
6383
|
+
"CompactionJob::Run():Start"}});
|
|
6384
|
+
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
6385
|
+
"FindIntraL0Compaction",
|
|
6386
|
+
[&](void* /*arg*/) { pick_intra_l0_count.fetch_add(1); });
|
|
6387
|
+
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
|
6388
|
+
// Make 6 L0 sst.
|
|
6836
6389
|
for (int i = 0; i < 6; ++i) {
|
|
6837
6390
|
if (i % 2 == 0) {
|
|
6838
|
-
IngestOneKeyValue(dbfull(), Key(i),
|
|
6391
|
+
IngestOneKeyValue(dbfull(), Key(i), value, options);
|
|
6839
6392
|
} else {
|
|
6840
|
-
ASSERT_OK(Put(Key(i),
|
|
6393
|
+
ASSERT_OK(Put(Key(i), value));
|
|
6841
6394
|
ASSERT_OK(Flush());
|
|
6842
6395
|
}
|
|
6843
6396
|
}
|
|
6397
|
+
|
|
6844
6398
|
ASSERT_EQ(6, NumTableFilesAtLevel(0));
|
|
6845
6399
|
|
|
6846
|
-
//
|
|
6400
|
+
// Stop run flush job
|
|
6401
|
+
env_->SetBackgroundThreads(1, Env::HIGH);
|
|
6402
|
+
test::SleepingBackgroundTask sleeping_tasks;
|
|
6403
|
+
env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_tasks,
|
|
6404
|
+
Env::Priority::HIGH);
|
|
6405
|
+
sleeping_tasks.WaitUntilSleeping();
|
|
6406
|
+
|
|
6407
|
+
// Put many keys to make memtable request to flush
|
|
6847
6408
|
for (int i = 0; i < 6; ++i) {
|
|
6848
|
-
ASSERT_OK(Put(Key(i),
|
|
6409
|
+
ASSERT_OK(Put(Key(i), bigvalue));
|
|
6849
6410
|
}
|
|
6850
|
-
ASSERT_EQ(6, NumTableFilesAtLevel(0));
|
|
6851
6411
|
|
|
6852
|
-
|
|
6853
|
-
|
|
6412
|
+
ASSERT_EQ(6, NumTableFilesAtLevel(0));
|
|
6413
|
+
TEST_SYNC_POINT(
|
|
6414
|
+
"DBCompactionTestWithParam::"
|
|
6415
|
+
"IntraL0CompactionAfterFlushCheckConsistencyFail:L0ToL1Ready");
|
|
6416
|
+
// ingest file to trigger IntraL0Compaction
|
|
6417
|
+
for (int i = 6; i < 10; ++i) {
|
|
6854
6418
|
ASSERT_EQ(i, NumTableFilesAtLevel(0));
|
|
6855
|
-
IngestOneKeyValue(dbfull(), Key(i),
|
|
6419
|
+
IngestOneKeyValue(dbfull(), Key(i), value2, options);
|
|
6856
6420
|
}
|
|
6857
6421
|
|
|
6858
|
-
|
|
6859
|
-
|
|
6860
|
-
|
|
6422
|
+
// Wake up flush job
|
|
6423
|
+
sleeping_tasks.WakeUp();
|
|
6424
|
+
sleeping_tasks.WaitUntilDone();
|
|
6861
6425
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
6426
|
+
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
|
6862
6427
|
|
|
6863
|
-
|
|
6864
|
-
|
|
6865
|
-
|
|
6866
|
-
|
|
6867
|
-
//
|
|
6868
|
-
// memtable: m1[ 5:new@12 .. 1:new@8, 0:new@7]
|
|
6869
|
-
// L0: s7[6:new@13, 5:old@6 .. 0:old@1]
|
|
6870
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
6871
|
-
SequenceNumber compact_output_file_largest_seqno =
|
|
6872
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
6873
|
-
|
|
6874
|
-
ASSERT_OK(Flush());
|
|
6875
|
-
// After flush, we have LSM tree:
|
|
6876
|
-
//
|
|
6877
|
-
// L0: s8[5:new@12 .. 0:new@7],s7[6:new@13, 5:old@5 .. 0:old@1]
|
|
6878
|
-
ASSERT_EQ(2, NumTableFilesAtLevel(0));
|
|
6879
|
-
SequenceNumber flushed_file_largest_seqno =
|
|
6880
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
6881
|
-
|
|
6882
|
-
// To verify there isn't any file misorder leading to returning a old value
|
|
6883
|
-
// of Key(0) - Key(5) , which is caused by flushed table s8 has a
|
|
6884
|
-
// smaller largest seqno than the compaction output file s7's largest seqno
|
|
6885
|
-
// while the flushed table has the newer version of the values than the
|
|
6886
|
-
// compaction output file's.
|
|
6887
|
-
ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno);
|
|
6428
|
+
uint64_t error_count = 0;
|
|
6429
|
+
db_->GetIntProperty("rocksdb.background-errors", &error_count);
|
|
6430
|
+
ASSERT_EQ(error_count, 0);
|
|
6431
|
+
ASSERT_GT(pick_intra_l0_count.load(), 0);
|
|
6888
6432
|
for (int i = 0; i < 6; ++i) {
|
|
6889
|
-
ASSERT_EQ(
|
|
6433
|
+
ASSERT_EQ(bigvalue, Get(Key(i)));
|
|
6890
6434
|
}
|
|
6891
|
-
for (int i = 6; i <
|
|
6892
|
-
ASSERT_EQ(
|
|
6435
|
+
for (int i = 6; i < 10; ++i) {
|
|
6436
|
+
ASSERT_EQ(value2, Get(Key(i)));
|
|
6893
6437
|
}
|
|
6894
6438
|
}
|
|
6895
6439
|
|
|
6896
|
-
TEST_F(DBCompactionTestL0FilesMisorderCorruption,
|
|
6897
|
-
FlushAfterIntraL0UniversalCompactionWithIngestedFile) {
|
|
6898
|
-
for (const std::string compaction_path_to_test :
|
|
6899
|
-
{"PickPeriodicCompaction", "PickCompactionToReduceSizeAmp",
|
|
6900
|
-
"PickCompactionToReduceSortedRuns", "PickDeleteTriggeredCompaction"}) {
|
|
6901
|
-
SetupOptions(CompactionStyle::kCompactionStyleUniversal,
|
|
6902
|
-
compaction_path_to_test);
|
|
6903
|
-
DestroyAndReopen(options_);
|
|
6904
|
-
|
|
6905
|
-
// To get accurate NumTableFilesAtLevel(0) when the number reaches
|
|
6906
|
-
// options_.level0_file_num_compaction_trigger
|
|
6907
|
-
PauseCompactionThread();
|
|
6908
|
-
|
|
6909
|
-
// To create below LSM tree
|
|
6910
|
-
// (key:value@n indicates key-value pair has seqno "n", L0 is sorted):
|
|
6911
|
-
//
|
|
6912
|
-
// memtable: m1 [ k2:new@8, k1:new@7]
|
|
6913
|
-
// L0: s4[k9:dummy@10], s3[k8:dummy@9],
|
|
6914
|
-
// s2[k7:old@6, k6:old@5].. s0[k3:old@2, k1:old@1]
|
|
6915
|
-
//
|
|
6916
|
-
// (1) Create 3 existing SST file (i.e, s0 - s2)
|
|
6917
|
-
ASSERT_OK(Put("k1", "old"));
|
|
6918
|
-
ASSERT_OK(Put("k3", "old"));
|
|
6919
|
-
ASSERT_OK(Flush());
|
|
6920
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
6921
|
-
ASSERT_OK(Put("k4", "old"));
|
|
6922
|
-
ASSERT_OK(Put("k5", "old"));
|
|
6923
|
-
ASSERT_OK(Flush());
|
|
6924
|
-
ASSERT_EQ(2, NumTableFilesAtLevel(0));
|
|
6925
|
-
ASSERT_OK(Put("k6", "old"));
|
|
6926
|
-
ASSERT_OK(Put("k7", "old"));
|
|
6927
|
-
ASSERT_OK(Flush());
|
|
6928
|
-
ASSERT_EQ(3, NumTableFilesAtLevel(0));
|
|
6929
|
-
|
|
6930
|
-
// (2) Create m1. Noted that it contains a overlaped key with s0
|
|
6931
|
-
ASSERT_OK(Put("k1", "new")); // overlapped key
|
|
6932
|
-
ASSERT_OK(Put("k2", "new"));
|
|
6933
|
-
|
|
6934
|
-
// (3) Ingest two SST files s3, s4
|
|
6935
|
-
IngestOneKeyValue(dbfull(), "k8", "dummy", options_);
|
|
6936
|
-
IngestOneKeyValue(dbfull(), "k9", "dummy", options_);
|
|
6937
|
-
// Up to now, L0 contains s0 - s4
|
|
6938
|
-
ASSERT_EQ(5, NumTableFilesAtLevel(0));
|
|
6939
|
-
|
|
6940
|
-
if (compaction_path_to_test == "PickPeriodicCompaction") {
|
|
6941
|
-
AddFilesMarkedForPeriodicCompaction(5);
|
|
6942
|
-
} else if (compaction_path_to_test == "PickDeleteTriggeredCompaction") {
|
|
6943
|
-
AddFilesMarkedForCompaction(5);
|
|
6944
|
-
}
|
|
6945
|
-
|
|
6946
|
-
SetupSyncPoints(compaction_path_to_test);
|
|
6947
|
-
ResumeCompactionThread();
|
|
6948
|
-
|
|
6949
|
-
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
6950
|
-
|
|
6951
|
-
ASSERT_TRUE(SyncPointsCalled())
|
|
6952
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
6953
|
-
DisableSyncPoints();
|
|
6954
|
-
|
|
6955
|
-
// After compaction, we have LSM tree:
|
|
6956
|
-
//
|
|
6957
|
-
// memtable: m1[ k2:new@8, k1:new@7]
|
|
6958
|
-
// L0: s5[k9:dummy@10, k8@dummy@9, k7:old@6 .. k3:old@2, k1:old@1]
|
|
6959
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0))
|
|
6960
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
6961
|
-
SequenceNumber compact_output_file_largest_seqno =
|
|
6962
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
6963
|
-
|
|
6964
|
-
ASSERT_OK(Flush()) << "failed for compaction path to test: "
|
|
6965
|
-
<< compaction_path_to_test;
|
|
6966
|
-
// After flush, we have LSM tree:
|
|
6967
|
-
//
|
|
6968
|
-
// L0: s6[k2:new@8, k1:new@7],
|
|
6969
|
-
// s5[k9:dummy@10, k8@dummy@9, k7:old@6 .. k3:old@2, k1:old@1]
|
|
6970
|
-
ASSERT_EQ(2, NumTableFilesAtLevel(0))
|
|
6971
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
6972
|
-
SequenceNumber flushed_file_largest_seqno =
|
|
6973
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
6974
|
-
|
|
6975
|
-
// To verify there isn't any file misorder leading to returning a old
|
|
6976
|
-
// value of "k1" , which is caused by flushed table s6 has a
|
|
6977
|
-
// smaller largest seqno than the compaction output file s5's largest seqno
|
|
6978
|
-
// while the flushed table has the newer version of the value
|
|
6979
|
-
// than the compaction output file's.
|
|
6980
|
-
ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno)
|
|
6981
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
6982
|
-
EXPECT_EQ(Get("k1"), "new")
|
|
6983
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
6984
|
-
}
|
|
6985
|
-
|
|
6986
|
-
Destroy(options_);
|
|
6987
|
-
}
|
|
6988
|
-
|
|
6989
|
-
TEST_F(DBCompactionTestL0FilesMisorderCorruption,
|
|
6990
|
-
FlushAfterIntraL0FIFOCompactionWithIngestedFile) {
|
|
6991
|
-
for (const std::string compaction_path_to_test : {"FindIntraL0Compaction"}) {
|
|
6992
|
-
SetupOptions(CompactionStyle::kCompactionStyleFIFO,
|
|
6993
|
-
compaction_path_to_test);
|
|
6994
|
-
DestroyAndReopen(options_);
|
|
6995
|
-
|
|
6996
|
-
// To create below LSM tree
|
|
6997
|
-
// (key:value@n indicates key-value pair has seqno "n", L0 is sorted):
|
|
6998
|
-
//
|
|
6999
|
-
// memtable: m1 [ k2:new@4, k1:new@3]
|
|
7000
|
-
// L0: s2[k5:dummy@6], s1[k4:dummy@5], s0[k3:old@2, k1:old@1]
|
|
7001
|
-
//
|
|
7002
|
-
// (1) Create an existing SST file s0
|
|
7003
|
-
ASSERT_OK(Put("k1", "old"));
|
|
7004
|
-
ASSERT_OK(Put("k3", "old"));
|
|
7005
|
-
ASSERT_OK(Flush());
|
|
7006
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
7007
|
-
|
|
7008
|
-
// (2) Create memtable m1. Noted that it contains a overlaped key with s0
|
|
7009
|
-
ASSERT_OK(Put("k1", "new")); // overlapped key
|
|
7010
|
-
ASSERT_OK(Put("k2", "new"));
|
|
7011
|
-
|
|
7012
|
-
// To get accurate NumTableFilesAtLevel(0) when the number reaches
|
|
7013
|
-
// options_.level0_file_num_compaction_trigger
|
|
7014
|
-
PauseCompactionThread();
|
|
7015
|
-
|
|
7016
|
-
// (3) Ingest two SST files s1, s2
|
|
7017
|
-
IngestOneKeyValue(dbfull(), "k4", "dummy", options_);
|
|
7018
|
-
IngestOneKeyValue(dbfull(), "k5", "dummy", options_);
|
|
7019
|
-
// Up to now, L0 contains s0, s1, s2
|
|
7020
|
-
ASSERT_EQ(3, NumTableFilesAtLevel(0));
|
|
7021
|
-
|
|
7022
|
-
SetupSyncPoints(compaction_path_to_test);
|
|
7023
|
-
ResumeCompactionThread();
|
|
7024
|
-
|
|
7025
|
-
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
7026
|
-
|
|
7027
|
-
ASSERT_TRUE(SyncPointsCalled())
|
|
7028
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
7029
|
-
DisableSyncPoints();
|
|
7030
|
-
// After compaction, we have LSM tree:
|
|
7031
|
-
//
|
|
7032
|
-
// memtable: m1 [ k2:new@4, k1:new@3]
|
|
7033
|
-
// L0: s3[k5:dummy@6, k4:dummy@5, k3:old@2, k1:old@1]
|
|
7034
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0))
|
|
7035
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
7036
|
-
SequenceNumber compact_output_file_largest_seqno =
|
|
7037
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
7038
|
-
|
|
7039
|
-
ASSERT_OK(Flush()) << "failed for compaction path to test: "
|
|
7040
|
-
<< compaction_path_to_test;
|
|
7041
|
-
// After flush, we have LSM tree:
|
|
7042
|
-
//
|
|
7043
|
-
// L0: s4[k2:new@4, k1:new@3], s3[k5:dummy@6, k4:dummy@5, k3:old@2,
|
|
7044
|
-
// k1:old@1]
|
|
7045
|
-
ASSERT_EQ(2, NumTableFilesAtLevel(0))
|
|
7046
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
7047
|
-
SequenceNumber flushed_file_largest_seqno =
|
|
7048
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
7049
|
-
|
|
7050
|
-
// To verify there isn't any file misorder leading to returning a old
|
|
7051
|
-
// value of "k1" , which is caused by flushed table s4 has a
|
|
7052
|
-
// smaller largest seqno than the compaction output file s3's largest seqno
|
|
7053
|
-
// while the flushed table has the newer version of the value
|
|
7054
|
-
// than the compaction output file's.
|
|
7055
|
-
ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno)
|
|
7056
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
7057
|
-
EXPECT_EQ(Get("k1"), "new")
|
|
7058
|
-
<< "failed for compaction path to test: " << compaction_path_to_test;
|
|
7059
|
-
}
|
|
7060
|
-
|
|
7061
|
-
Destroy(options_);
|
|
7062
|
-
}
|
|
7063
|
-
|
|
7064
|
-
class DBCompactionTestL0FilesMisorderCorruptionWithParam
|
|
7065
|
-
: public DBCompactionTestL0FilesMisorderCorruption,
|
|
7066
|
-
public testing::WithParamInterface<CompactionStyle> {
|
|
7067
|
-
public:
|
|
7068
|
-
DBCompactionTestL0FilesMisorderCorruptionWithParam()
|
|
7069
|
-
: DBCompactionTestL0FilesMisorderCorruption() {}
|
|
7070
|
-
};
|
|
7071
|
-
|
|
7072
|
-
// TODO: add `CompactionStyle::kCompactionStyleLevel` to testing parameter,
|
|
7073
|
-
// which requires careful unit test
|
|
7074
|
-
// design for ingesting file to L0 and CompactRange()/CompactFile() to L0
|
|
7075
|
-
INSTANTIATE_TEST_CASE_P(
|
|
7076
|
-
DBCompactionTestL0FilesMisorderCorruptionWithParam,
|
|
7077
|
-
DBCompactionTestL0FilesMisorderCorruptionWithParam,
|
|
7078
|
-
::testing::Values(CompactionStyle::kCompactionStyleUniversal,
|
|
7079
|
-
CompactionStyle::kCompactionStyleFIFO));
|
|
7080
|
-
|
|
7081
|
-
TEST_P(DBCompactionTestL0FilesMisorderCorruptionWithParam,
|
|
7082
|
-
FlushAfterIntraL0CompactFileWithIngestedFile) {
|
|
7083
|
-
SetupOptions(GetParam(), "CompactFile");
|
|
7084
|
-
DestroyAndReopen(options_);
|
|
7085
|
-
|
|
7086
|
-
// To create below LSM tree
|
|
7087
|
-
// (key:value@n indicates key-value pair has seqno "n", L0 is sorted):
|
|
7088
|
-
//
|
|
7089
|
-
// memtable: m1 [ k2:new@4, k1:new@3]
|
|
7090
|
-
// L0: s2[k5:dummy@6], s1[k4:dummy@5], s0[k3:old@2, k1:old@1]
|
|
7091
|
-
//
|
|
7092
|
-
// (1) Create an existing SST file s0
|
|
7093
|
-
ASSERT_OK(Put("k1", "old"));
|
|
7094
|
-
ASSERT_OK(Put("k3", "old"));
|
|
7095
|
-
ASSERT_OK(Flush());
|
|
7096
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
7097
|
-
|
|
7098
|
-
// (2) Create memtable m1. Noted that it contains a overlaped key with s0
|
|
7099
|
-
ASSERT_OK(Put("k1", "new")); // overlapped key
|
|
7100
|
-
ASSERT_OK(Put("k2", "new"));
|
|
7101
|
-
|
|
7102
|
-
// (3) Ingest two SST files s1, s2
|
|
7103
|
-
IngestOneKeyValue(dbfull(), "k4", "dummy", options_);
|
|
7104
|
-
IngestOneKeyValue(dbfull(), "k5", "dummy", options_);
|
|
7105
|
-
// Up to now, L0 contains s0, s1, s2
|
|
7106
|
-
ASSERT_EQ(3, NumTableFilesAtLevel(0));
|
|
7107
|
-
|
|
7108
|
-
ColumnFamilyMetaData cf_meta_data;
|
|
7109
|
-
db_->GetColumnFamilyMetaData(&cf_meta_data);
|
|
7110
|
-
ASSERT_EQ(cf_meta_data.levels[0].files.size(), 3);
|
|
7111
|
-
std::vector<std::string> input_files;
|
|
7112
|
-
for (const auto& file : cf_meta_data.levels[0].files) {
|
|
7113
|
-
input_files.push_back(file.name);
|
|
7114
|
-
}
|
|
7115
|
-
ASSERT_EQ(input_files.size(), 3);
|
|
7116
|
-
|
|
7117
|
-
Status s = db_->CompactFiles(CompactionOptions(), input_files, 0);
|
|
7118
|
-
// After compaction, we have LSM tree:
|
|
7119
|
-
//
|
|
7120
|
-
// memtable: m1 [ k2:new@4, k1:new@3]
|
|
7121
|
-
// L0: s3[k5:dummy@6, k4:dummy@5, k3:old@2, k1:old@1]
|
|
7122
|
-
ASSERT_OK(s);
|
|
7123
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
7124
|
-
SequenceNumber compact_output_file_largest_seqno =
|
|
7125
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
7126
|
-
|
|
7127
|
-
ASSERT_OK(Flush());
|
|
7128
|
-
// After flush, we have LSM tree:
|
|
7129
|
-
//
|
|
7130
|
-
// L0: s4[k2:new@4, k1:new@3], s3[k5:dummy@6, k4:dummy@5, k3:old@2,
|
|
7131
|
-
// k1:old@1]
|
|
7132
|
-
ASSERT_EQ(2, NumTableFilesAtLevel(0));
|
|
7133
|
-
SequenceNumber flushed_file_largest_seqno =
|
|
7134
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
7135
|
-
|
|
7136
|
-
// To verify there isn't any file misorder leading to returning a old value
|
|
7137
|
-
// of "1" , which is caused by flushed table s4 has a smaller
|
|
7138
|
-
// largest seqno than the compaction output file s3's largest seqno while the
|
|
7139
|
-
// flushed table has the newer version of the value than the
|
|
7140
|
-
// compaction output file's.
|
|
7141
|
-
ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno);
|
|
7142
|
-
EXPECT_EQ(Get("k1"), "new");
|
|
7143
|
-
|
|
7144
|
-
Destroy(options_);
|
|
7145
|
-
}
|
|
7146
|
-
|
|
7147
|
-
TEST_P(DBCompactionTestL0FilesMisorderCorruptionWithParam,
|
|
7148
|
-
FlushAfterIntraL0CompactRangeWithIngestedFile) {
|
|
7149
|
-
SetupOptions(GetParam(), "CompactRange");
|
|
7150
|
-
DestroyAndReopen(options_);
|
|
7151
|
-
|
|
7152
|
-
// To create below LSM tree
|
|
7153
|
-
// (key:value@n indicates key-value pair has seqno "n", L0 is sorted):
|
|
7154
|
-
//
|
|
7155
|
-
// memtable: m1 [ k2:new@4, k1:new@3]
|
|
7156
|
-
// L0: s2[k5:dummy@6], s1[k4:dummy@5], s0[k3:old@2, k1:old@1]
|
|
7157
|
-
//
|
|
7158
|
-
// (1) Create an existing SST file s0
|
|
7159
|
-
ASSERT_OK(Put("k1", "old"));
|
|
7160
|
-
ASSERT_OK(Put("k3", "old"));
|
|
7161
|
-
ASSERT_OK(Flush());
|
|
7162
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
7163
|
-
|
|
7164
|
-
// (2) Create memtable m1. Noted that it contains a overlaped key with s0
|
|
7165
|
-
ASSERT_OK(Put("k1", "new")); // overlapped key
|
|
7166
|
-
ASSERT_OK(Put("k2", "new"));
|
|
7167
|
-
|
|
7168
|
-
// (3) Ingest two SST files s1, s2
|
|
7169
|
-
IngestOneKeyValue(dbfull(), "k4", "dummy", options_);
|
|
7170
|
-
IngestOneKeyValue(dbfull(), "k5", "dummy", options_);
|
|
7171
|
-
// Up to now, L0 contains s0, s1, s2
|
|
7172
|
-
ASSERT_EQ(3, NumTableFilesAtLevel(0));
|
|
7173
|
-
|
|
7174
|
-
if (options_.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
7175
|
-
SetupSyncPoints("CompactRange");
|
|
7176
|
-
}
|
|
7177
|
-
// `start` and `end` is carefully chosen so that compact range:
|
|
7178
|
-
// (1) doesn't overlap with memtable therefore the memtable won't be flushed
|
|
7179
|
-
// (2) should target at compacting s0 with s1 and s2
|
|
7180
|
-
Slice start("k3"), end("k5");
|
|
7181
|
-
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end));
|
|
7182
|
-
// After compaction, we have LSM tree:
|
|
7183
|
-
//
|
|
7184
|
-
// memtable: m1 [ k2:new@4, k1:new@3]
|
|
7185
|
-
// L0: s3[k5:dummy@6, k4:dummy@5, k3:old@2, k1:old@1]
|
|
7186
|
-
if (options_.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
|
|
7187
|
-
ASSERT_TRUE(SyncPointsCalled());
|
|
7188
|
-
DisableSyncPoints();
|
|
7189
|
-
}
|
|
7190
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
7191
|
-
SequenceNumber compact_output_file_largest_seqno =
|
|
7192
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
7193
|
-
|
|
7194
|
-
ASSERT_OK(Flush());
|
|
7195
|
-
// After flush, we have LSM tree:
|
|
7196
|
-
//
|
|
7197
|
-
// L0: s4[k2:new@4, k1:new@3], s3[k5:dummy@6, k4:dummy@5, k3:old@2,
|
|
7198
|
-
// k1:old@1]
|
|
7199
|
-
ASSERT_EQ(2, NumTableFilesAtLevel(0));
|
|
7200
|
-
SequenceNumber flushed_file_largest_seqno =
|
|
7201
|
-
GetLatestL0FileLargestSeqnoHelper();
|
|
7202
|
-
|
|
7203
|
-
// To verify there isn't any file misorder leading to returning a old value
|
|
7204
|
-
// of "k1" , which is caused by flushed table s4 has a smaller
|
|
7205
|
-
// largest seqno than the compaction output file s3's largest seqno while the
|
|
7206
|
-
// flushed table has the newer version of the value than the
|
|
7207
|
-
// compaction output file's.
|
|
7208
|
-
ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno);
|
|
7209
|
-
EXPECT_EQ(Get("k1"), "new");
|
|
7210
|
-
|
|
7211
|
-
Destroy(options_);
|
|
7212
|
-
}
|
|
7213
|
-
|
|
7214
6440
|
TEST_P(DBCompactionTestWithBottommostParam, SequenceKeysManualCompaction) {
|
|
7215
6441
|
constexpr int kSstNum = 10;
|
|
7216
6442
|
Options options = CurrentOptions();
|