@nxtedition/rocksdb 13.5.9 → 13.5.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deps/rocksdb/rocksdb/BUCK +2 -1
- package/deps/rocksdb/rocksdb/CMakeLists.txt +2 -1
- package/deps/rocksdb/rocksdb/Makefile +1 -1
- package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.cc +4 -5
- package/deps/rocksdb/rocksdb/db/c.cc +13 -0
- package/deps/rocksdb/rocksdb/db/c_test.c +0 -12
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +8 -8
- package/deps/rocksdb/rocksdb/db/compaction/compaction.h +2 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +5 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +10 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +11 -6
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +10 -16
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +2 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +12 -17
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +164 -0
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +74 -3
- package/deps/rocksdb/rocksdb/db/db_encryption_test.cc +39 -4
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +2 -83
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +0 -4
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +11 -11
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +0 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +0 -9
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +16 -54
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +0 -6
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +186 -0
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +3 -40
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +0 -54
- package/deps/rocksdb/rocksdb/db/db_test.cc +0 -292
- package/deps/rocksdb/rocksdb/db/db_test2.cc +0 -1235
- package/deps/rocksdb/rocksdb/db/db_test_util.h +8 -0
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +11 -4
- package/deps/rocksdb/rocksdb/db/log_reader.cc +11 -11
- package/deps/rocksdb/rocksdb/db/merge_helper.h +1 -1
- package/deps/rocksdb/rocksdb/db/multi_scan.cc +70 -0
- package/deps/rocksdb/rocksdb/db/version_set.cc +15 -8
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +4 -0
- package/deps/rocksdb/rocksdb/env/composite_env.cc +4 -0
- package/deps/rocksdb/rocksdb/env/env.cc +4 -0
- package/deps/rocksdb/rocksdb/env/env_encryption.cc +38 -3
- package/deps/rocksdb/rocksdb/env/env_test.cc +36 -1
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +20 -4
- package/deps/rocksdb/rocksdb/env/io_posix.cc +16 -0
- package/deps/rocksdb/rocksdb/env/io_posix.h +3 -0
- package/deps/rocksdb/rocksdb/env/mock_env.cc +5 -0
- package/deps/rocksdb/rocksdb/file/readahead_raf.cc +4 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +33 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +25 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/env.h +10 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/env_encryption.h +9 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +12 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +12 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/multi_scan.h +29 -28
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +26 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +9 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/tool_hooks.h +3 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/user_defined_index.h +142 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/db_ttl.h +2 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/write_batch_with_index.h +2 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/options/options_helper.h +3 -0
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +2 -0
- package/deps/rocksdb/rocksdb/port/win/io_win.cc +20 -0
- package/deps/rocksdb/rocksdb/port/win/io_win.h +4 -0
- package/deps/rocksdb/rocksdb/src.mk +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/block.cc +31 -34
- package/deps/rocksdb/rocksdb/table/block_based/block.h +2 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +43 -7
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +6 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +367 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +69 -23
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +54 -6
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +27 -5
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +167 -3
- package/deps/rocksdb/rocksdb/table/block_based/block_builder.cc +6 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +6 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +12 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_type.h +1 -0
- package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +0 -3
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +10 -7
- package/deps/rocksdb/rocksdb/table/block_based/user_defined_index_wrapper.h +244 -0
- package/deps/rocksdb/rocksdb/table/external_table.cc +1 -1
- package/deps/rocksdb/rocksdb/table/format.cc +51 -33
- package/deps/rocksdb/rocksdb/table/format.h +1 -1
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +13 -8
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +1 -3
- package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +5 -1
- package/deps/rocksdb/rocksdb/table/table_test.cc +629 -1
- package/deps/rocksdb/rocksdb/test_util/testutil.cc +0 -1
- package/deps/rocksdb/rocksdb/test_util/testutil.h +5 -0
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +183 -94
- package/deps/rocksdb/rocksdb/tools/sst_dump_test.cc +71 -0
- package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +37 -22
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.cc +308 -0
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.h +189 -0
- package/deps/rocksdb/rocksdb/util/cast_util.h +22 -11
- package/deps/rocksdb/rocksdb/util/coding.h +4 -3
- package/deps/rocksdb/rocksdb/util/compression.cc +2 -0
- package/deps/rocksdb/rocksdb/util/compression.h +16 -6
- package/deps/rocksdb/rocksdb/util/compression_test.cc +1679 -15
- package/deps/rocksdb/rocksdb/util/stop_watch.h +17 -7
- package/deps/rocksdb/rocksdb/util/timer_queue_test.cc +17 -3
- package/deps/rocksdb/rocksdb/utilities/env_mirror.cc +10 -0
- package/deps/rocksdb/rocksdb/utilities/fault_injection_env.cc +5 -0
- package/deps/rocksdb/rocksdb/utilities/fault_injection_env.h +2 -0
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +18 -2
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +22 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.cc +5 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +22 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +15 -4
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +61 -0
- package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.cc +18 -0
- package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.h +3 -0
- package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +3 -0
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +9 -3
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +9 -0
- package/deps/rocksdb/rocksdb.gyp +15 -1
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
- package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
- package/deps/rocksdb/rocksdb/util/auto_skip_compressor.cc +0 -131
- package/deps/rocksdb/rocksdb/util/auto_skip_compressor.h +0 -90
|
@@ -88,6 +88,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
|
|
|
88
88
|
"db/memtable_list.cc",
|
|
89
89
|
"db/merge_helper.cc",
|
|
90
90
|
"db/merge_operator.cc",
|
|
91
|
+
"db/multi_scan.cc",
|
|
91
92
|
"db/output_validator.cc",
|
|
92
93
|
"db/periodic_task_scheduler.cc",
|
|
93
94
|
"db/range_del_aggregator.cc",
|
|
@@ -249,7 +250,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
|
|
|
249
250
|
"trace_replay/trace_record_result.cc",
|
|
250
251
|
"trace_replay/trace_replay.cc",
|
|
251
252
|
"util/async_file_reader.cc",
|
|
252
|
-
"util/
|
|
253
|
+
"util/auto_tune_compressor.cc",
|
|
253
254
|
"util/build_version.cc",
|
|
254
255
|
"util/cleanable.cc",
|
|
255
256
|
"util/coding.cc",
|
|
@@ -721,6 +721,7 @@ set(SOURCES
|
|
|
721
721
|
db/memtable_list.cc
|
|
722
722
|
db/merge_helper.cc
|
|
723
723
|
db/merge_operator.cc
|
|
724
|
+
db/multi_scan.cc
|
|
724
725
|
db/output_validator.cc
|
|
725
726
|
db/periodic_task_scheduler.cc
|
|
726
727
|
db/range_del_aggregator.cc
|
|
@@ -874,7 +875,7 @@ set(SOURCES
|
|
|
874
875
|
trace_replay/trace_record.cc
|
|
875
876
|
trace_replay/trace_replay.cc
|
|
876
877
|
util/async_file_reader.cc
|
|
877
|
-
util/
|
|
878
|
+
util/auto_tune_compressor.cc
|
|
878
879
|
util/cleanable.cc
|
|
879
880
|
util/coding.cc
|
|
880
881
|
util/compaction_job_stats_impl.cc
|
|
@@ -2492,7 +2492,7 @@ checkout_folly:
|
|
|
2492
2492
|
fi
|
|
2493
2493
|
@# Pin to a particular version for public CI, so that PR authors don't
|
|
2494
2494
|
@# need to worry about folly breaking our integration. Update periodically
|
|
2495
|
-
cd third-party/folly && git reset --hard
|
|
2495
|
+
cd third-party/folly && git reset --hard 5c626dd6a028a02e461edb5396694d48305e9284
|
|
2496
2496
|
@# Apparently missing include
|
|
2497
2497
|
perl -pi -e 's/(#include <atomic>)/$$1\n#include <cstring>/' third-party/folly/folly/lang/Exception.h
|
|
2498
2498
|
@# Warning-as-error on memcpy
|
|
@@ -130,7 +130,7 @@ CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() {
|
|
|
130
130
|
"Secondary cache reserved: %zu\n",
|
|
131
131
|
pri_cache_res_->GetTotalMemoryUsed(), sec_capacity,
|
|
132
132
|
sec_reserved_);
|
|
133
|
-
assert(pri_cache_res_mismatch);
|
|
133
|
+
assert(!pri_cache_res_mismatch);
|
|
134
134
|
}
|
|
135
135
|
}
|
|
136
136
|
#endif // NDEBUG
|
|
@@ -489,12 +489,10 @@ const char* CacheWithSecondaryAdapter::Name() const {
|
|
|
489
489
|
// as well. At the moment, we don't have a good way of handling the case
|
|
490
490
|
// where the new capacity < total cache reservations.
|
|
491
491
|
void CacheWithSecondaryAdapter::SetCapacity(size_t capacity) {
|
|
492
|
-
size_t sec_capacity = static_cast<size_t>(
|
|
493
|
-
capacity * (distribute_cache_res_ ? sec_cache_res_ratio_ : 0.0));
|
|
494
|
-
size_t old_sec_capacity = 0;
|
|
495
|
-
|
|
496
492
|
if (distribute_cache_res_) {
|
|
497
493
|
MutexLock m(&cache_res_mutex_);
|
|
494
|
+
size_t sec_capacity = static_cast<size_t>(capacity * sec_cache_res_ratio_);
|
|
495
|
+
size_t old_sec_capacity = 0;
|
|
498
496
|
|
|
499
497
|
Status s = secondary_cache_->GetCapacity(old_sec_capacity);
|
|
500
498
|
if (!s.ok()) {
|
|
@@ -613,6 +611,7 @@ Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
|
|
|
613
611
|
// cache utilization (increase in capacity - increase in share of cache
|
|
614
612
|
// reservation)
|
|
615
613
|
// 3. Increase secondary cache capacity
|
|
614
|
+
assert(new_sec_reserved >= sec_reserved_);
|
|
616
615
|
s = secondary_cache_->Deflate(new_sec_reserved - sec_reserved_);
|
|
617
616
|
assert(s.ok());
|
|
618
617
|
s = pri_cache_res_->UpdateCacheReservation(
|
|
@@ -925,6 +925,10 @@ void rocksdb_backup_engine_options_destroy(
|
|
|
925
925
|
delete options;
|
|
926
926
|
}
|
|
927
927
|
|
|
928
|
+
void rocksdb_status_ptr_get_error(rocksdb_status_ptr_t* status, char** errptr) {
|
|
929
|
+
SaveError(errptr, *(status->rep));
|
|
930
|
+
}
|
|
931
|
+
|
|
928
932
|
rocksdb_checkpoint_t* rocksdb_checkpoint_object_create(rocksdb_t* db,
|
|
929
933
|
char** errptr) {
|
|
930
934
|
Checkpoint* checkpoint;
|
|
@@ -3061,6 +3065,10 @@ uint64_t rocksdb_flushjobinfo_smallest_seqno(
|
|
|
3061
3065
|
return info->rep.smallest_seqno;
|
|
3062
3066
|
}
|
|
3063
3067
|
|
|
3068
|
+
uint32_t rocksdb_flushjobinfo_flush_reason(const rocksdb_flushjobinfo_t* info) {
|
|
3069
|
+
return static_cast<uint32_t>(info->rep.flush_reason);
|
|
3070
|
+
}
|
|
3071
|
+
|
|
3064
3072
|
void rocksdb_reset_status(rocksdb_status_ptr_t* status_ptr) {
|
|
3065
3073
|
auto ptr = status_ptr->rep;
|
|
3066
3074
|
*ptr = Status::OK();
|
|
@@ -3192,6 +3200,11 @@ int rocksdb_subcompactionjobinfo_output_level(
|
|
|
3192
3200
|
return info->rep.output_level;
|
|
3193
3201
|
}
|
|
3194
3202
|
|
|
3203
|
+
uint32_t rocksdb_subcompactionjobinfo_compaction_reason(
|
|
3204
|
+
const rocksdb_subcompactionjobinfo_t* info) {
|
|
3205
|
+
return static_cast<uint32_t>(info->rep.compaction_reason);
|
|
3206
|
+
}
|
|
3207
|
+
|
|
3195
3208
|
/* ExternalFileIngestionInfo */
|
|
3196
3209
|
|
|
3197
3210
|
const char* rocksdb_externalfileingestioninfo_cf_name(
|
|
@@ -2230,10 +2230,6 @@ int main(int argc, char** argv) {
|
|
|
2230
2230
|
rocksdb_options_set_skip_stats_update_on_db_open(o, 1);
|
|
2231
2231
|
CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));
|
|
2232
2232
|
|
|
2233
|
-
rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(o, 1);
|
|
2234
|
-
CheckCondition(
|
|
2235
|
-
1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o));
|
|
2236
|
-
|
|
2237
2233
|
rocksdb_options_set_max_write_buffer_number(o, 97);
|
|
2238
2234
|
CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o));
|
|
2239
2235
|
|
|
@@ -2493,8 +2489,6 @@ int main(int argc, char** argv) {
|
|
|
2493
2489
|
CheckCondition(2.0 ==
|
|
2494
2490
|
rocksdb_options_get_max_bytes_for_level_multiplier(copy));
|
|
2495
2491
|
CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(copy));
|
|
2496
|
-
CheckCondition(
|
|
2497
|
-
1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy));
|
|
2498
2492
|
CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(copy));
|
|
2499
2493
|
CheckCondition(23 ==
|
|
2500
2494
|
rocksdb_options_get_min_write_buffer_number_to_merge(copy));
|
|
@@ -2681,12 +2675,6 @@ int main(int argc, char** argv) {
|
|
|
2681
2675
|
CheckCondition(0 == rocksdb_options_get_skip_stats_update_on_db_open(copy));
|
|
2682
2676
|
CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));
|
|
2683
2677
|
|
|
2684
|
-
rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(copy, 0);
|
|
2685
|
-
CheckCondition(
|
|
2686
|
-
0 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy));
|
|
2687
|
-
CheckCondition(
|
|
2688
|
-
1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o));
|
|
2689
|
-
|
|
2690
2678
|
rocksdb_options_set_max_write_buffer_number(copy, 2000);
|
|
2691
2679
|
CheckCondition(2000 == rocksdb_options_get_max_write_buffer_number(copy));
|
|
2692
2680
|
CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o));
|
|
@@ -284,9 +284,9 @@ Compaction::Compaction(
|
|
|
284
284
|
CompressionOptions _compression_opts, Temperature _output_temperature,
|
|
285
285
|
uint32_t _max_subcompactions, std::vector<FileMetaData*> _grandparents,
|
|
286
286
|
std::optional<SequenceNumber> _earliest_snapshot,
|
|
287
|
-
const SnapshotChecker* _snapshot_checker,
|
|
288
|
-
const std::string& _trim_ts,
|
|
289
|
-
|
|
287
|
+
const SnapshotChecker* _snapshot_checker,
|
|
288
|
+
CompactionReason _compaction_reason, const std::string& _trim_ts,
|
|
289
|
+
double _score, bool l0_files_might_overlap,
|
|
290
290
|
BlobGarbageCollectionPolicy _blob_garbage_collection_policy,
|
|
291
291
|
double _blob_garbage_collection_age_cutoff)
|
|
292
292
|
: input_vstorage_(vstorage),
|
|
@@ -304,7 +304,9 @@ Compaction::Compaction(
|
|
|
304
304
|
output_compression_(_compression),
|
|
305
305
|
output_compression_opts_(_compression_opts),
|
|
306
306
|
output_temperature_(_output_temperature),
|
|
307
|
-
deletion_compaction_(
|
|
307
|
+
deletion_compaction_(_compaction_reason == CompactionReason::kFIFOTtl ||
|
|
308
|
+
_compaction_reason ==
|
|
309
|
+
CompactionReason::kFIFOMaxSize),
|
|
308
310
|
l0_files_might_overlap_(l0_files_might_overlap),
|
|
309
311
|
inputs_(PopulateWithAtomicBoundaries(vstorage, std::move(_inputs))),
|
|
310
312
|
grandparents_(std::move(_grandparents)),
|
|
@@ -321,7 +323,8 @@ Compaction::Compaction(
|
|
|
321
323
|
? false
|
|
322
324
|
: IsBottommostLevel(output_level_, vstorage, inputs_)),
|
|
323
325
|
is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
|
|
324
|
-
is_manual_compaction_(
|
|
326
|
+
is_manual_compaction_(_compaction_reason ==
|
|
327
|
+
CompactionReason::kManualCompaction),
|
|
325
328
|
trim_ts_(_trim_ts),
|
|
326
329
|
is_trivial_move_(false),
|
|
327
330
|
compaction_reason_(_compaction_reason),
|
|
@@ -349,9 +352,6 @@ Compaction::Compaction(
|
|
|
349
352
|
immutable_options_, start_level_,
|
|
350
353
|
output_level_)) {
|
|
351
354
|
MarkFilesBeingCompacted(true);
|
|
352
|
-
if (is_manual_compaction_) {
|
|
353
|
-
compaction_reason_ = CompactionReason::kManualCompaction;
|
|
354
|
-
}
|
|
355
355
|
if (max_subcompactions_ == 0) {
|
|
356
356
|
max_subcompactions_ = _mutable_db_options.max_subcompactions;
|
|
357
357
|
}
|
|
@@ -94,10 +94,9 @@ class Compaction {
|
|
|
94
94
|
std::vector<FileMetaData*> grandparents,
|
|
95
95
|
std::optional<SequenceNumber> earliest_snapshot,
|
|
96
96
|
const SnapshotChecker* snapshot_checker,
|
|
97
|
-
|
|
98
|
-
|
|
97
|
+
CompactionReason compaction_reason,
|
|
98
|
+
const std::string& trim_ts = "", double score = -1,
|
|
99
99
|
bool l0_files_might_overlap = true,
|
|
100
|
-
CompactionReason compaction_reason = CompactionReason::kUnknown,
|
|
101
100
|
BlobGarbageCollectionPolicy blob_garbage_collection_policy =
|
|
102
101
|
BlobGarbageCollectionPolicy::kUseDefault,
|
|
103
102
|
double blob_garbage_collection_age_cutoff = -1);
|
|
@@ -1736,14 +1736,11 @@ Status CompactionJob::FinishCompactionOutputFile(
|
|
|
1736
1736
|
if (s.ok()) {
|
|
1737
1737
|
tp = outputs.GetTableProperties();
|
|
1738
1738
|
}
|
|
1739
|
-
|
|
1740
1739
|
if (s.ok() && current_entries == 0 && tp.num_range_deletions == 0) {
|
|
1741
1740
|
// If there is nothing to output, no necessary to generate a sst file.
|
|
1742
1741
|
// This happens when the output level is bottom level, at the same time
|
|
1743
1742
|
// the sub_compact output nothing.
|
|
1744
|
-
std::string fname =
|
|
1745
|
-
TableFileName(sub_compact->compaction->immutable_options().cf_paths,
|
|
1746
|
-
meta->fd.GetNumber(), meta->fd.GetPathId());
|
|
1743
|
+
std::string fname = GetTableFileName(meta->fd.GetNumber());
|
|
1747
1744
|
|
|
1748
1745
|
// TODO(AR) it is not clear if there are any larger implications if
|
|
1749
1746
|
// DeleteFile fails here
|
|
@@ -1942,6 +1939,10 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
|
|
|
1942
1939
|
|
|
1943
1940
|
// no need to lock because VersionSet::next_file_number_ is atomic
|
|
1944
1941
|
uint64_t file_number = versions_->NewFileNumber();
|
|
1942
|
+
#ifndef NDEBUG
|
|
1943
|
+
TEST_SYNC_POINT_CALLBACK(
|
|
1944
|
+
"CompactionJob::OpenCompactionOutputFile::NewFileNumber", &file_number);
|
|
1945
|
+
#endif
|
|
1945
1946
|
std::string fname = GetTableFileName(file_number);
|
|
1946
1947
|
// Fire events.
|
|
1947
1948
|
ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
|
|
@@ -651,7 +651,8 @@ class CompactionJobTestBase : public testing::Test {
|
|
|
651
651
|
mutable_cf_options_.max_compaction_bytes, 0, kNoCompression,
|
|
652
652
|
cfd->GetLatestMutableCFOptions().compression_opts,
|
|
653
653
|
Temperature::kUnknown, max_subcompactions, grandparents,
|
|
654
|
-
/*earliest_snapshot*/ std::nullopt, /*snapshot_checker*/ nullptr,
|
|
654
|
+
/*earliest_snapshot*/ std::nullopt, /*snapshot_checker*/ nullptr,
|
|
655
|
+
CompactionReason::kManualCompaction);
|
|
655
656
|
compaction.FinalizeInputInfo(cfd->current());
|
|
656
657
|
|
|
657
658
|
assert(db_options_.info_log);
|
|
@@ -337,7 +337,9 @@ Compaction* CompactionPicker::CompactFiles(
|
|
|
337
337
|
const CompactionOptions& compact_options,
|
|
338
338
|
const std::vector<CompactionInputFiles>& input_files, int output_level,
|
|
339
339
|
VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
|
|
340
|
-
const MutableDBOptions& mutable_db_options, uint32_t output_path_id
|
|
340
|
+
const MutableDBOptions& mutable_db_options, uint32_t output_path_id,
|
|
341
|
+
std::optional<SequenceNumber> earliest_snapshot,
|
|
342
|
+
const SnapshotChecker* snapshot_checker) {
|
|
341
343
|
#ifndef NDEBUG
|
|
342
344
|
assert(input_files.size());
|
|
343
345
|
// This compaction output should not overlap with a running compaction as
|
|
@@ -380,8 +382,8 @@ Compaction* CompactionPicker::CompactFiles(
|
|
|
380
382
|
GetCompressionOptions(mutable_cf_options, vstorage, output_level),
|
|
381
383
|
mutable_cf_options.default_write_temperature,
|
|
382
384
|
compact_options.max_subcompactions,
|
|
383
|
-
/* grandparents */ {},
|
|
384
|
-
|
|
385
|
+
/* grandparents */ {}, earliest_snapshot, snapshot_checker,
|
|
386
|
+
CompactionReason::kManualCompaction);
|
|
385
387
|
RegisterCompaction(c);
|
|
386
388
|
return c;
|
|
387
389
|
}
|
|
@@ -680,10 +682,9 @@ Compaction* CompactionPicker::CompactRange(
|
|
|
680
682
|
mutable_cf_options.default_write_temperature,
|
|
681
683
|
compact_range_options.max_subcompactions,
|
|
682
684
|
/* grandparents */ {}, /* earliest_snapshot */ std::nullopt,
|
|
683
|
-
/* snapshot_checker */ nullptr,
|
|
684
|
-
|
|
685
|
-
/*
|
|
686
|
-
CompactionReason::kUnknown,
|
|
685
|
+
/* snapshot_checker */ nullptr, CompactionReason::kManualCompaction,
|
|
686
|
+
trim_ts, /* score */ -1,
|
|
687
|
+
/* l0_files_might_overlap */ true,
|
|
687
688
|
compact_range_options.blob_garbage_collection_policy,
|
|
688
689
|
compact_range_options.blob_garbage_collection_age_cutoff);
|
|
689
690
|
|
|
@@ -873,9 +874,8 @@ Compaction* CompactionPicker::CompactRange(
|
|
|
873
874
|
mutable_cf_options.default_write_temperature,
|
|
874
875
|
compact_range_options.max_subcompactions, std::move(grandparents),
|
|
875
876
|
/* earliest_snapshot */ std::nullopt, /* snapshot_checker */ nullptr,
|
|
876
|
-
|
|
877
|
-
/*
|
|
878
|
-
CompactionReason::kUnknown,
|
|
877
|
+
CompactionReason::kManualCompaction, trim_ts, /* score */ -1,
|
|
878
|
+
/* l0_files_might_overlap */ true,
|
|
879
879
|
compact_range_options.blob_garbage_collection_policy,
|
|
880
880
|
compact_range_options.blob_garbage_collection_age_cutoff);
|
|
881
881
|
|
|
@@ -117,12 +117,17 @@ class CompactionPicker {
|
|
|
117
117
|
// Caller must provide a set of input files that has been passed through
|
|
118
118
|
// `SanitizeAndConvertCompactionInputFiles` earlier. The lock should not be
|
|
119
119
|
// released between that call and this one.
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
120
|
+
//
|
|
121
|
+
// TODO - Remove default values for earliest_snapshot and snapshot_checker
|
|
122
|
+
// and require all callers to pass them in so that DB::CompactFiles() can
|
|
123
|
+
// also benefit from Standalone Range Tombstone Optimization
|
|
124
|
+
Compaction* CompactFiles(
|
|
125
|
+
const CompactionOptions& compact_options,
|
|
126
|
+
const std::vector<CompactionInputFiles>& input_files, int output_level,
|
|
127
|
+
VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
|
|
128
|
+
const MutableDBOptions& mutable_db_options, uint32_t output_path_id,
|
|
129
|
+
std::optional<SequenceNumber> earliest_snapshot = std::nullopt,
|
|
130
|
+
const SnapshotChecker* snapshot_checker = nullptr);
|
|
126
131
|
|
|
127
132
|
// Converts a set of compaction input file numbers into
|
|
128
133
|
// a list of CompactionInputFiles.
|
|
@@ -127,11 +127,9 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
|
|
|
127
127
|
mutable_cf_options.compression_opts,
|
|
128
128
|
mutable_cf_options.default_write_temperature,
|
|
129
129
|
/* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
|
|
130
|
-
/* snapshot_checker */ nullptr,
|
|
131
|
-
/* is manual */ false,
|
|
130
|
+
/* snapshot_checker */ nullptr, CompactionReason::kFIFOTtl,
|
|
132
131
|
/* trim_ts */ "", vstorage->CompactionScore(0),
|
|
133
|
-
/*
|
|
134
|
-
CompactionReason::kFIFOTtl);
|
|
132
|
+
/* l0_files_might_overlap */ true);
|
|
135
133
|
return c;
|
|
136
134
|
}
|
|
137
135
|
|
|
@@ -200,11 +198,10 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
|
|
|
200
198
|
mutable_cf_options.default_write_temperature,
|
|
201
199
|
0 /* max_subcompactions */, {},
|
|
202
200
|
/* earliest_snapshot */ std::nullopt,
|
|
203
|
-
/* snapshot_checker */ nullptr,
|
|
201
|
+
/* snapshot_checker */ nullptr,
|
|
202
|
+
CompactionReason::kFIFOReduceNumFiles,
|
|
204
203
|
/* trim_ts */ "", vstorage->CompactionScore(0),
|
|
205
|
-
/*
|
|
206
|
-
/* l0_files_might_overlap */ true,
|
|
207
|
-
CompactionReason::kFIFOReduceNumFiles);
|
|
204
|
+
/* l0_files_might_overlap */ true);
|
|
208
205
|
return c;
|
|
209
206
|
}
|
|
210
207
|
}
|
|
@@ -297,11 +294,9 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
|
|
|
297
294
|
mutable_cf_options.compression_opts,
|
|
298
295
|
mutable_cf_options.default_write_temperature,
|
|
299
296
|
/* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
|
|
300
|
-
/* snapshot_checker */ nullptr,
|
|
301
|
-
/* is manual */ false,
|
|
297
|
+
/* snapshot_checker */ nullptr, CompactionReason::kFIFOMaxSize,
|
|
302
298
|
/* trim_ts */ "", vstorage->CompactionScore(0),
|
|
303
|
-
/*
|
|
304
|
-
/* l0_files_might_overlap */ true, CompactionReason::kFIFOMaxSize);
|
|
299
|
+
/* l0_files_might_overlap */ true);
|
|
305
300
|
return c;
|
|
306
301
|
}
|
|
307
302
|
|
|
@@ -416,10 +411,9 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
|
|
|
416
411
|
mutable_cf_options.compression, mutable_cf_options.compression_opts,
|
|
417
412
|
compaction_target_temp,
|
|
418
413
|
/* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
|
|
419
|
-
/* snapshot_checker */ nullptr,
|
|
420
|
-
/*
|
|
421
|
-
/*
|
|
422
|
-
CompactionReason::kChangeTemperature);
|
|
414
|
+
/* snapshot_checker */ nullptr, CompactionReason::kChangeTemperature,
|
|
415
|
+
/* trim_ts */ "", vstorage->CompactionScore(0),
|
|
416
|
+
/* l0_files_might_overlap */ true);
|
|
423
417
|
return c;
|
|
424
418
|
}
|
|
425
419
|
|
|
@@ -145,7 +145,6 @@ class LevelCompactionBuilder {
|
|
|
145
145
|
int parent_index_ = -1;
|
|
146
146
|
int base_index_ = -1;
|
|
147
147
|
double start_level_score_ = 0;
|
|
148
|
-
bool is_manual_ = false;
|
|
149
148
|
bool is_l0_trivial_move_ = false;
|
|
150
149
|
CompactionInputFiles start_level_inputs_;
|
|
151
150
|
std::vector<CompactionInputFiles> compaction_inputs_;
|
|
@@ -561,9 +560,8 @@ Compaction* LevelCompactionBuilder::GetCompaction() {
|
|
|
561
560
|
mutable_cf_options_.default_write_temperature,
|
|
562
561
|
/* max_subcompactions */ 0, std::move(grandparents_),
|
|
563
562
|
/* earliest_snapshot */ std::nullopt, /* snapshot_checker */ nullptr,
|
|
564
|
-
|
|
565
|
-
/* trim_ts */ "", start_level_score_,
|
|
566
|
-
l0_files_might_overlap, compaction_reason_);
|
|
563
|
+
compaction_reason_,
|
|
564
|
+
/* trim_ts */ "", start_level_score_, l0_files_might_overlap);
|
|
567
565
|
|
|
568
566
|
// If it's level 0 compaction, make sure we don't execute any other level 0
|
|
569
567
|
// compactions in parallel
|
|
@@ -1097,10 +1097,9 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
|
|
|
1097
1097
|
mutable_cf_options_.default_write_temperature,
|
|
1098
1098
|
/* max_subcompactions */ 0, grandparents,
|
|
1099
1099
|
/* earliest_snapshot */ std::nullopt,
|
|
1100
|
-
/* snapshot_checker */ nullptr,
|
|
1101
|
-
/*
|
|
1102
|
-
|
|
1103
|
-
/* l0_files_might_overlap */ true, compaction_reason);
|
|
1100
|
+
/* snapshot_checker */ nullptr, compaction_reason,
|
|
1101
|
+
/* trim_ts */ "", score_,
|
|
1102
|
+
/* l0_files_might_overlap */ true);
|
|
1104
1103
|
}
|
|
1105
1104
|
|
|
1106
1105
|
// Look at overall size amplification. If size amplification
|
|
@@ -1447,10 +1446,9 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
|
|
|
1447
1446
|
/* max_subcompactions */ 0, /* grandparents */ {},
|
|
1448
1447
|
/* earliest_snapshot */ std::nullopt,
|
|
1449
1448
|
/* snapshot_checker */ nullptr,
|
|
1450
|
-
|
|
1451
|
-
/* trim_ts */ "", score_,
|
|
1452
|
-
/* l0_files_might_overlap */ true
|
|
1453
|
-
CompactionReason::kUniversalSizeAmplification);
|
|
1449
|
+
CompactionReason::kUniversalSizeAmplification,
|
|
1450
|
+
/* trim_ts */ "", score_,
|
|
1451
|
+
/* l0_files_might_overlap */ true);
|
|
1454
1452
|
}
|
|
1455
1453
|
|
|
1456
1454
|
// Pick files marked for compaction. Typically, files are marked by
|
|
@@ -1600,11 +1598,9 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
|
|
|
1600
1598
|
GetCompressionOptions(mutable_cf_options_, vstorage_, output_level),
|
|
1601
1599
|
mutable_cf_options_.default_write_temperature,
|
|
1602
1600
|
/* max_subcompactions */ 0, grandparents, earliest_snapshot_,
|
|
1603
|
-
snapshot_checker_,
|
|
1604
|
-
/*
|
|
1605
|
-
/*
|
|
1606
|
-
/* l0_files_might_overlap */ true,
|
|
1607
|
-
CompactionReason::kFilesMarkedForCompaction);
|
|
1601
|
+
snapshot_checker_, CompactionReason::kFilesMarkedForCompaction,
|
|
1602
|
+
/* trim_ts */ "", score_,
|
|
1603
|
+
/* l0_files_might_overlap */ true);
|
|
1608
1604
|
}
|
|
1609
1605
|
|
|
1610
1606
|
Compaction* UniversalCompactionBuilder::PickCompactionToOldest(
|
|
@@ -1700,10 +1696,9 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
|
|
|
1700
1696
|
mutable_cf_options_.default_write_temperature,
|
|
1701
1697
|
/* max_subcompactions */ 0, /* grandparents */ {},
|
|
1702
1698
|
/* earliest_snapshot */ std::nullopt,
|
|
1703
|
-
/* snapshot_checker */ nullptr,
|
|
1704
|
-
/*
|
|
1705
|
-
/*
|
|
1706
|
-
/* l0_files_might_overlap */ true, compaction_reason);
|
|
1699
|
+
/* snapshot_checker */ nullptr, compaction_reason,
|
|
1700
|
+
/* trim_ts */ "", score_,
|
|
1701
|
+
/* l0_files_might_overlap */ true);
|
|
1707
1702
|
}
|
|
1708
1703
|
|
|
1709
1704
|
Compaction* UniversalCompactionBuilder::PickPeriodicCompaction() {
|
|
@@ -461,6 +461,97 @@ TEST_F(CompactionServiceTest, ManualCompaction) {
|
|
|
461
461
|
ASSERT_EQ(handles_[1]->GetName(), info.cf_name);
|
|
462
462
|
}
|
|
463
463
|
|
|
464
|
+
TEST_F(CompactionServiceTest, StandaloneDeleteRangeTombstoneOptimization) {
|
|
465
|
+
Options options = CurrentOptions();
|
|
466
|
+
options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
|
|
467
|
+
ReopenWithCompactionService(&options);
|
|
468
|
+
|
|
469
|
+
size_t num_files_after_filtered = 0;
|
|
470
|
+
SyncPoint::GetInstance()->SetCallBack(
|
|
471
|
+
"VersionSet::MakeInputIterator:NewCompactionMergingIterator",
|
|
472
|
+
[&](void* arg) {
|
|
473
|
+
num_files_after_filtered = *static_cast<size_t*>(arg);
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
SyncPoint::GetInstance()->EnableProcessing();
|
|
477
|
+
|
|
478
|
+
std::vector<std::string> files;
|
|
479
|
+
{
|
|
480
|
+
// Writes first version of data in range partitioned files.
|
|
481
|
+
SstFileWriter sst_file_writer(EnvOptions(), options);
|
|
482
|
+
std::string file1 = dbname_ + "file1.sst";
|
|
483
|
+
ASSERT_OK(sst_file_writer.Open(file1));
|
|
484
|
+
ASSERT_OK(sst_file_writer.Put("a", "a1"));
|
|
485
|
+
ASSERT_OK(sst_file_writer.Put("b", "b1"));
|
|
486
|
+
ExternalSstFileInfo file1_info;
|
|
487
|
+
ASSERT_OK(sst_file_writer.Finish(&file1_info));
|
|
488
|
+
files.push_back(std::move(file1));
|
|
489
|
+
|
|
490
|
+
std::string file2 = dbname_ + "file2.sst";
|
|
491
|
+
ASSERT_OK(sst_file_writer.Open(file2));
|
|
492
|
+
ASSERT_OK(sst_file_writer.Put("x", "x1"));
|
|
493
|
+
ASSERT_OK(sst_file_writer.Put("y", "y1"));
|
|
494
|
+
ExternalSstFileInfo file2_info;
|
|
495
|
+
ASSERT_OK(sst_file_writer.Finish(&file2_info));
|
|
496
|
+
files.push_back(std::move(file2));
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
IngestExternalFileOptions ifo;
|
|
500
|
+
ASSERT_OK(db_->IngestExternalFile(files, ifo));
|
|
501
|
+
ASSERT_EQ(Get("a"), "a1");
|
|
502
|
+
ASSERT_EQ(Get("b"), "b1");
|
|
503
|
+
ASSERT_EQ(Get("x"), "x1");
|
|
504
|
+
ASSERT_EQ(Get("y"), "y1");
|
|
505
|
+
ASSERT_EQ(2, NumTableFilesAtLevel(6));
|
|
506
|
+
|
|
507
|
+
auto my_cs = GetCompactionService();
|
|
508
|
+
uint64_t comp_num = my_cs->GetCompactionNum();
|
|
509
|
+
|
|
510
|
+
{
|
|
511
|
+
// Atomically delete old version of data with one range delete file.
|
|
512
|
+
// And a new batch of range partitioned files with new version of data.
|
|
513
|
+
files.clear();
|
|
514
|
+
SstFileWriter sst_file_writer(EnvOptions(), options);
|
|
515
|
+
std::string file2 = dbname_ + "file2.sst";
|
|
516
|
+
ASSERT_OK(sst_file_writer.Open(file2));
|
|
517
|
+
ASSERT_OK(sst_file_writer.DeleteRange("a", "z"));
|
|
518
|
+
ExternalSstFileInfo file2_info;
|
|
519
|
+
ASSERT_OK(sst_file_writer.Finish(&file2_info));
|
|
520
|
+
files.push_back(std::move(file2));
|
|
521
|
+
|
|
522
|
+
std::string file3 = dbname_ + "file3.sst";
|
|
523
|
+
ASSERT_OK(sst_file_writer.Open(file3));
|
|
524
|
+
ASSERT_OK(sst_file_writer.Put("a", "a2"));
|
|
525
|
+
ASSERT_OK(sst_file_writer.Put("b", "b2"));
|
|
526
|
+
ExternalSstFileInfo file3_info;
|
|
527
|
+
ASSERT_OK(sst_file_writer.Finish(&file3_info));
|
|
528
|
+
files.push_back(std::move(file3));
|
|
529
|
+
|
|
530
|
+
std::string file4 = dbname_ + "file4.sst";
|
|
531
|
+
ASSERT_OK(sst_file_writer.Open(file4));
|
|
532
|
+
ASSERT_OK(sst_file_writer.Put("x", "x2"));
|
|
533
|
+
ASSERT_OK(sst_file_writer.Put("y", "y2"));
|
|
534
|
+
ExternalSstFileInfo file4_info;
|
|
535
|
+
ASSERT_OK(sst_file_writer.Finish(&file4_info));
|
|
536
|
+
files.push_back(std::move(file4));
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
ASSERT_OK(db_->IngestExternalFile(files, ifo));
|
|
540
|
+
ASSERT_OK(db_->WaitForCompact(WaitForCompactOptions()));
|
|
541
|
+
ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
|
|
542
|
+
|
|
543
|
+
CompactionServiceResult result;
|
|
544
|
+
my_cs->GetResult(&result);
|
|
545
|
+
ASSERT_OK(result.status);
|
|
546
|
+
ASSERT_TRUE(result.stats.is_manual_compaction);
|
|
547
|
+
ASSERT_TRUE(result.stats.is_remote_compaction);
|
|
548
|
+
|
|
549
|
+
ASSERT_EQ(num_files_after_filtered, 1);
|
|
550
|
+
|
|
551
|
+
Close();
|
|
552
|
+
SyncPoint::GetInstance()->DisableProcessing();
|
|
553
|
+
}
|
|
554
|
+
|
|
464
555
|
TEST_F(CompactionServiceTest, CompactionOutputFileIOError) {
|
|
465
556
|
Options options = CurrentOptions();
|
|
466
557
|
options.disable_auto_compactions = true;
|
|
@@ -793,6 +884,79 @@ TEST_F(CompactionServiceTest, VerifyInputRecordCount) {
|
|
|
793
884
|
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
794
885
|
}
|
|
795
886
|
|
|
887
|
+
TEST_F(CompactionServiceTest, EmptyResult) {
|
|
888
|
+
Options options = CurrentOptions();
|
|
889
|
+
options.disable_auto_compactions = true;
|
|
890
|
+
ReopenWithCompactionService(&options);
|
|
891
|
+
GenerateTestData();
|
|
892
|
+
|
|
893
|
+
auto my_cs = GetCompactionService();
|
|
894
|
+
|
|
895
|
+
uint64_t comp_num = my_cs->GetCompactionNum();
|
|
896
|
+
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
897
|
+
ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
|
|
898
|
+
|
|
899
|
+
// Delete range to cover entire range
|
|
900
|
+
ASSERT_OK(db_->DeleteRange(WriteOptions(), "key", "keyz"));
|
|
901
|
+
ASSERT_OK(Flush());
|
|
902
|
+
|
|
903
|
+
// In this unit test, both remote compaction and primary db instance are
|
|
904
|
+
// running in the same process, so NewFileNumber will never have a collision.
|
|
905
|
+
// In the real-world remote compactions, when the compaction is indeed running
|
|
906
|
+
// in another process, this is not going to be the case.
|
|
907
|
+
// To simulate the SST file with the same name created in the tmp directory,
|
|
908
|
+
// override the file number in remote compaction to re-use old SST file
|
|
909
|
+
// number.
|
|
910
|
+
bool need_to_override_file_number = false;
|
|
911
|
+
SyncPoint::GetInstance()->SetCallBack(
|
|
912
|
+
"DBImplSecondary::OpenAndCompact::BeforeLoadingOptions:0",
|
|
913
|
+
[&](void*) { need_to_override_file_number = true; });
|
|
914
|
+
|
|
915
|
+
SyncPoint::GetInstance()->SetCallBack(
|
|
916
|
+
"CompactionJob::OpenCompactionOutputFile::NewFileNumber",
|
|
917
|
+
[&](void* file_number) {
|
|
918
|
+
if (need_to_override_file_number) {
|
|
919
|
+
auto n = static_cast<uint64_t*>(file_number);
|
|
920
|
+
ColumnFamilyMetaData cf_meta;
|
|
921
|
+
db_->GetColumnFamilyMetaData(&cf_meta);
|
|
922
|
+
for (const auto& level : cf_meta.levels) {
|
|
923
|
+
for (const auto& file : level.files) {
|
|
924
|
+
// Use one of the existing file name
|
|
925
|
+
*n = test::GetFileNumber(file.name);
|
|
926
|
+
need_to_override_file_number = false;
|
|
927
|
+
return;
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
});
|
|
932
|
+
|
|
933
|
+
// Inject failure, so that the remote compaction fails after
|
|
934
|
+
// ProcessKeyValueCompaction()
|
|
935
|
+
SyncPoint::GetInstance()->SetCallBack(
|
|
936
|
+
"DBImplSecondary::CompactWithoutInstallation::End", [&](void* status) {
|
|
937
|
+
// override job status
|
|
938
|
+
auto s = static_cast<Status*>(status);
|
|
939
|
+
*s = Status::Aborted("MyTestCompactionService failed to compact!");
|
|
940
|
+
});
|
|
941
|
+
SyncPoint::GetInstance()->EnableProcessing();
|
|
942
|
+
|
|
943
|
+
// Compaction should fail and SST files in the primary db should exist
|
|
944
|
+
{
|
|
945
|
+
ASSERT_NOK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
946
|
+
ColumnFamilyMetaData meta;
|
|
947
|
+
db_->GetColumnFamilyMetaData(&meta);
|
|
948
|
+
for (const auto& level : meta.levels) {
|
|
949
|
+
for (const auto& file : level.files) {
|
|
950
|
+
std::string fname = file.db_path + "/" + file.name;
|
|
951
|
+
ASSERT_OK(db_->GetEnv()->FileExists(fname));
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
Close();
|
|
956
|
+
SyncPoint::GetInstance()->DisableProcessing();
|
|
957
|
+
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
958
|
+
}
|
|
959
|
+
|
|
796
960
|
TEST_F(CompactionServiceTest, CorruptedOutput) {
|
|
797
961
|
Options options = CurrentOptions();
|
|
798
962
|
options.disable_auto_compactions = true;
|