@nxtedition/rocksdb 7.1.14 → 7.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +1 -0
- package/deps/rocksdb/rocksdb/CMakeLists.txt +72 -18
- package/deps/rocksdb/rocksdb/Makefile +91 -11
- package/deps/rocksdb/rocksdb/TARGETS +8 -4
- package/deps/rocksdb/rocksdb/cache/cache.cc +5 -0
- package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +13 -8
- package/deps/rocksdb/rocksdb/cache/cache_entry_roles.cc +2 -0
- package/deps/rocksdb/rocksdb/cache/cache_test.cc +116 -57
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +958 -459
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +407 -622
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +104 -40
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h +23 -8
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +350 -184
- package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc +12 -2
- package/deps/rocksdb/rocksdb/cache/fast_lru_cache.h +2 -0
- package/deps/rocksdb/rocksdb/cache/lru_cache.cc +130 -43
- package/deps/rocksdb/rocksdb/cache/lru_cache.h +24 -2
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +423 -98
- package/deps/rocksdb/rocksdb/cache/sharded_cache.cc +19 -2
- package/deps/rocksdb/rocksdb/cache/sharded_cache.h +10 -7
- package/deps/rocksdb/rocksdb/crash_test.mk +2 -2
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +46 -26
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +9 -3
- package/deps/rocksdb/rocksdb/db/blob/blob_contents.cc +90 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_contents.h +56 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +23 -10
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +64 -59
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.h +11 -8
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +92 -62
- package/deps/rocksdb/rocksdb/db/blob/blob_source.cc +159 -136
- package/deps/rocksdb/rocksdb/db/blob/blob_source.h +13 -13
- package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +129 -57
- package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc +81 -3
- package/deps/rocksdb/rocksdb/db/c.cc +29 -0
- package/deps/rocksdb/rocksdb/db/column_family.cc +10 -1
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +21 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +42 -36
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +344 -102
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +163 -28
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +52 -17
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +35 -30
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +8 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +167 -11
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +8 -8
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +10 -13
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +0 -117
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +6 -49
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +29 -4
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +18 -11
- package/deps/rocksdb/rocksdb/db/db_compaction_filter_test.cc +4 -10
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +12 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +144 -93
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +28 -32
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +5 -9
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +2 -33
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +3 -5
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +11 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +1 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +8 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +2 -1
- package/deps/rocksdb/rocksdb/db/db_iter.cc +76 -138
- package/deps/rocksdb/rocksdb/db/db_iter.h +26 -23
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +931 -0
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +6 -0
- package/deps/rocksdb/rocksdb/db/db_test2.cc +44 -22
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +6 -14
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +155 -0
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +45 -0
- package/deps/rocksdb/rocksdb/db/dbformat.h +2 -1
- package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +8 -0
- package/deps/rocksdb/rocksdb/db/experimental.cc +5 -1
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +24 -12
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +7 -1
- package/deps/rocksdb/rocksdb/db/internal_stats.h +3 -0
- package/deps/rocksdb/rocksdb/db/memtable.cc +79 -18
- package/deps/rocksdb/rocksdb/db/memtable.h +5 -0
- package/deps/rocksdb/rocksdb/db/memtable_list.cc +26 -4
- package/deps/rocksdb/rocksdb/db/memtable_list.h +2 -1
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +113 -0
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.h +110 -0
- package/deps/rocksdb/rocksdb/db/{periodic_work_scheduler_test.cc → periodic_task_scheduler_test.cc} +33 -39
- package/deps/rocksdb/rocksdb/db/range_del_aggregator.cc +12 -20
- package/deps/rocksdb/rocksdb/db/range_del_aggregator.h +6 -5
- package/deps/rocksdb/rocksdb/db/range_del_aggregator_test.cc +12 -8
- package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.cc +20 -5
- package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.h +14 -0
- package/deps/rocksdb/rocksdb/db/repair.cc +17 -8
- package/deps/rocksdb/rocksdb/db/repair_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +49 -66
- package/deps/rocksdb/rocksdb/db/table_cache.cc +92 -63
- package/deps/rocksdb/rocksdb/db/table_cache.h +16 -9
- package/deps/rocksdb/rocksdb/db/table_cache_sync_and_async.h +2 -2
- package/deps/rocksdb/rocksdb/db/table_properties_collector.cc +2 -2
- package/deps/rocksdb/rocksdb/db/table_properties_collector.h +3 -3
- package/deps/rocksdb/rocksdb/db/table_properties_collector_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/version_builder.cc +1 -1
- package/deps/rocksdb/rocksdb/db/version_edit.h +1 -2
- package/deps/rocksdb/rocksdb/db/version_set.cc +379 -145
- package/deps/rocksdb/rocksdb/db/version_set.h +26 -24
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +9 -9
- package/deps/rocksdb/rocksdb/db/version_util.h +3 -2
- package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +10 -2
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.cc +2 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/batched_ops_stress.cc +5 -8
- package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +5 -8
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress.cc +2 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +71 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +14 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +23 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +26 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +105 -34
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +16 -8
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +6 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +4 -8
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.h +4 -8
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +282 -25
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +6 -4
- package/deps/rocksdb/rocksdb/env/io_posix.cc +3 -1
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +367 -177
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +144 -56
- package/deps/rocksdb/rocksdb/file/filename.cc +3 -3
- package/deps/rocksdb/rocksdb/file/filename.h +4 -2
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +415 -0
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +2 -0
- package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +36 -45
- package/deps/rocksdb/rocksdb/file/writable_file_writer.h +21 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +11 -11
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +15 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +163 -68
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +26 -12
- package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +23 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +21 -17
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +17 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/persistent_cache.h +3 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/secondary_cache.h +17 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +3 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +20 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +3 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/option_change_migration.h +4 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/wide_columns.h +3 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/write_batch_base.h +2 -1
- package/deps/rocksdb/rocksdb/logging/env_logger.h +2 -2
- package/deps/rocksdb/rocksdb/monitoring/histogram.cc +4 -2
- package/deps/rocksdb/rocksdb/monitoring/histogram.h +2 -0
- package/deps/rocksdb/rocksdb/monitoring/histogram_test.cc +15 -1
- package/deps/rocksdb/rocksdb/monitoring/instrumented_mutex.cc +17 -0
- package/deps/rocksdb/rocksdb/monitoring/instrumented_mutex.h +14 -3
- package/deps/rocksdb/rocksdb/monitoring/iostats_context_imp.h +3 -0
- package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +50 -0
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +1 -0
- package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +31 -32
- package/deps/rocksdb/rocksdb/options/customizable_test.cc +4 -1
- package/deps/rocksdb/rocksdb/options/options.cc +2 -2
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +2 -1
- package/deps/rocksdb/rocksdb/port/jemalloc_helper.h +1 -0
- package/deps/rocksdb/rocksdb/src.mk +4 -2
- package/deps/rocksdb/rocksdb/table/block_based/block.h +9 -8
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +110 -99
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +12 -10
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +11 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +138 -83
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +25 -24
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +31 -30
- package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +16 -13
- package/deps/rocksdb/rocksdb/table/block_based/cachable_entry.h +4 -4
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +3 -3
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +3 -3
- package/deps/rocksdb/rocksdb/table/block_fetcher.cc +17 -19
- package/deps/rocksdb/rocksdb/table/block_fetcher.h +1 -1
- package/deps/rocksdb/rocksdb/table/format.cc +26 -29
- package/deps/rocksdb/rocksdb/table/format.h +44 -26
- package/deps/rocksdb/rocksdb/table/get_context.cc +17 -12
- package/deps/rocksdb/rocksdb/table/internal_iterator.h +7 -0
- package/deps/rocksdb/rocksdb/table/iterator_wrapper.h +4 -0
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +950 -104
- package/deps/rocksdb/rocksdb/table/merging_iterator.h +28 -1
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +3 -2
- package/deps/rocksdb/rocksdb/table/meta_blocks.h +1 -1
- package/deps/rocksdb/rocksdb/table/persistent_cache_helper.cc +10 -9
- package/deps/rocksdb/rocksdb/table/persistent_cache_helper.h +22 -20
- package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +1 -1
- package/deps/rocksdb/rocksdb/table/sst_file_writer_collectors.h +1 -1
- package/deps/rocksdb/rocksdb/table/table_builder.h +9 -21
- package/deps/rocksdb/rocksdb/table/table_test.cc +12 -12
- package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_pysim_test.py +4 -4
- package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_plot.py +1 -0
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +116 -34
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +6 -1
- package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.cc +1 -1
- package/deps/rocksdb/rocksdb/util/autovector.h +12 -0
- package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc +3 -2
- package/deps/rocksdb/rocksdb/util/stderr_logger.cc +30 -0
- package/deps/rocksdb/rocksdb/util/stderr_logger.h +5 -18
- package/deps/rocksdb/rocksdb/util/timer.h +2 -3
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +9 -2
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.cc +34 -53
- package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.h +9 -14
- package/deps/rocksdb/rocksdb/utilities/debug.cc +2 -4
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +4 -0
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/fault_injection_secondary_cache.cc +4 -3
- package/deps/rocksdb/rocksdb/utilities/fault_injection_secondary_cache.h +3 -1
- package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc +26 -8
- package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration_test.cc +114 -16
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_test.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_test.cc +59 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +3 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/timestamped_snapshot_test.cc +39 -0
- package/deps/rocksdb/rocksdb.gyp +0 -1
- package/index.js +6 -10
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
- package/deps/rocksdb/rocksdb/db/periodic_work_scheduler.cc +0 -168
- package/deps/rocksdb/rocksdb/db/periodic_work_scheduler.h +0 -90
|
@@ -37,7 +37,6 @@ TruncatedRangeDelIterator::TruncatedRangeDelIterator(
|
|
|
37
37
|
false /* log_err_key */); // TODO
|
|
38
38
|
pik_status.PermitUncheckedError();
|
|
39
39
|
assert(pik_status.ok());
|
|
40
|
-
|
|
41
40
|
smallest_ = &parsed_smallest;
|
|
42
41
|
}
|
|
43
42
|
if (largest != nullptr) {
|
|
@@ -69,12 +68,16 @@ TruncatedRangeDelIterator::TruncatedRangeDelIterator(
|
|
|
69
68
|
// the truncated end key can cover the largest key in this sstable, reduce
|
|
70
69
|
// its sequence number by 1.
|
|
71
70
|
parsed_largest.sequence -= 1;
|
|
71
|
+
// This line is not needed for correctness, but it ensures that the
|
|
72
|
+
// truncated end key is not covering keys from the next SST file.
|
|
73
|
+
parsed_largest.type = kValueTypeForSeek;
|
|
72
74
|
}
|
|
73
75
|
largest_ = &parsed_largest;
|
|
74
76
|
}
|
|
75
77
|
}
|
|
76
78
|
|
|
77
79
|
bool TruncatedRangeDelIterator::Valid() const {
|
|
80
|
+
assert(iter_ != nullptr);
|
|
78
81
|
return iter_->Valid() &&
|
|
79
82
|
(smallest_ == nullptr ||
|
|
80
83
|
icmp_->Compare(*smallest_, iter_->parsed_end_key()) < 0) &&
|
|
@@ -82,12 +85,6 @@ bool TruncatedRangeDelIterator::Valid() const {
|
|
|
82
85
|
icmp_->Compare(iter_->parsed_start_key(), *largest_) < 0);
|
|
83
86
|
}
|
|
84
87
|
|
|
85
|
-
void TruncatedRangeDelIterator::Next() { iter_->TopNext(); }
|
|
86
|
-
|
|
87
|
-
void TruncatedRangeDelIterator::Prev() { iter_->TopPrev(); }
|
|
88
|
-
|
|
89
|
-
void TruncatedRangeDelIterator::InternalNext() { iter_->Next(); }
|
|
90
|
-
|
|
91
88
|
// NOTE: target is a user key
|
|
92
89
|
void TruncatedRangeDelIterator::Seek(const Slice& target) {
|
|
93
90
|
if (largest_ != nullptr &&
|
|
@@ -149,9 +146,8 @@ TruncatedRangeDelIterator::SplitBySnapshot(
|
|
|
149
146
|
std::for_each(
|
|
150
147
|
split_untruncated_iters.begin(), split_untruncated_iters.end(),
|
|
151
148
|
[&](FragmentedIterPair& iter_pair) {
|
|
152
|
-
std::
|
|
153
|
-
|
|
154
|
-
smallest_ikey_, largest_ikey_));
|
|
149
|
+
auto truncated_iter = std::make_unique<TruncatedRangeDelIterator>(
|
|
150
|
+
std::move(iter_pair.second), icmp_, smallest_ikey_, largest_ikey_);
|
|
155
151
|
split_truncated_iters.emplace(iter_pair.first,
|
|
156
152
|
std::move(truncated_iter));
|
|
157
153
|
});
|
|
@@ -322,9 +318,8 @@ void ReadRangeDelAggregator::AddTombstones(
|
|
|
322
318
|
if (input_iter == nullptr || input_iter->empty()) {
|
|
323
319
|
return;
|
|
324
320
|
}
|
|
325
|
-
rep_.AddTombstones(
|
|
326
|
-
std::
|
|
327
|
-
std::move(input_iter), icmp_, smallest, largest)));
|
|
321
|
+
rep_.AddTombstones(std::make_unique<TruncatedRangeDelIterator>(
|
|
322
|
+
std::move(input_iter), icmp_, smallest, largest));
|
|
328
323
|
}
|
|
329
324
|
|
|
330
325
|
bool ReadRangeDelAggregator::ShouldDeleteImpl(const ParsedInternalKey& parsed,
|
|
@@ -471,19 +466,16 @@ CompactionRangeDelAggregator::NewIterator(const Slice* lower_bound,
|
|
|
471
466
|
const Slice* upper_bound,
|
|
472
467
|
bool upper_bound_inclusive) {
|
|
473
468
|
InvalidateRangeDelMapPositions();
|
|
474
|
-
std::
|
|
475
|
-
|
|
476
|
-
upper_bound_inclusive, parent_iters_));
|
|
469
|
+
auto merging_iter = std::make_unique<TruncatedRangeDelMergingIter>(
|
|
470
|
+
icmp_, lower_bound, upper_bound, upper_bound_inclusive, parent_iters_);
|
|
477
471
|
|
|
478
472
|
auto fragmented_tombstone_list =
|
|
479
473
|
std::make_shared<FragmentedRangeTombstoneList>(
|
|
480
474
|
std::move(merging_iter), *icmp_, true /* for_compaction */,
|
|
481
475
|
*snapshots_);
|
|
482
476
|
|
|
483
|
-
return std::
|
|
484
|
-
|
|
485
|
-
fragmented_tombstone_list, *icmp_,
|
|
486
|
-
kMaxSequenceNumber /* upper_bound */));
|
|
477
|
+
return std::make_unique<FragmentedRangeTombstoneIterator>(
|
|
478
|
+
fragmented_tombstone_list, *icmp_, kMaxSequenceNumber /* upper_bound */);
|
|
487
479
|
}
|
|
488
480
|
|
|
489
481
|
} // namespace ROCKSDB_NAMESPACE
|
|
@@ -38,14 +38,15 @@ class TruncatedRangeDelIterator {
|
|
|
38
38
|
|
|
39
39
|
bool Valid() const;
|
|
40
40
|
|
|
41
|
-
void Next();
|
|
42
|
-
void Prev();
|
|
41
|
+
void Next() { iter_->TopNext(); }
|
|
42
|
+
void Prev() { iter_->TopPrev(); }
|
|
43
43
|
|
|
44
|
-
void InternalNext();
|
|
44
|
+
void InternalNext() { iter_->Next(); }
|
|
45
45
|
|
|
46
46
|
// Seeks to the tombstone with the highest visible sequence number that covers
|
|
47
47
|
// target (a user key). If no such tombstone exists, the position will be at
|
|
48
48
|
// the earliest tombstone that ends after target.
|
|
49
|
+
// REQUIRES: target is a user key.
|
|
49
50
|
void Seek(const Slice& target);
|
|
50
51
|
|
|
51
52
|
// Seeks to the tombstone with the highest visible sequence number that covers
|
|
@@ -281,11 +282,11 @@ class RangeDelAggregator {
|
|
|
281
282
|
const InternalKey* smallest = nullptr,
|
|
282
283
|
const InternalKey* largest = nullptr) = 0;
|
|
283
284
|
|
|
284
|
-
bool ShouldDelete(const Slice&
|
|
285
|
+
bool ShouldDelete(const Slice& ikey, RangeDelPositioningMode mode) {
|
|
285
286
|
ParsedInternalKey parsed;
|
|
286
287
|
|
|
287
288
|
Status pik_status =
|
|
288
|
-
ParseInternalKey(
|
|
289
|
+
ParseInternalKey(ikey, &parsed, false /* log_err_key */); // TODO
|
|
289
290
|
assert(pik_status.ok());
|
|
290
291
|
if (!pik_status.ok()) {
|
|
291
292
|
return false;
|
|
@@ -76,8 +76,9 @@ ParsedInternalKey UncutEndpoint(const Slice& s) {
|
|
|
76
76
|
return ParsedInternalKey(s, kMaxSequenceNumber, kTypeRangeDeletion);
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
-
ParsedInternalKey InternalValue(const Slice& key, SequenceNumber seq
|
|
80
|
-
|
|
79
|
+
ParsedInternalKey InternalValue(const Slice& key, SequenceNumber seq,
|
|
80
|
+
ValueType type = kTypeValue) {
|
|
81
|
+
return ParsedInternalKey(key, seq, type);
|
|
81
82
|
}
|
|
82
83
|
|
|
83
84
|
void VerifyIterator(
|
|
@@ -292,16 +293,18 @@ TEST_F(RangeDelAggregatorTest, TruncatedIterPartiallyCutTombstones) {
|
|
|
292
293
|
TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp,
|
|
293
294
|
&smallest, &largest);
|
|
294
295
|
|
|
295
|
-
VerifyIterator(
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
296
|
+
VerifyIterator(
|
|
297
|
+
&iter, bytewise_icmp,
|
|
298
|
+
{{InternalValue("d", 7), UncutEndpoint("e"), 10},
|
|
299
|
+
{UncutEndpoint("e"), UncutEndpoint("g"), 8},
|
|
300
|
+
{UncutEndpoint("j"), InternalValue("m", 8, kValueTypeForSeek), 4}});
|
|
299
301
|
|
|
300
302
|
VerifySeek(
|
|
301
303
|
&iter, bytewise_icmp,
|
|
302
304
|
{{"d", InternalValue("d", 7), UncutEndpoint("e"), 10},
|
|
303
305
|
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8},
|
|
304
|
-
{"ia", UncutEndpoint("j"), InternalValue("m", 8), 4
|
|
306
|
+
{"ia", UncutEndpoint("j"), InternalValue("m", 8, kValueTypeForSeek), 4,
|
|
307
|
+
false /* invalid */},
|
|
305
308
|
{"n", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */},
|
|
306
309
|
{"", InternalValue("d", 7), UncutEndpoint("e"), 10}});
|
|
307
310
|
|
|
@@ -310,7 +313,8 @@ TEST_F(RangeDelAggregatorTest, TruncatedIterPartiallyCutTombstones) {
|
|
|
310
313
|
{{"d", InternalValue("d", 7), UncutEndpoint("e"), 10},
|
|
311
314
|
{"e", UncutEndpoint("e"), UncutEndpoint("g"), 8},
|
|
312
315
|
{"ia", UncutEndpoint("e"), UncutEndpoint("g"), 8},
|
|
313
|
-
{"n", UncutEndpoint("j"), InternalValue("m", 8), 4
|
|
316
|
+
{"n", UncutEndpoint("j"), InternalValue("m", 8, kValueTypeForSeek), 4,
|
|
317
|
+
false /* invalid */},
|
|
314
318
|
{"", UncutEndpoint(""), UncutEndpoint(""), 0, true /* invalid */}});
|
|
315
319
|
}
|
|
316
320
|
|
|
@@ -67,8 +67,8 @@ FragmentedRangeTombstoneList::FragmentedRangeTombstoneList(
|
|
|
67
67
|
unfragmented_tombstones->value().size());
|
|
68
68
|
}
|
|
69
69
|
// VectorIterator implicitly sorts by key during construction.
|
|
70
|
-
auto iter = std::
|
|
71
|
-
|
|
70
|
+
auto iter = std::make_unique<VectorIterator>(std::move(keys),
|
|
71
|
+
std::move(values), &icmp);
|
|
72
72
|
FragmentTombstones(std::move(iter), icmp, for_compaction, snapshots);
|
|
73
73
|
}
|
|
74
74
|
|
|
@@ -251,6 +251,22 @@ FragmentedRangeTombstoneIterator::FragmentedRangeTombstoneIterator(
|
|
|
251
251
|
Invalidate();
|
|
252
252
|
}
|
|
253
253
|
|
|
254
|
+
FragmentedRangeTombstoneIterator::FragmentedRangeTombstoneIterator(
|
|
255
|
+
const std::shared_ptr<FragmentedRangeTombstoneListCache>& tombstones_cache,
|
|
256
|
+
const InternalKeyComparator& icmp, SequenceNumber _upper_bound,
|
|
257
|
+
SequenceNumber _lower_bound)
|
|
258
|
+
: tombstone_start_cmp_(icmp.user_comparator()),
|
|
259
|
+
tombstone_end_cmp_(icmp.user_comparator()),
|
|
260
|
+
icmp_(&icmp),
|
|
261
|
+
ucmp_(icmp.user_comparator()),
|
|
262
|
+
tombstones_cache_ref_(tombstones_cache),
|
|
263
|
+
tombstones_(tombstones_cache_ref_->tombstones.get()),
|
|
264
|
+
upper_bound_(_upper_bound),
|
|
265
|
+
lower_bound_(_lower_bound) {
|
|
266
|
+
assert(tombstones_ != nullptr);
|
|
267
|
+
Invalidate();
|
|
268
|
+
}
|
|
269
|
+
|
|
254
270
|
void FragmentedRangeTombstoneIterator::SeekToFirst() {
|
|
255
271
|
pos_ = tombstones_->begin();
|
|
256
272
|
seq_pos_ = tombstones_->seq_begin();
|
|
@@ -433,9 +449,8 @@ FragmentedRangeTombstoneIterator::SplitBySnapshot(
|
|
|
433
449
|
upper = snapshots[i];
|
|
434
450
|
}
|
|
435
451
|
if (tombstones_->ContainsRange(lower, upper)) {
|
|
436
|
-
splits.emplace(upper, std::
|
|
437
|
-
|
|
438
|
-
tombstones_, *icmp_, upper, lower)));
|
|
452
|
+
splits.emplace(upper, std::make_unique<FragmentedRangeTombstoneIterator>(
|
|
453
|
+
tombstones_, *icmp_, upper, lower));
|
|
439
454
|
}
|
|
440
455
|
lower = upper + 1;
|
|
441
456
|
}
|
|
@@ -17,6 +17,15 @@
|
|
|
17
17
|
#include "table/internal_iterator.h"
|
|
18
18
|
|
|
19
19
|
namespace ROCKSDB_NAMESPACE {
|
|
20
|
+
struct FragmentedRangeTombstoneList;
|
|
21
|
+
|
|
22
|
+
struct FragmentedRangeTombstoneListCache {
|
|
23
|
+
// ensure only the first reader needs to initialize l
|
|
24
|
+
std::mutex reader_mutex;
|
|
25
|
+
std::unique_ptr<FragmentedRangeTombstoneList> tombstones = nullptr;
|
|
26
|
+
// readers will first check this bool to avoid
|
|
27
|
+
std::atomic<bool> initialized = false;
|
|
28
|
+
};
|
|
20
29
|
|
|
21
30
|
struct FragmentedRangeTombstoneList {
|
|
22
31
|
public:
|
|
@@ -113,6 +122,10 @@ class FragmentedRangeTombstoneIterator : public InternalIterator {
|
|
|
113
122
|
const std::shared_ptr<const FragmentedRangeTombstoneList>& tombstones,
|
|
114
123
|
const InternalKeyComparator& icmp, SequenceNumber upper_bound,
|
|
115
124
|
SequenceNumber lower_bound = 0);
|
|
125
|
+
FragmentedRangeTombstoneIterator(
|
|
126
|
+
const std::shared_ptr<FragmentedRangeTombstoneListCache>& tombstones,
|
|
127
|
+
const InternalKeyComparator& icmp, SequenceNumber upper_bound,
|
|
128
|
+
SequenceNumber lower_bound = 0);
|
|
116
129
|
|
|
117
130
|
void SeekToFirst() override;
|
|
118
131
|
void SeekToLast() override;
|
|
@@ -260,6 +273,7 @@ class FragmentedRangeTombstoneIterator : public InternalIterator {
|
|
|
260
273
|
const InternalKeyComparator* icmp_;
|
|
261
274
|
const Comparator* ucmp_;
|
|
262
275
|
std::shared_ptr<const FragmentedRangeTombstoneList> tombstones_ref_;
|
|
276
|
+
std::shared_ptr<FragmentedRangeTombstoneListCache> tombstones_cache_ref_;
|
|
263
277
|
const FragmentedRangeTombstoneList* tombstones_;
|
|
264
278
|
SequenceNumber upper_bound_;
|
|
265
279
|
SequenceNumber lower_bound_;
|
|
@@ -162,9 +162,13 @@ class Repairer {
|
|
|
162
162
|
edit.AddColumnFamily(cf_name);
|
|
163
163
|
|
|
164
164
|
mutex_.Lock();
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
165
|
+
std::unique_ptr<FSDirectory> db_dir;
|
|
166
|
+
Status status = env_->GetFileSystem()->NewDirectory(dbname_, IOOptions(),
|
|
167
|
+
&db_dir, nullptr);
|
|
168
|
+
if (status.ok()) {
|
|
169
|
+
status = vset_.LogAndApply(cfd, mut_cf_opts, &edit, &mutex_, db_dir.get(),
|
|
170
|
+
false /* new_descriptor_log */, cf_opts);
|
|
171
|
+
}
|
|
168
172
|
mutex_.Unlock();
|
|
169
173
|
return status;
|
|
170
174
|
}
|
|
@@ -504,8 +508,8 @@ class Repairer {
|
|
|
504
508
|
file_size);
|
|
505
509
|
std::shared_ptr<const TableProperties> props;
|
|
506
510
|
if (status.ok()) {
|
|
507
|
-
status = table_cache_->GetTableProperties(file_options_, icmp_,
|
|
508
|
-
|
|
511
|
+
status = table_cache_->GetTableProperties(file_options_, icmp_, t->meta,
|
|
512
|
+
&props);
|
|
509
513
|
}
|
|
510
514
|
if (status.ok()) {
|
|
511
515
|
auto s =
|
|
@@ -656,9 +660,14 @@ class Repairer {
|
|
|
656
660
|
assert(next_file_number_ > 0);
|
|
657
661
|
vset_.MarkFileNumberUsed(next_file_number_ - 1);
|
|
658
662
|
mutex_.Lock();
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
663
|
+
std::unique_ptr<FSDirectory> db_dir;
|
|
664
|
+
Status status = env_->GetFileSystem()->NewDirectory(dbname_, IOOptions(),
|
|
665
|
+
&db_dir, nullptr);
|
|
666
|
+
if (status.ok()) {
|
|
667
|
+
status = vset_.LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
|
|
668
|
+
&edit, &mutex_, db_dir.get(),
|
|
669
|
+
false /* new_descriptor_log */);
|
|
670
|
+
}
|
|
662
671
|
mutex_.Unlock();
|
|
663
672
|
if (!status.ok()) {
|
|
664
673
|
return status;
|
|
@@ -48,7 +48,7 @@ class RepairTest : public DBTestBase {
|
|
|
48
48
|
void ReopenWithSstIdVerify() {
|
|
49
49
|
std::atomic_int verify_passed{0};
|
|
50
50
|
SyncPoint::GetInstance()->SetCallBack(
|
|
51
|
-
"
|
|
51
|
+
"BlockBasedTable::Open::PassedVerifyUniqueId", [&](void* arg) {
|
|
52
52
|
// override job status
|
|
53
53
|
auto id = static_cast<UniqueId64x2*>(arg);
|
|
54
54
|
assert(*id != kNullUniqueId64x2);
|
|
@@ -60,6 +60,7 @@ class RepairTest : public DBTestBase {
|
|
|
60
60
|
Reopen(options);
|
|
61
61
|
|
|
62
62
|
ASSERT_GT(verify_passed, 0);
|
|
63
|
+
SyncPoint::GetInstance()->DisableProcessing();
|
|
63
64
|
}
|
|
64
65
|
};
|
|
65
66
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// (found in the LICENSE.Apache file in the root directory).
|
|
6
6
|
|
|
7
7
|
#include "db/db_test_util.h"
|
|
8
|
-
#include "db/
|
|
8
|
+
#include "db/periodic_task_scheduler.h"
|
|
9
9
|
#include "db/seqno_to_time_mapping.h"
|
|
10
10
|
#include "port/stack_trace.h"
|
|
11
11
|
#include "rocksdb/iostats_context.h"
|
|
@@ -29,11 +29,10 @@ class SeqnoTimeTest : public DBTestBase {
|
|
|
29
29
|
void SetUp() override {
|
|
30
30
|
mock_clock_->InstallTimedWaitFixCallback();
|
|
31
31
|
SyncPoint::GetInstance()->SetCallBack(
|
|
32
|
-
"DBImpl::
|
|
33
|
-
auto
|
|
34
|
-
reinterpret_cast<
|
|
35
|
-
|
|
36
|
-
PeriodicWorkTestScheduler::Default(mock_clock_);
|
|
32
|
+
"DBImpl::StartPeriodicTaskScheduler:Init", [&](void* arg) {
|
|
33
|
+
auto periodic_task_scheduler_ptr =
|
|
34
|
+
reinterpret_cast<PeriodicTaskScheduler*>(arg);
|
|
35
|
+
periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get());
|
|
37
36
|
});
|
|
38
37
|
}
|
|
39
38
|
|
|
@@ -80,7 +79,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
|
|
|
80
79
|
|
|
81
80
|
// pass some time first, otherwise the first a few keys write time are going
|
|
82
81
|
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime
|
|
83
|
-
dbfull()->
|
|
82
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
84
83
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
|
|
85
84
|
|
|
86
85
|
int sst_num = 0;
|
|
@@ -88,7 +87,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
|
|
|
88
87
|
for (; sst_num < kNumTrigger; sst_num++) {
|
|
89
88
|
for (int i = 0; i < kNumKeys; i++) {
|
|
90
89
|
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
|
|
91
|
-
dbfull()->
|
|
90
|
+
dbfull()->TEST_WaitForPeridicTaskRun([&] {
|
|
92
91
|
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
|
|
93
92
|
});
|
|
94
93
|
}
|
|
@@ -110,7 +109,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
|
|
|
110
109
|
for (; sst_num < kNumTrigger * 2; sst_num++) {
|
|
111
110
|
for (int i = 0; i < kNumKeys; i++) {
|
|
112
111
|
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
|
|
113
|
-
dbfull()->
|
|
112
|
+
dbfull()->TEST_WaitForPeridicTaskRun([&] {
|
|
114
113
|
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
|
|
115
114
|
});
|
|
116
115
|
}
|
|
@@ -124,7 +123,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
|
|
|
124
123
|
for (; sst_num < kNumTrigger * 3; sst_num++) {
|
|
125
124
|
for (int i = 0; i < kNumKeys; i++) {
|
|
126
125
|
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
|
|
127
|
-
dbfull()->
|
|
126
|
+
dbfull()->TEST_WaitForPeridicTaskRun([&] {
|
|
128
127
|
mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
|
|
129
128
|
});
|
|
130
129
|
}
|
|
@@ -142,36 +141,31 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
|
|
|
142
141
|
// the first a few key should be cold
|
|
143
142
|
AssertKetTemperature(20, Temperature::kCold);
|
|
144
143
|
|
|
145
|
-
// Wait some time, each time after compaction, the cold data size is
|
|
146
|
-
// increasing and hot data size is decreasing
|
|
147
144
|
for (int i = 0; i < 30; i++) {
|
|
148
|
-
dbfull()->
|
|
145
|
+
dbfull()->TEST_WaitForPeridicTaskRun([&] {
|
|
149
146
|
mock_clock_->MockSleepForSeconds(static_cast<int>(20 * kKeyPerSec));
|
|
150
147
|
});
|
|
151
148
|
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
|
|
152
|
-
uint64_t pre_hot = hot_data_size;
|
|
153
|
-
uint64_t pre_cold = cold_data_size;
|
|
154
|
-
hot_data_size = GetSstSizeHelper(Temperature::kUnknown);
|
|
155
|
-
cold_data_size = GetSstSizeHelper(Temperature::kCold);
|
|
156
|
-
ASSERT_LT(hot_data_size, pre_hot);
|
|
157
|
-
ASSERT_GT(cold_data_size, pre_cold);
|
|
158
149
|
|
|
159
150
|
// the hot/cold data cut off range should be between i * 20 + 200 -> 250
|
|
160
151
|
AssertKetTemperature(i * 20 + 250, Temperature::kUnknown);
|
|
161
152
|
AssertKetTemperature(i * 20 + 200, Temperature::kCold);
|
|
162
153
|
}
|
|
163
154
|
|
|
164
|
-
|
|
155
|
+
ASSERT_LT(GetSstSizeHelper(Temperature::kUnknown), hot_data_size);
|
|
156
|
+
ASSERT_GT(GetSstSizeHelper(Temperature::kCold), cold_data_size);
|
|
157
|
+
|
|
158
|
+
// Wait again, the most of the data should be cold after that
|
|
159
|
+
// but it may not be all cold, because if there's no new data write to SST,
|
|
160
|
+
// the compaction will not get the new seqno->time sampling to decide the last
|
|
161
|
+
// a few data's time.
|
|
165
162
|
for (int i = 0; i < 5; i++) {
|
|
166
|
-
dbfull()->
|
|
163
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
167
164
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });
|
|
168
165
|
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
|
|
169
166
|
}
|
|
170
167
|
|
|
171
|
-
|
|
172
|
-
ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
|
|
173
|
-
|
|
174
|
-
// any random data should be cold
|
|
168
|
+
// any random data close to the end should be cold
|
|
175
169
|
AssertKetTemperature(1000, Temperature::kCold);
|
|
176
170
|
|
|
177
171
|
// close explicitly, because the env is local variable which will be released
|
|
@@ -197,7 +191,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
|
|
|
197
191
|
|
|
198
192
|
// pass some time first, otherwise the first a few keys write time are going
|
|
199
193
|
// to be zero, and internally zero has special meaning: kUnknownSeqnoTime
|
|
200
|
-
dbfull()->
|
|
194
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
201
195
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
|
|
202
196
|
|
|
203
197
|
int sst_num = 0;
|
|
@@ -205,7 +199,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
|
|
|
205
199
|
for (; sst_num < 4; sst_num++) {
|
|
206
200
|
for (int i = 0; i < kNumKeys; i++) {
|
|
207
201
|
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
|
|
208
|
-
dbfull()->
|
|
202
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
209
203
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
|
|
210
204
|
}
|
|
211
205
|
ASSERT_OK(Flush());
|
|
@@ -227,7 +221,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
|
|
|
227
221
|
for (; sst_num < 14; sst_num++) {
|
|
228
222
|
for (int i = 0; i < kNumKeys; i++) {
|
|
229
223
|
ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
|
|
230
|
-
dbfull()->
|
|
224
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
231
225
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
|
|
232
226
|
}
|
|
233
227
|
ASSERT_OK(Flush());
|
|
@@ -248,7 +242,7 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
|
|
|
248
242
|
// Wait some time, with each wait, the cold data is increasing and hot data is
|
|
249
243
|
// decreasing
|
|
250
244
|
for (int i = 0; i < 30; i++) {
|
|
251
|
-
dbfull()->
|
|
245
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
252
246
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(200)); });
|
|
253
247
|
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
|
|
254
248
|
uint64_t pre_hot = hot_data_size;
|
|
@@ -263,17 +257,16 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
|
|
|
263
257
|
AssertKetTemperature(i * 20 + 400, Temperature::kCold);
|
|
264
258
|
}
|
|
265
259
|
|
|
266
|
-
// Wait again,
|
|
260
|
+
// Wait again, the most of the data should be cold after that
|
|
261
|
+
// hot data might not be empty, because if we don't write new data, there's
|
|
262
|
+
// no seqno->time sampling available to the compaction
|
|
267
263
|
for (int i = 0; i < 5; i++) {
|
|
268
|
-
dbfull()->
|
|
264
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
269
265
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });
|
|
270
266
|
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
|
|
271
267
|
}
|
|
272
268
|
|
|
273
|
-
|
|
274
|
-
ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
|
|
275
|
-
|
|
276
|
-
// any random data should be cold
|
|
269
|
+
// any random data close to the end should be cold
|
|
277
270
|
AssertKetTemperature(1000, Temperature::kCold);
|
|
278
271
|
|
|
279
272
|
Close();
|
|
@@ -291,7 +284,7 @@ TEST_F(SeqnoTimeTest, BasicSeqnoToTimeMapping) {
|
|
|
291
284
|
// Write a key every 10 seconds
|
|
292
285
|
for (int i = 0; i < 200; i++) {
|
|
293
286
|
ASSERT_OK(Put(Key(i), "value"));
|
|
294
|
-
dbfull()->
|
|
287
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
295
288
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
|
|
296
289
|
}
|
|
297
290
|
ASSERT_OK(Flush());
|
|
@@ -322,7 +315,7 @@ TEST_F(SeqnoTimeTest, BasicSeqnoToTimeMapping) {
|
|
|
322
315
|
// Write a key every 1 seconds
|
|
323
316
|
for (int i = 0; i < 200; i++) {
|
|
324
317
|
ASSERT_OK(Put(Key(i + 190), "value"));
|
|
325
|
-
dbfull()->
|
|
318
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
326
319
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1)); });
|
|
327
320
|
}
|
|
328
321
|
seq_end = dbfull()->GetLatestSequenceNumber();
|
|
@@ -358,7 +351,7 @@ TEST_F(SeqnoTimeTest, BasicSeqnoToTimeMapping) {
|
|
|
358
351
|
// Write a key every 200 seconds
|
|
359
352
|
for (int i = 0; i < 200; i++) {
|
|
360
353
|
ASSERT_OK(Put(Key(i + 380), "value"));
|
|
361
|
-
dbfull()->
|
|
354
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
362
355
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(200)); });
|
|
363
356
|
}
|
|
364
357
|
seq_end = dbfull()->GetLatestSequenceNumber();
|
|
@@ -400,7 +393,7 @@ TEST_F(SeqnoTimeTest, BasicSeqnoToTimeMapping) {
|
|
|
400
393
|
// Write a key every 100 seconds
|
|
401
394
|
for (int i = 0; i < 200; i++) {
|
|
402
395
|
ASSERT_OK(Put(Key(i + 570), "value"));
|
|
403
|
-
dbfull()->
|
|
396
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
404
397
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
|
|
405
398
|
}
|
|
406
399
|
seq_end = dbfull()->GetLatestSequenceNumber();
|
|
@@ -464,9 +457,7 @@ TEST_F(SeqnoTimeTest, BasicSeqnoToTimeMapping) {
|
|
|
464
457
|
ASSERT_OK(db_->Close());
|
|
465
458
|
}
|
|
466
459
|
|
|
467
|
-
|
|
468
|
-
// is fixed
|
|
469
|
-
TEST_F(SeqnoTimeTest, DISABLED_MultiCFs) {
|
|
460
|
+
TEST_F(SeqnoTimeTest, MultiCFs) {
|
|
470
461
|
Options options = CurrentOptions();
|
|
471
462
|
options.preclude_last_level_data_seconds = 0;
|
|
472
463
|
options.env = mock_env_.get();
|
|
@@ -474,14 +465,14 @@ TEST_F(SeqnoTimeTest, DISABLED_MultiCFs) {
|
|
|
474
465
|
options.stats_persist_period_sec = 0;
|
|
475
466
|
ReopenWithColumnFamilies({"default"}, options);
|
|
476
467
|
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
468
|
+
const PeriodicTaskScheduler& scheduler =
|
|
469
|
+
dbfull()->TEST_GetPeriodicTaskScheduler();
|
|
470
|
+
ASSERT_FALSE(scheduler.TEST_HasTask(PeriodicTaskType::kRecordSeqnoTime));
|
|
480
471
|
|
|
481
472
|
// Write some data and increase the current time
|
|
482
473
|
for (int i = 0; i < 200; i++) {
|
|
483
474
|
ASSERT_OK(Put(Key(i), "value"));
|
|
484
|
-
dbfull()->
|
|
475
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
485
476
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
|
|
486
477
|
}
|
|
487
478
|
ASSERT_OK(Flush());
|
|
@@ -496,26 +487,20 @@ TEST_F(SeqnoTimeTest, DISABLED_MultiCFs) {
|
|
|
496
487
|
Options options_1 = options;
|
|
497
488
|
options_1.preclude_last_level_data_seconds = 10000; // 10k
|
|
498
489
|
CreateColumnFamilies({"one"}, options_1);
|
|
499
|
-
ASSERT_TRUE(scheduler
|
|
500
|
-
dbfull(), PeriodicWorkTaskNames::kRecordSeqnoTime));
|
|
490
|
+
ASSERT_TRUE(scheduler.TEST_HasTask(PeriodicTaskType::kRecordSeqnoTime));
|
|
501
491
|
|
|
502
492
|
// Write some data to the default CF (without preclude_last_level feature)
|
|
503
493
|
for (int i = 0; i < 200; i++) {
|
|
504
494
|
ASSERT_OK(Put(Key(i), "value"));
|
|
505
|
-
dbfull()->
|
|
495
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
506
496
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
|
|
507
497
|
}
|
|
508
498
|
ASSERT_OK(Flush());
|
|
509
499
|
|
|
510
|
-
// in memory mapping won't increase because CFs with preclude_last_level
|
|
511
|
-
// feature doesn't have memtable
|
|
512
|
-
auto queue = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping();
|
|
513
|
-
ASSERT_LT(queue.size(), 5);
|
|
514
|
-
|
|
515
500
|
// Write some data to the CF one
|
|
516
501
|
for (int i = 0; i < 20; i++) {
|
|
517
502
|
ASSERT_OK(Put(1, Key(i), "value"));
|
|
518
|
-
dbfull()->
|
|
503
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
519
504
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
|
|
520
505
|
}
|
|
521
506
|
ASSERT_OK(Flush(1));
|
|
@@ -539,7 +524,7 @@ TEST_F(SeqnoTimeTest, DISABLED_MultiCFs) {
|
|
|
539
524
|
// Add more data to CF "two" to fill the in memory mapping
|
|
540
525
|
for (int i = 0; i < 2000; i++) {
|
|
541
526
|
ASSERT_OK(Put(2, Key(i), "value"));
|
|
542
|
-
dbfull()->
|
|
527
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
543
528
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
|
|
544
529
|
}
|
|
545
530
|
seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping();
|
|
@@ -563,11 +548,10 @@ TEST_F(SeqnoTimeTest, DISABLED_MultiCFs) {
|
|
|
563
548
|
// enabled have flushed, the in-memory seqno->time mapping should be cleared
|
|
564
549
|
for (int i = 0; i < 10; i++) {
|
|
565
550
|
ASSERT_OK(Put(0, Key(i), "value"));
|
|
566
|
-
dbfull()->
|
|
551
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
567
552
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
|
|
568
553
|
}
|
|
569
554
|
seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping();
|
|
570
|
-
ASSERT_LE(seqs.size(), 5);
|
|
571
555
|
ASSERT_OK(Flush(0));
|
|
572
556
|
|
|
573
557
|
// trigger compaction for CF "two" and make sure the compaction output has
|
|
@@ -575,7 +559,7 @@ TEST_F(SeqnoTimeTest, DISABLED_MultiCFs) {
|
|
|
575
559
|
for (int j = 0; j < 3; j++) {
|
|
576
560
|
for (int i = 0; i < 200; i++) {
|
|
577
561
|
ASSERT_OK(Put(2, Key(i), "value"));
|
|
578
|
-
dbfull()->
|
|
562
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
579
563
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
|
|
580
564
|
}
|
|
581
565
|
ASSERT_OK(Flush(2));
|
|
@@ -595,7 +579,7 @@ TEST_F(SeqnoTimeTest, DISABLED_MultiCFs) {
|
|
|
595
579
|
for (int j = 0; j < 2; j++) {
|
|
596
580
|
for (int i = 0; i < 200; i++) {
|
|
597
581
|
ASSERT_OK(Put(0, Key(i), "value"));
|
|
598
|
-
dbfull()->
|
|
582
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
599
583
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
|
|
600
584
|
}
|
|
601
585
|
ASSERT_OK(Flush(0));
|
|
@@ -610,7 +594,7 @@ TEST_F(SeqnoTimeTest, DISABLED_MultiCFs) {
|
|
|
610
594
|
// Write some data to CF "two", but don't flush to accumulate
|
|
611
595
|
for (int i = 0; i < 1000; i++) {
|
|
612
596
|
ASSERT_OK(Put(2, Key(i), "value"));
|
|
613
|
-
dbfull()->
|
|
597
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
614
598
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
|
|
615
599
|
}
|
|
616
600
|
ASSERT_GE(
|
|
@@ -630,8 +614,7 @@ TEST_F(SeqnoTimeTest, DISABLED_MultiCFs) {
|
|
|
630
614
|
0);
|
|
631
615
|
|
|
632
616
|
// And the timer worker is stopped
|
|
633
|
-
ASSERT_FALSE(scheduler
|
|
634
|
-
dbfull(), PeriodicWorkTaskNames::kRecordSeqnoTime));
|
|
617
|
+
ASSERT_FALSE(scheduler.TEST_HasTask(PeriodicTaskType::kRecordSeqnoTime));
|
|
635
618
|
Close();
|
|
636
619
|
}
|
|
637
620
|
|
|
@@ -655,7 +638,7 @@ TEST_F(SeqnoTimeTest, MultiInstancesBasic) {
|
|
|
655
638
|
WriteOptions wo;
|
|
656
639
|
for (int i = 0; i < 200; i++) {
|
|
657
640
|
ASSERT_OK(dbi->Put(wo, Key(i), "value"));
|
|
658
|
-
dbfull()->
|
|
641
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
659
642
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
|
|
660
643
|
}
|
|
661
644
|
SeqnoToTimeMapping seqno_to_time_mapping = dbi->TEST_GetSeqnoToTimeMapping();
|
|
@@ -678,7 +661,7 @@ TEST_F(SeqnoTimeTest, SeqnoToTimeMappingUniversal) {
|
|
|
678
661
|
for (int j = 0; j < 3; j++) {
|
|
679
662
|
for (int i = 0; i < 100; i++) {
|
|
680
663
|
ASSERT_OK(Put(Key(i), "value"));
|
|
681
|
-
dbfull()->
|
|
664
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
682
665
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
|
|
683
666
|
}
|
|
684
667
|
ASSERT_OK(Flush());
|
|
@@ -700,7 +683,7 @@ TEST_F(SeqnoTimeTest, SeqnoToTimeMappingUniversal) {
|
|
|
700
683
|
// Trigger a compaction
|
|
701
684
|
for (int i = 0; i < 100; i++) {
|
|
702
685
|
ASSERT_OK(Put(Key(i), "value"));
|
|
703
|
-
dbfull()->
|
|
686
|
+
dbfull()->TEST_WaitForPeridicTaskRun(
|
|
704
687
|
[&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
|
|
705
688
|
}
|
|
706
689
|
ASSERT_OK(Flush());
|