@nxtedition/rocksdb 15.4.0 → 15.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +24 -19
- package/cache.js +1 -1
- package/chained-batch.js +12 -3
- package/deps/rocksdb/rocksdb/.clang-tidy +86 -0
- package/deps/rocksdb/rocksdb/BUCK +42 -0
- package/deps/rocksdb/rocksdb/CMakeLists.txt +11 -0
- package/deps/rocksdb/rocksdb/Makefile +59 -32
- package/deps/rocksdb/rocksdb/cache/cache.cc +0 -5
- package/deps/rocksdb/rocksdb/cache/cache_entry_stats.h +9 -9
- package/deps/rocksdb/rocksdb/cache/cache_key.cc +3 -3
- package/deps/rocksdb/rocksdb/cache/cache_key.h +5 -5
- package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.h +16 -16
- package/deps/rocksdb/rocksdb/cache/cache_test.cc +1 -1
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +258 -294
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +98 -49
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +1 -5
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +2 -3
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +18 -18
- package/deps/rocksdb/rocksdb/crash_test.mk +5 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +23 -22
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.h +6 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +14 -16
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +38 -26
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.h +5 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +101 -18
- package/deps/rocksdb/rocksdb/db/blob/blob_index.h +12 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +6 -9
- package/deps/rocksdb/rocksdb/db/builder.cc +23 -0
- package/deps/rocksdb/rocksdb/db/builder.h +7 -0
- package/deps/rocksdb/rocksdb/db/c.cc +373 -57
- package/deps/rocksdb/rocksdb/db/c_test.c +101 -1
- package/deps/rocksdb/rocksdb/db/column_family.cc +31 -3
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +10 -13
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +35 -48
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +13 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +201 -39
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +15 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +7 -7
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +2 -455
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +4 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +19 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +72 -9
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +12 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +405 -83
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +25 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +23 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +1 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +1410 -106
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +12 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +19 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +505 -45
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +2 -2
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +9 -1
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +7 -9
- package/deps/rocksdb/rocksdb/db/convenience.cc +4 -4
- package/deps/rocksdb/rocksdb/db/convenience_impl.h +2 -1
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +60 -88
- package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +10 -12
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +471 -40
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +116 -2
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +5 -15
- package/deps/rocksdb/rocksdb/db/db_compaction_abort_test.cc +993 -0
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +329 -29
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +155 -13
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +54 -31
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +1 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +232 -70
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +57 -9
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +224 -31
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +5 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +4 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +164 -8
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +6 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +5 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +47 -35
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +22 -9
- package/deps/rocksdb/rocksdb/db/db_iter.cc +9 -0
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +371 -6
- package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +7 -5
- package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +22 -23
- package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +0 -2
- package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_options_test.cc +40 -0
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +32 -13
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_readonly_with_timestamp_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +68 -15
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +2 -3
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +6 -21
- package/deps/rocksdb/rocksdb/db/db_test.cc +644 -128
- package/deps/rocksdb/rocksdb/db/db_test2.cc +198 -81
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +35 -10
- package/deps/rocksdb/rocksdb/db/db_test_util.h +8 -2
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +36 -32
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +11 -7
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +499 -0
- package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +284 -20
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/dbformat.h +0 -5
- package/deps/rocksdb/rocksdb/db/error_handler.cc +24 -0
- package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +12 -14
- package/deps/rocksdb/rocksdb/db/experimental.cc +13 -10
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +22 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +21 -15
- package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +4 -6
- package/deps/rocksdb/rocksdb/db/flush_job.cc +11 -3
- package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +5 -6
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +4 -2
- package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +17 -17
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +13 -0
- package/deps/rocksdb/rocksdb/db/internal_stats.h +2 -0
- package/deps/rocksdb/rocksdb/db/listener_test.cc +154 -27
- package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/memtable.cc +197 -51
- package/deps/rocksdb/rocksdb/db/memtable.h +6 -0
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/merge_test.cc +37 -35
- package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/options_file_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +9 -11
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +10 -1
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +292 -15
- package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +10 -17
- package/deps/rocksdb/rocksdb/db/prefix_test.cc +6 -8
- package/deps/rocksdb/rocksdb/db/repair.cc +10 -10
- package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +5 -5
- package/deps/rocksdb/rocksdb/db/table_cache.cc +142 -135
- package/deps/rocksdb/rocksdb/db/table_cache.h +30 -6
- package/deps/rocksdb/rocksdb/db/table_cache_sync_and_async.h +7 -7
- package/deps/rocksdb/rocksdb/db/version_builder.cc +11 -50
- package/deps/rocksdb/rocksdb/db/version_builder.h +2 -1
- package/deps/rocksdb/rocksdb/db/version_builder_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/version_edit.cc +51 -2
- package/deps/rocksdb/rocksdb/db/version_edit.h +91 -29
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -7
- package/deps/rocksdb/rocksdb/db/version_set.cc +211 -50
- package/deps/rocksdb/rocksdb/db/version_set.h +40 -3
- package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +5 -0
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +294 -21
- package/deps/rocksdb/rocksdb/db/version_util.cc +96 -0
- package/deps/rocksdb/rocksdb/db/version_util.h +24 -0
- package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +5 -5
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.cc +647 -31
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.h +219 -1
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization_test.cc +549 -12
- package/deps/rocksdb/rocksdb/db/write_callback_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +1 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +19 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +21 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +32 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +74 -22
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +9 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +143 -61
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +15 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +76 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +92 -72
- package/deps/rocksdb/rocksdb/env/env.cc +1 -0
- package/deps/rocksdb/rocksdb/env/env_test.cc +365 -2
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +31 -30
- package/deps/rocksdb/rocksdb/env/io_posix.cc +8 -11
- package/deps/rocksdb/rocksdb/env/io_posix.h +30 -1
- package/deps/rocksdb/rocksdb/env/io_posix_test.cc +43 -0
- package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +1 -1
- package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +108 -0
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +32 -4
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +4 -4
- package/deps/rocksdb/rocksdb/file/file_util.cc +8 -2
- package/deps/rocksdb/rocksdb/file/file_util.h +2 -1
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +331 -12
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +52 -35
- package/deps/rocksdb/rocksdb/folly.mk +22 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_cache.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +100 -54
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +67 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +149 -13
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +1 -12
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +78 -97
- package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +3 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/external_table.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +17 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/functor_wrapper.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/io_dispatcher.h +358 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +43 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +20 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +63 -21
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +10 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +2 -7
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +3 -14
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +49 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/status.h +8 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +77 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +15 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/tool_hooks.h +16 -10
- package/deps/rocksdb/rocksdb/include/rocksdb/unique_id.h +5 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +2 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/user_defined_index.h +106 -46
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/db_ttl.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +14 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/memory_util.h +5 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +7 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +1 -2
- package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc +2 -2
- package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +226 -8
- package/deps/rocksdb/rocksdb/memtable/inlineskiplist_test.cc +490 -0
- package/deps/rocksdb/rocksdb/memtable/skiplist.h +3 -3
- package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +11 -0
- package/deps/rocksdb/rocksdb/microbench/db_basic_bench.cc +4 -12
- package/deps/rocksdb/rocksdb/microbench/ribbon_bench.cc +5 -5
- package/deps/rocksdb/rocksdb/monitoring/file_read_sample.h +21 -4
- package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +9 -3
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +21 -2
- package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +2 -2
- package/deps/rocksdb/rocksdb/options/cf_options.cc +21 -1
- package/deps/rocksdb/rocksdb/options/cf_options.h +2 -0
- package/deps/rocksdb/rocksdb/options/customizable_test.cc +0 -2
- package/deps/rocksdb/rocksdb/options/db_options.cc +26 -5
- package/deps/rocksdb/rocksdb/options/db_options.h +3 -1
- package/deps/rocksdb/rocksdb/options/options.cc +5 -1
- package/deps/rocksdb/rocksdb/options/options_helper.cc +7 -2
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +109 -103
- package/deps/rocksdb/rocksdb/options/options_test.cc +14 -0
- package/deps/rocksdb/rocksdb/port/jemalloc_helper.h +15 -17
- package/deps/rocksdb/rocksdb/port/lang.h +4 -0
- package/deps/rocksdb/rocksdb/port/port_example.h +0 -23
- package/deps/rocksdb/rocksdb/port/stack_trace.cc +36 -0
- package/deps/rocksdb/rocksdb/port/stack_trace.h +9 -0
- package/deps/rocksdb/rocksdb/src.mk +12 -0
- package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +1 -2
- package/deps/rocksdb/rocksdb/table/block_based/binary_search_index_reader.cc +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/block.cc +571 -292
- package/deps/rocksdb/rocksdb/table/block_based/block.h +143 -53
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +154 -90
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +5 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +51 -14
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h +0 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +147 -734
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +30 -233
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +178 -108
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +13 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +17 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +5 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +70 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_builder.cc +168 -24
- package/deps/rocksdb/rocksdb/table/block_based/block_builder.h +25 -9
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +7 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +9 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +548 -169
- package/deps/rocksdb/rocksdb/table/block_based/block_type.h +30 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_util.h +156 -0
- package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.cc +73 -30
- package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.h +74 -7
- package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +20 -14
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +22 -12
- package/deps/rocksdb/rocksdb/table/block_based/mock_block_based_table.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.cc +332 -0
- package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.h +133 -0
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +4 -2
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/reader_common.cc +3 -2
- package/deps/rocksdb/rocksdb/table/block_based/reader_common.h +4 -1
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +0 -1
- package/deps/rocksdb/rocksdb/table/block_based/user_defined_index_wrapper.h +126 -46
- package/deps/rocksdb/rocksdb/table/block_fetcher.cc +31 -3
- package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +1 -2
- package/deps/rocksdb/rocksdb/table/cleanable_test.cc +3 -1
- package/deps/rocksdb/rocksdb/table/external_table.cc +25 -4
- package/deps/rocksdb/rocksdb/table/format.cc +27 -15
- package/deps/rocksdb/rocksdb/table/format.h +41 -15
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +1 -0
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +22 -12
- package/deps/rocksdb/rocksdb/table/meta_blocks.h +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +7 -21
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +88 -13
- package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +53 -42
- package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +3 -12
- package/deps/rocksdb/rocksdb/table/table_builder.h +0 -4
- package/deps/rocksdb/rocksdb/table/table_properties.cc +18 -0
- package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +2 -3
- package/deps/rocksdb/rocksdb/table/table_test.cc +848 -172
- package/deps/rocksdb/rocksdb/table/unique_id.cc +24 -20
- package/deps/rocksdb/rocksdb/table/unique_id_impl.h +8 -8
- package/deps/rocksdb/rocksdb/test_util/sync_point.h +5 -4
- package/deps/rocksdb/rocksdb/test_util/testutil.cc +2 -1
- package/deps/rocksdb/rocksdb/test_util/testutil.h +2 -2
- package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +2 -1
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +238 -120
- package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +2 -2
- package/deps/rocksdb/rocksdb/tools/db_sanity_test.cc +2 -4
- package/deps/rocksdb/rocksdb/tools/dump/db_dump_tool.cc +4 -8
- package/deps/rocksdb/rocksdb/tools/dump/rocksdb_undump.cc +1 -1
- package/deps/rocksdb/rocksdb/tools/io_tracer_parser_test.cc +2 -3
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +82 -20
- package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +41 -47
- package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +9 -0
- package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +5 -6
- package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +1 -1
- package/deps/rocksdb/rocksdb/tools/tool_hooks.cc +6 -5
- package/deps/rocksdb/rocksdb/tools/trace_analyzer_test.cc +4 -4
- package/deps/rocksdb/rocksdb/tools/write_stress.cc +1 -3
- package/deps/rocksdb/rocksdb/util/atomic.h +30 -23
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.cc +6 -7
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.h +3 -3
- package/deps/rocksdb/rocksdb/util/bit_fields.h +68 -46
- package/deps/rocksdb/rocksdb/util/bloom_impl.h +16 -16
- package/deps/rocksdb/rocksdb/util/coding.h +14 -27
- package/deps/rocksdb/rocksdb/util/compression.cc +365 -207
- package/deps/rocksdb/rocksdb/util/compression.h +16 -1298
- package/deps/rocksdb/rocksdb/util/compression_test.cc +347 -61
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +8 -9
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.h +1 -1
- package/deps/rocksdb/rocksdb/util/crc32c_ppc.h +1 -1
- package/deps/rocksdb/rocksdb/util/dynamic_bloom_test.cc +3 -3
- package/deps/rocksdb/rocksdb/util/filter_bench.cc +18 -18
- package/deps/rocksdb/rocksdb/util/gflags_compat.h +3 -3
- package/deps/rocksdb/rocksdb/util/hash_test.cc +19 -7
- package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.cc +1099 -0
- package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.h +36 -0
- package/deps/rocksdb/rocksdb/util/io_dispatcher_test.cc +1919 -0
- package/deps/rocksdb/rocksdb/util/math.h +3 -1
- package/deps/rocksdb/rocksdb/util/mutexlock.h +19 -19
- package/deps/rocksdb/rocksdb/util/ribbon_alg.h +25 -25
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +5 -7
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +4 -5
- package/deps/rocksdb/rocksdb/util/slice.cc +0 -10
- package/deps/rocksdb/rocksdb/util/slice_test.cc +35 -1
- package/deps/rocksdb/rocksdb/util/slice_transform_test.cc +5 -7
- package/deps/rocksdb/rocksdb/util/status.cc +3 -1
- package/deps/rocksdb/rocksdb/util/stop_watch.h +2 -0
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +4 -1
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +123 -78
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +12 -93
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.h +1 -4
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.cc +0 -21
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +6 -48
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +94 -307
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +12 -58
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +2 -8
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +2 -3
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +205 -811
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +18 -9
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.cc +2 -7
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.h +1 -9
- package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_functional_test.cc +17 -11
- package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +68 -61
- package/deps/rocksdb/rocksdb/utilities/debug.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +105 -59
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +274 -7
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs_test.cc +94 -0
- package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +13 -17
- package/deps/rocksdb/rocksdb/utilities/memory/memory_util.cc +16 -3
- package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc +25 -25
- package/deps/rocksdb/rocksdb/utilities/object_registry.cc +40 -40
- package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc +2 -5
- package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +17 -19
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.h +2 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.h +4 -13
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +3 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +6 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_seqno_test.cc +431 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +1 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.h +91 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.cc +562 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.h +615 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.cc +2575 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.h +685 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_db_test.cc +2843 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.cc +567 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.h +275 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_test.cc +5183 -0
- package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.cc +4 -3
- package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +3 -3
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +93 -88
- package/deps/rocksdb/rocksdb.gyp +7 -0
- package/index.js +11 -2
- package/iterator.js +15 -7
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
- package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_custom_library.h +0 -43
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_util.h +0 -55
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
#include <string>
|
|
8
8
|
#include <utility>
|
|
9
9
|
|
|
10
|
+
#include "db/blob/blob_file_meta.h"
|
|
11
|
+
#include "db/column_family.h"
|
|
10
12
|
#include "db/compaction/compaction.h"
|
|
11
13
|
#include "db/compaction/compaction_picker_fifo.h"
|
|
12
14
|
#include "db/compaction/compaction_picker_level.h"
|
|
@@ -17,6 +19,7 @@
|
|
|
17
19
|
#include "table/unique_id_impl.h"
|
|
18
20
|
#include "test_util/testharness.h"
|
|
19
21
|
#include "test_util/testutil.h"
|
|
22
|
+
#include "util/random.h"
|
|
20
23
|
#include "util/string_util.h"
|
|
21
24
|
|
|
22
25
|
namespace ROCKSDB_NAMESPACE {
|
|
@@ -160,14 +163,23 @@ class CompactionPickerTestBase : public testing::Test {
|
|
|
160
163
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
161
164
|
kUnknownFileCreationTime, epoch_number, kUnknownFileChecksum,
|
|
162
165
|
kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0,
|
|
163
|
-
true /* user_defined_timestamps_persisted
|
|
166
|
+
true /* user_defined_timestamps_persisted */, "" /* min timestamp */,
|
|
167
|
+
"" /* max timestamp */);
|
|
164
168
|
f->compensated_file_size =
|
|
165
169
|
(compensated_file_size != 0) ? compensated_file_size : file_size;
|
|
166
170
|
// oldest_ancester_time is only used if newest_key_time is not available
|
|
167
171
|
f->oldest_ancester_time = oldest_ancestor_time;
|
|
172
|
+
// Set min/max timestamps for UDT support
|
|
173
|
+
if (!ts_of_smallest.empty()) {
|
|
174
|
+
f->min_timestamp = ts_of_smallest.ToString();
|
|
175
|
+
}
|
|
176
|
+
if (!ts_of_largest.empty()) {
|
|
177
|
+
f->max_timestamp = ts_of_largest.ToString();
|
|
178
|
+
}
|
|
168
179
|
TableProperties tp;
|
|
169
180
|
tp.newest_key_time = newest_key_time;
|
|
170
|
-
f->fd.
|
|
181
|
+
f->fd.pinned_reader.TEST_SetReader(
|
|
182
|
+
new mock::MockTableReader(mock::KVVector{}, tp));
|
|
171
183
|
|
|
172
184
|
vstorage->AddFile(level, f);
|
|
173
185
|
files_.emplace_back(f);
|
|
@@ -195,6 +207,11 @@ class CompactionPickerTestBase : public testing::Test {
|
|
|
195
207
|
}
|
|
196
208
|
|
|
197
209
|
void UpdateVersionStorageInfo() {
|
|
210
|
+
UpdateVersionStorageInfoWithTsLow(/*full_history_ts_low=*/"");
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
void UpdateVersionStorageInfoWithTsLow(
|
|
214
|
+
const std::string& full_history_ts_low) {
|
|
198
215
|
if (temp_vstorage_) {
|
|
199
216
|
VersionBuilder builder(FileOptions(), &ioptions_, nullptr,
|
|
200
217
|
vstorage_.get(), nullptr);
|
|
@@ -202,10 +219,51 @@ class CompactionPickerTestBase : public testing::Test {
|
|
|
202
219
|
vstorage_ = std::move(temp_vstorage_);
|
|
203
220
|
}
|
|
204
221
|
vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_);
|
|
205
|
-
vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_
|
|
222
|
+
vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_,
|
|
223
|
+
full_history_ts_low);
|
|
206
224
|
vstorage_->SetFinalized();
|
|
207
225
|
}
|
|
208
226
|
|
|
227
|
+
void AddBlobFile(uint64_t blob_file_number, uint64_t total_blob_bytes,
|
|
228
|
+
BlobFileMetaData::LinkedSsts linked_ssts = {}) {
|
|
229
|
+
auto shared_meta = SharedBlobFileMetaData::Create(
|
|
230
|
+
blob_file_number, /*total_blob_count=*/1, total_blob_bytes,
|
|
231
|
+
/*checksum_method=*/"", /*checksum_value=*/"");
|
|
232
|
+
auto meta =
|
|
233
|
+
BlobFileMetaData::Create(std::move(shared_meta), std::move(linked_ssts),
|
|
234
|
+
/*garbage_blob_count=*/0,
|
|
235
|
+
/*garbage_blob_bytes=*/0);
|
|
236
|
+
vstorage_->AddBlobFile(std::move(meta));
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// Helper to set up FIFO ratio-based compaction options and version storage.
|
|
240
|
+
// Call before Add()/AddBlobFile(), then create FIFOCompactionPicker after.
|
|
241
|
+
void SetupFIFORatioBased(uint64_t max_table_files_size,
|
|
242
|
+
uint64_t max_data_files_size, int trigger,
|
|
243
|
+
bool allow_compaction = true,
|
|
244
|
+
bool use_kv_ratio = true, int num_levels = 1) {
|
|
245
|
+
ioptions_.compaction_style = kCompactionStyleFIFO;
|
|
246
|
+
NewVersionStorage(num_levels, kCompactionStyleFIFO);
|
|
247
|
+
mutable_cf_options_.compaction_options_fifo.max_table_files_size =
|
|
248
|
+
max_table_files_size;
|
|
249
|
+
mutable_cf_options_.compaction_options_fifo.max_data_files_size =
|
|
250
|
+
max_data_files_size;
|
|
251
|
+
mutable_cf_options_.compaction_options_fifo.allow_compaction =
|
|
252
|
+
allow_compaction;
|
|
253
|
+
mutable_cf_options_.compaction_options_fifo.use_kv_ratio_compaction =
|
|
254
|
+
use_kv_ratio;
|
|
255
|
+
mutable_cf_options_.level0_file_num_compaction_trigger = trigger;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Helper to finalize version storage and pick a FIFO compaction.
|
|
259
|
+
std::unique_ptr<Compaction> PickFIFOCompaction(FIFOCompactionPicker& picker) {
|
|
260
|
+
UpdateVersionStorageInfo();
|
|
261
|
+
return std::unique_ptr<Compaction>(picker.PickCompaction(
|
|
262
|
+
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
263
|
+
/*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr,
|
|
264
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
265
|
+
}
|
|
266
|
+
|
|
209
267
|
private:
|
|
210
268
|
Options CreateOptions(const Comparator* ucmp) const {
|
|
211
269
|
Options opts;
|
|
@@ -215,8 +273,9 @@ class CompactionPickerTestBase : public testing::Test {
|
|
|
215
273
|
|
|
216
274
|
void ClearFiles() {
|
|
217
275
|
for (auto& file : files_) {
|
|
218
|
-
|
|
219
|
-
|
|
276
|
+
TableReader* t = file->fd.pinned_reader.Get();
|
|
277
|
+
if (t != nullptr) {
|
|
278
|
+
delete t;
|
|
220
279
|
}
|
|
221
280
|
}
|
|
222
281
|
files_.clear();
|
|
@@ -242,6 +301,60 @@ class CompactionPickerU64TsTest : public CompactionPickerTestBase {
|
|
|
242
301
|
: CompactionPickerTestBase(test::BytewiseComparatorWithU64TsWrapper()) {}
|
|
243
302
|
|
|
244
303
|
~CompactionPickerU64TsTest() override = default;
|
|
304
|
+
|
|
305
|
+
protected:
|
|
306
|
+
// Helper to create a U64 timestamp string from a uint64_t value
|
|
307
|
+
static std::string MakeU64Timestamp(uint64_t ts) {
|
|
308
|
+
std::string result;
|
|
309
|
+
PutFixed64(&result, ts);
|
|
310
|
+
return result;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// Helper to add a bottommost file with timestamps and setup version storage
|
|
314
|
+
// for testing bottommost file marking behavior
|
|
315
|
+
void SetupBottommostFileWithTimestamps(uint64_t min_ts, uint64_t max_ts,
|
|
316
|
+
uint64_t full_history_ts_low_val,
|
|
317
|
+
SequenceNumber oldest_snapshot_seqnum,
|
|
318
|
+
std::string* out_full_history_ts_low) {
|
|
319
|
+
std::string ts_small = MakeU64Timestamp(min_ts);
|
|
320
|
+
std::string ts_large = MakeU64Timestamp(max_ts);
|
|
321
|
+
|
|
322
|
+
Add(5, 1U, "100", "200", /*file_size=*/1000, /*path_id=*/0,
|
|
323
|
+
/*smallest_seq=*/10, /*largest_seq=*/40,
|
|
324
|
+
/*compensated_file_size=*/1000,
|
|
325
|
+
/*marked_for_compact=*/false, Temperature::kUnknown,
|
|
326
|
+
kUnknownOldestAncesterTime, kUnknownNewestKeyTime, ts_small, ts_large);
|
|
327
|
+
|
|
328
|
+
std::string full_history_ts_low = MakeU64Timestamp(full_history_ts_low_val);
|
|
329
|
+
|
|
330
|
+
UpdateVersionStorageInfoWithTsLow(full_history_ts_low);
|
|
331
|
+
|
|
332
|
+
vstorage_->UpdateOldestSnapshot(oldest_snapshot_seqnum,
|
|
333
|
+
/*allow_ingest_behind=*/false,
|
|
334
|
+
/*ucmp=*/ucmp_, full_history_ts_low);
|
|
335
|
+
|
|
336
|
+
if (out_full_history_ts_low) {
|
|
337
|
+
*out_full_history_ts_low = full_history_ts_low;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// Helper to add L0 files with timestamps for compaction trigger tests
|
|
342
|
+
void AddL0FilesWithTimestamps(uint64_t ts1_val, uint64_t ts2_val,
|
|
343
|
+
uint64_t file_size = 1U) {
|
|
344
|
+
std::string ts1 = MakeU64Timestamp(ts1_val);
|
|
345
|
+
std::string ts2 = MakeU64Timestamp(ts2_val);
|
|
346
|
+
|
|
347
|
+
Add(0, 1U, "100", "200", file_size, /*path_id=*/0,
|
|
348
|
+
/*smallest_seq=*/100, /*largest_seq=*/100,
|
|
349
|
+
/*compensated_file_size=*/file_size,
|
|
350
|
+
/*marked_for_compact=*/false, Temperature::kUnknown,
|
|
351
|
+
kUnknownOldestAncesterTime, kUnknownNewestKeyTime, ts1, ts2);
|
|
352
|
+
Add(0, 2U, "150", "250", file_size, /*path_id=*/0,
|
|
353
|
+
/*smallest_seq=*/200, /*largest_seq=*/200,
|
|
354
|
+
/*compensated_file_size=*/file_size,
|
|
355
|
+
/*marked_for_compact=*/false, Temperature::kUnknown,
|
|
356
|
+
kUnknownOldestAncesterTime, kUnknownNewestKeyTime, ts1, ts2);
|
|
357
|
+
}
|
|
245
358
|
};
|
|
246
359
|
|
|
247
360
|
TEST_F(CompactionPickerTest, Empty) {
|
|
@@ -250,7 +363,7 @@ TEST_F(CompactionPickerTest, Empty) {
|
|
|
250
363
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
251
364
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
252
365
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
253
|
-
vstorage_.get(), &log_buffer_));
|
|
366
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
254
367
|
ASSERT_TRUE(compaction.get() == nullptr);
|
|
255
368
|
}
|
|
256
369
|
|
|
@@ -263,7 +376,7 @@ TEST_F(CompactionPickerTest, Single) {
|
|
|
263
376
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
264
377
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
265
378
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
266
|
-
vstorage_.get(), &log_buffer_));
|
|
379
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
267
380
|
ASSERT_TRUE(compaction.get() == nullptr);
|
|
268
381
|
}
|
|
269
382
|
|
|
@@ -278,7 +391,7 @@ TEST_F(CompactionPickerTest, Level0Trigger) {
|
|
|
278
391
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
279
392
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
280
393
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
281
|
-
vstorage_.get(), &log_buffer_));
|
|
394
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
282
395
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
283
396
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
284
397
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
@@ -293,7 +406,7 @@ TEST_F(CompactionPickerTest, Level1Trigger) {
|
|
|
293
406
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
294
407
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
295
408
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
296
|
-
vstorage_.get(), &log_buffer_));
|
|
409
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
297
410
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
298
411
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
299
412
|
ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
|
|
@@ -313,7 +426,7 @@ TEST_F(CompactionPickerTest, Level1Trigger2) {
|
|
|
313
426
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
314
427
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
315
428
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
316
|
-
vstorage_.get(), &log_buffer_));
|
|
429
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
317
430
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
318
431
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
319
432
|
ASSERT_EQ(2U, compaction->num_input_files(1));
|
|
@@ -346,7 +459,7 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
|
|
|
346
459
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
347
460
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
348
461
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
349
|
-
vstorage_.get(), &log_buffer_));
|
|
462
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
350
463
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
351
464
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
352
465
|
ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
|
|
@@ -395,7 +508,7 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic) {
|
|
|
395
508
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
396
509
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
397
510
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
398
|
-
vstorage_.get(), &log_buffer_));
|
|
511
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
399
512
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
400
513
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
401
514
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
@@ -421,7 +534,7 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic2) {
|
|
|
421
534
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
422
535
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
423
536
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
424
|
-
vstorage_.get(), &log_buffer_));
|
|
537
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
425
538
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
426
539
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
427
540
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
@@ -448,7 +561,7 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic3) {
|
|
|
448
561
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
449
562
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
450
563
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
451
|
-
vstorage_.get(), &log_buffer_));
|
|
564
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
452
565
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
453
566
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
454
567
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
@@ -479,7 +592,7 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic4) {
|
|
|
479
592
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
480
593
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
481
594
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
482
|
-
vstorage_.get(), &log_buffer_));
|
|
595
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
483
596
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
484
597
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
485
598
|
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
|
|
@@ -513,7 +626,7 @@ TEST_F(CompactionPickerTest, LevelTriggerDynamic4) {
|
|
|
513
626
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
514
627
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
515
628
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
516
|
-
vstorage_.get(), &log_buffer_));
|
|
629
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
517
630
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
518
631
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
519
632
|
ASSERT_EQ(5U, compaction->input(0, 0)->fd.GetNumber());
|
|
@@ -575,7 +688,7 @@ TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
|
|
|
575
688
|
universal_compaction_picker.PickCompaction(
|
|
576
689
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
577
690
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
578
|
-
vstorage_.get(), &log_buffer_));
|
|
691
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
579
692
|
|
|
580
693
|
// output level should be the one above the bottom-most
|
|
581
694
|
ASSERT_EQ(1, compaction->output_level());
|
|
@@ -620,7 +733,7 @@ TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
|
|
|
620
733
|
universal_compaction_picker.PickCompaction(
|
|
621
734
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
622
735
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
623
|
-
vstorage_.get(), &log_buffer_));
|
|
736
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
624
737
|
|
|
625
738
|
ASSERT_TRUE(!compaction->is_trivial_move());
|
|
626
739
|
}
|
|
@@ -648,7 +761,7 @@ TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) {
|
|
|
648
761
|
universal_compaction_picker.PickCompaction(
|
|
649
762
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
650
763
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
651
|
-
vstorage_.get(), &log_buffer_));
|
|
764
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
652
765
|
|
|
653
766
|
ASSERT_TRUE(compaction->is_trivial_move());
|
|
654
767
|
}
|
|
@@ -678,7 +791,7 @@ TEST_F(CompactionPickerTest, UniversalPeriodicCompaction1) {
|
|
|
678
791
|
universal_compaction_picker.PickCompaction(
|
|
679
792
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
680
793
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
681
|
-
vstorage_.get(), &log_buffer_));
|
|
794
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
682
795
|
|
|
683
796
|
ASSERT_TRUE(compaction);
|
|
684
797
|
ASSERT_EQ(4, compaction->output_level());
|
|
@@ -710,7 +823,7 @@ TEST_F(CompactionPickerTest, UniversalPeriodicCompaction2) {
|
|
|
710
823
|
universal_compaction_picker.PickCompaction(
|
|
711
824
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
712
825
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
713
|
-
vstorage_.get(), &log_buffer_));
|
|
826
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
714
827
|
|
|
715
828
|
ASSERT_FALSE(compaction);
|
|
716
829
|
}
|
|
@@ -738,7 +851,7 @@ TEST_F(CompactionPickerTest, UniversalPeriodicCompaction3) {
|
|
|
738
851
|
universal_compaction_picker.PickCompaction(
|
|
739
852
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
740
853
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
741
|
-
vstorage_.get(), &log_buffer_));
|
|
854
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
742
855
|
|
|
743
856
|
ASSERT_FALSE(compaction);
|
|
744
857
|
}
|
|
@@ -770,7 +883,7 @@ TEST_F(CompactionPickerTest, UniversalPeriodicCompaction4) {
|
|
|
770
883
|
universal_compaction_picker.PickCompaction(
|
|
771
884
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
772
885
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
773
|
-
vstorage_.get(), &log_buffer_));
|
|
886
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
774
887
|
ASSERT_TRUE(!compaction ||
|
|
775
888
|
compaction->start_level() != compaction->output_level());
|
|
776
889
|
}
|
|
@@ -792,7 +905,7 @@ TEST_F(CompactionPickerTest, UniversalPeriodicCompaction5) {
|
|
|
792
905
|
universal_compaction_picker.PickCompaction(
|
|
793
906
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
794
907
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
795
|
-
vstorage_.get(), &log_buffer_));
|
|
908
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
796
909
|
ASSERT_TRUE(compaction);
|
|
797
910
|
ASSERT_EQ(0, compaction->start_level());
|
|
798
911
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
@@ -818,7 +931,7 @@ TEST_F(CompactionPickerTest, UniversalPeriodicCompaction6) {
|
|
|
818
931
|
universal_compaction_picker.PickCompaction(
|
|
819
932
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
820
933
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
821
|
-
vstorage_.get(), &log_buffer_));
|
|
934
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
822
935
|
ASSERT_TRUE(compaction);
|
|
823
936
|
ASSERT_EQ(4, compaction->start_level());
|
|
824
937
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
@@ -857,7 +970,7 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace1) {
|
|
|
857
970
|
universal_compaction_picker.PickCompaction(
|
|
858
971
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
859
972
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
860
|
-
vstorage_.get(), &log_buffer_));
|
|
973
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
861
974
|
ASSERT_TRUE(compaction);
|
|
862
975
|
ASSERT_EQ(4, compaction->output_level());
|
|
863
976
|
ASSERT_EQ(3, compaction->start_level());
|
|
@@ -900,7 +1013,7 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace2) {
|
|
|
900
1013
|
universal_compaction_picker.PickCompaction(
|
|
901
1014
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
902
1015
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
903
|
-
vstorage_.get(), &log_buffer_));
|
|
1016
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
904
1017
|
ASSERT_TRUE(compaction);
|
|
905
1018
|
ASSERT_EQ(4, compaction->output_level());
|
|
906
1019
|
ASSERT_EQ(2, compaction->start_level());
|
|
@@ -943,7 +1056,7 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace3) {
|
|
|
943
1056
|
universal_compaction_picker.PickCompaction(
|
|
944
1057
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
945
1058
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
946
|
-
vstorage_.get(), &log_buffer_));
|
|
1059
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
947
1060
|
ASSERT_TRUE(compaction);
|
|
948
1061
|
ASSERT_EQ(4, compaction->output_level());
|
|
949
1062
|
ASSERT_EQ(2, compaction->start_level());
|
|
@@ -992,7 +1105,7 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace4) {
|
|
|
992
1105
|
universal_compaction_picker.PickCompaction(
|
|
993
1106
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
994
1107
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
995
|
-
vstorage_.get(), &log_buffer_));
|
|
1108
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
996
1109
|
ASSERT_TRUE(compaction);
|
|
997
1110
|
ASSERT_EQ(4, compaction->output_level());
|
|
998
1111
|
ASSERT_EQ(3, compaction->start_level());
|
|
@@ -1037,7 +1150,7 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace5) {
|
|
|
1037
1150
|
universal_compaction_picker.PickCompaction(
|
|
1038
1151
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1039
1152
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1040
|
-
vstorage_.get(), &log_buffer_));
|
|
1153
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1041
1154
|
ASSERT_TRUE(compaction);
|
|
1042
1155
|
ASSERT_EQ(4, compaction->output_level());
|
|
1043
1156
|
ASSERT_EQ(3, compaction->start_level());
|
|
@@ -1090,7 +1203,7 @@ TEST_F(CompactionPickerTest,
|
|
|
1090
1203
|
universal_compaction_picker.PickCompaction(
|
|
1091
1204
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1092
1205
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1093
|
-
vstorage_.get(), &log_buffer_));
|
|
1206
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1094
1207
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1095
1208
|
ASSERT_EQ(compaction->compaction_reason(),
|
|
1096
1209
|
CompactionReason::kUniversalSizeAmplification);
|
|
@@ -1174,7 +1287,7 @@ TEST_F(CompactionPickerTest, FIFOToCold1) {
|
|
|
1174
1287
|
fifo_compaction_picker.PickCompaction(
|
|
1175
1288
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1176
1289
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1177
|
-
vstorage_.get(), &log_buffer_));
|
|
1290
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1178
1291
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1179
1292
|
ASSERT_EQ(compaction->compaction_reason(),
|
|
1180
1293
|
CompactionReason::kChangeTemperature);
|
|
@@ -1243,7 +1356,7 @@ TEST_F(CompactionPickerTest, FIFOToColdMaxCompactionSize) {
|
|
|
1243
1356
|
fifo_compaction_picker.PickCompaction(
|
|
1244
1357
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1245
1358
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1246
|
-
vstorage_.get(), &log_buffer_));
|
|
1359
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1247
1360
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1248
1361
|
ASSERT_EQ(compaction->compaction_reason(),
|
|
1249
1362
|
CompactionReason::kChangeTemperature);
|
|
@@ -1311,7 +1424,7 @@ TEST_F(CompactionPickerTest, FIFOToColdWithExistingCold) {
|
|
|
1311
1424
|
fifo_compaction_picker.PickCompaction(
|
|
1312
1425
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1313
1426
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1314
|
-
vstorage_.get(), &log_buffer_));
|
|
1427
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1315
1428
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1316
1429
|
ASSERT_EQ(compaction->compaction_reason(),
|
|
1317
1430
|
CompactionReason::kChangeTemperature);
|
|
@@ -1379,7 +1492,7 @@ TEST_F(CompactionPickerTest, FIFOToColdWithHotBetweenCold) {
|
|
|
1379
1492
|
fifo_compaction_picker.PickCompaction(
|
|
1380
1493
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1381
1494
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1382
|
-
vstorage_.get(), &log_buffer_));
|
|
1495
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1383
1496
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1384
1497
|
ASSERT_EQ(compaction->compaction_reason(),
|
|
1385
1498
|
CompactionReason::kChangeTemperature);
|
|
@@ -1459,7 +1572,7 @@ TEST_F(CompactionPickerTest, FIFOToHotAndWarm) {
|
|
|
1459
1572
|
fifo_compaction_picker.PickCompaction(
|
|
1460
1573
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1461
1574
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1462
|
-
vstorage_.get(), &log_buffer_));
|
|
1575
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1463
1576
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1464
1577
|
ASSERT_EQ(compaction->compaction_reason(),
|
|
1465
1578
|
CompactionReason::kChangeTemperature);
|
|
@@ -1516,7 +1629,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
|
|
|
1516
1629
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1517
1630
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1518
1631
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1519
|
-
vstorage_.get(), &log_buffer_));
|
|
1632
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1520
1633
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1521
1634
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
1522
1635
|
// Pick file 8 because it overlaps with 0 files on level 3.
|
|
@@ -1550,7 +1663,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
|
|
|
1550
1663
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1551
1664
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1552
1665
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1553
|
-
vstorage_.get(), &log_buffer_));
|
|
1666
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1554
1667
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1555
1668
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
1556
1669
|
// Picking file 7 because overlapping ratio is the biggest.
|
|
@@ -1579,7 +1692,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
|
|
|
1579
1692
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1580
1693
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1581
1694
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1582
|
-
vstorage_.get(), &log_buffer_));
|
|
1695
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1583
1696
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1584
1697
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
1585
1698
|
// Picking file 8 because overlapping ratio is the biggest.
|
|
@@ -1608,7 +1721,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping4) {
|
|
|
1608
1721
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1609
1722
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1610
1723
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1611
|
-
vstorage_.get(), &log_buffer_));
|
|
1724
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1612
1725
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1613
1726
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
1614
1727
|
// Picking file 6 because overlapping ratio is the biggest.
|
|
@@ -1645,7 +1758,7 @@ TEST_F(CompactionPickerTest, CompactionPriRoundRobin) {
|
|
|
1645
1758
|
local_level_compaction_picker.PickCompaction(
|
|
1646
1759
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1647
1760
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1648
|
-
vstorage_.get(), &log_buffer_));
|
|
1761
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1649
1762
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1650
1763
|
// Since the max bytes for level 2 is 120M, picking one file to compact
|
|
1651
1764
|
// makes the post-compaction level size less than 120M, there is exactly one
|
|
@@ -1686,7 +1799,7 @@ TEST_F(CompactionPickerTest, CompactionPriMultipleFilesRoundRobin1) {
|
|
|
1686
1799
|
local_level_compaction_picker.PickCompaction(
|
|
1687
1800
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1688
1801
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1689
|
-
vstorage_.get(), &log_buffer_));
|
|
1802
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1690
1803
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1691
1804
|
|
|
1692
1805
|
// The maximum compaction bytes is very large in this case so we can igore its
|
|
@@ -1730,7 +1843,7 @@ TEST_F(CompactionPickerTest, CompactionPriMultipleFilesRoundRobin2) {
|
|
|
1730
1843
|
local_level_compaction_picker.PickCompaction(
|
|
1731
1844
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1732
1845
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1733
|
-
vstorage_.get(), &log_buffer_));
|
|
1846
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1734
1847
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1735
1848
|
|
|
1736
1849
|
// The maximum compaction bytes is only 2500 bytes now. Even though we are
|
|
@@ -1775,7 +1888,7 @@ TEST_F(CompactionPickerTest, CompactionPriMultipleFilesRoundRobin3) {
|
|
|
1775
1888
|
local_level_compaction_picker.PickCompaction(
|
|
1776
1889
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1777
1890
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1778
|
-
vstorage_.get(), &log_buffer_));
|
|
1891
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1779
1892
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1780
1893
|
|
|
1781
1894
|
// Cannot pick more files since we reach the last file in level 2
|
|
@@ -1835,7 +1948,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlappingManyFiles) {
|
|
|
1835
1948
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1836
1949
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1837
1950
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1838
|
-
vstorage_.get(), &log_buffer_));
|
|
1951
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1839
1952
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1840
1953
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
1841
1954
|
// Picking file 8 because overlapping ratio is the biggest.
|
|
@@ -1864,7 +1977,7 @@ TEST_F(CompactionPickerTest, ParentIndexResetBug) {
|
|
|
1864
1977
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1865
1978
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1866
1979
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1867
|
-
vstorage_.get(), &log_buffer_));
|
|
1980
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1868
1981
|
}
|
|
1869
1982
|
|
|
1870
1983
|
// This test checks ExpandWhileOverlapping() by having overlapping user keys
|
|
@@ -1883,7 +1996,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys) {
|
|
|
1883
1996
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1884
1997
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1885
1998
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1886
|
-
vstorage_.get(), &log_buffer_));
|
|
1999
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1887
2000
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1888
2001
|
ASSERT_EQ(1U, compaction->num_input_levels());
|
|
1889
2002
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
@@ -1904,7 +2017,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys2) {
|
|
|
1904
2017
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1905
2018
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1906
2019
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1907
|
-
vstorage_.get(), &log_buffer_));
|
|
2020
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1908
2021
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1909
2022
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
1910
2023
|
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
@@ -1933,7 +2046,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys3) {
|
|
|
1933
2046
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1934
2047
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1935
2048
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1936
|
-
vstorage_.get(), &log_buffer_));
|
|
2049
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1937
2050
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1938
2051
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
1939
2052
|
ASSERT_EQ(5U, compaction->num_input_files(0));
|
|
@@ -1965,7 +2078,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys4) {
|
|
|
1965
2078
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1966
2079
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1967
2080
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1968
|
-
vstorage_.get(), &log_buffer_));
|
|
2081
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1969
2082
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
1970
2083
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
1971
2084
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
@@ -1990,7 +2103,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys5) {
|
|
|
1990
2103
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
1991
2104
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
1992
2105
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
1993
|
-
vstorage_.get(), &log_buffer_));
|
|
2106
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
1994
2107
|
ASSERT_TRUE(compaction.get() == nullptr);
|
|
1995
2108
|
}
|
|
1996
2109
|
|
|
@@ -2013,7 +2126,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys6) {
|
|
|
2013
2126
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2014
2127
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2015
2128
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2016
|
-
vstorage_.get(), &log_buffer_));
|
|
2129
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2017
2130
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2018
2131
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
2019
2132
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
@@ -2035,7 +2148,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys7) {
|
|
|
2035
2148
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2036
2149
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2037
2150
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2038
|
-
vstorage_.get(), &log_buffer_));
|
|
2151
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2039
2152
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2040
2153
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
2041
2154
|
ASSERT_GE(1U, compaction->num_input_files(0));
|
|
@@ -2065,7 +2178,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys8) {
|
|
|
2065
2178
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2066
2179
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2067
2180
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2068
|
-
vstorage_.get(), &log_buffer_));
|
|
2181
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2069
2182
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2070
2183
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
2071
2184
|
ASSERT_EQ(3U, compaction->num_input_files(0));
|
|
@@ -2099,7 +2212,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys9) {
|
|
|
2099
2212
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2100
2213
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2101
2214
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2102
|
-
vstorage_.get(), &log_buffer_));
|
|
2215
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2103
2216
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2104
2217
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
2105
2218
|
ASSERT_EQ(5U, compaction->num_input_files(0));
|
|
@@ -2141,7 +2254,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys10) {
|
|
|
2141
2254
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2142
2255
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2143
2256
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2144
|
-
vstorage_.get(), &log_buffer_));
|
|
2257
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2145
2258
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2146
2259
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
2147
2260
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
@@ -2181,7 +2294,7 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys11) {
|
|
|
2181
2294
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2182
2295
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2183
2296
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2184
|
-
vstorage_.get(), &log_buffer_));
|
|
2297
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2185
2298
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2186
2299
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
2187
2300
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
@@ -2289,7 +2402,7 @@ TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
|
|
|
2289
2402
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2290
2403
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2291
2404
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2292
|
-
vstorage_.get(), &log_buffer_));
|
|
2405
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2293
2406
|
ASSERT_TRUE(compaction.get() == nullptr);
|
|
2294
2407
|
}
|
|
2295
2408
|
|
|
@@ -2321,7 +2434,7 @@ TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri2) {
|
|
|
2321
2434
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2322
2435
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2323
2436
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2324
|
-
vstorage_.get(), &log_buffer_));
|
|
2437
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2325
2438
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2326
2439
|
}
|
|
2327
2440
|
|
|
@@ -2356,7 +2469,7 @@ TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri3) {
|
|
|
2356
2469
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2357
2470
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2358
2471
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2359
|
-
vstorage_.get(), &log_buffer_));
|
|
2472
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2360
2473
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2361
2474
|
}
|
|
2362
2475
|
|
|
@@ -2658,7 +2771,7 @@ TEST_F(CompactionPickerTest, CompactionLimitWhenAddFileFromInputLevel) {
|
|
|
2658
2771
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2659
2772
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2660
2773
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2661
|
-
vstorage_.get(), &log_buffer_));
|
|
2774
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2662
2775
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2663
2776
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
2664
2777
|
ASSERT_EQ(4U, compaction->num_input_files(0));
|
|
@@ -2694,7 +2807,7 @@ TEST_F(CompactionPickerTest, HitCompactionLimitWhenAddFileFromInputLevel) {
|
|
|
2694
2807
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2695
2808
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2696
2809
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2697
|
-
vstorage_.get(), &log_buffer_));
|
|
2810
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2698
2811
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2699
2812
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
2700
2813
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
@@ -2726,7 +2839,7 @@ TEST_F(CompactionPickerTest, CompactRangeMaxCompactionBytes) {
|
|
|
2726
2839
|
/*compact_range_options*/ {}, /*begin=*/nullptr, /*end=*/nullptr,
|
|
2727
2840
|
&manual_end_ptr, &manual_conflict,
|
|
2728
2841
|
/*max_file_num_to_ignore=*/std::numeric_limits<uint64_t>::max(),
|
|
2729
|
-
/*trim_ts=*/""));
|
|
2842
|
+
/*trim_ts=*/"", /*full_history_ts_low=*/""));
|
|
2730
2843
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2731
2844
|
ASSERT_EQ(1U, compaction->num_input_levels());
|
|
2732
2845
|
ASSERT_EQ(2, compaction->output_level());
|
|
@@ -2755,7 +2868,7 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOn) {
|
|
|
2755
2868
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2756
2869
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2757
2870
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2758
|
-
vstorage_.get(), &log_buffer_));
|
|
2871
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2759
2872
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2760
2873
|
ASSERT_TRUE(compaction->IsTrivialMove());
|
|
2761
2874
|
}
|
|
@@ -2781,7 +2894,7 @@ TEST_F(CompactionPickerTest, L0TrivialMove1) {
|
|
|
2781
2894
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2782
2895
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2783
2896
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2784
|
-
vstorage_.get(), &log_buffer_));
|
|
2897
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2785
2898
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2786
2899
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
2787
2900
|
ASSERT_EQ(2, compaction->num_input_files(0));
|
|
@@ -2811,7 +2924,7 @@ TEST_F(CompactionPickerTest, L0TrivialMoveOneFile) {
|
|
|
2811
2924
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2812
2925
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2813
2926
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2814
|
-
vstorage_.get(), &log_buffer_));
|
|
2927
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2815
2928
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2816
2929
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
2817
2930
|
ASSERT_EQ(1, compaction->num_input_files(0));
|
|
@@ -2838,7 +2951,7 @@ TEST_F(CompactionPickerTest, L0TrivialMoveWholeL0) {
|
|
|
2838
2951
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2839
2952
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2840
2953
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2841
|
-
vstorage_.get(), &log_buffer_));
|
|
2954
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2842
2955
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2843
2956
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
2844
2957
|
ASSERT_EQ(4, compaction->num_input_files(0));
|
|
@@ -2867,7 +2980,7 @@ TEST_F(CompactionPickerTest, NonL0TrivialMoveExtendBothDirection) {
|
|
|
2867
2980
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2868
2981
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2869
2982
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2870
|
-
vstorage_.get(), &log_buffer_));
|
|
2983
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2871
2984
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2872
2985
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
2873
2986
|
ASSERT_EQ(3, compaction->num_input_files(0));
|
|
@@ -2898,7 +3011,7 @@ TEST_F(CompactionPickerTest, L0TrivialMoveToEmptyLevel) {
|
|
|
2898
3011
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2899
3012
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2900
3013
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2901
|
-
vstorage_.get(), &log_buffer_));
|
|
3014
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2902
3015
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2903
3016
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
2904
3017
|
ASSERT_EQ(1, compaction->num_input_files(0));
|
|
@@ -2927,7 +3040,7 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOffSstPartitioned) {
|
|
|
2927
3040
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2928
3041
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2929
3042
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2930
|
-
vstorage_.get(), &log_buffer_));
|
|
3043
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2931
3044
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2932
3045
|
// No trivial move, because partitioning is applied
|
|
2933
3046
|
ASSERT_TRUE(!compaction->IsTrivialMove());
|
|
@@ -2951,7 +3064,7 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
|
|
|
2951
3064
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2952
3065
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2953
3066
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2954
|
-
vstorage_.get(), &log_buffer_));
|
|
3067
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2955
3068
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2956
3069
|
ASSERT_FALSE(compaction->IsTrivialMove());
|
|
2957
3070
|
}
|
|
@@ -2981,7 +3094,7 @@ TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles1) {
|
|
|
2981
3094
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
2982
3095
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
2983
3096
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
2984
|
-
vstorage_.get(), &log_buffer_));
|
|
3097
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
2985
3098
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
2986
3099
|
ASSERT_TRUE(compaction->IsTrivialMove());
|
|
2987
3100
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
@@ -3016,7 +3129,7 @@ TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles2) {
|
|
|
3016
3129
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
3017
3130
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3018
3131
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3019
|
-
vstorage_.get(), &log_buffer_));
|
|
3132
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3020
3133
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
3021
3134
|
ASSERT_TRUE(compaction->IsTrivialMove());
|
|
3022
3135
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
@@ -3050,7 +3163,7 @@ TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles3) {
|
|
|
3050
3163
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
3051
3164
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3052
3165
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3053
|
-
vstorage_.get(), &log_buffer_));
|
|
3166
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3054
3167
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
3055
3168
|
ASSERT_TRUE(compaction->IsTrivialMove());
|
|
3056
3169
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
@@ -3077,7 +3190,7 @@ TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles4) {
|
|
|
3077
3190
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
3078
3191
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3079
3192
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3080
|
-
vstorage_.get(), &log_buffer_));
|
|
3193
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3081
3194
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
3082
3195
|
ASSERT_TRUE(compaction->IsTrivialMove());
|
|
3083
3196
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
@@ -3108,7 +3221,7 @@ TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles5) {
|
|
|
3108
3221
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
3109
3222
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3110
3223
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3111
|
-
vstorage_.get(), &log_buffer_));
|
|
3224
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3112
3225
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
3113
3226
|
ASSERT_TRUE(compaction->IsTrivialMove());
|
|
3114
3227
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
@@ -3143,7 +3256,7 @@ TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles6) {
|
|
|
3143
3256
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
3144
3257
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3145
3258
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3146
|
-
vstorage_.get(), &log_buffer_));
|
|
3259
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3147
3260
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
3148
3261
|
ASSERT_TRUE(compaction->IsTrivialMove());
|
|
3149
3262
|
ASSERT_EQ(1, compaction->num_input_levels());
|
|
@@ -3179,7 +3292,7 @@ TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
|
|
|
3179
3292
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
3180
3293
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3181
3294
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3182
|
-
vstorage_.get(), &log_buffer_));
|
|
3295
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3183
3296
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
3184
3297
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
3185
3298
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
@@ -3190,7 +3303,7 @@ TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
|
|
|
3190
3303
|
compaction.reset(level_compaction_picker.PickCompaction(
|
|
3191
3304
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3192
3305
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3193
|
-
vstorage_.get(), &log_buffer_));
|
|
3306
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3194
3307
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
3195
3308
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
3196
3309
|
ASSERT_EQ(1U, compaction->num_input_files(0));
|
|
@@ -3201,7 +3314,7 @@ TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
|
|
|
3201
3314
|
compaction.reset(level_compaction_picker.PickCompaction(
|
|
3202
3315
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3203
3316
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3204
|
-
vstorage_.get(), &log_buffer_));
|
|
3317
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3205
3318
|
ASSERT_TRUE(compaction.get() == nullptr);
|
|
3206
3319
|
ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
|
|
3207
3320
|
}
|
|
@@ -3228,7 +3341,7 @@ TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesNotHit) {
|
|
|
3228
3341
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
3229
3342
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3230
3343
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3231
|
-
vstorage_.get(), &log_buffer_));
|
|
3344
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3232
3345
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
3233
3346
|
ASSERT_EQ(1U, compaction->num_input_levels());
|
|
3234
3347
|
ASSERT_EQ(5U, compaction->num_input_files(0));
|
|
@@ -3260,7 +3373,7 @@ TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesHit) {
|
|
|
3260
3373
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
3261
3374
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3262
3375
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3263
|
-
vstorage_.get(), &log_buffer_));
|
|
3376
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3264
3377
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
3265
3378
|
ASSERT_EQ(1U, compaction->num_input_levels());
|
|
3266
3379
|
ASSERT_EQ(4U, compaction->num_input_files(0));
|
|
@@ -3310,7 +3423,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
|
|
|
3310
3423
|
universal_compaction_picker.PickCompaction(
|
|
3311
3424
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3312
3425
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3313
|
-
vstorage_.get(), &log_buffer_));
|
|
3426
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3314
3427
|
|
|
3315
3428
|
ASSERT_TRUE(compaction);
|
|
3316
3429
|
// Validate that its a compaction to reduce sorted runs
|
|
@@ -3334,7 +3447,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
|
|
|
3334
3447
|
universal_compaction_picker.PickCompaction(
|
|
3335
3448
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3336
3449
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3337
|
-
vstorage_.get(), &log_buffer_));
|
|
3450
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3338
3451
|
ASSERT_FALSE(compaction2);
|
|
3339
3452
|
}
|
|
3340
3453
|
|
|
@@ -3365,7 +3478,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap2) {
|
|
|
3365
3478
|
universal_compaction_picker.PickCompaction(
|
|
3366
3479
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3367
3480
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3368
|
-
vstorage_.get(), &log_buffer_));
|
|
3481
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3369
3482
|
|
|
3370
3483
|
ASSERT_TRUE(compaction);
|
|
3371
3484
|
// Validate that its a delete triggered compaction
|
|
@@ -3396,7 +3509,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap2) {
|
|
|
3396
3509
|
universal_compaction_picker.PickCompaction(
|
|
3397
3510
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3398
3511
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3399
|
-
vstorage_.get(), &log_buffer_));
|
|
3512
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3400
3513
|
ASSERT_FALSE(compaction2);
|
|
3401
3514
|
}
|
|
3402
3515
|
|
|
@@ -3438,7 +3551,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionStartOutputOverlap) {
|
|
|
3438
3551
|
universal_compaction_picker.PickCompaction(
|
|
3439
3552
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3440
3553
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3441
|
-
vstorage_.get(), &log_buffer_));
|
|
3554
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3442
3555
|
|
|
3443
3556
|
ASSERT_TRUE(compaction);
|
|
3444
3557
|
// Validate that its a delete triggered compaction
|
|
@@ -3463,14 +3576,15 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionStartOutputOverlap) {
|
|
|
3463
3576
|
ASSERT_EQ(1U, compaction->num_input_files(1));
|
|
3464
3577
|
}
|
|
3465
3578
|
|
|
3466
|
-
vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_
|
|
3579
|
+
vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_,
|
|
3580
|
+
/*full_history_ts_low=*/"");
|
|
3467
3581
|
// After recomputing the compaction score, only one marked file will remain
|
|
3468
3582
|
random_index = 0;
|
|
3469
3583
|
std::unique_ptr<Compaction> compaction2(
|
|
3470
3584
|
universal_compaction_picker.PickCompaction(
|
|
3471
3585
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3472
3586
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3473
|
-
vstorage_.get(), &log_buffer_));
|
|
3587
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3474
3588
|
ASSERT_FALSE(compaction2);
|
|
3475
3589
|
DeleteVersionStorage();
|
|
3476
3590
|
}
|
|
@@ -3497,7 +3611,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0NoOverlap) {
|
|
|
3497
3611
|
universal_compaction_picker.PickCompaction(
|
|
3498
3612
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3499
3613
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3500
|
-
vstorage_.get(), &log_buffer_));
|
|
3614
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3501
3615
|
|
|
3502
3616
|
ASSERT_TRUE(compaction);
|
|
3503
3617
|
// Validate that its a delete triggered compaction
|
|
@@ -3535,7 +3649,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0WithOverlap) {
|
|
|
3535
3649
|
universal_compaction_picker.PickCompaction(
|
|
3536
3650
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3537
3651
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3538
|
-
vstorage_.get(), &log_buffer_));
|
|
3652
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3539
3653
|
|
|
3540
3654
|
ASSERT_TRUE(compaction);
|
|
3541
3655
|
// Validate that its a delete triggered compaction
|
|
@@ -3593,7 +3707,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0Overlap2) {
|
|
|
3593
3707
|
universal_compaction_picker.PickCompaction(
|
|
3594
3708
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3595
3709
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3596
|
-
vstorage_.get(), &log_buffer_));
|
|
3710
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3597
3711
|
|
|
3598
3712
|
ASSERT_TRUE(compaction);
|
|
3599
3713
|
// Validate that its a delete triggered compaction
|
|
@@ -3627,7 +3741,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0Overlap2) {
|
|
|
3627
3741
|
universal_compaction_picker.PickCompaction(
|
|
3628
3742
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3629
3743
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3630
|
-
vstorage_.get(), &log_buffer_));
|
|
3744
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3631
3745
|
ASSERT_TRUE(compaction2);
|
|
3632
3746
|
ASSERT_EQ(3U, compaction->num_input_files(0));
|
|
3633
3747
|
ASSERT_TRUE(file_map_[1].first->being_compacted);
|
|
@@ -3662,7 +3776,8 @@ TEST_F(CompactionPickerTest, UniversalMarkedManualCompaction) {
|
|
|
3662
3776
|
cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
|
|
3663
3777
|
ColumnFamilyData::kCompactAllLevels, 6, CompactRangeOptions(),
|
|
3664
3778
|
nullptr, nullptr, &manual_end, &manual_conflict,
|
|
3665
|
-
std::numeric_limits<uint64_t>::max(), ""
|
|
3779
|
+
std::numeric_limits<uint64_t>::max(), "",
|
|
3780
|
+
/*full_history_ts_low=*/""));
|
|
3666
3781
|
|
|
3667
3782
|
ASSERT_TRUE(compaction);
|
|
3668
3783
|
|
|
@@ -3707,7 +3822,7 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNonLastLevel) {
|
|
|
3707
3822
|
universal_compaction_picker.PickCompaction(
|
|
3708
3823
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3709
3824
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3710
|
-
vstorage_.get(), &log_buffer_));
|
|
3825
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3711
3826
|
|
|
3712
3827
|
// Make sure it's a size amp compaction and includes all files
|
|
3713
3828
|
ASSERT_EQ(compaction->compaction_reason(),
|
|
@@ -3744,7 +3859,7 @@ TEST_F(CompactionPickerTest, UniversalSizeRatioTierCompactionLastLevel) {
|
|
|
3744
3859
|
universal_compaction_picker.PickCompaction(
|
|
3745
3860
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3746
3861
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3747
|
-
vstorage_.get(), &log_buffer_));
|
|
3862
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3748
3863
|
|
|
3749
3864
|
// Internally, size amp compaction is evaluated before size ratio compaction.
|
|
3750
3865
|
// Here to make sure it's size ratio compaction instead of size amp
|
|
@@ -3785,7 +3900,7 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNotSuport) {
|
|
|
3785
3900
|
universal_compaction_picker.PickCompaction(
|
|
3786
3901
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3787
3902
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3788
|
-
vstorage_.get(), &log_buffer_));
|
|
3903
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3789
3904
|
|
|
3790
3905
|
// size amp compaction is still triggered even preclude_last_level is set
|
|
3791
3906
|
ASSERT_EQ(compaction->compaction_reason(),
|
|
@@ -3820,7 +3935,7 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionLastLevel) {
|
|
|
3820
3935
|
universal_compaction_picker.PickCompaction(
|
|
3821
3936
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3822
3937
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3823
|
-
vstorage_.get(), &log_buffer_));
|
|
3938
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3824
3939
|
|
|
3825
3940
|
// It's a Size Amp compaction, but doesn't include the last level file and
|
|
3826
3941
|
// output to the proximal level.
|
|
@@ -3933,7 +4048,7 @@ TEST_F(CompactionPickerU64TsTest, CannotTrivialMoveUniversal) {
|
|
|
3933
4048
|
universal_compaction_picker.PickCompaction(
|
|
3934
4049
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
3935
4050
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
3936
|
-
vstorage_.get(), &log_buffer_));
|
|
4051
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
3937
4052
|
assert(compaction);
|
|
3938
4053
|
ASSERT_TRUE(!compaction->is_trivial_move());
|
|
3939
4054
|
}
|
|
@@ -4478,7 +4593,7 @@ TEST_F(CompactionPickerTest,
|
|
|
4478
4593
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
4479
4594
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
4480
4595
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
4481
|
-
vstorage_.get(), &log_buffer_));
|
|
4596
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
4482
4597
|
ASSERT_TRUE(compaction);
|
|
4483
4598
|
ASSERT_EQ(num_levels - 2, compaction->start_level());
|
|
4484
4599
|
ASSERT_EQ(num_levels - 1, compaction->output_level());
|
|
@@ -4489,7 +4604,7 @@ TEST_F(CompactionPickerTest,
|
|
|
4489
4604
|
level_compaction_picker.PickCompaction(
|
|
4490
4605
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
4491
4606
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
4492
|
-
vstorage_.get(), &log_buffer_));
|
|
4607
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
4493
4608
|
ASSERT_TRUE(second_compaction);
|
|
4494
4609
|
ASSERT_EQ(num_levels - 1, compaction->output_level());
|
|
4495
4610
|
ASSERT_EQ(num_levels - 2, compaction->start_level());
|
|
@@ -4536,7 +4651,7 @@ TEST_F(CompactionPickerTest,
|
|
|
4536
4651
|
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
4537
4652
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
4538
4653
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
4539
|
-
vstorage_.get(), &log_buffer_));
|
|
4654
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
4540
4655
|
ASSERT_TRUE(compaction);
|
|
4541
4656
|
ASSERT_EQ(num_levels - 3, compaction->start_level());
|
|
4542
4657
|
ASSERT_EQ(num_levels - 2, compaction->output_level());
|
|
@@ -4586,7 +4701,7 @@ TEST_F(CompactionPickerTest, IntraL0WhenL0IsSmall) {
|
|
|
4586
4701
|
std::unique_ptr<Compaction> compaction(compaction_picker.PickCompaction(
|
|
4587
4702
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
4588
4703
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
4589
|
-
vstorage_.get(), &log_buffer_));
|
|
4704
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
4590
4705
|
ASSERT_TRUE(compaction.get() != nullptr);
|
|
4591
4706
|
ASSERT_EQ(CompactionReason::kLevelL0FilesNum,
|
|
4592
4707
|
compaction->compaction_reason());
|
|
@@ -4663,7 +4778,7 @@ TEST_F(CompactionPickerTest, UniversalMaxReadAmpLargeDB) {
|
|
|
4663
4778
|
universal_compaction_picker.PickCompaction(
|
|
4664
4779
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
4665
4780
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
4666
|
-
vstorage_.get(), &log_buffer_));
|
|
4781
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
4667
4782
|
if (i == kMaxRuns) {
|
|
4668
4783
|
// There are in total i + 1 > kMaxRuns sorted runs.
|
|
4669
4784
|
// This triggers compaction ignoring size_ratio.
|
|
@@ -4711,7 +4826,7 @@ TEST_F(CompactionPickerTest, UniversalMaxReadAmpSmallDB) {
|
|
|
4711
4826
|
universal_compaction_picker.PickCompaction(
|
|
4712
4827
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
4713
4828
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
4714
|
-
vstorage_.get(), &log_buffer_));
|
|
4829
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
4715
4830
|
ASSERT_EQ(nullptr, compaction);
|
|
4716
4831
|
}
|
|
4717
4832
|
}
|
|
@@ -4746,7 +4861,7 @@ TEST_F(CompactionPickerTest, StandaloneRangeDeletionOnlyPicksOlderFiles) {
|
|
|
4746
4861
|
universal_compaction_picker.PickCompaction(
|
|
4747
4862
|
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
4748
4863
|
/*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
|
|
4749
|
-
vstorage_.get(), &log_buffer_));
|
|
4864
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
4750
4865
|
|
|
4751
4866
|
ASSERT_NE(nullptr, compaction);
|
|
4752
4867
|
ASSERT_EQ(2U, compaction->num_input_levels());
|
|
@@ -4763,6 +4878,1195 @@ TEST_F(CompactionPickerTest, StandaloneRangeDeletionOnlyPicksOlderFiles) {
|
|
|
4763
4878
|
ASSERT_EQ(10U, compaction->input(1, 0)->fd.GetNumber());
|
|
4764
4879
|
}
|
|
4765
4880
|
|
|
4881
|
+
// Tests for full_history_ts_low parameter in compaction picker.
|
|
4882
|
+
// The full_history_ts_low parameter is used to control bottommost file marking
|
|
4883
|
+
// for compaction when user-defined timestamps (UDT) are enabled.
|
|
4884
|
+
|
|
4885
|
+
// Level compaction tests for full_history_ts_low:
|
|
4886
|
+
// These tests verify that bottommost files are correctly marked/unmarked
|
|
4887
|
+
// for compaction based on their max timestamp relative to full_history_ts_low.
|
|
4888
|
+
|
|
4889
|
+
TEST_F(CompactionPickerU64TsTest,
|
|
4890
|
+
BottommostNotMarkedWhenTimestampAboveFullHistoryTsLow) {
|
|
4891
|
+
// Test that bottommost files are NOT marked for compaction when their
|
|
4892
|
+
// max timestamp is >= full_history_ts_low. This prevents infinite
|
|
4893
|
+
// compaction loops where timestamp could not be collapsed.
|
|
4894
|
+
NewVersionStorage(6, kCompactionStyleLevel);
|
|
4895
|
+
|
|
4896
|
+
// File has max_ts = 1000, full_history_ts_low = 500
|
|
4897
|
+
// Since 1000 >= 500, the file should NOT be marked for compaction.
|
|
4898
|
+
SetupBottommostFileWithTimestamps(
|
|
4899
|
+
/*min_ts=*/500, /*max_ts=*/1000, /*full_history_ts_low_val=*/500,
|
|
4900
|
+
/*oldest_snapshot_seqnum=*/50, /*out_full_history_ts_low=*/nullptr);
|
|
4901
|
+
|
|
4902
|
+
// File's max_ts (1000) >= full_history_ts_low (500), so it should NOT
|
|
4903
|
+
// be marked for bottommost compaction
|
|
4904
|
+
ASSERT_TRUE(vstorage_->BottommostFilesMarkedForCompaction().empty());
|
|
4905
|
+
}
|
|
4906
|
+
|
|
4907
|
+
TEST_F(CompactionPickerU64TsTest,
|
|
4908
|
+
BottommostMarkedWhenTimestampBelowFullHistoryTsLow) {
|
|
4909
|
+
// Test that bottommost files ARE marked for compaction when their
|
|
4910
|
+
// max timestamp is < full_history_ts_low.
|
|
4911
|
+
NewVersionStorage(6, kCompactionStyleLevel);
|
|
4912
|
+
|
|
4913
|
+
// File has max_ts = 100, full_history_ts_low = 500
|
|
4914
|
+
// Since 100 < 500, the file SHOULD be marked for compaction.
|
|
4915
|
+
SetupBottommostFileWithTimestamps(
|
|
4916
|
+
/*min_ts=*/50, /*max_ts=*/100, /*full_history_ts_low_val=*/500,
|
|
4917
|
+
/*oldest_snapshot_seqnum=*/50, /*out_full_history_ts_low=*/nullptr);
|
|
4918
|
+
|
|
4919
|
+
// File's max_ts (100) < full_history_ts_low (500), so it SHOULD be
|
|
4920
|
+
// marked for bottommost compaction
|
|
4921
|
+
ASSERT_EQ(1U, vstorage_->BottommostFilesMarkedForCompaction().size());
|
|
4922
|
+
ASSERT_EQ(5, vstorage_->BottommostFilesMarkedForCompaction()[0].first);
|
|
4923
|
+
ASSERT_EQ(1U, vstorage_->BottommostFilesMarkedForCompaction()[0]
|
|
4924
|
+
.second->fd.GetNumber());
|
|
4925
|
+
}
|
|
4926
|
+
|
|
4927
|
+
TEST_F(CompactionPickerU64TsTest,
|
|
4928
|
+
BottommostNotMarkedWithEmptyFullHistoryTsLow) {
|
|
4929
|
+
// Test that when full_history_ts_low is empty, files are still marked
|
|
4930
|
+
// based on seqno condition (backward compatibility behavior).
|
|
4931
|
+
NewVersionStorage(6, kCompactionStyleLevel);
|
|
4932
|
+
|
|
4933
|
+
std::string ts_small = MakeU64Timestamp(500);
|
|
4934
|
+
std::string ts_large = MakeU64Timestamp(1000);
|
|
4935
|
+
|
|
4936
|
+
// Add a file at bottommost level with seqno < oldest_snapshot
|
|
4937
|
+
Add(5, 1U, "100", "200", /*file_size=*/1000, /*path_id=*/0,
|
|
4938
|
+
/*smallest_seq=*/10, /*largest_seq=*/40,
|
|
4939
|
+
/*compensated_file_size=*/1000,
|
|
4940
|
+
/*marked_for_compact=*/false, Temperature::kUnknown,
|
|
4941
|
+
kUnknownOldestAncesterTime, kUnknownNewestKeyTime, ts_small, ts_large);
|
|
4942
|
+
|
|
4943
|
+
// Update version storage with empty full_history_ts_low
|
|
4944
|
+
UpdateVersionStorageInfo();
|
|
4945
|
+
|
|
4946
|
+
// Update oldest snapshot with empty full_history_ts_low
|
|
4947
|
+
vstorage_->UpdateOldestSnapshot(
|
|
4948
|
+
/*oldest_snapshot_seqnum=*/50,
|
|
4949
|
+
/*allow_ingest_behind=*/false,
|
|
4950
|
+
/*ucmp=*/ucmp_,
|
|
4951
|
+
/*full_history_ts_low=*/"");
|
|
4952
|
+
|
|
4953
|
+
// With empty full_history_ts_low and UDT enabled, the file should NOT be
|
|
4954
|
+
// marked. When full_history_ts_low is empty, it means it was never set,
|
|
4955
|
+
// effectively 0, which is smaller than any valid timestamp. Since the file's
|
|
4956
|
+
// max_timestamp would be >= full_history_ts_low, it won't be marked.
|
|
4957
|
+
ASSERT_EQ(0U, vstorage_->BottommostFilesMarkedForCompaction().size());
|
|
4958
|
+
}
|
|
4959
|
+
|
|
4960
|
+
TEST_F(CompactionPickerU64TsTest, LevelPickCompactionWithFullHistoryTsLow) {
|
|
4961
|
+
// Test that level compaction correctly passes full_history_ts_low
|
|
4962
|
+
// and picks compaction appropriately
|
|
4963
|
+
NewVersionStorage(6, kCompactionStyleLevel);
|
|
4964
|
+
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
|
4965
|
+
|
|
4966
|
+
AddL0FilesWithTimestamps(/*ts1_val=*/100, /*ts2_val=*/200);
|
|
4967
|
+
|
|
4968
|
+
UpdateVersionStorageInfo();
|
|
4969
|
+
|
|
4970
|
+
std::string full_history_ts_low = MakeU64Timestamp(150);
|
|
4971
|
+
|
|
4972
|
+
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
|
4973
|
+
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
4974
|
+
/*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr, vstorage_.get(),
|
|
4975
|
+
&log_buffer_, full_history_ts_low, /*require_max_output_level=*/false));
|
|
4976
|
+
|
|
4977
|
+
// Compaction should be picked for L0 files
|
|
4978
|
+
ASSERT_NE(nullptr, compaction);
|
|
4979
|
+
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
4980
|
+
ASSERT_EQ(0, compaction->start_level());
|
|
4981
|
+
}
|
|
4982
|
+
|
|
4983
|
+
TEST_F(CompactionPickerU64TsTest, UniversalPickCompactionWithFullHistoryTsLow) {
|
|
4984
|
+
// Test that universal compaction correctly accepts full_history_ts_low
|
|
4985
|
+
constexpr uint64_t kFileSize = 100000;
|
|
4986
|
+
|
|
4987
|
+
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
|
4988
|
+
NewVersionStorage(1, kCompactionStyleUniversal);
|
|
4989
|
+
UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
|
|
4990
|
+
|
|
4991
|
+
AddL0FilesWithTimestamps(/*ts1_val=*/100, /*ts2_val=*/200, kFileSize);
|
|
4992
|
+
|
|
4993
|
+
UpdateVersionStorageInfo();
|
|
4994
|
+
|
|
4995
|
+
std::string full_history_ts_low = MakeU64Timestamp(150);
|
|
4996
|
+
|
|
4997
|
+
std::unique_ptr<Compaction> compaction(
|
|
4998
|
+
universal_compaction_picker.PickCompaction(
|
|
4999
|
+
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
5000
|
+
/*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr,
|
|
5001
|
+
vstorage_.get(), &log_buffer_, full_history_ts_low,
|
|
5002
|
+
/*require_max_output_level=*/false));
|
|
5003
|
+
|
|
5004
|
+
// Universal compaction should be picked
|
|
5005
|
+
ASSERT_NE(nullptr, compaction);
|
|
5006
|
+
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
5007
|
+
}
|
|
5008
|
+
|
|
5009
|
+
// ============================================================================
|
|
5010
|
+
// FIFO Ratio-Based Compaction Picker Unit Tests
|
|
5011
|
+
// Tests the actual FIFOCompactionPicker with use_kv_ratio_compaction option
|
|
5012
|
+
// (PickRatioBasedIntraL0Compaction path).
|
|
5013
|
+
// ============================================================================
|
|
5014
|
+
|
|
5015
|
+
TEST_F(CompactionPickerTest, FIFORatioBasedCompactionFileCountThreshold) {
|
|
5016
|
+
// Test three file count scenarios relative to trigger (= 4):
|
|
5017
|
+
// - fewer than trigger: no compaction
|
|
5018
|
+
// - exactly trigger: compaction fires
|
|
5019
|
+
// - more than trigger: compaction fires, picks >= 2 files
|
|
5020
|
+
|
|
5021
|
+
// Sub-test 1: fewer than trigger (3 files < trigger 4) -> no compaction
|
|
5022
|
+
{
|
|
5023
|
+
SetupFIFORatioBased(10 * 1024 * 1024, 1ULL * 1024 * 1024 * 1024, 4);
|
|
5024
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5025
|
+
Add(0, 1U, "100", "200", 64 * 1024);
|
|
5026
|
+
Add(0, 2U, "200", "300", 64 * 1024);
|
|
5027
|
+
Add(0, 3U, "300", "400", 64 * 1024);
|
|
5028
|
+
AddBlobFile(100, 64ULL * 1024 * 1024);
|
|
5029
|
+
AddBlobFile(101, 64ULL * 1024 * 1024);
|
|
5030
|
+
AddBlobFile(102, 64ULL * 1024 * 1024);
|
|
5031
|
+
|
|
5032
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5033
|
+
ASSERT_EQ(nullptr, compaction.get())
|
|
5034
|
+
<< "Should not compact when file count < trigger";
|
|
5035
|
+
}
|
|
5036
|
+
|
|
5037
|
+
// Sub-test 2: exactly trigger (4 files = trigger 4) -> compaction fires
|
|
5038
|
+
{
|
|
5039
|
+
SetupFIFORatioBased(10 * 1024 * 1024, 1ULL * 1024 * 1024 * 1024, 4);
|
|
5040
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5041
|
+
Add(0, 1U, "100", "200", 64 * 1024);
|
|
5042
|
+
Add(0, 2U, "200", "300", 32 * 1024);
|
|
5043
|
+
Add(0, 3U, "300", "400", 48 * 1024);
|
|
5044
|
+
Add(0, 4U, "400", "500", 96 * 1024);
|
|
5045
|
+
// sst_ratio ~ 240KB/256MB ~ 0.001, target ~ 250KB
|
|
5046
|
+
AddBlobFile(100, 64ULL * 1024 * 1024);
|
|
5047
|
+
AddBlobFile(101, 64ULL * 1024 * 1024);
|
|
5048
|
+
AddBlobFile(102, 64ULL * 1024 * 1024);
|
|
5049
|
+
AddBlobFile(103, 64ULL * 1024 * 1024);
|
|
5050
|
+
|
|
5051
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5052
|
+
ASSERT_NE(nullptr, compaction.get())
|
|
5053
|
+
<< "Should compact when file count == trigger";
|
|
5054
|
+
ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
|
|
5055
|
+
compaction->compaction_reason());
|
|
5056
|
+
ASSERT_EQ(0, compaction->output_level());
|
|
5057
|
+
}
|
|
5058
|
+
|
|
5059
|
+
// Sub-test 3: more than trigger (8 files > trigger 4) -> compaction fires
|
|
5060
|
+
{
|
|
5061
|
+
SetupFIFORatioBased(100 * 1024 * 1024, 500ULL * 1024 * 1024, 4);
|
|
5062
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5063
|
+
Add(0, 1U, "100", "199", 64 * 1024);
|
|
5064
|
+
Add(0, 2U, "200", "299", 32 * 1024);
|
|
5065
|
+
Add(0, 3U, "300", "399", 48 * 1024);
|
|
5066
|
+
Add(0, 4U, "400", "499", 96 * 1024);
|
|
5067
|
+
Add(0, 5U, "500", "599", 64 * 1024);
|
|
5068
|
+
Add(0, 6U, "600", "699", 48 * 1024);
|
|
5069
|
+
Add(0, 7U, "700", "799", 64 * 1024);
|
|
5070
|
+
Add(0, 8U, "800", "899", 64 * 1024);
|
|
5071
|
+
for (uint64_t i = 0; i < 8; i++) {
|
|
5072
|
+
AddBlobFile(100 + i, 50ULL * 1024 * 1024);
|
|
5073
|
+
}
|
|
5074
|
+
|
|
5075
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5076
|
+
ASSERT_NE(nullptr, compaction.get())
|
|
5077
|
+
<< "Should compact when file count > trigger";
|
|
5078
|
+
ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
|
|
5079
|
+
compaction->compaction_reason());
|
|
5080
|
+
ASSERT_GE(compaction->num_input_files(0), 2);
|
|
5081
|
+
}
|
|
5082
|
+
}
|
|
5083
|
+
|
|
5084
|
+
TEST_F(CompactionPickerTest, FIFORatioBasedCompactionNoBlobsFallback) {
|
|
5085
|
+
// When total_blob == 0, sst_ratio = 1.0 and target becomes huge
|
|
5086
|
+
// (max_data_files_size / trigger). With the tiered algorithm, the tier
|
|
5087
|
+
// boundaries descend from target, and the lowest boundary where files
|
|
5088
|
+
// can accumulate will be found. The algorithm should still work
|
|
5089
|
+
// correctly (not crash) and produce a compaction at a low tier boundary.
|
|
5090
|
+
SetupFIFORatioBased(10 * 1024 * 1024, 10ULL * 1024 * 1024 * 1024, 4);
|
|
5091
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5092
|
+
|
|
5093
|
+
// Small SST files, no blob files
|
|
5094
|
+
Add(0, 1U, "100", "200", 64 * 1024);
|
|
5095
|
+
Add(0, 2U, "200", "300", 64 * 1024);
|
|
5096
|
+
Add(0, 3U, "300", "400", 64 * 1024);
|
|
5097
|
+
Add(0, 4U, "400", "500", 64 * 1024);
|
|
5098
|
+
|
|
5099
|
+
// No blob files added -- total_blob == 0
|
|
5100
|
+
|
|
5101
|
+
// With sst_ratio=1.0 and 10GB cap, target = 10GB/4 = 2.5GB.
|
|
5102
|
+
// Tiered boundaries descend: 2.5GB, 625MB, ..., ~152KB, ~38KB, ...
|
|
5103
|
+
// At boundary ~152KB, 4 files of 64KB accumulate to 256KB >= 152KB.
|
|
5104
|
+
// The tiered algorithm finds a viable batch and compacts.
|
|
5105
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5106
|
+
ASSERT_NE(nullptr, compaction.get());
|
|
5107
|
+
ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
|
|
5108
|
+
compaction->compaction_reason());
|
|
5109
|
+
ASSERT_GE(compaction->num_input_files(0), 2);
|
|
5110
|
+
}
|
|
5111
|
+
|
|
5112
|
+
TEST_F(CompactionPickerTest, FIFORatioBasedCompactionNoRecompaction) {
|
|
5113
|
+
// When all files are at or above the target size (graduated),
|
|
5114
|
+
// no re-compaction should happen. Files >= target are skipped at every
|
|
5115
|
+
// tier boundary.
|
|
5116
|
+
SetupFIFORatioBased(100 * 1024 * 1024, 500ULL * 1024 * 1024, 4);
|
|
5117
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5118
|
+
// Use max_compaction_bytes to set an explicit target of 256KB.
|
|
5119
|
+
// Make all files >= 256KB so they are "graduated" (at or above target).
|
|
5120
|
+
mutable_cf_options_.max_compaction_bytes = 256 * 1024;
|
|
5121
|
+
|
|
5122
|
+
// All files at 300KB, which is >= target (256KB) -> graduated
|
|
5123
|
+
Add(0, 1U, "100", "199", 300 * 1024);
|
|
5124
|
+
Add(0, 2U, "200", "299", 300 * 1024);
|
|
5125
|
+
Add(0, 3U, "300", "399", 300 * 1024);
|
|
5126
|
+
Add(0, 4U, "400", "499", 300 * 1024);
|
|
5127
|
+
|
|
5128
|
+
// All files are at/above target -> graduated -> no compaction.
|
|
5129
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5130
|
+
ASSERT_EQ(nullptr, compaction.get());
|
|
5131
|
+
}
|
|
5132
|
+
|
|
5133
|
+
TEST_F(CompactionPickerTest,
|
|
5134
|
+
FIFORatioBasedCompactionWithExplicitMaxCompactionBytes) {
|
|
5135
|
+
// When max_compaction_bytes > 0, it overrides the auto-calculated target.
|
|
5136
|
+
SetupFIFORatioBased(100 * 1024 * 1024, 10ULL * 1024 * 1024 * 1024, 4);
|
|
5137
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5138
|
+
// Explicitly set target to 256KB
|
|
5139
|
+
mutable_cf_options_.max_compaction_bytes = 256 * 1024;
|
|
5140
|
+
|
|
5141
|
+
// 6 small SST files
|
|
5142
|
+
Add(0, 1U, "100", "199", 64 * 1024);
|
|
5143
|
+
Add(0, 2U, "200", "299", 64 * 1024);
|
|
5144
|
+
Add(0, 3U, "300", "399", 64 * 1024);
|
|
5145
|
+
Add(0, 4U, "400", "499", 64 * 1024);
|
|
5146
|
+
Add(0, 5U, "500", "599", 64 * 1024);
|
|
5147
|
+
Add(0, 6U, "600", "699", 64 * 1024);
|
|
5148
|
+
|
|
5149
|
+
// No blob files needed when max_compaction_bytes is explicitly set
|
|
5150
|
+
|
|
5151
|
+
// target = max_compaction_bytes = 256KB.
|
|
5152
|
+
// Tier boundaries descend from 256KB: [25KB, 256KB] (trigger=4, floor=10KB).
|
|
5153
|
+
// At boundary 25KB: each 64KB file >= 25KB -> skipped.
|
|
5154
|
+
// At boundary 256KB: all 64KB files < 256KB -> accumulated until >= 256KB.
|
|
5155
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5156
|
+
ASSERT_NE(nullptr, compaction.get());
|
|
5157
|
+
ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
|
|
5158
|
+
compaction->compaction_reason());
|
|
5159
|
+
ASSERT_EQ(0, compaction->output_level());
|
|
5160
|
+
}
|
|
5161
|
+
|
|
5162
|
+
TEST_F(CompactionPickerTest, FIFORatioBasedCompactionFallbackToOldPath) {
|
|
5163
|
+
// When use_kv_ratio_compaction is false, PickIntraL0Compaction should
|
|
5164
|
+
// fall through to the old PickCostBasedIntraL0Compaction path.
|
|
5165
|
+
|
|
5166
|
+
// Sub-test 1: allow_compaction = false -> no intra-L0 at all
|
|
5167
|
+
{
|
|
5168
|
+
SetupFIFORatioBased(10 * 1024 * 1024, 0, 4,
|
|
5169
|
+
/*allow_compaction=*/false, /*use_kv_ratio=*/false);
|
|
5170
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5171
|
+
Add(0, 1U, "100", "200", 64 * 1024);
|
|
5172
|
+
Add(0, 2U, "200", "300", 64 * 1024);
|
|
5173
|
+
Add(0, 3U, "300", "400", 64 * 1024);
|
|
5174
|
+
Add(0, 4U, "400", "500", 64 * 1024);
|
|
5175
|
+
|
|
5176
|
+
// Total size (256KB) < max_table_files_size (10MB), so no deletion.
|
|
5177
|
+
// allow_compaction=false, so no intra-L0 either.
|
|
5178
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5179
|
+
ASSERT_EQ(nullptr, compaction.get());
|
|
5180
|
+
}
|
|
5181
|
+
|
|
5182
|
+
// Sub-test 2: allow_compaction = true, use_kv_ratio = false
|
|
5183
|
+
// -> falls through to old PickCostBasedIntraL0Compaction path
|
|
5184
|
+
{
|
|
5185
|
+
SetupFIFORatioBased(10 * 1024 * 1024, 0, 4,
|
|
5186
|
+
/*allow_compaction=*/true, /*use_kv_ratio=*/false);
|
|
5187
|
+
// The old path uses max_compaction_bytes to cap total input size.
|
|
5188
|
+
// In production this is sanitized to target_file_size_base * 25,
|
|
5189
|
+
// but tests bypass sanitization, so set it explicitly.
|
|
5190
|
+
mutable_cf_options_.max_compaction_bytes = 64 * 1024 * 1024; // 64MB
|
|
5191
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5192
|
+
Add(0, 1U, "100", "200", 64 * 1024);
|
|
5193
|
+
Add(0, 2U, "200", "300", 64 * 1024);
|
|
5194
|
+
Add(0, 3U, "300", "400", 64 * 1024);
|
|
5195
|
+
Add(0, 4U, "400", "500", 64 * 1024);
|
|
5196
|
+
|
|
5197
|
+
// Total size (256KB) < max_table_files_size (10MB), so no deletion.
|
|
5198
|
+
// allow_compaction=true and use_kv_ratio=false -> old path.
|
|
5199
|
+
// 4 files >= trigger(4), per_del = 256KB/3 ~ 85KB < 1.1*WBS -> passes.
|
|
5200
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5201
|
+
ASSERT_NE(nullptr, compaction.get())
|
|
5202
|
+
<< "Old path should compact when allow_compaction=true";
|
|
5203
|
+
ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
|
|
5204
|
+
compaction->compaction_reason());
|
|
5205
|
+
}
|
|
5206
|
+
}
|
|
5207
|
+
|
|
5208
|
+
// ============================================================================
|
|
5209
|
+
// FIFO Option Validation Tests
|
|
5210
|
+
// Tests that ColumnFamilyData::ValidateOptions accepts kv_ratio configs
|
|
5211
|
+
// (prerequisites are checked at runtime in the picker, not at validation time).
|
|
5212
|
+
// ============================================================================
|
|
5213
|
+
|
|
5214
|
+
TEST_F(CompactionPickerTest, FIFOOptionValidation) {
|
|
5215
|
+
auto validate = [](std::function<void(ColumnFamilyOptions&)> configure) {
|
|
5216
|
+
ColumnFamilyOptions cf_opts;
|
|
5217
|
+
cf_opts.compaction_style = kCompactionStyleFIFO;
|
|
5218
|
+
cf_opts.compaction_options_fifo.allow_compaction = true;
|
|
5219
|
+
cf_opts.compaction_options_fifo.use_kv_ratio_compaction = true;
|
|
5220
|
+
cf_opts.compaction_options_fifo.max_data_files_size =
|
|
5221
|
+
1ULL * 1024 * 1024 * 1024;
|
|
5222
|
+
cf_opts.num_levels = 1;
|
|
5223
|
+
configure(cf_opts);
|
|
5224
|
+
return ColumnFamilyData::ValidateOptions(DBOptions(), cf_opts);
|
|
5225
|
+
};
|
|
5226
|
+
|
|
5227
|
+
// These used to be InvalidArgument, now accepted (handled at runtime
|
|
5228
|
+
// in the picker with fallback to cost-based intra-L0 compaction):
|
|
5229
|
+
|
|
5230
|
+
// use_kv_ratio_compaction with non-FIFO compaction style
|
|
5231
|
+
ASSERT_OK(
|
|
5232
|
+
validate([](auto& o) { o.compaction_style = kCompactionStyleLevel; }));
|
|
5233
|
+
|
|
5234
|
+
// use_kv_ratio_compaction without allow_compaction
|
|
5235
|
+
ASSERT_OK(validate(
|
|
5236
|
+
[](auto& o) { o.compaction_options_fifo.allow_compaction = false; }));
|
|
5237
|
+
|
|
5238
|
+
// use_kv_ratio_compaction without max_data_files_size
|
|
5239
|
+
ASSERT_OK(validate(
|
|
5240
|
+
[](auto& o) { o.compaction_options_fifo.max_data_files_size = 0; }));
|
|
5241
|
+
|
|
5242
|
+
// max_data_files_size < max_table_files_size
|
|
5243
|
+
ASSERT_OK(validate([](auto& o) {
|
|
5244
|
+
o.compaction_options_fifo.use_kv_ratio_compaction = false;
|
|
5245
|
+
o.compaction_options_fifo.max_data_files_size = 0;
|
|
5246
|
+
o.compaction_options_fifo.max_table_files_size = 1ULL * 1024 * 1024 * 1024;
|
|
5247
|
+
o.compaction_options_fifo.max_data_files_size = 500ULL * 1024 * 1024;
|
|
5248
|
+
}));
|
|
5249
|
+
|
|
5250
|
+
// These should still pass:
|
|
5251
|
+
|
|
5252
|
+
// Accepts multi-level (for migration from level/universal to FIFO)
|
|
5253
|
+
ASSERT_OK(validate([](auto& o) { o.num_levels = 4; }));
|
|
5254
|
+
|
|
5255
|
+
// Accepts valid single-level config
|
|
5256
|
+
ASSERT_OK(validate([](auto& /*o*/) {}));
|
|
5257
|
+
|
|
5258
|
+
// max_data_files_size == max_table_files_size is valid
|
|
5259
|
+
ASSERT_OK(validate([](auto& o) {
|
|
5260
|
+
o.compaction_options_fifo.use_kv_ratio_compaction = false;
|
|
5261
|
+
o.compaction_options_fifo.max_data_files_size = 0;
|
|
5262
|
+
o.compaction_options_fifo.max_table_files_size = 1ULL * 1024 * 1024 * 1024;
|
|
5263
|
+
o.compaction_options_fifo.max_data_files_size = 1ULL * 1024 * 1024 * 1024;
|
|
5264
|
+
}));
|
|
5265
|
+
}
|
|
5266
|
+
|
|
5267
|
+
TEST_F(CompactionPickerTest, FIFORatioBasedFallbackOnInvalidConfig) {
|
|
5268
|
+
// When use_kv_ratio_compaction is true but prerequisites are not met,
|
|
5269
|
+
// the picker should fall back to cost-based intra-L0 compaction.
|
|
5270
|
+
|
|
5271
|
+
// Sub-test 1: max_data_files_size == 0 -> falls back to cost-based
|
|
5272
|
+
{
|
|
5273
|
+
SetupFIFORatioBased(/*max_table_files_size=*/10 * 1024 * 1024,
|
|
5274
|
+
/*max_data_files_size=*/0,
|
|
5275
|
+
/*trigger=*/4,
|
|
5276
|
+
/*allow_compaction=*/true,
|
|
5277
|
+
/*use_kv_ratio=*/true);
|
|
5278
|
+
mutable_cf_options_.max_compaction_bytes = 10ULL * 1024 * 1024;
|
|
5279
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5280
|
+
Add(0, 1U, "100", "200", 64 * 1024);
|
|
5281
|
+
Add(0, 2U, "200", "300", 64 * 1024);
|
|
5282
|
+
Add(0, 3U, "300", "400", 64 * 1024);
|
|
5283
|
+
Add(0, 4U, "400", "500", 64 * 1024);
|
|
5284
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5285
|
+
// kv_ratio skipped, cost-based should pick these files
|
|
5286
|
+
ASSERT_NE(nullptr, compaction.get());
|
|
5287
|
+
ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
|
|
5288
|
+
compaction->compaction_reason());
|
|
5289
|
+
}
|
|
5290
|
+
|
|
5291
|
+
// Sub-test 2: max_data_files_size < max_table_files_size -> falls back
|
|
5292
|
+
{
|
|
5293
|
+
SetupFIFORatioBased(/*max_table_files_size=*/1ULL * 1024 * 1024 * 1024,
|
|
5294
|
+
/*max_data_files_size=*/500ULL * 1024 * 1024,
|
|
5295
|
+
/*trigger=*/4,
|
|
5296
|
+
/*allow_compaction=*/true,
|
|
5297
|
+
/*use_kv_ratio=*/true);
|
|
5298
|
+
mutable_cf_options_.max_compaction_bytes = 10ULL * 1024 * 1024;
|
|
5299
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5300
|
+
Add(0, 1U, "100", "200", 64 * 1024);
|
|
5301
|
+
Add(0, 2U, "200", "300", 64 * 1024);
|
|
5302
|
+
Add(0, 3U, "300", "400", 64 * 1024);
|
|
5303
|
+
Add(0, 4U, "400", "500", 64 * 1024);
|
|
5304
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5305
|
+
ASSERT_NE(nullptr, compaction.get());
|
|
5306
|
+
ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
|
|
5307
|
+
compaction->compaction_reason());
|
|
5308
|
+
}
|
|
5309
|
+
}
|
|
5310
|
+
|
|
5311
|
+
// ============================================================================
|
|
5312
|
+
// FIFO Ratio-Based Compaction: Multi-Level Migration Graceful Skip
|
|
5313
|
+
// Tests that PickRatioBasedIntraL0Compaction gracefully skips when non-L0
|
|
5314
|
+
// levels still contain files (e.g., during migration from level/universal
|
|
5315
|
+
// to FIFO), and resumes once all data has been drained to L0.
|
|
5316
|
+
// ============================================================================
|
|
5317
|
+
|
|
5318
|
+
TEST_F(CompactionPickerTest, FIFORatioBasedMultiLevelMigration) {
|
|
5319
|
+
// Sub-case 1: During migration (non-L0 levels have files).
|
|
5320
|
+
// Ratio-based intra-L0 compaction should be skipped.
|
|
5321
|
+
{
|
|
5322
|
+
SetupFIFORatioBased(/*max_table_files_size=*/100 * 1024 * 1024,
|
|
5323
|
+
/*max_data_files_size=*/1ULL * 1024 * 1024 * 1024,
|
|
5324
|
+
/*trigger=*/4,
|
|
5325
|
+
/*allow_compaction=*/true,
|
|
5326
|
+
/*use_kv_ratio=*/true,
|
|
5327
|
+
/*num_levels=*/4);
|
|
5328
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5329
|
+
|
|
5330
|
+
Add(0, 1U, "100", "200", 64 * 1024);
|
|
5331
|
+
Add(0, 2U, "200", "300", 64 * 1024);
|
|
5332
|
+
Add(0, 3U, "300", "400", 64 * 1024);
|
|
5333
|
+
Add(0, 4U, "400", "500", 64 * 1024);
|
|
5334
|
+
Add(0, 5U, "500", "600", 64 * 1024);
|
|
5335
|
+
Add(2, 10U, "100", "600", 50 * 1024 * 1024);
|
|
5336
|
+
AddBlobFile(100, 64ULL * 1024 * 1024);
|
|
5337
|
+
AddBlobFile(101, 64ULL * 1024 * 1024);
|
|
5338
|
+
|
|
5339
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5340
|
+
if (compaction != nullptr) {
|
|
5341
|
+
if (compaction->compaction_reason() ==
|
|
5342
|
+
CompactionReason::kFIFOReduceNumFiles) {
|
|
5343
|
+
// Cost-based path is fine; verify it's not ratio-based.
|
|
5344
|
+
ASSERT_EQ(16 * 1024 * 1024, compaction->max_output_file_size());
|
|
5345
|
+
}
|
|
5346
|
+
}
|
|
5347
|
+
}
|
|
5348
|
+
|
|
5349
|
+
// Sub-case 2: After migration (only L0 has files).
|
|
5350
|
+
// Ratio-based compaction should resume normally.
|
|
5351
|
+
{
|
|
5352
|
+
SetupFIFORatioBased(/*max_table_files_size=*/100 * 1024 * 1024,
|
|
5353
|
+
/*max_data_files_size=*/1ULL * 1024 * 1024 * 1024,
|
|
5354
|
+
/*trigger=*/4,
|
|
5355
|
+
/*allow_compaction=*/true,
|
|
5356
|
+
/*use_kv_ratio=*/true,
|
|
5357
|
+
/*num_levels=*/4);
|
|
5358
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5359
|
+
|
|
5360
|
+
Add(0, 1U, "100", "200", 64 * 1024);
|
|
5361
|
+
Add(0, 2U, "200", "300", 32 * 1024);
|
|
5362
|
+
Add(0, 3U, "300", "400", 48 * 1024);
|
|
5363
|
+
Add(0, 4U, "400", "500", 96 * 1024);
|
|
5364
|
+
AddBlobFile(100, 64ULL * 1024 * 1024);
|
|
5365
|
+
AddBlobFile(101, 64ULL * 1024 * 1024);
|
|
5366
|
+
AddBlobFile(102, 64ULL * 1024 * 1024);
|
|
5367
|
+
AddBlobFile(103, 64ULL * 1024 * 1024);
|
|
5368
|
+
|
|
5369
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5370
|
+
ASSERT_NE(nullptr, compaction.get())
|
|
5371
|
+
<< "Should compact when non-L0 levels are empty (migration complete)";
|
|
5372
|
+
ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
|
|
5373
|
+
compaction->compaction_reason());
|
|
5374
|
+
ASSERT_EQ(0, compaction->output_level());
|
|
5375
|
+
}
|
|
5376
|
+
}
|
|
5377
|
+
|
|
5378
|
+
// ============================================================================
|
|
5379
|
+
// FIFO TTL Compaction with Blob-Aware Estimation Tests
|
|
5380
|
+
// Tests that PickTTLCompaction correctly estimates remaining data (SST + blob)
|
|
5381
|
+
// in both single-level and multi-level FIFO configurations.
|
|
5382
|
+
// ============================================================================
|
|
5383
|
+
|
|
5384
|
+
TEST_F(CompactionPickerTest, FIFOTTLBlobEstimationSingleLevel) {
|
|
5385
|
+
// Single-level FIFO with TTL and max_data_files_size.
|
|
5386
|
+
// After dropping expired L0 SSTs, the blob estimate should be proportional
|
|
5387
|
+
// to the remaining SST fraction.
|
|
5388
|
+
//
|
|
5389
|
+
// Common setup: L0 = 4 files x 50KB = 200KB, files 3,4 expired.
|
|
5390
|
+
// Remaining SST after drop = 100KB = 50%.
|
|
5391
|
+
|
|
5392
|
+
auto run = [&](uint64_t blob_total, uint64_t limit, bool expect_ttl_fires) {
|
|
5393
|
+
ioptions_.compaction_style = kCompactionStyleFIFO;
|
|
5394
|
+
NewVersionStorage(1, kCompactionStyleFIFO);
|
|
5395
|
+
mutable_cf_options_.compaction_options_fifo.max_table_files_size = limit;
|
|
5396
|
+
mutable_cf_options_.compaction_options_fifo.max_data_files_size = limit;
|
|
5397
|
+
mutable_cf_options_.compaction_options_fifo.allow_compaction = true;
|
|
5398
|
+
mutable_cf_options_.ttl = 3600;
|
|
5399
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5400
|
+
|
|
5401
|
+
uint64_t recent_time = static_cast<uint64_t>(time(nullptr));
|
|
5402
|
+
Add(0, 1U, "100", "200", 50 * 1024, 0, 100, 100, 0, false,
|
|
5403
|
+
Temperature::kUnknown, kUnknownOldestAncesterTime, recent_time);
|
|
5404
|
+
Add(0, 2U, "200", "300", 50 * 1024, 0, 100, 100, 0, false,
|
|
5405
|
+
Temperature::kUnknown, kUnknownOldestAncesterTime, recent_time);
|
|
5406
|
+
Add(0, 3U, "300", "400", 50 * 1024, 0, 100, 100, 0, false,
|
|
5407
|
+
Temperature::kUnknown, kUnknownOldestAncesterTime, 1);
|
|
5408
|
+
Add(0, 4U, "400", "500", 50 * 1024, 0, 100, 100, 0, false,
|
|
5409
|
+
Temperature::kUnknown, kUnknownOldestAncesterTime, 1);
|
|
5410
|
+
if (blob_total > 0) {
|
|
5411
|
+
AddBlobFile(100, blob_total / 2);
|
|
5412
|
+
AddBlobFile(101, blob_total / 2);
|
|
5413
|
+
}
|
|
5414
|
+
|
|
5415
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5416
|
+
if (expect_ttl_fires) {
|
|
5417
|
+
ASSERT_NE(nullptr, compaction.get())
|
|
5418
|
+
<< "TTL compaction should fire when remaining data < limit";
|
|
5419
|
+
ASSERT_EQ(CompactionReason::kFIFOTtl, compaction->compaction_reason());
|
|
5420
|
+
ASSERT_EQ(2U, compaction->num_input_files(0));
|
|
5421
|
+
} else {
|
|
5422
|
+
if (compaction != nullptr) {
|
|
5423
|
+
ASSERT_NE(CompactionReason::kFIFOTtl, compaction->compaction_reason())
|
|
5424
|
+
<< "TTL should not fire when remaining data still exceeds limit";
|
|
5425
|
+
}
|
|
5426
|
+
}
|
|
5427
|
+
};
|
|
5428
|
+
|
|
5429
|
+
// Sub-case 1: Under limit after drop.
|
|
5430
|
+
// blob=400KB, limit=500KB.
|
|
5431
|
+
// effective = 100KB + (100KB/200KB)*400KB = 300KB < 500KB -> fires.
|
|
5432
|
+
run(400 * 1024, 500 * 1024, /*expect_ttl_fires=*/true);
|
|
5433
|
+
|
|
5434
|
+
// Sub-case 2: Over limit after drop.
|
|
5435
|
+
// blob=4MB, limit=100KB.
|
|
5436
|
+
// effective = 100KB + (100KB/200KB)*4MB ~ 2MB >> 100KB -> does NOT fire.
|
|
5437
|
+
run(4ULL * 1024 * 1024, 100 * 1024, /*expect_ttl_fires=*/false);
|
|
5438
|
+
|
|
5439
|
+
// Sub-case 3: No blob files. Falls back to SST-only estimation.
|
|
5440
|
+
// blob=0, limit=150KB. remaining SST = 100KB < 150KB -> fires.
|
|
5441
|
+
run(0, 150 * 1024, /*expect_ttl_fires=*/true);
|
|
5442
|
+
}
|
|
5443
|
+
|
|
5444
|
+
TEST_F(CompactionPickerTest, FIFOTTLBlobEstimationMultiLevel) {
|
|
5445
|
+
// Multi-level FIFO (migration) with TTL and max_data_files_size.
|
|
5446
|
+
// This is the ritical bug fix scenario:
|
|
5447
|
+
// - L0 has some SSTs, L2 has legacy SSTs from migration
|
|
5448
|
+
// - Blob files cover ALL levels
|
|
5449
|
+
// - The estimation must use total SST across ALL levels (not just L0)
|
|
5450
|
+
// to avoid inflating the blob proportion.
|
|
5451
|
+
//
|
|
5452
|
+
// Setup:
|
|
5453
|
+
// L0: 4 files x 50KB = 200KB SST (files 3,4 expired)
|
|
5454
|
+
// L2: 1 file x 200KB SST (legacy migration data)
|
|
5455
|
+
// Total SST = 400KB
|
|
5456
|
+
// Blob: 800KB total
|
|
5457
|
+
// max_data_files_size = 1000KB
|
|
5458
|
+
// Remaining SST after TTL drop = 400KB - 100KB = 300KB
|
|
5459
|
+
//
|
|
5460
|
+
// CORRECT (fixed): effective = 300KB + (300KB/400KB)*800KB = 300+600 =
|
|
5461
|
+
// 900KB < 1000KB -> fires BUG (old): effective = 100KB +
|
|
5462
|
+
// (100KB/200KB)*800KB = 100+400 = 500KB < 1000KB -> fires
|
|
5463
|
+
// (coincidentally fires too, but with wrong estimate)
|
|
5464
|
+
//
|
|
5465
|
+
// To distinguish correct vs buggy behavior, use a limit that triggers the
|
|
5466
|
+
// difference: set max_data_files_size = 850KB.
|
|
5467
|
+
// CORRECT: effective = 300KB + (300KB/400KB)*800KB = 900KB > 850KB -> does
|
|
5468
|
+
// NOT fire BUG: effective = 100KB + (100KB/200KB)*800KB = 500KB < 850KB
|
|
5469
|
+
// -> fires (wrong!)
|
|
5470
|
+
ioptions_.compaction_style = kCompactionStyleFIFO;
|
|
5471
|
+
NewVersionStorage(4, kCompactionStyleFIFO);
|
|
5472
|
+
mutable_cf_options_.compaction_options_fifo.max_table_files_size =
|
|
5473
|
+
850 * 1024; // match max_data_files_size
|
|
5474
|
+
mutable_cf_options_.compaction_options_fifo.max_data_files_size = 850 * 1024;
|
|
5475
|
+
mutable_cf_options_.compaction_options_fifo.allow_compaction = true;
|
|
5476
|
+
mutable_cf_options_.ttl = 3600;
|
|
5477
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5478
|
+
|
|
5479
|
+
uint64_t recent_time = static_cast<uint64_t>(time(nullptr));
|
|
5480
|
+
// L0 files: 2 recent, 2 expired
|
|
5481
|
+
Add(0, 1U, "100", "200", 50 * 1024, 0, 100, 100, 0, false,
|
|
5482
|
+
Temperature::kUnknown, kUnknownOldestAncesterTime, recent_time);
|
|
5483
|
+
Add(0, 2U, "200", "300", 50 * 1024, 0, 100, 100, 0, false,
|
|
5484
|
+
Temperature::kUnknown, kUnknownOldestAncesterTime, recent_time);
|
|
5485
|
+
Add(0, 3U, "300", "400", 50 * 1024, 0, 100, 100, 0, false,
|
|
5486
|
+
Temperature::kUnknown, kUnknownOldestAncesterTime, 1);
|
|
5487
|
+
Add(0, 4U, "400", "500", 50 * 1024, 0, 100, 100, 0, false,
|
|
5488
|
+
Temperature::kUnknown, kUnknownOldestAncesterTime, 1);
|
|
5489
|
+
// L2 legacy migration file
|
|
5490
|
+
Add(2, 10U, "100", "600", 200 * 1024);
|
|
5491
|
+
// Blob files (associated with ALL levels)
|
|
5492
|
+
AddBlobFile(100, 400 * 1024);
|
|
5493
|
+
AddBlobFile(101, 400 * 1024);
|
|
5494
|
+
|
|
5495
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5496
|
+
// With correct all-levels estimation:
|
|
5497
|
+
// remaining_sst_all = 400KB - 100KB(dropped) = 300KB
|
|
5498
|
+
// effective = 300KB + (300KB/400KB)*800KB = 900KB > 850KB
|
|
5499
|
+
// -> TTL should NOT fire (falls through to size-based)
|
|
5500
|
+
if (compaction != nullptr) {
|
|
5501
|
+
ASSERT_NE(CompactionReason::kFIFOTtl, compaction->compaction_reason())
|
|
5502
|
+
<< "Multi-level FIFO: TTL should not fire when correct all-levels "
|
|
5503
|
+
"blob estimation shows data still exceeds limit";
|
|
5504
|
+
}
|
|
5505
|
+
}
|
|
5506
|
+
|
|
5507
|
+
TEST_F(CompactionPickerTest, FIFOBlobAwareSizeDropping) {
|
|
5508
|
+
// PickSizeCompaction with max_data_files_size should account for blob data.
|
|
5509
|
+
//
|
|
5510
|
+
// Sub-case 1: Single-level. SST = 200KB, blob = 500MB, limit = 200MB.
|
|
5511
|
+
// effective_size ~ 500MB >> 200MB -> drops from L0.
|
|
5512
|
+
{
|
|
5513
|
+
SetupFIFORatioBased(/*max_table=*/200ULL * 1024 * 1024,
|
|
5514
|
+
/*max_data=*/200ULL * 1024 * 1024,
|
|
5515
|
+
/*trigger=*/4,
|
|
5516
|
+
/*allow_compaction=*/true,
|
|
5517
|
+
/*use_kv_ratio=*/false);
|
|
5518
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5519
|
+
|
|
5520
|
+
Add(0, 1U, "100", "199", 40 * 1024);
|
|
5521
|
+
Add(0, 2U, "200", "299", 40 * 1024);
|
|
5522
|
+
Add(0, 3U, "300", "399", 40 * 1024);
|
|
5523
|
+
Add(0, 4U, "400", "499", 40 * 1024);
|
|
5524
|
+
Add(0, 5U, "500", "599", 40 * 1024);
|
|
5525
|
+
AddBlobFile(100, 100ULL * 1024 * 1024);
|
|
5526
|
+
AddBlobFile(101, 100ULL * 1024 * 1024);
|
|
5527
|
+
AddBlobFile(102, 100ULL * 1024 * 1024);
|
|
5528
|
+
AddBlobFile(103, 100ULL * 1024 * 1024);
|
|
5529
|
+
AddBlobFile(104, 100ULL * 1024 * 1024);
|
|
5530
|
+
|
|
5531
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5532
|
+
ASSERT_NE(nullptr, compaction.get());
|
|
5533
|
+
ASSERT_EQ(CompactionReason::kFIFOMaxSize, compaction->compaction_reason());
|
|
5534
|
+
ASSERT_GE(compaction->num_input_files(0), 1);
|
|
5535
|
+
}
|
|
5536
|
+
|
|
5537
|
+
// Sub-case 2: Multi-level (migration). L0=100KB, L2=150KB, blob=500KB.
|
|
5538
|
+
// effective_size = 250KB + 500KB = 750KB > 400KB -> drops from L2.
|
|
5539
|
+
{
|
|
5540
|
+
ioptions_.compaction_style = kCompactionStyleFIFO;
|
|
5541
|
+
NewVersionStorage(4, kCompactionStyleFIFO);
|
|
5542
|
+
mutable_cf_options_.compaction_options_fifo.max_table_files_size =
|
|
5543
|
+
400 * 1024;
|
|
5544
|
+
mutable_cf_options_.compaction_options_fifo.max_data_files_size =
|
|
5545
|
+
400 * 1024;
|
|
5546
|
+
mutable_cf_options_.compaction_options_fifo.allow_compaction = true;
|
|
5547
|
+
mutable_cf_options_.ttl = 0;
|
|
5548
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5549
|
+
|
|
5550
|
+
Add(0, 1U, "100", "200", 50 * 1024);
|
|
5551
|
+
Add(0, 2U, "200", "300", 50 * 1024);
|
|
5552
|
+
Add(2, 10U, "100", "300", 50 * 1024);
|
|
5553
|
+
Add(2, 11U, "300", "500", 50 * 1024);
|
|
5554
|
+
Add(2, 12U, "500", "700", 50 * 1024);
|
|
5555
|
+
AddBlobFile(100, 250 * 1024);
|
|
5556
|
+
AddBlobFile(101, 250 * 1024);
|
|
5557
|
+
|
|
5558
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5559
|
+
ASSERT_NE(nullptr, compaction.get());
|
|
5560
|
+
ASSERT_EQ(CompactionReason::kFIFOMaxSize, compaction->compaction_reason());
|
|
5561
|
+
ASSERT_EQ(2, compaction->start_level());
|
|
5562
|
+
ASSERT_GE(compaction->num_input_files(0), 1U);
|
|
5563
|
+
}
|
|
5564
|
+
|
|
5565
|
+
// Sub-case 3: Under limit. SST = 256KB, blob = 200MB, limit = 1GB.
|
|
5566
|
+
// effective_size ~ 200MB < 1GB -> no dropping.
|
|
5567
|
+
{
|
|
5568
|
+
SetupFIFORatioBased(/*max_table=*/1ULL * 1024 * 1024 * 1024,
|
|
5569
|
+
/*max_data=*/1ULL * 1024 * 1024 * 1024,
|
|
5570
|
+
/*trigger=*/4,
|
|
5571
|
+
/*allow_compaction=*/true,
|
|
5572
|
+
/*use_kv_ratio=*/true);
|
|
5573
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5574
|
+
|
|
5575
|
+
Add(0, 1U, "100", "199", 64 * 1024);
|
|
5576
|
+
Add(0, 2U, "200", "299", 64 * 1024);
|
|
5577
|
+
Add(0, 3U, "300", "399", 64 * 1024);
|
|
5578
|
+
Add(0, 4U, "400", "499", 64 * 1024);
|
|
5579
|
+
AddBlobFile(100, 50ULL * 1024 * 1024);
|
|
5580
|
+
AddBlobFile(101, 50ULL * 1024 * 1024);
|
|
5581
|
+
AddBlobFile(102, 50ULL * 1024 * 1024);
|
|
5582
|
+
AddBlobFile(103, 50ULL * 1024 * 1024);
|
|
5583
|
+
|
|
5584
|
+
auto compaction = PickFIFOCompaction(picker);
|
|
5585
|
+
if (compaction) {
|
|
5586
|
+
ASSERT_NE(CompactionReason::kFIFOMaxSize,
|
|
5587
|
+
compaction->compaction_reason());
|
|
5588
|
+
}
|
|
5589
|
+
}
|
|
5590
|
+
}
|
|
5591
|
+
|
|
5592
|
+
// ============================================================================
|
|
5593
|
+
// FIFO Blob-Aware Score Computation Test
|
|
5594
|
+
// Tests that ComputeCompactionScore includes blob sizes when
|
|
5595
|
+
// max_data_files_size > 0.
|
|
5596
|
+
// ============================================================================
|
|
5597
|
+
|
|
5598
|
+
TEST_F(CompactionPickerTest, FIFOBlobAwareScoreComputation) {
|
|
5599
|
+
// Sub-case 1: With max_data_files_size, score includes blob sizes.
|
|
5600
|
+
// SST = 100KB, blob = 500MB, max_data = 200MB -> score ~ 2.5
|
|
5601
|
+
{
|
|
5602
|
+
ioptions_.compaction_style = kCompactionStyleFIFO;
|
|
5603
|
+
NewVersionStorage(1, kCompactionStyleFIFO);
|
|
5604
|
+
mutable_cf_options_.compaction_options_fifo.max_table_files_size =
|
|
5605
|
+
200ULL * 1024 * 1024;
|
|
5606
|
+
mutable_cf_options_.compaction_options_fifo.max_data_files_size =
|
|
5607
|
+
200ULL * 1024 * 1024;
|
|
5608
|
+
mutable_cf_options_.compaction_options_fifo.allow_compaction = false;
|
|
5609
|
+
mutable_cf_options_.level0_file_num_compaction_trigger = 4;
|
|
5610
|
+
|
|
5611
|
+
Add(0, 1U, "100", "199", 25 * 1024);
|
|
5612
|
+
Add(0, 2U, "200", "299", 25 * 1024);
|
|
5613
|
+
Add(0, 3U, "300", "399", 25 * 1024);
|
|
5614
|
+
Add(0, 4U, "400", "499", 25 * 1024);
|
|
5615
|
+
AddBlobFile(100, 500ULL * 1024 * 1024);
|
|
5616
|
+
UpdateVersionStorageInfo();
|
|
5617
|
+
|
|
5618
|
+
double score = vstorage_->CompactionScore(0);
|
|
5619
|
+
ASSERT_GT(score, 2.0) << "Score should reflect 500MB/200MB ~ 2.5";
|
|
5620
|
+
}
|
|
5621
|
+
|
|
5622
|
+
// Sub-case 2: Without max_data_files_size, score ignores blobs.
|
|
5623
|
+
// SST = 400KB < 1MB, blob = 500MB ignored -> score ~ 0.4
|
|
5624
|
+
{
|
|
5625
|
+
ioptions_.compaction_style = kCompactionStyleFIFO;
|
|
5626
|
+
NewVersionStorage(1, kCompactionStyleFIFO);
|
|
5627
|
+
mutable_cf_options_.compaction_options_fifo.max_table_files_size =
|
|
5628
|
+
1ULL * 1024 * 1024;
|
|
5629
|
+
mutable_cf_options_.compaction_options_fifo.max_data_files_size = 0;
|
|
5630
|
+
mutable_cf_options_.compaction_options_fifo.allow_compaction = false;
|
|
5631
|
+
mutable_cf_options_.level0_file_num_compaction_trigger = 4;
|
|
5632
|
+
|
|
5633
|
+
Add(0, 1U, "100", "199", 100 * 1024);
|
|
5634
|
+
Add(0, 2U, "200", "299", 100 * 1024);
|
|
5635
|
+
Add(0, 3U, "300", "399", 100 * 1024);
|
|
5636
|
+
Add(0, 4U, "400", "499", 100 * 1024);
|
|
5637
|
+
AddBlobFile(100, 500ULL * 1024 * 1024);
|
|
5638
|
+
UpdateVersionStorageInfo();
|
|
5639
|
+
|
|
5640
|
+
double score = vstorage_->CompactionScore(0);
|
|
5641
|
+
ASSERT_LT(score, 1.0)
|
|
5642
|
+
<< "Score should be < 1 when only SST sizes are counted";
|
|
5643
|
+
}
|
|
5644
|
+
}
|
|
5645
|
+
|
|
5646
|
+
// ============================================================================
|
|
5647
|
+
// FIFO + BlobDB Intra-L0 Compaction Picking Tests
|
|
5648
|
+
//
|
|
5649
|
+
// These tests validate the tiered intra-L0 compaction picking algorithm
|
|
5650
|
+
// over multiple flush/compaction cycles. Each round:
|
|
5651
|
+
// 1. Add a flush file to the L0 file list
|
|
5652
|
+
// 2. Rebuild VersionStorageInfo and call FIFOCompactionPicker::PickCompaction
|
|
5653
|
+
// 3. If compaction is picked, update the file list accordingly
|
|
5654
|
+
// 4. Repeat
|
|
5655
|
+
//
|
|
5656
|
+
// The compaction PICKING uses the real FIFOCompactionPicker -- this ensures
|
|
5657
|
+
// the tests always match the production picking logic. The rest of the
|
|
5658
|
+
// system (compaction execution, file metadata updates, FIFO dropping) is
|
|
5659
|
+
// handled by test helpers, since wiring up the full compaction execution
|
|
5660
|
+
// pipeline (CompactionJob, VersionEdit, etc.) would add significant
|
|
5661
|
+
// complexity without testing the picking logic more thoroughly.
|
|
5662
|
+
//
|
|
5663
|
+
// ============================================================================
|
|
5664
|
+
|
|
5665
|
+
class FIFORatioBasedCompactionPickingTest : public CompactionPickerTest {
|
|
5666
|
+
protected:
|
|
5667
|
+
struct L0File {
|
|
5668
|
+
uint64_t size; // SST file size in bytes
|
|
5669
|
+
uint64_t blob_size; // Associated blob data size
|
|
5670
|
+
uint64_t age; // Creation order (lower = older)
|
|
5671
|
+
bool is_compacted; // Created by compaction (vs flush)
|
|
5672
|
+
};
|
|
5673
|
+
|
|
5674
|
+
// Pick compaction using FIFOCompactionPicker.
|
|
5675
|
+
//
|
|
5676
|
+
// Rebuilds VersionStorageInfo from the files vector and calls
|
|
5677
|
+
// PickCompaction on the given picker. Maps the returned
|
|
5678
|
+
// Compaction's input files back to vector indices.
|
|
5679
|
+
//
|
|
5680
|
+
// Returns the picked indices, or empty if no compaction.
|
|
5681
|
+
// Also returns the compaction reason via out-parameter.
|
|
5682
|
+
std::vector<size_t> PickCompactionFromFiles(
|
|
5683
|
+
FIFOCompactionPicker& picker, const std::vector<L0File>& files,
|
|
5684
|
+
uint64_t max_table_files_size, uint64_t max_data_files_size, int trigger,
|
|
5685
|
+
CompactionReason* out_reason = nullptr) {
|
|
5686
|
+
// Rebuild VersionStorageInfo from the current file list
|
|
5687
|
+
NewVersionStorage(1, kCompactionStyleFIFO);
|
|
5688
|
+
mutable_cf_options_.compaction_options_fifo.max_table_files_size =
|
|
5689
|
+
max_table_files_size;
|
|
5690
|
+
mutable_cf_options_.compaction_options_fifo.max_data_files_size =
|
|
5691
|
+
max_data_files_size;
|
|
5692
|
+
mutable_cf_options_.compaction_options_fifo.allow_compaction = true;
|
|
5693
|
+
mutable_cf_options_.compaction_options_fifo.use_kv_ratio_compaction = true;
|
|
5694
|
+
mutable_cf_options_.level0_file_num_compaction_trigger = trigger;
|
|
5695
|
+
|
|
5696
|
+
// Add files: newest first. Use descending file numbers so L0 sort
|
|
5697
|
+
// (newest-first by epoch/seqno/file_number) matches our order.
|
|
5698
|
+
uint32_t base_fn = static_cast<uint32_t>(files.size());
|
|
5699
|
+
for (size_t i = 0; i < files.size(); i++) {
|
|
5700
|
+
uint32_t fn = base_fn - static_cast<uint32_t>(i);
|
|
5701
|
+
std::string smallest = "k" + std::to_string(10000 + fn * 10);
|
|
5702
|
+
std::string largest = "k" + std::to_string(10000 + fn * 10 + 9);
|
|
5703
|
+
Add(0, fn, smallest.c_str(), largest.c_str(), files[i].size);
|
|
5704
|
+
}
|
|
5705
|
+
|
|
5706
|
+
// Add one blob file with the total blob size
|
|
5707
|
+
uint64_t total_blob = 0;
|
|
5708
|
+
for (const auto& f : files) {
|
|
5709
|
+
total_blob += f.blob_size;
|
|
5710
|
+
}
|
|
5711
|
+
if (total_blob > 0) {
|
|
5712
|
+
AddBlobFile(9999, total_blob);
|
|
5713
|
+
}
|
|
5714
|
+
|
|
5715
|
+
UpdateVersionStorageInfo();
|
|
5716
|
+
|
|
5717
|
+
std::unique_ptr<Compaction> compaction(picker.PickCompaction(
|
|
5718
|
+
cf_name_, mutable_cf_options_, mutable_db_options_,
|
|
5719
|
+
/*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr,
|
|
5720
|
+
vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/""));
|
|
5721
|
+
|
|
5722
|
+
if (!compaction) return {};
|
|
5723
|
+
|
|
5724
|
+
if (out_reason) {
|
|
5725
|
+
*out_reason = compaction->compaction_reason();
|
|
5726
|
+
}
|
|
5727
|
+
|
|
5728
|
+
// For size-based dropping (kFIFOMaxSize / kFIFOTtl), map input files
|
|
5729
|
+
// back to sim indices, same as for intra-L0.
|
|
5730
|
+
std::vector<size_t> result;
|
|
5731
|
+
for (size_t j = 0; j < compaction->num_input_files(0); j++) {
|
|
5732
|
+
uint32_t fn =
|
|
5733
|
+
static_cast<uint32_t>(compaction->input(0, j)->fd.GetNumber());
|
|
5734
|
+
size_t idx = base_fn - fn;
|
|
5735
|
+
result.push_back(idx);
|
|
5736
|
+
}
|
|
5737
|
+
|
|
5738
|
+
// Unregister so the picker allows the next compaction
|
|
5739
|
+
picker.UnregisterCompaction(compaction.get());
|
|
5740
|
+
|
|
5741
|
+
return result;
|
|
5742
|
+
}
|
|
5743
|
+
|
|
5744
|
+
// Execute one compaction: merge input files into 1 output
|
|
5745
|
+
void ExecuteCompaction(std::vector<L0File>& files,
|
|
5746
|
+
const std::vector<size_t>& input_indices,
|
|
5747
|
+
uint64_t& global_age) {
|
|
5748
|
+
uint64_t output_size = 0;
|
|
5749
|
+
uint64_t output_blob = 0;
|
|
5750
|
+
for (size_t idx : input_indices) {
|
|
5751
|
+
output_size += files[idx].size;
|
|
5752
|
+
output_blob += files[idx].blob_size;
|
|
5753
|
+
}
|
|
5754
|
+
|
|
5755
|
+
size_t oldest_input_pos = 0;
|
|
5756
|
+
for (size_t idx : input_indices) {
|
|
5757
|
+
oldest_input_pos = std::max(oldest_input_pos, idx);
|
|
5758
|
+
}
|
|
5759
|
+
|
|
5760
|
+
std::vector<size_t> sorted_indices = input_indices;
|
|
5761
|
+
std::sort(sorted_indices.rbegin(), sorted_indices.rend());
|
|
5762
|
+
for (size_t idx : sorted_indices) {
|
|
5763
|
+
files.erase(files.begin() + idx);
|
|
5764
|
+
}
|
|
5765
|
+
|
|
5766
|
+
size_t insert_pos = oldest_input_pos;
|
|
5767
|
+
for (size_t idx : sorted_indices) {
|
|
5768
|
+
if (idx < oldest_input_pos) insert_pos--;
|
|
5769
|
+
}
|
|
5770
|
+
insert_pos = std::min(insert_pos, files.size());
|
|
5771
|
+
files.insert(files.begin() + insert_pos,
|
|
5772
|
+
{output_size, output_blob, global_age++, true});
|
|
5773
|
+
}
|
|
5774
|
+
|
|
5775
|
+
// Compute statistics about compacted file sizes
|
|
5776
|
+
struct FileStats {
|
|
5777
|
+
uint64_t count;
|
|
5778
|
+
uint64_t min_size;
|
|
5779
|
+
uint64_t max_size;
|
|
5780
|
+
double mean_size;
|
|
5781
|
+
double cv;
|
|
5782
|
+
};
|
|
5783
|
+
|
|
5784
|
+
FileStats ComputeStats(const std::vector<L0File>& files,
|
|
5785
|
+
bool compacted_only) {
|
|
5786
|
+
std::vector<uint64_t> sizes;
|
|
5787
|
+
for (const auto& f : files) {
|
|
5788
|
+
if (!compacted_only || f.is_compacted) {
|
|
5789
|
+
sizes.push_back(f.size);
|
|
5790
|
+
}
|
|
5791
|
+
}
|
|
5792
|
+
if (sizes.empty()) return {0, 0, 0, 0.0, 0.0};
|
|
5793
|
+
|
|
5794
|
+
uint64_t sum = 0;
|
|
5795
|
+
uint64_t min_s = UINT64_MAX, max_s = 0;
|
|
5796
|
+
for (uint64_t s : sizes) {
|
|
5797
|
+
sum += s;
|
|
5798
|
+
min_s = std::min(min_s, s);
|
|
5799
|
+
max_s = std::max(max_s, s);
|
|
5800
|
+
}
|
|
5801
|
+
double mean = static_cast<double>(sum) / sizes.size();
|
|
5802
|
+
|
|
5803
|
+
double variance = 0;
|
|
5804
|
+
for (uint64_t s : sizes) {
|
|
5805
|
+
double diff = static_cast<double>(s) - mean;
|
|
5806
|
+
variance += diff * diff;
|
|
5807
|
+
}
|
|
5808
|
+
variance /= sizes.size();
|
|
5809
|
+
double stddev = std::sqrt(variance);
|
|
5810
|
+
double cv = mean > 0 ? stddev / mean : 0;
|
|
5811
|
+
|
|
5812
|
+
return {sizes.size(), min_s, max_s, mean, cv};
|
|
5813
|
+
}
|
|
5814
|
+
|
|
5815
|
+
// Track write amplification
|
|
5816
|
+
struct WriteAmpTracker {
|
|
5817
|
+
uint64_t bytes_flushed = 0;
|
|
5818
|
+
uint64_t bytes_compacted = 0;
|
|
5819
|
+
|
|
5820
|
+
double sst_write_amp() const {
|
|
5821
|
+
return bytes_flushed > 0
|
|
5822
|
+
? static_cast<double>(bytes_flushed + bytes_compacted) /
|
|
5823
|
+
bytes_flushed
|
|
5824
|
+
: 1.0;
|
|
5825
|
+
}
|
|
5826
|
+
};
|
|
5827
|
+
|
|
5828
|
+
struct TestState {
|
|
5829
|
+
std::vector<L0File> files;
|
|
5830
|
+
uint64_t global_age = 0;
|
|
5831
|
+
WriteAmpTracker wa;
|
|
5832
|
+
int compaction_count = 0;
|
|
5833
|
+
uint64_t max_file_count_seen = 0;
|
|
5834
|
+
};
|
|
5835
|
+
|
|
5836
|
+
using FlushGenerator =
|
|
5837
|
+
std::function<std::pair<uint64_t, uint64_t>(int round)>;
|
|
5838
|
+
|
|
5839
|
+
// Core test loop: flush -> pick -> execute -> repeat.
|
|
5840
|
+
void RunFlushAndCompact(TestState& s, int num_rounds, int trigger,
|
|
5841
|
+
uint64_t max_data_files_size,
|
|
5842
|
+
const FlushGenerator& gen) {
|
|
5843
|
+
ioptions_.compaction_style = kCompactionStyleFIFO;
|
|
5844
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5845
|
+
|
|
5846
|
+
// Use max_data_files_size for both limits. When max_data_files_size > 0,
|
|
5847
|
+
// it takes precedence and max_table_files_size is ignored, but keeping
|
|
5848
|
+
// them consistent avoids contradictory configurations.
|
|
5849
|
+
const uint64_t max_table_files_size = max_data_files_size;
|
|
5850
|
+
|
|
5851
|
+
for (int round = 0; round < num_rounds; round++) {
|
|
5852
|
+
auto [sst_size, blob_size] = gen(round);
|
|
5853
|
+
s.files.insert(s.files.begin(),
|
|
5854
|
+
{sst_size, blob_size, s.global_age++, false});
|
|
5855
|
+
s.wa.bytes_flushed += sst_size;
|
|
5856
|
+
|
|
5857
|
+
// Pick compaction. Handle both dropping and intra-L0 results.
|
|
5858
|
+
CompactionReason reason;
|
|
5859
|
+
auto inputs =
|
|
5860
|
+
PickCompactionFromFiles(picker, s.files, max_table_files_size,
|
|
5861
|
+
max_data_files_size, trigger, &reason);
|
|
5862
|
+
if (!inputs.empty()) {
|
|
5863
|
+
if (reason == CompactionReason::kFIFOMaxSize ||
|
|
5864
|
+
reason == CompactionReason::kFIFOTtl) {
|
|
5865
|
+
// Size/TTL dropping: remove the picked files
|
|
5866
|
+
std::vector<size_t> sorted = inputs;
|
|
5867
|
+
std::sort(sorted.rbegin(), sorted.rend());
|
|
5868
|
+
for (size_t idx : sorted) {
|
|
5869
|
+
s.files.erase(s.files.begin() + idx);
|
|
5870
|
+
}
|
|
5871
|
+
} else {
|
|
5872
|
+
// Intra-L0 compaction: merge picked files
|
|
5873
|
+
uint64_t compaction_input = 0;
|
|
5874
|
+
for (size_t idx : inputs) {
|
|
5875
|
+
compaction_input += s.files[idx].size;
|
|
5876
|
+
}
|
|
5877
|
+
s.wa.bytes_compacted += compaction_input;
|
|
5878
|
+
ExecuteCompaction(s.files, inputs, s.global_age);
|
|
5879
|
+
s.compaction_count++;
|
|
5880
|
+
}
|
|
5881
|
+
}
|
|
5882
|
+
s.max_file_count_seen = std::max(s.max_file_count_seen,
|
|
5883
|
+
static_cast<uint64_t>(s.files.size()));
|
|
5884
|
+
}
|
|
5885
|
+
}
|
|
5886
|
+
|
|
5887
|
+
// Assertion helpers
|
|
5888
|
+
void AssertFileCountBounded(const std::vector<L0File>& files,
|
|
5889
|
+
uint64_t max_count, uint64_t multiplier = 3) {
|
|
5890
|
+
ASSERT_LE(files.size(), max_count * multiplier)
|
|
5891
|
+
<< "File count " << files.size() << " exceeds "
|
|
5892
|
+
<< max_count * multiplier;
|
|
5893
|
+
}
|
|
5894
|
+
|
|
5895
|
+
void AssertCompactedUniform(const std::vector<L0File>& files, double max_cv) {
|
|
5896
|
+
auto stats = ComputeStats(files, true);
|
|
5897
|
+
if (stats.count >= 2) {
|
|
5898
|
+
ASSERT_LE(stats.cv, max_cv)
|
|
5899
|
+
<< "Compacted CV=" << stats.cv << " exceeds " << max_cv
|
|
5900
|
+
<< " (min=" << stats.min_size << " max=" << stats.max_size
|
|
5901
|
+
<< " mean=" << stats.mean_size << " count=" << stats.count << ")";
|
|
5902
|
+
}
|
|
5903
|
+
}
|
|
5904
|
+
|
|
5905
|
+
void AssertLowWriteAmp(const WriteAmpTracker& wa, double max_wa = 3.0) {
|
|
5906
|
+
ASSERT_LE(wa.sst_write_amp(), max_wa)
|
|
5907
|
+
<< "Write amp=" << wa.sst_write_amp() << " exceeds " << max_wa;
|
|
5908
|
+
}
|
|
5909
|
+
|
|
5910
|
+
void AssertStandardGoals(const TestState& s, uint64_t max_count,
|
|
5911
|
+
double max_cv = 0.30, double max_wa = 3.0,
|
|
5912
|
+
uint64_t file_mult = 3) {
|
|
5913
|
+
AssertFileCountBounded(s.files, max_count, file_mult);
|
|
5914
|
+
AssertCompactedUniform(s.files, max_cv);
|
|
5915
|
+
AssertLowWriteAmp(s.wa, max_wa);
|
|
5916
|
+
}
|
|
5917
|
+
|
|
5918
|
+
// Verify that graduated files (>= target) are never picked for compaction.
|
|
5919
|
+
void AssertGraduatedNotPicked(const std::vector<L0File>& files, int trigger,
|
|
5920
|
+
uint64_t max_data_files_size) {
|
|
5921
|
+
ioptions_.compaction_style = kCompactionStyleFIFO;
|
|
5922
|
+
FIFOCompactionPicker picker(ioptions_, &icmp_);
|
|
5923
|
+
const uint64_t max_table_files_size = max_data_files_size;
|
|
5924
|
+
|
|
5925
|
+
CompactionReason reason;
|
|
5926
|
+
auto inputs =
|
|
5927
|
+
PickCompactionFromFiles(picker, files, max_table_files_size,
|
|
5928
|
+
max_data_files_size, trigger, &reason);
|
|
5929
|
+
if (!inputs.empty() && reason == CompactionReason::kFIFOReduceNumFiles) {
|
|
5930
|
+
// Compute target from the picker's perspective: we need to estimate
|
|
5931
|
+
// it the same way the picker does.
|
|
5932
|
+
uint64_t total_sst = 0, total_blob = 0;
|
|
5933
|
+
for (const auto& f : files) {
|
|
5934
|
+
total_sst += f.size;
|
|
5935
|
+
total_blob += f.blob_size;
|
|
5936
|
+
}
|
|
5937
|
+
double sst_ratio = total_blob > 0 ? static_cast<double>(total_sst) /
|
|
5938
|
+
(total_sst + total_blob)
|
|
5939
|
+
: 1.0;
|
|
5940
|
+
uint64_t target =
|
|
5941
|
+
static_cast<uint64_t>(max_data_files_size * sst_ratio) / trigger;
|
|
5942
|
+
|
|
5943
|
+
for (size_t idx : inputs) {
|
|
5944
|
+
ASSERT_LT(files[idx].size, target)
|
|
5945
|
+
<< "Should not re-compact graduated file at index " << idx
|
|
5946
|
+
<< " size=" << files[idx].size << " target=" << target;
|
|
5947
|
+
}
|
|
5948
|
+
}
|
|
5949
|
+
}
|
|
5950
|
+
};
|
|
5951
|
+
|
|
5952
|
+
// Variable flush + FIFO dropping -- the full scenario.
|
|
5953
|
+
// Variable SST sizes (32-128KB), variable blob sizes (32-96MB), with
|
|
5954
|
+
// FIFO size-based dropping active. This covers constant flush, variable
|
|
5955
|
+
// flush, and FIFO dropping behaviors in a single test.
|
|
5956
|
+
TEST_F(FIFORatioBasedCompactionPickingTest, VariableFlushWithFIFODropping) {
|
|
5957
|
+
const uint64_t kCap = 500ULL * 1024 * 1024;
|
|
5958
|
+
Random rng(42);
|
|
5959
|
+
TestState s;
|
|
5960
|
+
RunFlushAndCompact(s, 200, /*trigger=*/10, kCap, [&](int) {
|
|
5961
|
+
return std::make_pair((32 + rng.Next() % 97) * 1024ULL,
|
|
5962
|
+
(32 + rng.Next() % 65) * 1024ULL * 1024);
|
|
5963
|
+
});
|
|
5964
|
+
AssertStandardGoals(s, 10, /*max_cv=*/0.40);
|
|
5965
|
+
}
|
|
5966
|
+
|
|
5967
|
+
// Verify graduated files are never re-compacted.
|
|
5968
|
+
// With the tiered algorithm, intermediate compacted files CAN be merged
|
|
5969
|
+
// at higher tier boundaries (that's the whole point of tiering). But files
|
|
5970
|
+
// that have reached the target size ("graduated") should never be picked.
|
|
5971
|
+
TEST_F(FIFORatioBasedCompactionPickingTest, NoCascadingReCompaction) {
|
|
5972
|
+
const uint64_t kCap = 10ULL * 1024 * 1024 * 1024;
|
|
5973
|
+
TestState s;
|
|
5974
|
+
RunFlushAndCompact(s, 200, /*trigger=*/10, kCap, [](int) {
|
|
5975
|
+
return std::make_pair(64ULL * 1024, 64ULL * 1024 * 1024);
|
|
5976
|
+
});
|
|
5977
|
+
|
|
5978
|
+
AssertGraduatedNotPicked(s.files, 10, kCap);
|
|
5979
|
+
// Write amp should be bounded (k=2 tiers for this config, so wa <= 3+margin)
|
|
5980
|
+
AssertLowWriteAmp(s.wa, 4.0);
|
|
5981
|
+
}
|
|
5982
|
+
|
|
5983
|
+
// Early memtable flush -- very small flushes
|
|
5984
|
+
TEST_F(FIFORatioBasedCompactionPickingTest, EarlyMemtableFlush) {
|
|
5985
|
+
const uint64_t kCap = 1ULL * 1024 * 1024 * 1024;
|
|
5986
|
+
Random rng(123);
|
|
5987
|
+
TestState s;
|
|
5988
|
+
RunFlushAndCompact(s, 100, /*trigger=*/10, kCap, [&](int) {
|
|
5989
|
+
uint64_t sst = (rng.Next() % 5 == 0) ? (64 + rng.Next() % 65) * 1024ULL
|
|
5990
|
+
: (8 + rng.Next() % 25) * 1024ULL;
|
|
5991
|
+
return std::make_pair(sst, 32ULL * 1024 * 1024);
|
|
5992
|
+
});
|
|
5993
|
+
|
|
5994
|
+
AssertStandardGoals(s, 10, /*max_cv=*/0.50, /*max_wa=*/4.0,
|
|
5995
|
+
/*file_mult=*/5);
|
|
5996
|
+
}
|
|
5997
|
+
|
|
5998
|
+
// Blob compression variation -- data per flush varies, shifting
|
|
5999
|
+
// the SST/blob ratio. The target is recomputed on every PickCompaction call
|
|
6000
|
+
// (no caching), so the picker naturally adapts to ratio changes.
|
|
6001
|
+
TEST_F(FIFORatioBasedCompactionPickingTest, BlobCompressionVariation) {
|
|
6002
|
+
const uint64_t kCap = 300ULL * 1024 * 1024;
|
|
6003
|
+
Random rng(456);
|
|
6004
|
+
TestState s;
|
|
6005
|
+
RunFlushAndCompact(s, 150, /*trigger=*/10, kCap, [&](int) {
|
|
6006
|
+
return std::make_pair(64ULL * 1024,
|
|
6007
|
+
(20 + rng.Next() % 61) * 1024ULL * 1024);
|
|
6008
|
+
});
|
|
6009
|
+
AssertCompactedUniform(s.files, 0.30);
|
|
6010
|
+
}
|
|
6011
|
+
|
|
6012
|
+
// Large target/flush ratio -- verify logarithmic write amp with tiering
|
|
6013
|
+
TEST_F(FIFORatioBasedCompactionPickingTest, TieredLargeRatio) {
|
|
6014
|
+
// target/flush ~ 1000x with trigger=10 -> k=3 tiers, write amp ~ 4.
|
|
6015
|
+
// Without tiering (flat merge), write amp would be ~57x.
|
|
6016
|
+
const uint64_t kCap = 10ULL * 1024 * 1024 * 1024; // 10GB
|
|
6017
|
+
TestState s;
|
|
6018
|
+
// SST = 1KB, blob = 1MB. sst_ratio ~ 0.001.
|
|
6019
|
+
// target = 10GB * 0.001 / 10 = 1MB. ratio = 1MB/1KB = 1024.
|
|
6020
|
+
// k = ceil(log_10(1024)) = 4. Tier boundaries: ~10KB, ~100KB, 1MB.
|
|
6021
|
+
// (10KB floor means lowest boundary is 10KB, not 1KB)
|
|
6022
|
+
RunFlushAndCompact(s, 500, /*trigger=*/10, kCap, [](int) {
|
|
6023
|
+
return std::make_pair(1ULL * 1024, 1ULL * 1024 * 1024);
|
|
6024
|
+
});
|
|
6025
|
+
|
|
6026
|
+
// Write amp should be logarithmic: k+1 = 4 (with 10KB floor, 3 tiers).
|
|
6027
|
+
// Allow some margin for ramp-up and boundary effects.
|
|
6028
|
+
AssertLowWriteAmp(s.wa, 6.0);
|
|
6029
|
+
|
|
6030
|
+
// File count should be bounded: trigger * (k+1) ~ 10 * 4 = 40
|
|
6031
|
+
AssertFileCountBounded(s.files, 10, /*multiplier=*/6);
|
|
6032
|
+
}
|
|
6033
|
+
|
|
6034
|
+
// Tiered progression -- verify intermediate tiers form and merge up
|
|
6035
|
+
TEST_F(FIFORatioBasedCompactionPickingTest, TieredProgression) {
|
|
6036
|
+
// SST = 10KB, blob = 1MB, cap = 100MB, trigger=4.
|
|
6037
|
+
// sst_ratio ~ 10KB/1010KB ~ 0.0099.
|
|
6038
|
+
// target = 100MB * 0.0099 / 4 ~ 248KB. ratio ~ 25.
|
|
6039
|
+
// k = ceil(log_4(25)) = ceil(2.32) = 3. Boundaries: ~16KB, ~62KB, ~248KB.
|
|
6040
|
+
const uint64_t kCap = 100ULL * 1024 * 1024;
|
|
6041
|
+
TestState s;
|
|
6042
|
+
RunFlushAndCompact(s, 200, /*trigger=*/4, kCap, [](int) {
|
|
6043
|
+
return std::make_pair(10ULL * 1024, 1ULL * 1024 * 1024);
|
|
6044
|
+
});
|
|
6045
|
+
|
|
6046
|
+
// Should have compacted files at multiple tier sizes
|
|
6047
|
+
auto stats = ComputeStats(s.files, true);
|
|
6048
|
+
ASSERT_GE(stats.count, 1u) << "Should have at least one compacted file";
|
|
6049
|
+
|
|
6050
|
+
// Write amp should be bounded: k+1 = 4, plus margin
|
|
6051
|
+
AssertLowWriteAmp(s.wa, 5.0);
|
|
6052
|
+
}
|
|
6053
|
+
|
|
6054
|
+
// Graduated files should never be re-compacted
|
|
6055
|
+
TEST_F(FIFORatioBasedCompactionPickingTest, GraduatedFilesNotRecompacted) {
|
|
6056
|
+
// Build a state with graduated files (>= target), then verify they are
|
|
6057
|
+
// never selected for compaction.
|
|
6058
|
+
const uint64_t kCap = 500ULL * 1024 * 1024; // 500MB
|
|
6059
|
+
TestState s;
|
|
6060
|
+
// SST = 64KB, blob = 50MB. sst_ratio ~ 0.00125.
|
|
6061
|
+
// target = 500MB * 0.00125 / 4 ~ 156KB.
|
|
6062
|
+
// k = ceil(log_4(156/64)) = ceil(log_4(2.44)) = 1.
|
|
6063
|
+
RunFlushAndCompact(s, 60, /*trigger=*/4, kCap, [](int) {
|
|
6064
|
+
return std::make_pair(64ULL * 1024, 50ULL * 1024 * 1024);
|
|
6065
|
+
});
|
|
6066
|
+
|
|
6067
|
+
AssertGraduatedNotPicked(s.files, 4, kCap);
|
|
6068
|
+
}
|
|
6069
|
+
|
|
4766
6070
|
} // namespace ROCKSDB_NAMESPACE
|
|
4767
6071
|
|
|
4768
6072
|
int main(int argc, char** argv) {
|