@nxtedition/rocksdb 15.4.1 → 15.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +24 -15
- package/deps/rocksdb/rocksdb/.clang-tidy +86 -0
- package/deps/rocksdb/rocksdb/BUCK +42 -0
- package/deps/rocksdb/rocksdb/CMakeLists.txt +11 -0
- package/deps/rocksdb/rocksdb/Makefile +59 -32
- package/deps/rocksdb/rocksdb/cache/cache.cc +0 -5
- package/deps/rocksdb/rocksdb/cache/cache_entry_stats.h +9 -9
- package/deps/rocksdb/rocksdb/cache/cache_key.cc +3 -3
- package/deps/rocksdb/rocksdb/cache/cache_key.h +5 -5
- package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.h +16 -16
- package/deps/rocksdb/rocksdb/cache/cache_test.cc +1 -1
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +258 -294
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +98 -49
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +1 -5
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +2 -3
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +18 -18
- package/deps/rocksdb/rocksdb/crash_test.mk +5 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +23 -22
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.h +6 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +14 -16
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +38 -26
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.h +5 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +101 -18
- package/deps/rocksdb/rocksdb/db/blob/blob_index.h +12 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +6 -9
- package/deps/rocksdb/rocksdb/db/builder.cc +23 -0
- package/deps/rocksdb/rocksdb/db/builder.h +7 -0
- package/deps/rocksdb/rocksdb/db/c.cc +373 -57
- package/deps/rocksdb/rocksdb/db/c_test.c +101 -1
- package/deps/rocksdb/rocksdb/db/column_family.cc +31 -3
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +10 -13
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +35 -48
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +13 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +201 -39
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +15 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +7 -7
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +2 -455
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +4 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +19 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +72 -9
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +12 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +405 -83
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +25 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +23 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +1 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +1410 -106
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +12 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +19 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +505 -45
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +2 -2
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +9 -1
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +7 -9
- package/deps/rocksdb/rocksdb/db/convenience.cc +4 -4
- package/deps/rocksdb/rocksdb/db/convenience_impl.h +2 -1
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +60 -88
- package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +10 -12
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +471 -40
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +116 -2
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +5 -15
- package/deps/rocksdb/rocksdb/db/db_compaction_abort_test.cc +993 -0
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +329 -29
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +155 -13
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +54 -31
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +1 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +232 -70
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +57 -9
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +224 -31
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +5 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +4 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +164 -8
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +6 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +5 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +47 -35
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +22 -9
- package/deps/rocksdb/rocksdb/db/db_iter.cc +9 -0
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +371 -6
- package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +7 -5
- package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +22 -23
- package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +0 -2
- package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_options_test.cc +40 -0
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +32 -13
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_readonly_with_timestamp_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +68 -15
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +2 -3
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +6 -21
- package/deps/rocksdb/rocksdb/db/db_test.cc +644 -128
- package/deps/rocksdb/rocksdb/db/db_test2.cc +198 -81
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +35 -10
- package/deps/rocksdb/rocksdb/db/db_test_util.h +8 -2
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +36 -32
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +11 -7
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +499 -0
- package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +284 -20
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/dbformat.h +0 -5
- package/deps/rocksdb/rocksdb/db/error_handler.cc +24 -0
- package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +12 -14
- package/deps/rocksdb/rocksdb/db/experimental.cc +13 -10
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +22 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +21 -15
- package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +4 -6
- package/deps/rocksdb/rocksdb/db/flush_job.cc +11 -3
- package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +5 -6
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +4 -2
- package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +17 -17
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +13 -0
- package/deps/rocksdb/rocksdb/db/internal_stats.h +2 -0
- package/deps/rocksdb/rocksdb/db/listener_test.cc +154 -27
- package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/memtable.cc +197 -51
- package/deps/rocksdb/rocksdb/db/memtable.h +6 -0
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/merge_test.cc +37 -35
- package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/options_file_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +9 -11
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +10 -1
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +292 -15
- package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +10 -17
- package/deps/rocksdb/rocksdb/db/prefix_test.cc +6 -8
- package/deps/rocksdb/rocksdb/db/repair.cc +10 -10
- package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +5 -5
- package/deps/rocksdb/rocksdb/db/table_cache.cc +142 -135
- package/deps/rocksdb/rocksdb/db/table_cache.h +30 -6
- package/deps/rocksdb/rocksdb/db/table_cache_sync_and_async.h +7 -7
- package/deps/rocksdb/rocksdb/db/version_builder.cc +11 -50
- package/deps/rocksdb/rocksdb/db/version_builder.h +2 -1
- package/deps/rocksdb/rocksdb/db/version_builder_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/version_edit.cc +51 -2
- package/deps/rocksdb/rocksdb/db/version_edit.h +91 -29
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -7
- package/deps/rocksdb/rocksdb/db/version_set.cc +211 -50
- package/deps/rocksdb/rocksdb/db/version_set.h +40 -3
- package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +5 -0
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +294 -21
- package/deps/rocksdb/rocksdb/db/version_util.cc +96 -0
- package/deps/rocksdb/rocksdb/db/version_util.h +24 -0
- package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +5 -5
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.cc +647 -31
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.h +219 -1
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization_test.cc +549 -12
- package/deps/rocksdb/rocksdb/db/write_callback_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +1 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +19 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +21 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +32 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +74 -22
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +9 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +143 -61
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +15 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +76 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +92 -72
- package/deps/rocksdb/rocksdb/env/env.cc +1 -0
- package/deps/rocksdb/rocksdb/env/env_test.cc +365 -2
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +31 -30
- package/deps/rocksdb/rocksdb/env/io_posix.cc +8 -11
- package/deps/rocksdb/rocksdb/env/io_posix.h +30 -1
- package/deps/rocksdb/rocksdb/env/io_posix_test.cc +43 -0
- package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +1 -1
- package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +108 -0
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +32 -4
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +4 -4
- package/deps/rocksdb/rocksdb/file/file_util.cc +8 -2
- package/deps/rocksdb/rocksdb/file/file_util.h +2 -1
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +331 -12
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +52 -35
- package/deps/rocksdb/rocksdb/folly.mk +22 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_cache.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +100 -54
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +67 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +149 -13
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +1 -12
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +78 -97
- package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +3 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/external_table.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +17 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/functor_wrapper.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/io_dispatcher.h +358 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +43 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +20 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +63 -21
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +10 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +2 -7
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +3 -14
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +49 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/status.h +8 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +77 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +15 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/tool_hooks.h +16 -10
- package/deps/rocksdb/rocksdb/include/rocksdb/unique_id.h +5 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +2 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/user_defined_index.h +106 -46
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/db_ttl.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +14 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/memory_util.h +5 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +7 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +1 -2
- package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc +2 -2
- package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +226 -8
- package/deps/rocksdb/rocksdb/memtable/inlineskiplist_test.cc +490 -0
- package/deps/rocksdb/rocksdb/memtable/skiplist.h +3 -3
- package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +11 -0
- package/deps/rocksdb/rocksdb/microbench/db_basic_bench.cc +4 -12
- package/deps/rocksdb/rocksdb/microbench/ribbon_bench.cc +5 -5
- package/deps/rocksdb/rocksdb/monitoring/file_read_sample.h +21 -4
- package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +9 -3
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +21 -2
- package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +2 -2
- package/deps/rocksdb/rocksdb/options/cf_options.cc +21 -1
- package/deps/rocksdb/rocksdb/options/cf_options.h +2 -0
- package/deps/rocksdb/rocksdb/options/customizable_test.cc +0 -2
- package/deps/rocksdb/rocksdb/options/db_options.cc +26 -5
- package/deps/rocksdb/rocksdb/options/db_options.h +3 -1
- package/deps/rocksdb/rocksdb/options/options.cc +5 -1
- package/deps/rocksdb/rocksdb/options/options_helper.cc +7 -2
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +109 -103
- package/deps/rocksdb/rocksdb/options/options_test.cc +14 -0
- package/deps/rocksdb/rocksdb/port/jemalloc_helper.h +15 -17
- package/deps/rocksdb/rocksdb/port/lang.h +4 -0
- package/deps/rocksdb/rocksdb/port/port_example.h +0 -23
- package/deps/rocksdb/rocksdb/port/stack_trace.cc +36 -0
- package/deps/rocksdb/rocksdb/port/stack_trace.h +9 -0
- package/deps/rocksdb/rocksdb/src.mk +12 -0
- package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +1 -2
- package/deps/rocksdb/rocksdb/table/block_based/binary_search_index_reader.cc +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/block.cc +571 -292
- package/deps/rocksdb/rocksdb/table/block_based/block.h +143 -53
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +154 -90
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +5 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +51 -14
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h +0 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +147 -734
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +30 -233
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +178 -108
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +13 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +17 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +5 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +70 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_builder.cc +168 -24
- package/deps/rocksdb/rocksdb/table/block_based/block_builder.h +25 -9
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +7 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +9 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +548 -169
- package/deps/rocksdb/rocksdb/table/block_based/block_type.h +30 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_util.h +156 -0
- package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.cc +73 -30
- package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.h +74 -7
- package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +20 -14
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +22 -12
- package/deps/rocksdb/rocksdb/table/block_based/mock_block_based_table.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.cc +332 -0
- package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.h +133 -0
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +4 -2
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/reader_common.cc +3 -2
- package/deps/rocksdb/rocksdb/table/block_based/reader_common.h +4 -1
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +0 -1
- package/deps/rocksdb/rocksdb/table/block_based/user_defined_index_wrapper.h +126 -46
- package/deps/rocksdb/rocksdb/table/block_fetcher.cc +31 -3
- package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +1 -2
- package/deps/rocksdb/rocksdb/table/cleanable_test.cc +3 -1
- package/deps/rocksdb/rocksdb/table/external_table.cc +25 -4
- package/deps/rocksdb/rocksdb/table/format.cc +27 -15
- package/deps/rocksdb/rocksdb/table/format.h +41 -15
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +1 -0
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +22 -12
- package/deps/rocksdb/rocksdb/table/meta_blocks.h +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +7 -21
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +88 -13
- package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +53 -42
- package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +3 -12
- package/deps/rocksdb/rocksdb/table/table_builder.h +0 -4
- package/deps/rocksdb/rocksdb/table/table_properties.cc +18 -0
- package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +2 -3
- package/deps/rocksdb/rocksdb/table/table_test.cc +848 -172
- package/deps/rocksdb/rocksdb/table/unique_id.cc +24 -20
- package/deps/rocksdb/rocksdb/table/unique_id_impl.h +8 -8
- package/deps/rocksdb/rocksdb/test_util/sync_point.h +5 -4
- package/deps/rocksdb/rocksdb/test_util/testutil.cc +2 -1
- package/deps/rocksdb/rocksdb/test_util/testutil.h +2 -2
- package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +2 -1
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +238 -120
- package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +2 -2
- package/deps/rocksdb/rocksdb/tools/db_sanity_test.cc +2 -4
- package/deps/rocksdb/rocksdb/tools/dump/db_dump_tool.cc +4 -8
- package/deps/rocksdb/rocksdb/tools/dump/rocksdb_undump.cc +1 -1
- package/deps/rocksdb/rocksdb/tools/io_tracer_parser_test.cc +2 -3
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +82 -20
- package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +41 -47
- package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +9 -0
- package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +5 -6
- package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +1 -1
- package/deps/rocksdb/rocksdb/tools/tool_hooks.cc +6 -5
- package/deps/rocksdb/rocksdb/tools/trace_analyzer_test.cc +4 -4
- package/deps/rocksdb/rocksdb/tools/write_stress.cc +1 -3
- package/deps/rocksdb/rocksdb/util/atomic.h +30 -23
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.cc +6 -7
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.h +3 -3
- package/deps/rocksdb/rocksdb/util/bit_fields.h +68 -46
- package/deps/rocksdb/rocksdb/util/bloom_impl.h +16 -16
- package/deps/rocksdb/rocksdb/util/coding.h +14 -27
- package/deps/rocksdb/rocksdb/util/compression.cc +365 -207
- package/deps/rocksdb/rocksdb/util/compression.h +16 -1298
- package/deps/rocksdb/rocksdb/util/compression_test.cc +347 -61
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +8 -9
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.h +1 -1
- package/deps/rocksdb/rocksdb/util/crc32c_ppc.h +1 -1
- package/deps/rocksdb/rocksdb/util/dynamic_bloom_test.cc +3 -3
- package/deps/rocksdb/rocksdb/util/filter_bench.cc +18 -18
- package/deps/rocksdb/rocksdb/util/gflags_compat.h +3 -3
- package/deps/rocksdb/rocksdb/util/hash_test.cc +19 -7
- package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.cc +1099 -0
- package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.h +36 -0
- package/deps/rocksdb/rocksdb/util/io_dispatcher_test.cc +1919 -0
- package/deps/rocksdb/rocksdb/util/math.h +3 -1
- package/deps/rocksdb/rocksdb/util/mutexlock.h +19 -19
- package/deps/rocksdb/rocksdb/util/ribbon_alg.h +25 -25
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +5 -7
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +4 -5
- package/deps/rocksdb/rocksdb/util/slice.cc +0 -10
- package/deps/rocksdb/rocksdb/util/slice_test.cc +35 -1
- package/deps/rocksdb/rocksdb/util/slice_transform_test.cc +5 -7
- package/deps/rocksdb/rocksdb/util/status.cc +3 -1
- package/deps/rocksdb/rocksdb/util/stop_watch.h +2 -0
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +4 -1
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +123 -78
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +12 -93
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.h +1 -4
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.cc +0 -21
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +6 -48
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +94 -307
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +12 -58
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +2 -8
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +2 -3
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +205 -811
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +18 -9
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.cc +2 -7
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.h +1 -9
- package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_functional_test.cc +17 -11
- package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +68 -61
- package/deps/rocksdb/rocksdb/utilities/debug.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +105 -59
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +274 -7
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs_test.cc +94 -0
- package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +13 -17
- package/deps/rocksdb/rocksdb/utilities/memory/memory_util.cc +16 -3
- package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc +25 -25
- package/deps/rocksdb/rocksdb/utilities/object_registry.cc +40 -40
- package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc +2 -5
- package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +17 -19
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.h +2 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.h +4 -13
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +3 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +6 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_seqno_test.cc +431 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +1 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.h +91 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.cc +562 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.h +615 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.cc +2575 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.h +685 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_db_test.cc +2843 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.cc +567 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.h +275 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_test.cc +5183 -0
- package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.cc +4 -3
- package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +3 -3
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +93 -88
- package/deps/rocksdb/rocksdb.gyp +7 -0
- package/iterator.js +2 -2
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
- package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_custom_library.h +0 -43
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_util.h +0 -55
|
@@ -195,9 +195,11 @@ class MyTestCompactionService : public CompactionService {
|
|
|
195
195
|
std::vector<std::shared_ptr<EventListener>> listeners_;
|
|
196
196
|
std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
|
|
197
197
|
table_properties_collector_factories_;
|
|
198
|
-
std::atomic_bool canceled_{false};
|
|
199
198
|
std::atomic<CompactionServiceJobStatus> final_updated_status_{
|
|
200
199
|
CompactionServiceJobStatus::kUseLocal};
|
|
200
|
+
|
|
201
|
+
protected:
|
|
202
|
+
std::atomic_bool canceled_{false};
|
|
201
203
|
};
|
|
202
204
|
|
|
203
205
|
class CompactionServiceTest : public DBTestBase {
|
|
@@ -407,6 +409,34 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
|
|
|
407
409
|
SyncPoint::GetInstance()->DisableProcessing();
|
|
408
410
|
}
|
|
409
411
|
|
|
412
|
+
TEST_F(CompactionServiceTest, SkipWALRecoveryInOpenAndCompact) {
|
|
413
|
+
// Verify that OpenAndCompact skips WAL recovery when opening the secondary
|
|
414
|
+
// instance. WAL replay is unnecessary for remote compaction since it only
|
|
415
|
+
// needs the LSM state from MANIFEST.
|
|
416
|
+
Options options = CurrentOptions();
|
|
417
|
+
ReopenWithCompactionService(&options);
|
|
418
|
+
|
|
419
|
+
// Track whether FindAndRecoverLogFiles is called during compaction
|
|
420
|
+
std::atomic_bool wal_recovery_called{false};
|
|
421
|
+
SyncPoint::GetInstance()->SetCallBack(
|
|
422
|
+
"DBImplSecondary::FindAndRecoverLogFiles:Begin",
|
|
423
|
+
[&](void* /* arg */) { wal_recovery_called.store(true); });
|
|
424
|
+
SyncPoint::GetInstance()->EnableProcessing();
|
|
425
|
+
|
|
426
|
+
// Generate data and trigger compaction (which uses OpenAndCompact)
|
|
427
|
+
GenerateTestData();
|
|
428
|
+
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
429
|
+
|
|
430
|
+
// WAL recovery should NOT have been called during OpenAndCompact
|
|
431
|
+
ASSERT_FALSE(wal_recovery_called.load());
|
|
432
|
+
|
|
433
|
+
// Data should still be correct (compaction worked without WAL recovery)
|
|
434
|
+
VerifyTestData();
|
|
435
|
+
|
|
436
|
+
SyncPoint::GetInstance()->DisableProcessing();
|
|
437
|
+
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
438
|
+
}
|
|
439
|
+
|
|
410
440
|
TEST_F(CompactionServiceTest, ManualCompaction) {
|
|
411
441
|
Options options = CurrentOptions();
|
|
412
442
|
options.disable_auto_compactions = true;
|
|
@@ -1111,6 +1141,8 @@ TEST_F(CompactionServiceTest, CorruptedOutputVerifyOutputFlags) {
|
|
|
1111
1141
|
VerifyOutputFlags::kVerifyBlockChecksum,
|
|
1112
1142
|
VerifyOutputFlags::kEnableForRemoteCompaction |
|
|
1113
1143
|
VerifyOutputFlags::kVerifyIteration,
|
|
1144
|
+
VerifyOutputFlags::kEnableForRemoteCompaction |
|
|
1145
|
+
VerifyOutputFlags::kVerifyFileChecksum,
|
|
1114
1146
|
VerifyOutputFlags::kVerifyAll}) {
|
|
1115
1147
|
SCOPED_TRACE(
|
|
1116
1148
|
"verify_output_flags=" +
|
|
@@ -1122,6 +1154,7 @@ TEST_F(CompactionServiceTest, CorruptedOutputVerifyOutputFlags) {
|
|
|
1122
1154
|
options.disable_auto_compactions = true;
|
|
1123
1155
|
options.paranoid_file_checks = false;
|
|
1124
1156
|
options.verify_output_flags = verify_output_flags;
|
|
1157
|
+
options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
|
|
1125
1158
|
ReopenWithCompactionService(&options);
|
|
1126
1159
|
GenerateTestData();
|
|
1127
1160
|
|
|
@@ -1157,10 +1190,13 @@ TEST_F(CompactionServiceTest, CorruptedOutputVerifyOutputFlags) {
|
|
|
1157
1190
|
!!(verify_output_flags & VerifyOutputFlags::kVerifyBlockChecksum);
|
|
1158
1191
|
const bool should_verify_iteration =
|
|
1159
1192
|
!!(verify_output_flags & VerifyOutputFlags::kVerifyIteration);
|
|
1193
|
+
const bool should_verify_file_checksum =
|
|
1194
|
+
!!(verify_output_flags & VerifyOutputFlags::kVerifyFileChecksum);
|
|
1160
1195
|
|
|
1161
1196
|
Status s = db_->CompactRange(CompactRangeOptions(), &start, &end);
|
|
1162
1197
|
if (is_enabled_for_remote_compaction &&
|
|
1163
|
-
(should_verify_block_checksum || should_verify_iteration
|
|
1198
|
+
(should_verify_block_checksum || should_verify_iteration ||
|
|
1199
|
+
should_verify_file_checksum)) {
|
|
1164
1200
|
ASSERT_NOK(s);
|
|
1165
1201
|
ASSERT_TRUE(s.IsCorruption());
|
|
1166
1202
|
} else {
|
|
@@ -1378,8 +1414,9 @@ TEST_F(CompactionServiceTest, CancelCompactionOnPrimarySide) {
|
|
|
1378
1414
|
|
|
1379
1415
|
// Primary DB calls CancelAllBackgroundWork() while the compaction is running
|
|
1380
1416
|
SyncPoint::GetInstance()->SetCallBack(
|
|
1381
|
-
"CompactionJob::Run():Inprogress",
|
|
1382
|
-
|
|
1417
|
+
"CompactionJob::Run():Inprogress", [&](void* /*arg*/) {
|
|
1418
|
+
CancelAllBackgroundWork(db_.get(), false /*wait*/);
|
|
1419
|
+
});
|
|
1383
1420
|
|
|
1384
1421
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
1385
1422
|
|
|
@@ -2137,6 +2174,12 @@ class ResumableCompactionService : public MyTestCompactionService {
|
|
|
2137
2174
|
{} /* table_properties_collector_factories */),
|
|
2138
2175
|
scenario_(scenario) {}
|
|
2139
2176
|
|
|
2177
|
+
// Set the user key where cancellation should happen.
|
|
2178
|
+
void SetCancelAtKey(const std::string& key, SequenceNumber seqno) {
|
|
2179
|
+
cancel_at_key_ = key;
|
|
2180
|
+
cancel_at_seqno_ = seqno;
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2140
2183
|
CompactionServiceJobStatus Wait(const std::string& scheduled_job_id,
|
|
2141
2184
|
std::string* result) override {
|
|
2142
2185
|
std::string compaction_input = ExtractCompactionInput(scheduled_job_id);
|
|
@@ -2149,25 +2192,50 @@ class ResumableCompactionService : public MyTestCompactionService {
|
|
|
2149
2192
|
// ASSUMPTION: This makes stats.count directly proportional to keys
|
|
2150
2193
|
// processed.
|
|
2151
2194
|
SyncPoint::GetInstance()->SetCallBack(
|
|
2152
|
-
"CompactionOutputs::ShouldStopBefore::manual_decision",
|
|
2195
|
+
"CompactionOutputs::ShouldStopBefore::manual_decision",
|
|
2196
|
+
[this](void* p) {
|
|
2153
2197
|
auto* pair = static_cast<std::pair<bool*, const Slice>*>(p);
|
|
2154
|
-
*(pair->first) = true;
|
|
2198
|
+
*(pair->first) = true; // Force file cut at every key
|
|
2199
|
+
|
|
2200
|
+
// If cancel_at_key_ is set, cancel when we encounter that key
|
|
2201
|
+
if (!cancel_at_key_.empty() && !already_canceled_) {
|
|
2202
|
+
ParsedInternalKey parsed_key;
|
|
2203
|
+
if (ParseInternalKey(pair->second, &parsed_key, true).ok()) {
|
|
2204
|
+
if (parsed_key.user_key.ToString() == cancel_at_key_) {
|
|
2205
|
+
// Check sequence number if specified
|
|
2206
|
+
if (cancel_at_seqno_ == kMaxSequenceNumber ||
|
|
2207
|
+
parsed_key.sequence == cancel_at_seqno_) {
|
|
2208
|
+
canceled_ = true;
|
|
2209
|
+
already_canceled_ = true;
|
|
2210
|
+
}
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
}
|
|
2155
2214
|
});
|
|
2215
|
+
|
|
2216
|
+
// If no cancel_at_key_ is set, use the original behavior:
|
|
2156
2217
|
// Simulate cancelled compaction by overriding status at completion. So
|
|
2157
2218
|
// compaction processes all keys before this point to make stats.count
|
|
2158
2219
|
// comparison straightforward.
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
*
|
|
2163
|
-
|
|
2220
|
+
if (cancel_at_key_.empty()) {
|
|
2221
|
+
SyncPoint::GetInstance()->SetCallBack(
|
|
2222
|
+
"DBImplSecondary::CompactWithoutInstallation::End",
|
|
2223
|
+
[&](void* status) {
|
|
2224
|
+
auto s = static_cast<Status*>(status);
|
|
2225
|
+
*s = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
|
|
2226
|
+
});
|
|
2227
|
+
}
|
|
2164
2228
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
2165
2229
|
|
|
2166
2230
|
// Phase 1: Run compaction with resumption enabled and cancel it
|
|
2167
|
-
// - Processes
|
|
2231
|
+
// - Processes input keys until cancellation point
|
|
2168
2232
|
// - Creates output files and saves progress
|
|
2169
2233
|
// - Status overridden to "paused"
|
|
2170
2234
|
open_and_compaction_options.allow_resumption = true;
|
|
2235
|
+
open_and_compaction_options.canceled = &canceled_;
|
|
2236
|
+
already_canceled_ = false;
|
|
2237
|
+
canceled_ = false;
|
|
2238
|
+
|
|
2171
2239
|
auto phase1_stats =
|
|
2172
2240
|
RunCancelledCompaction(open_and_compaction_options, scheduled_job_id,
|
|
2173
2241
|
compaction_input, override_options);
|
|
@@ -2188,6 +2256,9 @@ class ResumableCompactionService : public MyTestCompactionService {
|
|
|
2188
2256
|
EXPECT_TRUE(cleanup_status.ok());
|
|
2189
2257
|
EXPECT_OK(override_options.env->CreateDir(output_dir));
|
|
2190
2258
|
|
|
2259
|
+
already_canceled_ = false;
|
|
2260
|
+
canceled_ = false;
|
|
2261
|
+
|
|
2191
2262
|
phase2_stats =
|
|
2192
2263
|
RunCancelledCompaction(open_and_compaction_options, scheduled_job_id,
|
|
2193
2264
|
compaction_input, override_options);
|
|
@@ -2199,9 +2270,6 @@ class ResumableCompactionService : public MyTestCompactionService {
|
|
|
2199
2270
|
EXPECT_EQ(phase2_stats.count, phase1_stats.count);
|
|
2200
2271
|
}
|
|
2201
2272
|
|
|
2202
|
-
SyncPoint::GetInstance()->ClearCallBack(
|
|
2203
|
-
"DBImplSecondary::CompactWithoutInstallation::End");
|
|
2204
|
-
|
|
2205
2273
|
// Final phase: Run compaction to completion (no cancellation)
|
|
2206
2274
|
if (scenario_ == TestScenario::kMultipleCancelToggleResumption) {
|
|
2207
2275
|
// Attempt to resume but it ends up starting fresh
|
|
@@ -2220,6 +2288,12 @@ class ResumableCompactionService : public MyTestCompactionService {
|
|
|
2220
2288
|
EXPECT_OK(override_options.env->CreateDir(output_dir));
|
|
2221
2289
|
}
|
|
2222
2290
|
|
|
2291
|
+
// Prevent triggering of cancellation
|
|
2292
|
+
SyncPoint::GetInstance()->ClearCallBack(
|
|
2293
|
+
"DBImplSecondary::CompactWithoutInstallation::End");
|
|
2294
|
+
already_canceled_ = true;
|
|
2295
|
+
canceled_ = false;
|
|
2296
|
+
|
|
2223
2297
|
auto final_phase_stats =
|
|
2224
2298
|
RunCompaction(open_and_compaction_options, scheduled_job_id,
|
|
2225
2299
|
compaction_input, override_options, result);
|
|
@@ -2227,36 +2301,38 @@ class ResumableCompactionService : public MyTestCompactionService {
|
|
|
2227
2301
|
SyncPoint::GetInstance()->DisableProcessing();
|
|
2228
2302
|
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
2229
2303
|
|
|
2230
|
-
// Validate statistics based on scenario
|
|
2231
|
-
if (
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2304
|
+
// Validate statistics based on scenario (only when cancelling at end)
|
|
2305
|
+
if (cancel_at_key_.empty()) {
|
|
2306
|
+
if (scenario_ == TestScenario::kMultipleCancelToggleResumption) {
|
|
2307
|
+
// ASSUMPTION: Phase 1 processes all keys before cancellation
|
|
2308
|
+
EXPECT_GT(phase1_stats.count, 0);
|
|
2309
|
+
|
|
2310
|
+
// ASSUMPTION: Phase 2 runs with allow_resumption=false and an empty
|
|
2311
|
+
// folder. Phase 2 then creates its own output files (but doesn't save
|
|
2312
|
+
// progress). When Phase 3 starts with allow_resumption=true, it finds
|
|
2313
|
+
// no progress file exists, so it cannot resume and must start from
|
|
2314
|
+
// scratch, processing all input keys again. Result: Phase 3 does the
|
|
2315
|
+
// same amount of work as Phase 1.
|
|
2316
|
+
EXPECT_EQ(final_phase_stats.count, phase1_stats.count);
|
|
2317
|
+
|
|
2318
|
+
} else if (scenario_ == TestScenario::kCancelThenResume) {
|
|
2319
|
+
// ASSUMPTION: Phase 1 processes all keys before cancellation
|
|
2320
|
+
EXPECT_GT(phase1_stats.count, 0);
|
|
2321
|
+
|
|
2322
|
+
// ASSUMPTION: Phase 1 processes all keys and saves progress before
|
|
2323
|
+
// cancellation. Final phase resumes from Phase 1's saved progress.
|
|
2324
|
+
// Since Phase 1 completed all processing before being cancelled, the
|
|
2325
|
+
// final phase should do less work than Phase 1.
|
|
2326
|
+
EXPECT_LT(final_phase_stats.count, phase1_stats.count);
|
|
2327
|
+
|
|
2328
|
+
} else { // kCancelThenFreshStart
|
|
2329
|
+
// ASSUMPTION: Phase 1 processes all keys before cancellation
|
|
2330
|
+
EXPECT_GT(phase1_stats.count, 0);
|
|
2331
|
+
|
|
2332
|
+
// ASSUMPTION: Final phase starts fresh without resumption, so it
|
|
2333
|
+
// processes all input keys again and creates the same number of files
|
|
2334
|
+
EXPECT_EQ(final_phase_stats.count, phase1_stats.count);
|
|
2335
|
+
}
|
|
2260
2336
|
}
|
|
2261
2337
|
|
|
2262
2338
|
StoreResult(*result);
|
|
@@ -2326,6 +2402,9 @@ class ResumableCompactionService : public MyTestCompactionService {
|
|
|
2326
2402
|
}
|
|
2327
2403
|
|
|
2328
2404
|
TestScenario scenario_;
|
|
2405
|
+
std::string cancel_at_key_;
|
|
2406
|
+
SequenceNumber cancel_at_seqno_ = kMaxSequenceNumber;
|
|
2407
|
+
std::atomic<bool> already_canceled_{false};
|
|
2329
2408
|
};
|
|
2330
2409
|
|
|
2331
2410
|
class ResumableCompactionServiceTest : public CompactionServiceTest {
|
|
@@ -2432,6 +2511,387 @@ TEST_F(ResumableCompactionServiceTest,
|
|
|
2432
2511
|
RunCompactionCancelTest(ResumableCompactionService::TestScenario::
|
|
2433
2512
|
kMultipleCancelToggleResumption);
|
|
2434
2513
|
}
|
|
2514
|
+
|
|
2515
|
+
class ResumableCompactionKeyTypeTest : public CompactionServiceTest {
|
|
2516
|
+
public:
|
|
2517
|
+
explicit ResumableCompactionKeyTypeTest() : CompactionServiceTest() {}
|
|
2518
|
+
|
|
2519
|
+
protected:
|
|
2520
|
+
void SetupResumableCompactionService(
|
|
2521
|
+
Options& options, const std::string& cancel_at_key = "",
|
|
2522
|
+
SequenceNumber cancel_at_seqno = kMaxSequenceNumber) {
|
|
2523
|
+
options.disable_auto_compactions = true;
|
|
2524
|
+
statistics_ = CreateDBStatistics();
|
|
2525
|
+
|
|
2526
|
+
resume_cs_ = std::make_shared<ResumableCompactionService>(
|
|
2527
|
+
dbname_, options, statistics_,
|
|
2528
|
+
ResumableCompactionService::TestScenario::kCancelThenResume);
|
|
2529
|
+
|
|
2530
|
+
if (!cancel_at_key.empty()) {
|
|
2531
|
+
resume_cs_->SetCancelAtKey(cancel_at_key, cancel_at_seqno);
|
|
2532
|
+
}
|
|
2533
|
+
|
|
2534
|
+
options.compaction_service = resume_cs_;
|
|
2535
|
+
DestroyAndReopen(options);
|
|
2536
|
+
}
|
|
2537
|
+
|
|
2538
|
+
void ResetStatistics() { ASSERT_OK(statistics_->Reset()); }
|
|
2539
|
+
|
|
2540
|
+
void VerifyResumeBytes() {
|
|
2541
|
+
uint64_t resumed_bytes =
|
|
2542
|
+
statistics_->getTickerCount(REMOTE_COMPACT_RESUMED_BYTES);
|
|
2543
|
+
ASSERT_GT(resumed_bytes, 0);
|
|
2544
|
+
}
|
|
2545
|
+
|
|
2546
|
+
private:
|
|
2547
|
+
std::shared_ptr<ResumableCompactionService> resume_cs_;
|
|
2548
|
+
std::shared_ptr<Statistics> statistics_;
|
|
2549
|
+
};
|
|
2550
|
+
|
|
2551
|
+
// Cancel compaction right before processing key "c" to test resumption at a
|
|
2552
|
+
// deletion at the non-bottom level. When resumed, compaction will continue
|
|
2553
|
+
// from this deletion.
|
|
2554
|
+
TEST_F(ResumableCompactionKeyTypeTest,
|
|
2555
|
+
CancelAndResumeWithDeleteAtNonBottomLevel) {
|
|
2556
|
+
Options options = CurrentOptions();
|
|
2557
|
+
|
|
2558
|
+
SetupResumableCompactionService(options, "c");
|
|
2559
|
+
|
|
2560
|
+
ASSERT_OK(Put("c", "old_value"));
|
|
2561
|
+
ASSERT_OK(Put("c_placeholder", "placeholder"));
|
|
2562
|
+
ASSERT_OK(Flush());
|
|
2563
|
+
MoveFilesToLevel(options.num_levels - 1);
|
|
2564
|
+
|
|
2565
|
+
ASSERT_OK(Put("a", "val1"));
|
|
2566
|
+
ASSERT_OK(Put("b", "val2"));
|
|
2567
|
+
ASSERT_OK(Put("d", "val4"));
|
|
2568
|
+
ASSERT_OK(Flush());
|
|
2569
|
+
|
|
2570
|
+
ASSERT_OK(Delete("c"));
|
|
2571
|
+
ASSERT_OK(Flush());
|
|
2572
|
+
|
|
2573
|
+
std::vector<std::string> input_files;
|
|
2574
|
+
ColumnFamilyMetaData cf_meta;
|
|
2575
|
+
db_->GetColumnFamilyMetaData(&cf_meta);
|
|
2576
|
+
|
|
2577
|
+
for (const auto& file : cf_meta.levels[0].files) {
|
|
2578
|
+
input_files.push_back(file.name);
|
|
2579
|
+
}
|
|
2580
|
+
|
|
2581
|
+
ASSERT_EQ(input_files.size(), 2);
|
|
2582
|
+
|
|
2583
|
+
ResetStatistics();
|
|
2584
|
+
|
|
2585
|
+
CompactionOptions compact_options;
|
|
2586
|
+
ASSERT_OK(
|
|
2587
|
+
db_->CompactFiles(compact_options, input_files, 1 /* output_level*/));
|
|
2588
|
+
|
|
2589
|
+
ASSERT_EQ(Get("a"), "val1");
|
|
2590
|
+
ASSERT_EQ(Get("b"), "val2");
|
|
2591
|
+
ASSERT_EQ(Get("c"), "NOT_FOUND");
|
|
2592
|
+
ASSERT_EQ(Get("d"), "val4");
|
|
2593
|
+
|
|
2594
|
+
VerifyResumeBytes();
|
|
2595
|
+
}
|
|
2596
|
+
|
|
2597
|
+
// Cancel compaction right before processing key "c" to test resumption at a
|
|
2598
|
+
// deletion at the ottom level. When resumed, compaction will continue from
|
|
2599
|
+
// the last saved progress point before the delete.
|
|
2600
|
+
TEST_F(ResumableCompactionKeyTypeTest, CancelAndResumeWithDeleteAtBottomLevel) {
|
|
2601
|
+
Options options = CurrentOptions();
|
|
2602
|
+
|
|
2603
|
+
SetupResumableCompactionService(options, "c");
|
|
2604
|
+
|
|
2605
|
+
ASSERT_OK(Put("c", "old_value"));
|
|
2606
|
+
const Snapshot* snapshot = db_->GetSnapshot();
|
|
2607
|
+
ASSERT_OK(Delete("c"));
|
|
2608
|
+
ASSERT_OK(Flush());
|
|
2609
|
+
MoveFilesToLevel(options.num_levels - 1);
|
|
2610
|
+
|
|
2611
|
+
ASSERT_OK(Put("a", "val1"));
|
|
2612
|
+
ASSERT_OK(Put("b", "val2"));
|
|
2613
|
+
ASSERT_OK(Put("d", "val4"));
|
|
2614
|
+
ASSERT_OK(Flush());
|
|
2615
|
+
|
|
2616
|
+
ResetStatistics();
|
|
2617
|
+
|
|
2618
|
+
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
2619
|
+
|
|
2620
|
+
ASSERT_EQ(Get("a"), "val1");
|
|
2621
|
+
ASSERT_EQ(Get("b"), "val2");
|
|
2622
|
+
ASSERT_EQ(Get("c"), "NOT_FOUND");
|
|
2623
|
+
ASSERT_EQ(Get("c", snapshot), "old_value");
|
|
2624
|
+
ASSERT_EQ(Get("d"), "val4");
|
|
2625
|
+
db_->ReleaseSnapshot(snapshot);
|
|
2626
|
+
|
|
2627
|
+
VerifyResumeBytes();
|
|
2628
|
+
}
|
|
2629
|
+
|
|
2630
|
+
// Cancel compaction right before processing key "c" to test resumption at a
|
|
2631
|
+
// merge operand. When resumed, compaction will continue from the last saved
|
|
2632
|
+
// progress point before the merge operand.
|
|
2633
|
+
TEST_F(ResumableCompactionKeyTypeTest, CancelAndResumeWithMerge) {
|
|
2634
|
+
Options options = CurrentOptions();
|
|
2635
|
+
options.merge_operator = MergeOperators::CreateStringAppendOperator();
|
|
2636
|
+
|
|
2637
|
+
SetupResumableCompactionService(options, "c");
|
|
2638
|
+
|
|
2639
|
+
ASSERT_OK(Put("c", "old_value"));
|
|
2640
|
+
ASSERT_OK(Put("c_placeholder", "placeholder"));
|
|
2641
|
+
ASSERT_OK(Flush());
|
|
2642
|
+
MoveFilesToLevel(options.num_levels - 1);
|
|
2643
|
+
|
|
2644
|
+
ASSERT_OK(Put("a", "val1"));
|
|
2645
|
+
ASSERT_OK(Put("b", "val2"));
|
|
2646
|
+
ASSERT_OK(Put("d", "val4"));
|
|
2647
|
+
ASSERT_OK(Flush());
|
|
2648
|
+
|
|
2649
|
+
ASSERT_OK(Merge("c", "new_value"));
|
|
2650
|
+
ASSERT_OK(Flush());
|
|
2651
|
+
|
|
2652
|
+
std::vector<std::string> input_files;
|
|
2653
|
+
ColumnFamilyMetaData cf_meta;
|
|
2654
|
+
db_->GetColumnFamilyMetaData(&cf_meta);
|
|
2655
|
+
|
|
2656
|
+
for (const auto& file : cf_meta.levels[0].files) {
|
|
2657
|
+
input_files.push_back(file.name);
|
|
2658
|
+
}
|
|
2659
|
+
|
|
2660
|
+
ASSERT_EQ(input_files.size(), 2);
|
|
2661
|
+
|
|
2662
|
+
ResetStatistics();
|
|
2663
|
+
|
|
2664
|
+
CompactionOptions compact_options;
|
|
2665
|
+
ASSERT_OK(
|
|
2666
|
+
db_->CompactFiles(compact_options, input_files, 1 /* output_level*/));
|
|
2667
|
+
|
|
2668
|
+
ASSERT_EQ(Get("a"), "val1");
|
|
2669
|
+
ASSERT_EQ(Get("b"), "val2");
|
|
2670
|
+
ASSERT_EQ(Get("c"), "old_value,new_value");
|
|
2671
|
+
ASSERT_EQ(Get("d"), "val4");
|
|
2672
|
+
|
|
2673
|
+
VerifyResumeBytes();
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
// Cancel compaction right before processing key "c" to test resumption at a
|
|
2677
|
+
// single delete. When resumed, compaction will continue from the last saved
|
|
2678
|
+
// progress point before the single delete.
|
|
2679
|
+
TEST_F(ResumableCompactionKeyTypeTest, CancelAndResumeWithSingleDelete) {
|
|
2680
|
+
Options options = CurrentOptions();
|
|
2681
|
+
|
|
2682
|
+
SetupResumableCompactionService(options, "c");
|
|
2683
|
+
|
|
2684
|
+
ASSERT_OK(Put("c", "old_value"));
|
|
2685
|
+
ASSERT_OK(Put("c_placeholder", "placeholder"));
|
|
2686
|
+
ASSERT_OK(Flush());
|
|
2687
|
+
MoveFilesToLevel(options.num_levels - 1);
|
|
2688
|
+
|
|
2689
|
+
ASSERT_OK(Put("a", "val1"));
|
|
2690
|
+
ASSERT_OK(Put("b", "val2"));
|
|
2691
|
+
ASSERT_OK(Put("d", "val4"));
|
|
2692
|
+
ASSERT_OK(Flush());
|
|
2693
|
+
|
|
2694
|
+
ASSERT_OK(SingleDelete("c"));
|
|
2695
|
+
ASSERT_OK(Flush());
|
|
2696
|
+
|
|
2697
|
+
std::vector<std::string> input_files;
|
|
2698
|
+
ColumnFamilyMetaData cf_meta;
|
|
2699
|
+
db_->GetColumnFamilyMetaData(&cf_meta);
|
|
2700
|
+
|
|
2701
|
+
for (const auto& file : cf_meta.levels[0].files) {
|
|
2702
|
+
input_files.push_back(file.name);
|
|
2703
|
+
}
|
|
2704
|
+
|
|
2705
|
+
ASSERT_EQ(input_files.size(), 2);
|
|
2706
|
+
|
|
2707
|
+
ResetStatistics();
|
|
2708
|
+
|
|
2709
|
+
CompactionOptions compact_options;
|
|
2710
|
+
ASSERT_OK(
|
|
2711
|
+
db_->CompactFiles(compact_options, input_files, 1 /* output_level*/));
|
|
2712
|
+
|
|
2713
|
+
ASSERT_EQ(Get("a"), "val1");
|
|
2714
|
+
ASSERT_EQ(Get("b"), "val2");
|
|
2715
|
+
ASSERT_EQ(Get("c"), "NOT_FOUND");
|
|
2716
|
+
ASSERT_EQ(Get("d"), "val4");
|
|
2717
|
+
|
|
2718
|
+
VerifyResumeBytes();
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2721
|
+
// Cancel compaction right before processing key "c" to test resumption at a
|
|
2722
|
+
// range delete. When resumed, compaction will continue from the last saved
|
|
2723
|
+
// progress point before the range delete.
|
|
2724
|
+
TEST_F(ResumableCompactionKeyTypeTest, CancelAndResumeWithRangeDelete) {
|
|
2725
|
+
Options options = CurrentOptions();
|
|
2726
|
+
|
|
2727
|
+
SetupResumableCompactionService(options, "c");
|
|
2728
|
+
|
|
2729
|
+
ASSERT_OK(Put("c", "old_value"));
|
|
2730
|
+
ASSERT_OK(Put("c_placeholder", "placeholder"));
|
|
2731
|
+
ASSERT_OK(Flush());
|
|
2732
|
+
MoveFilesToLevel(options.num_levels - 1);
|
|
2733
|
+
|
|
2734
|
+
ASSERT_OK(Put("a", "val1"));
|
|
2735
|
+
ASSERT_OK(Put("b", "val2"));
|
|
2736
|
+
ASSERT_OK(Put("d", "val4"));
|
|
2737
|
+
ASSERT_OK(Flush());
|
|
2738
|
+
|
|
2739
|
+
ASSERT_OK(
|
|
2740
|
+
db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "c", "c_"));
|
|
2741
|
+
ASSERT_OK(Flush());
|
|
2742
|
+
|
|
2743
|
+
std::vector<std::string> input_files;
|
|
2744
|
+
ColumnFamilyMetaData cf_meta;
|
|
2745
|
+
db_->GetColumnFamilyMetaData(&cf_meta);
|
|
2746
|
+
|
|
2747
|
+
for (const auto& file : cf_meta.levels[0].files) {
|
|
2748
|
+
input_files.push_back(file.name);
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
ASSERT_EQ(input_files.size(), 2);
|
|
2752
|
+
|
|
2753
|
+
ResetStatistics();
|
|
2754
|
+
|
|
2755
|
+
CompactionOptions compact_options;
|
|
2756
|
+
ASSERT_OK(
|
|
2757
|
+
db_->CompactFiles(compact_options, input_files, 1 /* output_level*/));
|
|
2758
|
+
|
|
2759
|
+
ASSERT_EQ(Get("a"), "val1");
|
|
2760
|
+
ASSERT_EQ(Get("b"), "val2");
|
|
2761
|
+
ASSERT_EQ(Get("c"), "NOT_FOUND");
|
|
2762
|
+
ASSERT_EQ(Get("d"), "val4");
|
|
2763
|
+
|
|
2764
|
+
VerifyResumeBytes();
|
|
2765
|
+
}
|
|
2766
|
+
|
|
2767
|
+
// Test resumption when a key has multiple versions spanning across file
|
|
2768
|
+
// boundaries (i.e., the same key exists in multiple SST files).
|
|
2769
|
+
//
|
|
2770
|
+
// Scenario:
|
|
2771
|
+
// File 1 largest key: key "b"
|
|
2772
|
+
// File 2 smallest key: key "c" with seqno=4 (older version)
|
|
2773
|
+
// File 3 largest key: key "c" with seqno=5 (newer version)
|
|
2774
|
+
//
|
|
2775
|
+
// Cancel compaction right before processing the older version of key "c".
|
|
2776
|
+
// Upon resumption, compaction continues from the saved progress point "b" and
|
|
2777
|
+
// correctly processes both versions
|
|
2778
|
+
TEST_F(ResumableCompactionKeyTypeTest,
|
|
2779
|
+
CancelAndResumeWithKeySpanningFileBoundaries) {
|
|
2780
|
+
Options options = CurrentOptions();
|
|
2781
|
+
|
|
2782
|
+
// Set up cancellation at the older version of the key which will have
|
|
2783
|
+
// sequence number zero-ed out
|
|
2784
|
+
SetupResumableCompactionService(options, "c" /*cancel_at_key*/, 0 /*seqno*/);
|
|
2785
|
+
|
|
2786
|
+
ASSERT_OK(Put("a", "val1"));
|
|
2787
|
+
ASSERT_OK(Put("b", "val2"));
|
|
2788
|
+
ASSERT_OK(Put("d", "val4"));
|
|
2789
|
+
ASSERT_OK(Flush());
|
|
2790
|
+
|
|
2791
|
+
ASSERT_OK(Put("c", "old_value"));
|
|
2792
|
+
const Snapshot* snapshot = db_->GetSnapshot();
|
|
2793
|
+
ASSERT_OK(Put("c", "new_value"));
|
|
2794
|
+
ASSERT_OK(Flush());
|
|
2795
|
+
|
|
2796
|
+
ResetStatistics();
|
|
2797
|
+
|
|
2798
|
+
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
2799
|
+
|
|
2800
|
+
ASSERT_EQ(Get("a"), "val1");
|
|
2801
|
+
ASSERT_EQ(Get("b"), "val2");
|
|
2802
|
+
ASSERT_EQ(Get("c"), "new_value");
|
|
2803
|
+
ASSERT_EQ(Get("c", snapshot), "old_value");
|
|
2804
|
+
ASSERT_EQ(Get("d"), "val4");
|
|
2805
|
+
db_->ReleaseSnapshot(snapshot);
|
|
2806
|
+
|
|
2807
|
+
VerifyResumeBytes();
|
|
2808
|
+
}
|
|
2809
|
+
|
|
2810
|
+
// Cancel compaction right before processing key "c" to test resumption at a
|
|
2811
|
+
// wide column. When resumed, compaction will continue
|
|
2812
|
+
// from the wide column.
|
|
2813
|
+
TEST_F(ResumableCompactionKeyTypeTest, CancelAndResumeWithWideColumn) {
|
|
2814
|
+
Options options = CurrentOptions();
|
|
2815
|
+
|
|
2816
|
+
SetupResumableCompactionService(options, "c" /*cancel_at_key*/);
|
|
2817
|
+
|
|
2818
|
+
ASSERT_OK(Put("a", "val1"));
|
|
2819
|
+
ASSERT_OK(Put("b", "val2"));
|
|
2820
|
+
ASSERT_OK(Put("d", "val4"));
|
|
2821
|
+
ASSERT_OK(Flush());
|
|
2822
|
+
|
|
2823
|
+
WideColumns columns{{"col1", "value1"}, {"col2", "value2"}};
|
|
2824
|
+
ASSERT_OK(
|
|
2825
|
+
db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), "c", columns));
|
|
2826
|
+
ASSERT_OK(Flush());
|
|
2827
|
+
|
|
2828
|
+
ResetStatistics();
|
|
2829
|
+
|
|
2830
|
+
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
2831
|
+
|
|
2832
|
+
ASSERT_EQ(Get("a"), "val1");
|
|
2833
|
+
ASSERT_EQ(Get("b"), "val2");
|
|
2834
|
+
|
|
2835
|
+
PinnableWideColumns result;
|
|
2836
|
+
ASSERT_OK(
|
|
2837
|
+
db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), "c", &result));
|
|
2838
|
+
WideColumns expected{{"col1", "value1"}, {"col2", "value2"}};
|
|
2839
|
+
ASSERT_EQ(result.columns(), expected);
|
|
2840
|
+
|
|
2841
|
+
ASSERT_EQ(Get("d"), "val4");
|
|
2842
|
+
|
|
2843
|
+
VerifyResumeBytes();
|
|
2844
|
+
}
|
|
2845
|
+
|
|
2846
|
+
// Cancel compaction right before processing key "c" to test resumption at a
|
|
2847
|
+
// timed put. When resumed, compaction will continue
|
|
2848
|
+
// from the timed put.
|
|
2849
|
+
TEST_F(ResumableCompactionKeyTypeTest, CancelAndResumeWithTimedPut) {
|
|
2850
|
+
Options options = CurrentOptions();
|
|
2851
|
+
options.preclude_last_level_data_seconds = 86400; // Enable TimedPut feature
|
|
2852
|
+
options.preserve_internal_time_seconds = 86400; // Preserve write time
|
|
2853
|
+
|
|
2854
|
+
SetupResumableCompactionService(options, "c" /*cancel_at_key*/);
|
|
2855
|
+
|
|
2856
|
+
ASSERT_OK(Put("c", "old_value"));
|
|
2857
|
+
ASSERT_OK(Put("c_placeholder", "placeholder"));
|
|
2858
|
+
ASSERT_OK(Flush());
|
|
2859
|
+
MoveFilesToLevel(options.num_levels - 1);
|
|
2860
|
+
|
|
2861
|
+
ASSERT_OK(Put("a", "val1"));
|
|
2862
|
+
ASSERT_OK(Put("b", "val2"));
|
|
2863
|
+
ASSERT_OK(Put("d", "val4"));
|
|
2864
|
+
ASSERT_OK(Flush());
|
|
2865
|
+
|
|
2866
|
+
// Use TimedPut for key "c" with current write time
|
|
2867
|
+
uint64_t write_time = env_->NowMicros() / 1000000;
|
|
2868
|
+
ASSERT_OK(TimedPut("c", "val3", write_time /*write_unix_time*/));
|
|
2869
|
+
ASSERT_OK(Put("d", "val4"));
|
|
2870
|
+
ASSERT_OK(Flush());
|
|
2871
|
+
|
|
2872
|
+
std::vector<std::string> input_files;
|
|
2873
|
+
ColumnFamilyMetaData cf_meta;
|
|
2874
|
+
db_->GetColumnFamilyMetaData(&cf_meta);
|
|
2875
|
+
|
|
2876
|
+
for (const auto& file : cf_meta.levels[0].files) {
|
|
2877
|
+
input_files.push_back(file.name);
|
|
2878
|
+
}
|
|
2879
|
+
|
|
2880
|
+
ASSERT_EQ(input_files.size(), 2);
|
|
2881
|
+
|
|
2882
|
+
ResetStatistics();
|
|
2883
|
+
|
|
2884
|
+
CompactionOptions compact_options;
|
|
2885
|
+
ASSERT_OK(
|
|
2886
|
+
db_->CompactFiles(compact_options, input_files, 1 /* output_level*/));
|
|
2887
|
+
|
|
2888
|
+
ASSERT_EQ(Get("a"), "val1");
|
|
2889
|
+
ASSERT_EQ(Get("b"), "val2");
|
|
2890
|
+
ASSERT_EQ(Get("c"), "val3");
|
|
2891
|
+
ASSERT_EQ(Get("d"), "val4");
|
|
2892
|
+
|
|
2893
|
+
VerifyResumeBytes();
|
|
2894
|
+
}
|
|
2435
2895
|
} // namespace ROCKSDB_NAMESPACE
|
|
2436
2896
|
|
|
2437
2897
|
int main(int argc, char** argv) {
|
|
@@ -38,11 +38,11 @@ OutputIterator SubcompactionState::GetOutputs() const {
|
|
|
38
38
|
compaction_outputs_.outputs_);
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
void SubcompactionState::Cleanup(Cache* cache) {
|
|
41
|
+
void SubcompactionState::Cleanup(Cache* cache, const Status& overall_status) {
|
|
42
42
|
proximal_level_outputs_.Cleanup();
|
|
43
43
|
compaction_outputs_.Cleanup();
|
|
44
44
|
|
|
45
|
-
if (!status.ok()) {
|
|
45
|
+
if (!status.ok() || !overall_status.ok()) {
|
|
46
46
|
for (const auto& out : GetOutputs()) {
|
|
47
47
|
// If this file was inserted into the table cache then remove it here
|
|
48
48
|
// because this compaction was not committed. This is not strictly
|