@nxtedition/rocksdb 15.4.0 → 15.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +24 -19
- package/cache.js +1 -1
- package/chained-batch.js +12 -3
- package/deps/rocksdb/rocksdb/.clang-tidy +86 -0
- package/deps/rocksdb/rocksdb/BUCK +42 -0
- package/deps/rocksdb/rocksdb/CMakeLists.txt +11 -0
- package/deps/rocksdb/rocksdb/Makefile +59 -32
- package/deps/rocksdb/rocksdb/cache/cache.cc +0 -5
- package/deps/rocksdb/rocksdb/cache/cache_entry_stats.h +9 -9
- package/deps/rocksdb/rocksdb/cache/cache_key.cc +3 -3
- package/deps/rocksdb/rocksdb/cache/cache_key.h +5 -5
- package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.h +16 -16
- package/deps/rocksdb/rocksdb/cache/cache_test.cc +1 -1
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +258 -294
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +98 -49
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +1 -5
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +2 -3
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +18 -18
- package/deps/rocksdb/rocksdb/crash_test.mk +5 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +23 -22
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.h +6 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +14 -16
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +38 -26
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.h +5 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +101 -18
- package/deps/rocksdb/rocksdb/db/blob/blob_index.h +12 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +6 -9
- package/deps/rocksdb/rocksdb/db/builder.cc +23 -0
- package/deps/rocksdb/rocksdb/db/builder.h +7 -0
- package/deps/rocksdb/rocksdb/db/c.cc +373 -57
- package/deps/rocksdb/rocksdb/db/c_test.c +101 -1
- package/deps/rocksdb/rocksdb/db/column_family.cc +31 -3
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +10 -13
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +35 -48
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +13 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +201 -39
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +15 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +7 -7
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +2 -455
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +4 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +19 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +72 -9
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +12 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +405 -83
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +25 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +23 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +1 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +1410 -106
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +12 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +19 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +505 -45
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +2 -2
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +9 -1
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +7 -9
- package/deps/rocksdb/rocksdb/db/convenience.cc +4 -4
- package/deps/rocksdb/rocksdb/db/convenience_impl.h +2 -1
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +60 -88
- package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +10 -12
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +471 -40
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +116 -2
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +5 -15
- package/deps/rocksdb/rocksdb/db/db_compaction_abort_test.cc +993 -0
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +329 -29
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +155 -13
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +54 -31
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +1 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +232 -70
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +57 -9
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +224 -31
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +5 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +4 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +164 -8
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +6 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +5 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +47 -35
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +22 -9
- package/deps/rocksdb/rocksdb/db/db_iter.cc +9 -0
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +371 -6
- package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +7 -5
- package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +22 -23
- package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +0 -2
- package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_options_test.cc +40 -0
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +32 -13
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_readonly_with_timestamp_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +68 -15
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +2 -3
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +6 -21
- package/deps/rocksdb/rocksdb/db/db_test.cc +644 -128
- package/deps/rocksdb/rocksdb/db/db_test2.cc +198 -81
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +35 -10
- package/deps/rocksdb/rocksdb/db/db_test_util.h +8 -2
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +36 -32
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +11 -7
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +499 -0
- package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +284 -20
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/dbformat.h +0 -5
- package/deps/rocksdb/rocksdb/db/error_handler.cc +24 -0
- package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +12 -14
- package/deps/rocksdb/rocksdb/db/experimental.cc +13 -10
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +22 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +21 -15
- package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +4 -6
- package/deps/rocksdb/rocksdb/db/flush_job.cc +11 -3
- package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +5 -6
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +4 -2
- package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +17 -17
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +13 -0
- package/deps/rocksdb/rocksdb/db/internal_stats.h +2 -0
- package/deps/rocksdb/rocksdb/db/listener_test.cc +154 -27
- package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/memtable.cc +197 -51
- package/deps/rocksdb/rocksdb/db/memtable.h +6 -0
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/merge_test.cc +37 -35
- package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/options_file_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +9 -11
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +10 -1
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +292 -15
- package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +10 -17
- package/deps/rocksdb/rocksdb/db/prefix_test.cc +6 -8
- package/deps/rocksdb/rocksdb/db/repair.cc +10 -10
- package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +5 -5
- package/deps/rocksdb/rocksdb/db/table_cache.cc +142 -135
- package/deps/rocksdb/rocksdb/db/table_cache.h +30 -6
- package/deps/rocksdb/rocksdb/db/table_cache_sync_and_async.h +7 -7
- package/deps/rocksdb/rocksdb/db/version_builder.cc +11 -50
- package/deps/rocksdb/rocksdb/db/version_builder.h +2 -1
- package/deps/rocksdb/rocksdb/db/version_builder_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/version_edit.cc +51 -2
- package/deps/rocksdb/rocksdb/db/version_edit.h +91 -29
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -7
- package/deps/rocksdb/rocksdb/db/version_set.cc +211 -50
- package/deps/rocksdb/rocksdb/db/version_set.h +40 -3
- package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +5 -0
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +294 -21
- package/deps/rocksdb/rocksdb/db/version_util.cc +96 -0
- package/deps/rocksdb/rocksdb/db/version_util.h +24 -0
- package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +5 -5
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.cc +647 -31
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.h +219 -1
- package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization_test.cc +549 -12
- package/deps/rocksdb/rocksdb/db/write_callback_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +1 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +19 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +21 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +32 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +74 -22
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +9 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +143 -61
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +15 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +76 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +92 -72
- package/deps/rocksdb/rocksdb/env/env.cc +1 -0
- package/deps/rocksdb/rocksdb/env/env_test.cc +365 -2
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +31 -30
- package/deps/rocksdb/rocksdb/env/io_posix.cc +8 -11
- package/deps/rocksdb/rocksdb/env/io_posix.h +30 -1
- package/deps/rocksdb/rocksdb/env/io_posix_test.cc +43 -0
- package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +1 -1
- package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +108 -0
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +32 -4
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +4 -4
- package/deps/rocksdb/rocksdb/file/file_util.cc +8 -2
- package/deps/rocksdb/rocksdb/file/file_util.h +2 -1
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +331 -12
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +52 -35
- package/deps/rocksdb/rocksdb/folly.mk +22 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_cache.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +100 -54
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +67 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +149 -13
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +1 -12
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +78 -97
- package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +3 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/external_table.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +17 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/functor_wrapper.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/io_dispatcher.h +358 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +43 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +20 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +63 -21
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +10 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +2 -7
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +3 -14
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +49 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/status.h +8 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +77 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +15 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/tool_hooks.h +16 -10
- package/deps/rocksdb/rocksdb/include/rocksdb/unique_id.h +5 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +2 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/user_defined_index.h +106 -46
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/db_ttl.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +14 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/memory_util.h +5 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +7 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +1 -2
- package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc +2 -2
- package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +226 -8
- package/deps/rocksdb/rocksdb/memtable/inlineskiplist_test.cc +490 -0
- package/deps/rocksdb/rocksdb/memtable/skiplist.h +3 -3
- package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +11 -0
- package/deps/rocksdb/rocksdb/microbench/db_basic_bench.cc +4 -12
- package/deps/rocksdb/rocksdb/microbench/ribbon_bench.cc +5 -5
- package/deps/rocksdb/rocksdb/monitoring/file_read_sample.h +21 -4
- package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +9 -3
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +21 -2
- package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +2 -2
- package/deps/rocksdb/rocksdb/options/cf_options.cc +21 -1
- package/deps/rocksdb/rocksdb/options/cf_options.h +2 -0
- package/deps/rocksdb/rocksdb/options/customizable_test.cc +0 -2
- package/deps/rocksdb/rocksdb/options/db_options.cc +26 -5
- package/deps/rocksdb/rocksdb/options/db_options.h +3 -1
- package/deps/rocksdb/rocksdb/options/options.cc +5 -1
- package/deps/rocksdb/rocksdb/options/options_helper.cc +7 -2
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +109 -103
- package/deps/rocksdb/rocksdb/options/options_test.cc +14 -0
- package/deps/rocksdb/rocksdb/port/jemalloc_helper.h +15 -17
- package/deps/rocksdb/rocksdb/port/lang.h +4 -0
- package/deps/rocksdb/rocksdb/port/port_example.h +0 -23
- package/deps/rocksdb/rocksdb/port/stack_trace.cc +36 -0
- package/deps/rocksdb/rocksdb/port/stack_trace.h +9 -0
- package/deps/rocksdb/rocksdb/src.mk +12 -0
- package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +1 -2
- package/deps/rocksdb/rocksdb/table/block_based/binary_search_index_reader.cc +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/block.cc +571 -292
- package/deps/rocksdb/rocksdb/table/block_based/block.h +143 -53
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +154 -90
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +5 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +51 -14
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h +0 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +147 -734
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +30 -233
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +178 -108
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +13 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +17 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +5 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +70 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_builder.cc +168 -24
- package/deps/rocksdb/rocksdb/table/block_based/block_builder.h +25 -9
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +7 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +9 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +548 -169
- package/deps/rocksdb/rocksdb/table/block_based/block_type.h +30 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_util.h +156 -0
- package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.cc +73 -30
- package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.h +74 -7
- package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +20 -14
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +22 -12
- package/deps/rocksdb/rocksdb/table/block_based/mock_block_based_table.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.cc +332 -0
- package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.h +133 -0
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +4 -2
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/reader_common.cc +3 -2
- package/deps/rocksdb/rocksdb/table/block_based/reader_common.h +4 -1
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +0 -1
- package/deps/rocksdb/rocksdb/table/block_based/user_defined_index_wrapper.h +126 -46
- package/deps/rocksdb/rocksdb/table/block_fetcher.cc +31 -3
- package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +1 -2
- package/deps/rocksdb/rocksdb/table/cleanable_test.cc +3 -1
- package/deps/rocksdb/rocksdb/table/external_table.cc +25 -4
- package/deps/rocksdb/rocksdb/table/format.cc +27 -15
- package/deps/rocksdb/rocksdb/table/format.h +41 -15
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +1 -0
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +22 -12
- package/deps/rocksdb/rocksdb/table/meta_blocks.h +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +7 -21
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +88 -13
- package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +53 -42
- package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +3 -12
- package/deps/rocksdb/rocksdb/table/table_builder.h +0 -4
- package/deps/rocksdb/rocksdb/table/table_properties.cc +18 -0
- package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +2 -3
- package/deps/rocksdb/rocksdb/table/table_test.cc +848 -172
- package/deps/rocksdb/rocksdb/table/unique_id.cc +24 -20
- package/deps/rocksdb/rocksdb/table/unique_id_impl.h +8 -8
- package/deps/rocksdb/rocksdb/test_util/sync_point.h +5 -4
- package/deps/rocksdb/rocksdb/test_util/testutil.cc +2 -1
- package/deps/rocksdb/rocksdb/test_util/testutil.h +2 -2
- package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +2 -1
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +238 -120
- package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +2 -2
- package/deps/rocksdb/rocksdb/tools/db_sanity_test.cc +2 -4
- package/deps/rocksdb/rocksdb/tools/dump/db_dump_tool.cc +4 -8
- package/deps/rocksdb/rocksdb/tools/dump/rocksdb_undump.cc +1 -1
- package/deps/rocksdb/rocksdb/tools/io_tracer_parser_test.cc +2 -3
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +82 -20
- package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +41 -47
- package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +9 -0
- package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +5 -6
- package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +1 -1
- package/deps/rocksdb/rocksdb/tools/tool_hooks.cc +6 -5
- package/deps/rocksdb/rocksdb/tools/trace_analyzer_test.cc +4 -4
- package/deps/rocksdb/rocksdb/tools/write_stress.cc +1 -3
- package/deps/rocksdb/rocksdb/util/atomic.h +30 -23
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.cc +6 -7
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.h +3 -3
- package/deps/rocksdb/rocksdb/util/bit_fields.h +68 -46
- package/deps/rocksdb/rocksdb/util/bloom_impl.h +16 -16
- package/deps/rocksdb/rocksdb/util/coding.h +14 -27
- package/deps/rocksdb/rocksdb/util/compression.cc +365 -207
- package/deps/rocksdb/rocksdb/util/compression.h +16 -1298
- package/deps/rocksdb/rocksdb/util/compression_test.cc +347 -61
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +8 -9
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.h +1 -1
- package/deps/rocksdb/rocksdb/util/crc32c_ppc.h +1 -1
- package/deps/rocksdb/rocksdb/util/dynamic_bloom_test.cc +3 -3
- package/deps/rocksdb/rocksdb/util/filter_bench.cc +18 -18
- package/deps/rocksdb/rocksdb/util/gflags_compat.h +3 -3
- package/deps/rocksdb/rocksdb/util/hash_test.cc +19 -7
- package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.cc +1099 -0
- package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.h +36 -0
- package/deps/rocksdb/rocksdb/util/io_dispatcher_test.cc +1919 -0
- package/deps/rocksdb/rocksdb/util/math.h +3 -1
- package/deps/rocksdb/rocksdb/util/mutexlock.h +19 -19
- package/deps/rocksdb/rocksdb/util/ribbon_alg.h +25 -25
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +5 -7
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +4 -5
- package/deps/rocksdb/rocksdb/util/slice.cc +0 -10
- package/deps/rocksdb/rocksdb/util/slice_test.cc +35 -1
- package/deps/rocksdb/rocksdb/util/slice_transform_test.cc +5 -7
- package/deps/rocksdb/rocksdb/util/status.cc +3 -1
- package/deps/rocksdb/rocksdb/util/stop_watch.h +2 -0
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +4 -1
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +123 -78
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +12 -93
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.h +1 -4
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.cc +0 -21
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +6 -48
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +94 -307
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +12 -58
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +2 -8
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +2 -3
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +205 -811
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +18 -9
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.cc +2 -7
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.h +1 -9
- package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_functional_test.cc +17 -11
- package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +68 -61
- package/deps/rocksdb/rocksdb/utilities/debug.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +105 -59
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +274 -7
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs_test.cc +94 -0
- package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +13 -17
- package/deps/rocksdb/rocksdb/utilities/memory/memory_util.cc +16 -3
- package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc +25 -25
- package/deps/rocksdb/rocksdb/utilities/object_registry.cc +40 -40
- package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc +2 -5
- package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +17 -19
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.h +2 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.h +4 -13
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +3 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +6 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_seqno_test.cc +431 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +1 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.h +91 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.cc +562 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.h +615 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.cc +2575 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.h +685 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_db_test.cc +2843 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.cc +567 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.h +275 -0
- package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_test.cc +5183 -0
- package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.cc +4 -3
- package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +3 -3
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +93 -88
- package/deps/rocksdb/rocksdb.gyp +7 -0
- package/index.js +11 -2
- package/iterator.js +15 -7
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
- package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_custom_library.h +0 -43
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_util.h +0 -55
|
@@ -84,6 +84,7 @@ class BlobDBTest : public testing::Test {
|
|
|
84
84
|
options.stats_dump_period_sec = 0;
|
|
85
85
|
options.stats_persist_period_sec = 0;
|
|
86
86
|
}
|
|
87
|
+
bdb_options_ = bdb_options;
|
|
87
88
|
return BlobDB::Open(options, bdb_options, dbname_, &blob_db_);
|
|
88
89
|
}
|
|
89
90
|
|
|
@@ -109,17 +110,16 @@ class BlobDBTest : public testing::Test {
|
|
|
109
110
|
void Destroy() {
|
|
110
111
|
if (blob_db_) {
|
|
111
112
|
Options options = blob_db_->GetOptions();
|
|
112
|
-
BlobDBOptions bdb_options = blob_db_->GetBlobDBOptions();
|
|
113
113
|
delete blob_db_;
|
|
114
114
|
blob_db_ = nullptr;
|
|
115
|
-
ASSERT_OK(DestroyBlobDB(dbname_, options,
|
|
115
|
+
ASSERT_OK(DestroyBlobDB(dbname_, options, bdb_options_));
|
|
116
116
|
}
|
|
117
117
|
}
|
|
118
118
|
|
|
119
|
-
BlobDBImpl
|
|
119
|
+
BlobDBImpl* blob_db_impl() { return static_cast<BlobDBImpl*>(blob_db_); }
|
|
120
120
|
|
|
121
|
-
Status Put(const Slice
|
|
122
|
-
std::map<std::string, std::string
|
|
121
|
+
Status Put(const Slice& key, const Slice& value,
|
|
122
|
+
std::map<std::string, std::string>* data = nullptr) {
|
|
123
123
|
Status s = blob_db_->Put(WriteOptions(), key, value);
|
|
124
124
|
if (data != nullptr) {
|
|
125
125
|
(*data)[key.ToString()] = value.ToString();
|
|
@@ -127,16 +127,16 @@ class BlobDBTest : public testing::Test {
|
|
|
127
127
|
return s;
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
-
void Delete(const std::string
|
|
131
|
-
std::map<std::string, std::string
|
|
130
|
+
void Delete(const std::string& key,
|
|
131
|
+
std::map<std::string, std::string>* data = nullptr) {
|
|
132
132
|
ASSERT_OK(blob_db_->Delete(WriteOptions(), key));
|
|
133
133
|
if (data != nullptr) {
|
|
134
134
|
data->erase(key);
|
|
135
135
|
}
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
-
Status PutWithTTL(const Slice
|
|
139
|
-
std::map<std::string, std::string
|
|
138
|
+
Status PutWithTTL(const Slice& key, const Slice& value, uint64_t ttl,
|
|
139
|
+
std::map<std::string, std::string>* data = nullptr) {
|
|
140
140
|
Status s = blob_db_->PutWithTTL(WriteOptions(), key, value, ttl);
|
|
141
141
|
if (data != nullptr) {
|
|
142
142
|
(*data)[key.ToString()] = value.ToString();
|
|
@@ -144,12 +144,8 @@ class BlobDBTest : public testing::Test {
|
|
|
144
144
|
return s;
|
|
145
145
|
}
|
|
146
146
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
void PutRandomWithTTL(const std::string &key, uint64_t ttl, Random *rnd,
|
|
152
|
-
std::map<std::string, std::string> *data = nullptr) {
|
|
147
|
+
void PutRandomWithTTL(const std::string& key, uint64_t ttl, Random* rnd,
|
|
148
|
+
std::map<std::string, std::string>* data = nullptr) {
|
|
153
149
|
int len = rnd->Next() % kMaxBlobSize + 1;
|
|
154
150
|
std::string value = rnd->HumanReadableString(len);
|
|
155
151
|
ASSERT_OK(
|
|
@@ -159,24 +155,13 @@ class BlobDBTest : public testing::Test {
|
|
|
159
155
|
}
|
|
160
156
|
}
|
|
161
157
|
|
|
162
|
-
void
|
|
163
|
-
|
|
164
|
-
int len = rnd->Next() % kMaxBlobSize + 1;
|
|
165
|
-
std::string value = rnd->HumanReadableString(len);
|
|
166
|
-
ASSERT_OK(blob_db_->PutUntil(WriteOptions(), Slice(key), Slice(value),
|
|
167
|
-
expiration));
|
|
168
|
-
if (data != nullptr) {
|
|
169
|
-
(*data)[key] = value;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
void PutRandom(const std::string &key, Random *rnd,
|
|
174
|
-
std::map<std::string, std::string> *data = nullptr) {
|
|
158
|
+
void PutRandom(const std::string& key, Random* rnd,
|
|
159
|
+
std::map<std::string, std::string>* data = nullptr) {
|
|
175
160
|
PutRandom(blob_db_, key, rnd, data);
|
|
176
161
|
}
|
|
177
162
|
|
|
178
|
-
void PutRandom(DB
|
|
179
|
-
std::map<std::string, std::string
|
|
163
|
+
void PutRandom(DB* db, const std::string& key, Random* rnd,
|
|
164
|
+
std::map<std::string, std::string>* data = nullptr) {
|
|
180
165
|
int len = rnd->Next() % kMaxBlobSize + 1;
|
|
181
166
|
std::string value = rnd->HumanReadableString(len);
|
|
182
167
|
ASSERT_OK(db->Put(WriteOptions(), Slice(key), Slice(value)));
|
|
@@ -186,8 +171,8 @@ class BlobDBTest : public testing::Test {
|
|
|
186
171
|
}
|
|
187
172
|
|
|
188
173
|
void PutRandomToWriteBatch(
|
|
189
|
-
const std::string
|
|
190
|
-
std::map<std::string, std::string
|
|
174
|
+
const std::string& key, Random* rnd, WriteBatch* batch,
|
|
175
|
+
std::map<std::string, std::string>* data = nullptr) {
|
|
191
176
|
int len = rnd->Next() % kMaxBlobSize + 1;
|
|
192
177
|
std::string value = rnd->HumanReadableString(len);
|
|
193
178
|
ASSERT_OK(batch->Put(key, value));
|
|
@@ -197,14 +182,14 @@ class BlobDBTest : public testing::Test {
|
|
|
197
182
|
}
|
|
198
183
|
|
|
199
184
|
// Verify blob db contain expected data and nothing more.
|
|
200
|
-
void VerifyDB(const std::map<std::string, std::string
|
|
185
|
+
void VerifyDB(const std::map<std::string, std::string>& data) {
|
|
201
186
|
VerifyDB(blob_db_, data);
|
|
202
187
|
}
|
|
203
188
|
|
|
204
|
-
void VerifyDB(DB
|
|
189
|
+
void VerifyDB(DB* db, const std::map<std::string, std::string>& data) {
|
|
205
190
|
// Verify normal Get
|
|
206
|
-
auto
|
|
207
|
-
for (auto
|
|
191
|
+
auto* cfh = db->DefaultColumnFamily();
|
|
192
|
+
for (auto& p : data) {
|
|
208
193
|
PinnableSlice value_slice;
|
|
209
194
|
ASSERT_OK(db->Get(ReadOptions(), cfh, p.first, &value_slice));
|
|
210
195
|
ASSERT_EQ(p.second, value_slice.ToString());
|
|
@@ -214,9 +199,9 @@ class BlobDBTest : public testing::Test {
|
|
|
214
199
|
}
|
|
215
200
|
|
|
216
201
|
// Verify iterators
|
|
217
|
-
Iterator
|
|
202
|
+
Iterator* iter = db->NewIterator(ReadOptions());
|
|
218
203
|
iter->SeekToFirst();
|
|
219
|
-
for (auto
|
|
204
|
+
for (auto& p : data) {
|
|
220
205
|
ASSERT_TRUE(iter->Valid());
|
|
221
206
|
ASSERT_EQ(p.first, iter->key().ToString());
|
|
222
207
|
ASSERT_EQ(p.second, iter->value().ToString());
|
|
@@ -228,16 +213,16 @@ class BlobDBTest : public testing::Test {
|
|
|
228
213
|
}
|
|
229
214
|
|
|
230
215
|
void VerifyBaseDB(
|
|
231
|
-
const std::map<std::string, KeyVersion
|
|
232
|
-
auto
|
|
233
|
-
DB
|
|
216
|
+
const std::map<std::string, KeyVersion>& expected_versions) {
|
|
217
|
+
auto* bdb_impl = static_cast<BlobDBImpl*>(blob_db_);
|
|
218
|
+
DB* db = blob_db_->GetRootDB();
|
|
234
219
|
const size_t kMaxKeys = 10000;
|
|
235
220
|
std::vector<KeyVersion> versions;
|
|
236
221
|
ASSERT_OK(GetAllKeyVersions(db, {}, {}, kMaxKeys, &versions));
|
|
237
222
|
ASSERT_EQ(expected_versions.size(), versions.size());
|
|
238
223
|
size_t i = 0;
|
|
239
|
-
for (auto
|
|
240
|
-
const KeyVersion
|
|
224
|
+
for (auto& key_version : expected_versions) {
|
|
225
|
+
const KeyVersion& expected_version = key_version.second;
|
|
241
226
|
ASSERT_EQ(expected_version.user_key, versions[i].user_key);
|
|
242
227
|
ASSERT_EQ(expected_version.sequence, versions[i].sequence);
|
|
243
228
|
ASSERT_EQ(expected_version.type, versions[i].type);
|
|
@@ -255,7 +240,7 @@ class BlobDBTest : public testing::Test {
|
|
|
255
240
|
}
|
|
256
241
|
|
|
257
242
|
void VerifyBaseDBBlobIndex(
|
|
258
|
-
const std::map<std::string, BlobIndexVersion
|
|
243
|
+
const std::map<std::string, BlobIndexVersion>& expected_versions) {
|
|
259
244
|
const size_t kMaxKeys = 10000;
|
|
260
245
|
std::vector<KeyVersion> versions;
|
|
261
246
|
ASSERT_OK(
|
|
@@ -263,8 +248,8 @@ class BlobDBTest : public testing::Test {
|
|
|
263
248
|
ASSERT_EQ(versions.size(), expected_versions.size());
|
|
264
249
|
|
|
265
250
|
size_t i = 0;
|
|
266
|
-
for (const auto
|
|
267
|
-
const BlobIndexVersion
|
|
251
|
+
for (const auto& expected_pair : expected_versions) {
|
|
252
|
+
const BlobIndexVersion& expected_version = expected_pair.second;
|
|
268
253
|
|
|
269
254
|
ASSERT_EQ(versions[i].user_key, expected_version.user_key);
|
|
270
255
|
ASSERT_EQ(versions[i].sequence, expected_version.sequence);
|
|
@@ -280,10 +265,7 @@ class BlobDBTest : public testing::Test {
|
|
|
280
265
|
BlobIndex blob_index;
|
|
281
266
|
ASSERT_OK(blob_index.DecodeFrom(versions[i].value));
|
|
282
267
|
|
|
283
|
-
|
|
284
|
-
? blob_index.file_number()
|
|
285
|
-
: kInvalidBlobFileNumber;
|
|
286
|
-
ASSERT_EQ(file_number, expected_version.file_number);
|
|
268
|
+
ASSERT_EQ(blob_index.file_number(), expected_version.file_number);
|
|
287
269
|
|
|
288
270
|
const uint64_t expiration =
|
|
289
271
|
blob_index.HasTTL() ? blob_index.expiration() : kNoExpiration;
|
|
@@ -312,13 +294,13 @@ class BlobDBTest : public testing::Test {
|
|
|
312
294
|
std::shared_ptr<MockSystemClock> mock_clock_;
|
|
313
295
|
std::unique_ptr<Env> mock_env_;
|
|
314
296
|
std::unique_ptr<FaultInjectionTestEnv> fault_injection_env_;
|
|
315
|
-
BlobDB
|
|
297
|
+
BlobDB* blob_db_;
|
|
298
|
+
BlobDBOptions bdb_options_;
|
|
316
299
|
}; // class BlobDBTest
|
|
317
300
|
|
|
318
301
|
TEST_F(BlobDBTest, Put) {
|
|
319
302
|
Random rnd(301);
|
|
320
303
|
BlobDBOptions bdb_options;
|
|
321
|
-
bdb_options.min_blob_size = 0;
|
|
322
304
|
bdb_options.disable_background_tasks = true;
|
|
323
305
|
Open(bdb_options);
|
|
324
306
|
std::map<std::string, std::string> data;
|
|
@@ -334,8 +316,6 @@ TEST_F(BlobDBTest, PutWithTTL) {
|
|
|
334
316
|
options.env = mock_env_.get();
|
|
335
317
|
BlobDBOptions bdb_options;
|
|
336
318
|
bdb_options.ttl_range_secs = 1000;
|
|
337
|
-
bdb_options.min_blob_size = 0;
|
|
338
|
-
bdb_options.blob_file_size = 256 * 1000 * 1000;
|
|
339
319
|
bdb_options.disable_background_tasks = true;
|
|
340
320
|
Open(bdb_options, options);
|
|
341
321
|
std::map<std::string, std::string> data;
|
|
@@ -346,33 +326,7 @@ TEST_F(BlobDBTest, PutWithTTL) {
|
|
|
346
326
|
(ttl <= 50 ? nullptr : &data));
|
|
347
327
|
}
|
|
348
328
|
mock_clock_->SetCurrentTime(100);
|
|
349
|
-
auto
|
|
350
|
-
auto blob_files = bdb_impl->TEST_GetBlobFiles();
|
|
351
|
-
ASSERT_EQ(1, blob_files.size());
|
|
352
|
-
ASSERT_TRUE(blob_files[0]->HasTTL());
|
|
353
|
-
ASSERT_OK(bdb_impl->TEST_CloseBlobFile(blob_files[0]));
|
|
354
|
-
VerifyDB(data);
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
TEST_F(BlobDBTest, PutUntil) {
|
|
358
|
-
Random rnd(301);
|
|
359
|
-
Options options;
|
|
360
|
-
options.env = mock_env_.get();
|
|
361
|
-
BlobDBOptions bdb_options;
|
|
362
|
-
bdb_options.ttl_range_secs = 1000;
|
|
363
|
-
bdb_options.min_blob_size = 0;
|
|
364
|
-
bdb_options.blob_file_size = 256 * 1000 * 1000;
|
|
365
|
-
bdb_options.disable_background_tasks = true;
|
|
366
|
-
Open(bdb_options, options);
|
|
367
|
-
std::map<std::string, std::string> data;
|
|
368
|
-
mock_clock_->SetCurrentTime(50);
|
|
369
|
-
for (size_t i = 0; i < 100; i++) {
|
|
370
|
-
uint64_t expiration = rnd.Next() % 100 + 50;
|
|
371
|
-
PutRandomUntil("key" + std::to_string(i), expiration, &rnd,
|
|
372
|
-
(expiration <= 100 ? nullptr : &data));
|
|
373
|
-
}
|
|
374
|
-
mock_clock_->SetCurrentTime(100);
|
|
375
|
-
auto *bdb_impl = static_cast<BlobDBImpl *>(blob_db_);
|
|
329
|
+
auto* bdb_impl = static_cast<BlobDBImpl*>(blob_db_);
|
|
376
330
|
auto blob_files = bdb_impl->TEST_GetBlobFiles();
|
|
377
331
|
ASSERT_EQ(1, blob_files.size());
|
|
378
332
|
ASSERT_TRUE(blob_files[0]->HasTTL());
|
|
@@ -383,7 +337,6 @@ TEST_F(BlobDBTest, PutUntil) {
|
|
|
383
337
|
TEST_F(BlobDBTest, StackableDBGet) {
|
|
384
338
|
Random rnd(301);
|
|
385
339
|
BlobDBOptions bdb_options;
|
|
386
|
-
bdb_options.min_blob_size = 0;
|
|
387
340
|
bdb_options.disable_background_tasks = true;
|
|
388
341
|
Open(bdb_options);
|
|
389
342
|
std::map<std::string, std::string> data;
|
|
@@ -391,8 +344,8 @@ TEST_F(BlobDBTest, StackableDBGet) {
|
|
|
391
344
|
PutRandom("key" + std::to_string(i), &rnd, &data);
|
|
392
345
|
}
|
|
393
346
|
for (size_t i = 0; i < 100; i++) {
|
|
394
|
-
StackableDB
|
|
395
|
-
ColumnFamilyHandle
|
|
347
|
+
StackableDB* db = blob_db_;
|
|
348
|
+
ColumnFamilyHandle* column_family = db->DefaultColumnFamily();
|
|
396
349
|
std::string key = "key" + std::to_string(i);
|
|
397
350
|
PinnableSlice pinnable_value;
|
|
398
351
|
ASSERT_OK(db->Get(ReadOptions(), column_family, key, &pinnable_value));
|
|
@@ -426,10 +379,9 @@ TEST_F(BlobDBTest, GetIOError) {
|
|
|
426
379
|
Options options;
|
|
427
380
|
options.env = fault_injection_env_.get();
|
|
428
381
|
BlobDBOptions bdb_options;
|
|
429
|
-
bdb_options.min_blob_size = 0; // Make sure value write to blob file
|
|
430
382
|
bdb_options.disable_background_tasks = true;
|
|
431
383
|
Open(bdb_options, options);
|
|
432
|
-
ColumnFamilyHandle
|
|
384
|
+
ColumnFamilyHandle* column_family = blob_db_->DefaultColumnFamily();
|
|
433
385
|
PinnableSlice value;
|
|
434
386
|
ASSERT_OK(Put("foo", "bar"));
|
|
435
387
|
fault_injection_env_->SetFilesystemActive(false, Status::IOError());
|
|
@@ -443,7 +395,6 @@ TEST_F(BlobDBTest, PutIOError) {
|
|
|
443
395
|
Options options;
|
|
444
396
|
options.env = fault_injection_env_.get();
|
|
445
397
|
BlobDBOptions bdb_options;
|
|
446
|
-
bdb_options.min_blob_size = 0; // Make sure value write to blob file
|
|
447
398
|
bdb_options.disable_background_tasks = true;
|
|
448
399
|
Open(bdb_options, options);
|
|
449
400
|
fault_injection_env_->SetFilesystemActive(false, Status::IOError());
|
|
@@ -455,7 +406,6 @@ TEST_F(BlobDBTest, PutIOError) {
|
|
|
455
406
|
TEST_F(BlobDBTest, WriteBatch) {
|
|
456
407
|
Random rnd(301);
|
|
457
408
|
BlobDBOptions bdb_options;
|
|
458
|
-
bdb_options.min_blob_size = 0;
|
|
459
409
|
bdb_options.disable_background_tasks = true;
|
|
460
410
|
Open(bdb_options);
|
|
461
411
|
std::map<std::string, std::string> data;
|
|
@@ -474,7 +424,6 @@ TEST_F(BlobDBTest, WriteBatch) {
|
|
|
474
424
|
TEST_F(BlobDBTest, Delete) {
|
|
475
425
|
Random rnd(301);
|
|
476
426
|
BlobDBOptions bdb_options;
|
|
477
|
-
bdb_options.min_blob_size = 0;
|
|
478
427
|
bdb_options.disable_background_tasks = true;
|
|
479
428
|
Open(bdb_options);
|
|
480
429
|
std::map<std::string, std::string> data;
|
|
@@ -490,7 +439,6 @@ TEST_F(BlobDBTest, Delete) {
|
|
|
490
439
|
TEST_F(BlobDBTest, DeleteBatch) {
|
|
491
440
|
Random rnd(301);
|
|
492
441
|
BlobDBOptions bdb_options;
|
|
493
|
-
bdb_options.min_blob_size = 0;
|
|
494
442
|
bdb_options.disable_background_tasks = true;
|
|
495
443
|
Open(bdb_options);
|
|
496
444
|
for (size_t i = 0; i < 100; i++) {
|
|
@@ -508,7 +456,6 @@ TEST_F(BlobDBTest, DeleteBatch) {
|
|
|
508
456
|
TEST_F(BlobDBTest, Override) {
|
|
509
457
|
Random rnd(301);
|
|
510
458
|
BlobDBOptions bdb_options;
|
|
511
|
-
bdb_options.min_blob_size = 0;
|
|
512
459
|
bdb_options.disable_background_tasks = true;
|
|
513
460
|
Open(bdb_options);
|
|
514
461
|
std::map<std::string, std::string> data;
|
|
@@ -522,208 +469,6 @@ TEST_F(BlobDBTest, Override) {
|
|
|
522
469
|
VerifyDB(data);
|
|
523
470
|
}
|
|
524
471
|
|
|
525
|
-
#ifdef SNAPPY
|
|
526
|
-
TEST_F(BlobDBTest, Compression) {
|
|
527
|
-
Random rnd(301);
|
|
528
|
-
BlobDBOptions bdb_options;
|
|
529
|
-
bdb_options.min_blob_size = 0;
|
|
530
|
-
bdb_options.disable_background_tasks = true;
|
|
531
|
-
bdb_options.compression = CompressionType::kSnappyCompression;
|
|
532
|
-
Open(bdb_options);
|
|
533
|
-
std::map<std::string, std::string> data;
|
|
534
|
-
for (size_t i = 0; i < 100; i++) {
|
|
535
|
-
PutRandom("put-key" + std::to_string(i), &rnd, &data);
|
|
536
|
-
}
|
|
537
|
-
for (int i = 0; i < 100; i++) {
|
|
538
|
-
WriteBatch batch;
|
|
539
|
-
for (size_t j = 0; j < 10; j++) {
|
|
540
|
-
PutRandomToWriteBatch("write-batch-key" + std::to_string(j * 100 + i),
|
|
541
|
-
&rnd, &batch, &data);
|
|
542
|
-
}
|
|
543
|
-
ASSERT_OK(blob_db_->Write(WriteOptions(), &batch));
|
|
544
|
-
}
|
|
545
|
-
VerifyDB(data);
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
TEST_F(BlobDBTest, DecompressAfterReopen) {
|
|
549
|
-
Random rnd(301);
|
|
550
|
-
BlobDBOptions bdb_options;
|
|
551
|
-
bdb_options.min_blob_size = 0;
|
|
552
|
-
bdb_options.disable_background_tasks = true;
|
|
553
|
-
bdb_options.compression = CompressionType::kSnappyCompression;
|
|
554
|
-
Open(bdb_options);
|
|
555
|
-
std::map<std::string, std::string> data;
|
|
556
|
-
for (size_t i = 0; i < 100; i++) {
|
|
557
|
-
PutRandom("put-key" + std::to_string(i), &rnd, &data);
|
|
558
|
-
}
|
|
559
|
-
VerifyDB(data);
|
|
560
|
-
bdb_options.compression = CompressionType::kNoCompression;
|
|
561
|
-
Reopen(bdb_options);
|
|
562
|
-
VerifyDB(data);
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
TEST_F(BlobDBTest, EnableDisableCompressionGC) {
|
|
566
|
-
Random rnd(301);
|
|
567
|
-
BlobDBOptions bdb_options;
|
|
568
|
-
bdb_options.min_blob_size = 0;
|
|
569
|
-
bdb_options.garbage_collection_cutoff = 1.0;
|
|
570
|
-
bdb_options.disable_background_tasks = true;
|
|
571
|
-
bdb_options.compression = kSnappyCompression;
|
|
572
|
-
Open(bdb_options);
|
|
573
|
-
std::map<std::string, std::string> data;
|
|
574
|
-
size_t data_idx = 0;
|
|
575
|
-
for (; data_idx < 100; data_idx++) {
|
|
576
|
-
PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
|
|
577
|
-
}
|
|
578
|
-
VerifyDB(data);
|
|
579
|
-
auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
580
|
-
ASSERT_EQ(1, blob_files.size());
|
|
581
|
-
ASSERT_EQ(kSnappyCompression, blob_files[0]->GetCompressionType());
|
|
582
|
-
|
|
583
|
-
// disable compression
|
|
584
|
-
bdb_options.compression = kNoCompression;
|
|
585
|
-
Reopen(bdb_options);
|
|
586
|
-
|
|
587
|
-
// Add more data with new compression type
|
|
588
|
-
for (; data_idx < 200; data_idx++) {
|
|
589
|
-
PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
|
|
590
|
-
}
|
|
591
|
-
VerifyDB(data);
|
|
592
|
-
|
|
593
|
-
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
594
|
-
ASSERT_EQ(2, blob_files.size());
|
|
595
|
-
ASSERT_EQ(kNoCompression, blob_files[1]->GetCompressionType());
|
|
596
|
-
|
|
597
|
-
// Enable GC. If we do it earlier the snapshot release triggered compaction
|
|
598
|
-
// may compact files and trigger GC before we can verify there are two files.
|
|
599
|
-
bdb_options.enable_garbage_collection = true;
|
|
600
|
-
Reopen(bdb_options);
|
|
601
|
-
|
|
602
|
-
// Trigger compaction
|
|
603
|
-
ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
604
|
-
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
605
|
-
VerifyDB(data);
|
|
606
|
-
|
|
607
|
-
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
608
|
-
for (const auto &bfile : blob_files) {
|
|
609
|
-
ASSERT_EQ(kNoCompression, bfile->GetCompressionType());
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
// enabling the compression again
|
|
613
|
-
bdb_options.compression = kSnappyCompression;
|
|
614
|
-
Reopen(bdb_options);
|
|
615
|
-
|
|
616
|
-
// Add more data with new compression type
|
|
617
|
-
for (; data_idx < 300; data_idx++) {
|
|
618
|
-
PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
|
|
619
|
-
}
|
|
620
|
-
VerifyDB(data);
|
|
621
|
-
|
|
622
|
-
// Trigger compaction
|
|
623
|
-
ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
624
|
-
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
625
|
-
VerifyDB(data);
|
|
626
|
-
|
|
627
|
-
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
628
|
-
for (const auto &bfile : blob_files) {
|
|
629
|
-
ASSERT_EQ(kSnappyCompression, bfile->GetCompressionType());
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
#ifdef LZ4
|
|
634
|
-
// Test switch compression types and run GC, it needs both Snappy and LZ4
|
|
635
|
-
// support.
|
|
636
|
-
TEST_F(BlobDBTest, ChangeCompressionGC) {
|
|
637
|
-
Random rnd(301);
|
|
638
|
-
BlobDBOptions bdb_options;
|
|
639
|
-
bdb_options.min_blob_size = 0;
|
|
640
|
-
bdb_options.garbage_collection_cutoff = 1.0;
|
|
641
|
-
bdb_options.disable_background_tasks = true;
|
|
642
|
-
bdb_options.compression = kLZ4Compression;
|
|
643
|
-
Open(bdb_options);
|
|
644
|
-
std::map<std::string, std::string> data;
|
|
645
|
-
size_t data_idx = 0;
|
|
646
|
-
for (; data_idx < 100; data_idx++) {
|
|
647
|
-
PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
|
|
648
|
-
}
|
|
649
|
-
VerifyDB(data);
|
|
650
|
-
auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
651
|
-
ASSERT_EQ(1, blob_files.size());
|
|
652
|
-
ASSERT_EQ(kLZ4Compression, blob_files[0]->GetCompressionType());
|
|
653
|
-
|
|
654
|
-
// Change compression type
|
|
655
|
-
bdb_options.compression = kSnappyCompression;
|
|
656
|
-
Reopen(bdb_options);
|
|
657
|
-
|
|
658
|
-
// Add more data with Snappy compression type
|
|
659
|
-
for (; data_idx < 200; data_idx++) {
|
|
660
|
-
PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
|
|
661
|
-
}
|
|
662
|
-
VerifyDB(data);
|
|
663
|
-
|
|
664
|
-
// Verify blob file compression type
|
|
665
|
-
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
666
|
-
ASSERT_EQ(2, blob_files.size());
|
|
667
|
-
ASSERT_EQ(kSnappyCompression, blob_files[1]->GetCompressionType());
|
|
668
|
-
|
|
669
|
-
// Enable GC. If we do it earlier the snapshot release triggered compaction
|
|
670
|
-
// may compact files and trigger GC before we can verify there are two files.
|
|
671
|
-
bdb_options.enable_garbage_collection = true;
|
|
672
|
-
Reopen(bdb_options);
|
|
673
|
-
|
|
674
|
-
ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
675
|
-
VerifyDB(data);
|
|
676
|
-
|
|
677
|
-
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
678
|
-
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
679
|
-
for (const auto &bfile : blob_files) {
|
|
680
|
-
ASSERT_EQ(kSnappyCompression, bfile->GetCompressionType());
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
// Disable compression
|
|
684
|
-
bdb_options.compression = kNoCompression;
|
|
685
|
-
Reopen(bdb_options);
|
|
686
|
-
for (; data_idx < 300; data_idx++) {
|
|
687
|
-
PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
|
|
688
|
-
}
|
|
689
|
-
VerifyDB(data);
|
|
690
|
-
|
|
691
|
-
ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
692
|
-
VerifyDB(data);
|
|
693
|
-
|
|
694
|
-
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
695
|
-
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
696
|
-
for (const auto &bfile : blob_files) {
|
|
697
|
-
ASSERT_EQ(kNoCompression, bfile->GetCompressionType());
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
// switching different compression types to generate mixed compression types
|
|
701
|
-
bdb_options.compression = kSnappyCompression;
|
|
702
|
-
Reopen(bdb_options);
|
|
703
|
-
for (; data_idx < 400; data_idx++) {
|
|
704
|
-
PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
|
|
705
|
-
}
|
|
706
|
-
VerifyDB(data);
|
|
707
|
-
|
|
708
|
-
bdb_options.compression = kLZ4Compression;
|
|
709
|
-
Reopen(bdb_options);
|
|
710
|
-
for (; data_idx < 500; data_idx++) {
|
|
711
|
-
PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
|
|
712
|
-
}
|
|
713
|
-
VerifyDB(data);
|
|
714
|
-
|
|
715
|
-
ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
716
|
-
VerifyDB(data);
|
|
717
|
-
|
|
718
|
-
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
719
|
-
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
720
|
-
for (const auto &bfile : blob_files) {
|
|
721
|
-
ASSERT_EQ(kLZ4Compression, bfile->GetCompressionType());
|
|
722
|
-
}
|
|
723
|
-
}
|
|
724
|
-
#endif // LZ4
|
|
725
|
-
#endif // SNAPPY
|
|
726
|
-
|
|
727
472
|
TEST_F(BlobDBTest, MultipleWriters) {
|
|
728
473
|
Open(BlobDBOptions());
|
|
729
474
|
|
|
@@ -760,21 +505,19 @@ TEST_F(BlobDBTest, SstFileManager) {
|
|
|
760
505
|
std::shared_ptr<SstFileManager> sst_file_manager(
|
|
761
506
|
NewSstFileManager(mock_env_.get()));
|
|
762
507
|
sst_file_manager->SetDeleteRateBytesPerSecond(1024 * 1024);
|
|
763
|
-
SstFileManagerImpl
|
|
764
|
-
static_cast<SstFileManagerImpl
|
|
508
|
+
SstFileManagerImpl* sfm =
|
|
509
|
+
static_cast<SstFileManagerImpl*>(sst_file_manager.get());
|
|
765
510
|
|
|
766
511
|
BlobDBOptions bdb_options;
|
|
767
|
-
bdb_options.min_blob_size = 0;
|
|
768
512
|
bdb_options.enable_garbage_collection = true;
|
|
769
|
-
bdb_options.garbage_collection_cutoff = 1.0;
|
|
770
513
|
Options db_options;
|
|
771
514
|
|
|
772
515
|
int files_scheduled_to_delete = 0;
|
|
773
516
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
774
|
-
"SstFileManagerImpl::ScheduleFileDeletion", [&](void
|
|
517
|
+
"SstFileManagerImpl::ScheduleFileDeletion", [&](void* arg) {
|
|
775
518
|
assert(arg);
|
|
776
|
-
const std::string
|
|
777
|
-
static_cast<const std::string
|
|
519
|
+
const std::string* const file_path =
|
|
520
|
+
static_cast<const std::string*>(arg);
|
|
778
521
|
if (file_path->find(".blob") != std::string::npos) {
|
|
779
522
|
++files_scheduled_to_delete;
|
|
780
523
|
}
|
|
@@ -784,12 +527,22 @@ TEST_F(BlobDBTest, SstFileManager) {
|
|
|
784
527
|
|
|
785
528
|
Open(bdb_options, db_options);
|
|
786
529
|
|
|
787
|
-
// Create
|
|
530
|
+
// Create 4 blob files. With GC cutoff of 0.25, the oldest file (file 1)
|
|
531
|
+
// will be in the GC zone: floor(0.25 * 4) = 1.
|
|
788
532
|
ASSERT_OK(blob_db_->Put(WriteOptions(), "foo", "bar"));
|
|
789
533
|
auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
790
534
|
ASSERT_EQ(1, blob_files.size());
|
|
791
535
|
std::shared_ptr<BlobFile> bfile = blob_files[0];
|
|
792
536
|
ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(bfile));
|
|
537
|
+
|
|
538
|
+
// Create 3 more blob files (files 2-4, outside GC zone).
|
|
539
|
+
for (int i = 1; i < 4; i++) {
|
|
540
|
+
ASSERT_OK(blob_db_->Put(WriteOptions(), "key" + std::to_string(i), "val"));
|
|
541
|
+
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
542
|
+
ASSERT_EQ(static_cast<size_t>(i + 1), blob_files.size());
|
|
543
|
+
ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_files[i]));
|
|
544
|
+
}
|
|
545
|
+
|
|
793
546
|
ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
794
547
|
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
795
548
|
|
|
@@ -797,7 +550,8 @@ TEST_F(BlobDBTest, SstFileManager) {
|
|
|
797
550
|
ASSERT_EQ(1, files_scheduled_to_delete);
|
|
798
551
|
Destroy();
|
|
799
552
|
// Make sure that DestroyBlobDB() also goes through delete scheduler.
|
|
800
|
-
|
|
553
|
+
// Remaining files: 3 original (files 2-4) + 1 GC output file = 4 files.
|
|
554
|
+
ASSERT_EQ(5, files_scheduled_to_delete);
|
|
801
555
|
SyncPoint::GetInstance()->DisableProcessing();
|
|
802
556
|
sfm->WaitForEmptyTrash();
|
|
803
557
|
}
|
|
@@ -805,10 +559,10 @@ TEST_F(BlobDBTest, SstFileManager) {
|
|
|
805
559
|
TEST_F(BlobDBTest, SstFileManagerRestart) {
|
|
806
560
|
int files_scheduled_to_delete = 0;
|
|
807
561
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
808
|
-
"SstFileManagerImpl::ScheduleFileDeletion", [&](void
|
|
562
|
+
"SstFileManagerImpl::ScheduleFileDeletion", [&](void* arg) {
|
|
809
563
|
assert(arg);
|
|
810
|
-
const std::string
|
|
811
|
-
static_cast<const std::string
|
|
564
|
+
const std::string* const file_path =
|
|
565
|
+
static_cast<const std::string*>(arg);
|
|
812
566
|
if (file_path->find(".blob") != std::string::npos) {
|
|
813
567
|
++files_scheduled_to_delete;
|
|
814
568
|
}
|
|
@@ -818,11 +572,10 @@ TEST_F(BlobDBTest, SstFileManagerRestart) {
|
|
|
818
572
|
std::shared_ptr<SstFileManager> sst_file_manager(
|
|
819
573
|
NewSstFileManager(mock_env_.get()));
|
|
820
574
|
sst_file_manager->SetDeleteRateBytesPerSecond(1024 * 1024);
|
|
821
|
-
SstFileManagerImpl
|
|
822
|
-
static_cast<SstFileManagerImpl
|
|
575
|
+
SstFileManagerImpl* sfm =
|
|
576
|
+
static_cast<SstFileManagerImpl*>(sst_file_manager.get());
|
|
823
577
|
|
|
824
578
|
BlobDBOptions bdb_options;
|
|
825
|
-
bdb_options.min_blob_size = 0;
|
|
826
579
|
Options db_options;
|
|
827
580
|
|
|
828
581
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
@@ -834,7 +587,7 @@ TEST_F(BlobDBTest, SstFileManagerRestart) {
|
|
|
834
587
|
Close();
|
|
835
588
|
|
|
836
589
|
// Create 3 dummy trash files under the blob_dir
|
|
837
|
-
const auto
|
|
590
|
+
const auto& fs = db_options.env->GetFileSystem();
|
|
838
591
|
ASSERT_OK(CreateFile(fs, blob_dir + "/000666.blob.trash", "", false));
|
|
839
592
|
ASSERT_OK(CreateFile(fs, blob_dir + "/000888.blob.trash", "", true));
|
|
840
593
|
ASSERT_OK(CreateFile(fs, blob_dir + "/something_not_match.trash", "", false));
|
|
@@ -849,7 +602,7 @@ TEST_F(BlobDBTest, SstFileManagerRestart) {
|
|
|
849
602
|
std::vector<std::string> all_files;
|
|
850
603
|
ASSERT_OK(db_options.env->GetChildren(blob_dir, &all_files));
|
|
851
604
|
int nfiles = 0;
|
|
852
|
-
for (const auto
|
|
605
|
+
for (const auto& f : all_files) {
|
|
853
606
|
assert(!f.empty());
|
|
854
607
|
if (f[0] == '.') {
|
|
855
608
|
continue;
|
|
@@ -863,22 +616,28 @@ TEST_F(BlobDBTest, SstFileManagerRestart) {
|
|
|
863
616
|
|
|
864
617
|
TEST_F(BlobDBTest, SnapshotAndGarbageCollection) {
|
|
865
618
|
BlobDBOptions bdb_options;
|
|
866
|
-
bdb_options.min_blob_size = 0;
|
|
867
619
|
bdb_options.enable_garbage_collection = true;
|
|
868
|
-
bdb_options.garbage_collection_cutoff = 1.0;
|
|
869
620
|
bdb_options.disable_background_tasks = true;
|
|
870
621
|
|
|
871
622
|
Options options;
|
|
872
623
|
options.disable_auto_compactions = true;
|
|
873
624
|
|
|
874
|
-
//
|
|
625
|
+
// This test verifies that snapshots protect blob files from deletion during
|
|
626
|
+
// garbage collection. With fixed GC cutoff of 0.25 and 8 immutable files,
|
|
627
|
+
// floor(0.25 * 8) = 2 files are in the GC zone (files 1 and 2).
|
|
628
|
+
//
|
|
629
|
+
// We run 4 iterations with different snapshot timing:
|
|
630
|
+
// i=0: snapshot after key1 (before key2) - protects file 1
|
|
631
|
+
// i=1: snapshot after key2 (before key3) - protects files 1 and 2
|
|
632
|
+
// i=2: snapshot after key9 (after all keys) - no protection needed
|
|
633
|
+
// i=3: snapshot after Delete(key2) - no protection needed
|
|
875
634
|
for (int i = 0; i < 4; i++) {
|
|
876
635
|
Destroy();
|
|
877
636
|
Open(bdb_options, options);
|
|
878
637
|
|
|
879
|
-
const Snapshot
|
|
638
|
+
const Snapshot* snapshot = nullptr;
|
|
880
639
|
|
|
881
|
-
//
|
|
640
|
+
// Create first blob file (will be in GC zone).
|
|
882
641
|
ASSERT_OK(Put("key1", "value"));
|
|
883
642
|
if (i == 0) {
|
|
884
643
|
snapshot = blob_db_->GetSnapshot();
|
|
@@ -888,7 +647,8 @@ TEST_F(BlobDBTest, SnapshotAndGarbageCollection) {
|
|
|
888
647
|
ASSERT_EQ(1, blob_files.size());
|
|
889
648
|
ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_files[0]));
|
|
890
649
|
|
|
891
|
-
//
|
|
650
|
+
// Create second blob file (will be in GC zone). We track this file
|
|
651
|
+
// to verify it becomes obsolete after GC relocates its blob.
|
|
892
652
|
ASSERT_OK(Put("key2", "value"));
|
|
893
653
|
if (i == 1) {
|
|
894
654
|
snapshot = blob_db_->GetSnapshot();
|
|
@@ -896,39 +656,66 @@ TEST_F(BlobDBTest, SnapshotAndGarbageCollection) {
|
|
|
896
656
|
|
|
897
657
|
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
898
658
|
ASSERT_EQ(2, blob_files.size());
|
|
899
|
-
auto
|
|
900
|
-
ASSERT_FALSE(
|
|
901
|
-
ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(
|
|
659
|
+
auto gc_target_file = blob_files[1];
|
|
660
|
+
ASSERT_FALSE(gc_target_file->Immutable());
|
|
661
|
+
ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(gc_target_file));
|
|
662
|
+
|
|
663
|
+
// Create files 3-8, all closed (these are outside GC zone).
|
|
664
|
+
for (int j = 3; j <= 8; j++) {
|
|
665
|
+
ASSERT_OK(Put("key" + std::to_string(j), "value"));
|
|
666
|
+
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
667
|
+
ASSERT_EQ(static_cast<size_t>(j), blob_files.size());
|
|
668
|
+
ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_files[j - 1]));
|
|
669
|
+
}
|
|
902
670
|
|
|
903
|
-
//
|
|
904
|
-
|
|
671
|
+
// Create file 9 but leave it open (mutable). Only immutable files are
|
|
672
|
+
// counted for GC cutoff calculation.
|
|
673
|
+
ASSERT_OK(Put("key9", "value"));
|
|
905
674
|
if (i == 2) {
|
|
906
675
|
snapshot = blob_db_->GetSnapshot();
|
|
907
676
|
}
|
|
908
677
|
|
|
678
|
+
// Verify we have 9 total files (8 immutable + 1 mutable).
|
|
679
|
+
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
680
|
+
ASSERT_EQ(9, blob_files.size());
|
|
681
|
+
|
|
682
|
+
// Trigger GC via compaction. Blobs in files 1 and 2 will be relocated
|
|
683
|
+
// to a new GC output file.
|
|
909
684
|
ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
910
|
-
|
|
685
|
+
|
|
686
|
+
// Verify gc_target_file (file 2) is now obsolete.
|
|
687
|
+
ASSERT_TRUE(gc_target_file->Obsolete());
|
|
688
|
+
// Verify the obsolete sequence matches the latest sequence number.
|
|
911
689
|
ASSERT_EQ(blob_db_->GetLatestSequenceNumber(),
|
|
912
|
-
|
|
690
|
+
gc_target_file->GetObsoleteSequence());
|
|
913
691
|
|
|
914
692
|
Delete("key2");
|
|
915
693
|
if (i == 3) {
|
|
916
694
|
snapshot = blob_db_->GetSnapshot();
|
|
917
695
|
}
|
|
918
696
|
|
|
919
|
-
|
|
697
|
+
// Verify we now have 10 files (9 original + 1 GC output file).
|
|
698
|
+
// Files 1 and 2 are obsolete but not yet deleted.
|
|
699
|
+
ASSERT_EQ(10, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
920
700
|
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
921
701
|
|
|
922
702
|
if (i >= 2) {
|
|
923
|
-
//
|
|
924
|
-
|
|
703
|
+
// Snapshot was taken after all keys were written, so it sees the
|
|
704
|
+
// post-compaction state where blob indexes point to the GC output file.
|
|
705
|
+
// Obsolete files 1 and 2 can be deleted immediately.
|
|
706
|
+
// Verify 8 files remain (10 - 2 obsolete files deleted).
|
|
707
|
+
ASSERT_EQ(8, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
925
708
|
blob_db_->ReleaseSnapshot(snapshot);
|
|
926
709
|
} else {
|
|
927
|
-
//
|
|
928
|
-
|
|
710
|
+
// Snapshot was taken before compaction completed, so it may still
|
|
711
|
+
// reference blobs in the obsolete files. Files cannot be deleted.
|
|
712
|
+
// Verify all 10 files still exist.
|
|
713
|
+
ASSERT_EQ(10, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
929
714
|
blob_db_->ReleaseSnapshot(snapshot);
|
|
930
715
|
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
931
|
-
|
|
716
|
+
// After releasing the snapshot, obsolete files can be deleted.
|
|
717
|
+
// Verify 8 files remain.
|
|
718
|
+
ASSERT_EQ(8, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
932
719
|
}
|
|
933
720
|
}
|
|
934
721
|
}
|
|
@@ -938,8 +725,8 @@ TEST_F(BlobDBTest, ColumnFamilyNotSupported) {
|
|
|
938
725
|
options.env = mock_env_.get();
|
|
939
726
|
mock_clock_->SetCurrentTime(0);
|
|
940
727
|
Open(BlobDBOptions(), options);
|
|
941
|
-
ColumnFamilyHandle
|
|
942
|
-
ColumnFamilyHandle
|
|
728
|
+
ColumnFamilyHandle* default_handle = blob_db_->DefaultColumnFamily();
|
|
729
|
+
ColumnFamilyHandle* handle = nullptr;
|
|
943
730
|
std::string value;
|
|
944
731
|
std::vector<std::string> values;
|
|
945
732
|
// The call simply pass through to base db. It should succeed.
|
|
@@ -948,8 +735,6 @@ TEST_F(BlobDBTest, ColumnFamilyNotSupported) {
|
|
|
948
735
|
ASSERT_TRUE(blob_db_->Put(WriteOptions(), handle, "k", "v").IsNotSupported());
|
|
949
736
|
ASSERT_TRUE(blob_db_->PutWithTTL(WriteOptions(), handle, "k", "v", 60)
|
|
950
737
|
.IsNotSupported());
|
|
951
|
-
ASSERT_TRUE(blob_db_->PutUntil(WriteOptions(), handle, "k", "v", 100)
|
|
952
|
-
.IsNotSupported());
|
|
953
738
|
WriteBatch batch;
|
|
954
739
|
ASSERT_OK(batch.Put("k1", "v1"));
|
|
955
740
|
ASSERT_OK(batch.Put(handle, "k2", "v2"));
|
|
@@ -970,10 +755,7 @@ TEST_F(BlobDBTest, GetLiveFilesMetaData) {
|
|
|
970
755
|
Random rnd(301);
|
|
971
756
|
|
|
972
757
|
BlobDBOptions bdb_options;
|
|
973
|
-
bdb_options.blob_dir = "blob_dir";
|
|
974
|
-
bdb_options.path_relative = true;
|
|
975
758
|
bdb_options.ttl_range_secs = 10;
|
|
976
|
-
bdb_options.min_blob_size = 0;
|
|
977
759
|
bdb_options.disable_background_tasks = true;
|
|
978
760
|
|
|
979
761
|
Options options;
|
|
@@ -986,8 +768,9 @@ TEST_F(BlobDBTest, GetLiveFilesMetaData) {
|
|
|
986
768
|
PutRandom("key" + std::to_string(i), &rnd, &data);
|
|
987
769
|
}
|
|
988
770
|
|
|
989
|
-
|
|
990
|
-
|
|
771
|
+
// At time 0, the stored expiration equals TTL
|
|
772
|
+
constexpr uint64_t ttl = 1000ULL;
|
|
773
|
+
PutRandomWithTTL("key100", ttl, &rnd, &data);
|
|
991
774
|
|
|
992
775
|
std::vector<LiveFileMetaData> metadata;
|
|
993
776
|
blob_db_->GetLiveFilesMetaData(&metadata);
|
|
@@ -1003,7 +786,7 @@ TEST_F(BlobDBTest, GetLiveFilesMetaData) {
|
|
|
1003
786
|
const std::string filename2("/blob_dir/000002.blob");
|
|
1004
787
|
ASSERT_EQ(filename2, metadata[1].name);
|
|
1005
788
|
ASSERT_EQ(2, metadata[1].file_number);
|
|
1006
|
-
ASSERT_EQ(
|
|
789
|
+
ASSERT_EQ(ttl, metadata[1].oldest_ancester_time);
|
|
1007
790
|
ASSERT_EQ(kDefaultColumnFamilyName, metadata[1].column_family_name);
|
|
1008
791
|
|
|
1009
792
|
std::vector<std::string> livefile;
|
|
@@ -1046,16 +829,15 @@ TEST_F(BlobDBTest, MigrateFromPlainRocksDB) {
|
|
|
1046
829
|
// Write to plain rocksdb.
|
|
1047
830
|
Options options;
|
|
1048
831
|
options.create_if_missing = true;
|
|
1049
|
-
DB
|
|
832
|
+
std::unique_ptr<DB> db;
|
|
1050
833
|
ASSERT_OK(DB::Open(options, dbname_, &db));
|
|
1051
834
|
for (size_t i = 0; i < kNumIteration; i++) {
|
|
1052
835
|
auto key_index = rnd.Next() % kNumKey;
|
|
1053
836
|
std::string key = "key" + std::to_string(key_index);
|
|
1054
|
-
PutRandom(db, key, &rnd, &data);
|
|
837
|
+
PutRandom(db.get(), key, &rnd, &data);
|
|
1055
838
|
}
|
|
1056
|
-
VerifyDB(db, data);
|
|
1057
|
-
|
|
1058
|
-
db = nullptr;
|
|
839
|
+
VerifyDB(db.get(), data);
|
|
840
|
+
db.reset();
|
|
1059
841
|
|
|
1060
842
|
// Open as blob db. Verify it can read existing data.
|
|
1061
843
|
Open();
|
|
@@ -1085,7 +867,6 @@ TEST_F(BlobDBTest, MigrateFromPlainRocksDB) {
|
|
|
1085
867
|
ASSERT_EQ(data[key], value);
|
|
1086
868
|
}
|
|
1087
869
|
}
|
|
1088
|
-
delete db;
|
|
1089
870
|
}
|
|
1090
871
|
|
|
1091
872
|
// Test to verify that a NoSpace IOError Status is returned on reaching
|
|
@@ -1096,7 +877,6 @@ TEST_F(BlobDBTest, OutOfSpace) {
|
|
|
1096
877
|
options.env = mock_env_.get();
|
|
1097
878
|
BlobDBOptions bdb_options;
|
|
1098
879
|
bdb_options.max_db_size = 200;
|
|
1099
|
-
bdb_options.is_fifo = false;
|
|
1100
880
|
bdb_options.disable_background_tasks = true;
|
|
1101
881
|
Open(bdb_options);
|
|
1102
882
|
|
|
@@ -1112,256 +892,13 @@ TEST_F(BlobDBTest, OutOfSpace) {
|
|
|
1112
892
|
ASSERT_TRUE(s.IsNoSpace());
|
|
1113
893
|
}
|
|
1114
894
|
|
|
1115
|
-
TEST_F(BlobDBTest, FIFOEviction) {
|
|
1116
|
-
BlobDBOptions bdb_options;
|
|
1117
|
-
bdb_options.max_db_size = 200;
|
|
1118
|
-
bdb_options.blob_file_size = 100;
|
|
1119
|
-
bdb_options.is_fifo = true;
|
|
1120
|
-
bdb_options.disable_background_tasks = true;
|
|
1121
|
-
Open(bdb_options);
|
|
1122
|
-
|
|
1123
|
-
std::atomic<int> evict_count{0};
|
|
1124
|
-
SyncPoint::GetInstance()->SetCallBack(
|
|
1125
|
-
"BlobDBImpl::EvictOldestBlobFile:Evicted",
|
|
1126
|
-
[&](void *) { evict_count++; });
|
|
1127
|
-
SyncPoint::GetInstance()->EnableProcessing();
|
|
1128
|
-
|
|
1129
|
-
// Each stored blob has an overhead of 32 bytes currently.
|
|
1130
|
-
// So a 100 byte blob should take up 132 bytes.
|
|
1131
|
-
std::string value(100, 'v');
|
|
1132
|
-
ASSERT_OK(blob_db_->PutWithTTL(WriteOptions(), "key1", value, 10));
|
|
1133
|
-
VerifyDB({{"key1", value}});
|
|
1134
|
-
|
|
1135
|
-
ASSERT_EQ(1, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
1136
|
-
|
|
1137
|
-
// Adding another 100 bytes blob would take the total size to 264 bytes
|
|
1138
|
-
// (2*132). max_db_size will be exceeded
|
|
1139
|
-
// than max_db_size and trigger FIFO eviction.
|
|
1140
|
-
ASSERT_OK(blob_db_->PutWithTTL(WriteOptions(), "key2", value, 60));
|
|
1141
|
-
ASSERT_EQ(1, evict_count);
|
|
1142
|
-
// key1 will exist until corresponding file be deleted.
|
|
1143
|
-
VerifyDB({{"key1", value}, {"key2", value}});
|
|
1144
|
-
|
|
1145
|
-
// Adding another 100 bytes blob without TTL.
|
|
1146
|
-
ASSERT_OK(blob_db_->Put(WriteOptions(), "key3", value));
|
|
1147
|
-
ASSERT_EQ(2, evict_count);
|
|
1148
|
-
// key1 and key2 will exist until corresponding file be deleted.
|
|
1149
|
-
VerifyDB({{"key1", value}, {"key2", value}, {"key3", value}});
|
|
1150
|
-
|
|
1151
|
-
// The fourth blob file, without TTL.
|
|
1152
|
-
ASSERT_OK(blob_db_->Put(WriteOptions(), "key4", value));
|
|
1153
|
-
ASSERT_EQ(3, evict_count);
|
|
1154
|
-
VerifyDB(
|
|
1155
|
-
{{"key1", value}, {"key2", value}, {"key3", value}, {"key4", value}});
|
|
1156
|
-
|
|
1157
|
-
auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
1158
|
-
ASSERT_EQ(4, blob_files.size());
|
|
1159
|
-
ASSERT_TRUE(blob_files[0]->Obsolete());
|
|
1160
|
-
ASSERT_TRUE(blob_files[1]->Obsolete());
|
|
1161
|
-
ASSERT_TRUE(blob_files[2]->Obsolete());
|
|
1162
|
-
ASSERT_FALSE(blob_files[3]->Obsolete());
|
|
1163
|
-
auto obsolete_files = blob_db_impl()->TEST_GetObsoleteFiles();
|
|
1164
|
-
ASSERT_EQ(3, obsolete_files.size());
|
|
1165
|
-
ASSERT_EQ(blob_files[0], obsolete_files[0]);
|
|
1166
|
-
ASSERT_EQ(blob_files[1], obsolete_files[1]);
|
|
1167
|
-
ASSERT_EQ(blob_files[2], obsolete_files[2]);
|
|
1168
|
-
|
|
1169
|
-
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
1170
|
-
obsolete_files = blob_db_impl()->TEST_GetObsoleteFiles();
|
|
1171
|
-
ASSERT_TRUE(obsolete_files.empty());
|
|
1172
|
-
VerifyDB({{"key4", value}});
|
|
1173
|
-
}
|
|
1174
|
-
|
|
1175
|
-
TEST_F(BlobDBTest, FIFOEviction_NoOldestFileToEvict) {
|
|
1176
|
-
Options options;
|
|
1177
|
-
BlobDBOptions bdb_options;
|
|
1178
|
-
bdb_options.max_db_size = 1000;
|
|
1179
|
-
bdb_options.blob_file_size = 5000;
|
|
1180
|
-
bdb_options.is_fifo = true;
|
|
1181
|
-
bdb_options.disable_background_tasks = true;
|
|
1182
|
-
Open(bdb_options);
|
|
1183
|
-
|
|
1184
|
-
std::atomic<int> evict_count{0};
|
|
1185
|
-
SyncPoint::GetInstance()->SetCallBack(
|
|
1186
|
-
"BlobDBImpl::EvictOldestBlobFile:Evicted",
|
|
1187
|
-
[&](void *) { evict_count++; });
|
|
1188
|
-
SyncPoint::GetInstance()->EnableProcessing();
|
|
1189
|
-
|
|
1190
|
-
std::string value(2000, 'v');
|
|
1191
|
-
ASSERT_TRUE(Put("foo", std::string(2000, 'v')).IsNoSpace());
|
|
1192
|
-
ASSERT_EQ(0, evict_count);
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
TEST_F(BlobDBTest, FIFOEviction_NoEnoughBlobFilesToEvict) {
|
|
1196
|
-
BlobDBOptions bdb_options;
|
|
1197
|
-
bdb_options.is_fifo = true;
|
|
1198
|
-
bdb_options.min_blob_size = 100;
|
|
1199
|
-
bdb_options.disable_background_tasks = true;
|
|
1200
|
-
Options options;
|
|
1201
|
-
// Use mock env to stop wall clock.
|
|
1202
|
-
options.env = mock_env_.get();
|
|
1203
|
-
options.disable_auto_compactions = true;
|
|
1204
|
-
auto statistics = CreateDBStatistics();
|
|
1205
|
-
options.statistics = statistics;
|
|
1206
|
-
Open(bdb_options, options);
|
|
1207
|
-
|
|
1208
|
-
SyncPoint::GetInstance()->LoadDependency(
|
|
1209
|
-
{{"DBImpl::NotifyOnFlushCompleted::PostAllOnFlushCompleted",
|
|
1210
|
-
"BlobDBTest.FIFOEviction_NoEnoughBlobFilesToEvict:AfterFlush"}});
|
|
1211
|
-
|
|
1212
|
-
SyncPoint::GetInstance()->EnableProcessing();
|
|
1213
|
-
|
|
1214
|
-
ASSERT_EQ(0, blob_db_impl()->TEST_live_sst_size());
|
|
1215
|
-
std::string small_value(50, 'v');
|
|
1216
|
-
std::map<std::string, std::string> data;
|
|
1217
|
-
// Insert some data into LSM tree to make sure FIFO eviction take SST
|
|
1218
|
-
// file size into account.
|
|
1219
|
-
for (int i = 0; i < 1000; i++) {
|
|
1220
|
-
ASSERT_OK(Put("key" + std::to_string(i), small_value, &data));
|
|
1221
|
-
}
|
|
1222
|
-
ASSERT_OK(blob_db_->Flush(FlushOptions()));
|
|
1223
|
-
|
|
1224
|
-
uint64_t live_sst_size = 0;
|
|
1225
|
-
ASSERT_TRUE(blob_db_->GetIntProperty(DB::Properties::kTotalSstFilesSize,
|
|
1226
|
-
&live_sst_size));
|
|
1227
|
-
ASSERT_TRUE(live_sst_size > 0);
|
|
1228
|
-
|
|
1229
|
-
TEST_SYNC_POINT(
|
|
1230
|
-
"BlobDBTest.FIFOEviction_NoEnoughBlobFilesToEvict:AfterFlush");
|
|
1231
|
-
|
|
1232
|
-
ASSERT_EQ(live_sst_size, blob_db_impl()->TEST_live_sst_size());
|
|
1233
|
-
|
|
1234
|
-
bdb_options.max_db_size = live_sst_size + 2000;
|
|
1235
|
-
Reopen(bdb_options, options);
|
|
1236
|
-
ASSERT_EQ(live_sst_size, blob_db_impl()->TEST_live_sst_size());
|
|
1237
|
-
|
|
1238
|
-
std::string value_1k(1000, 'v');
|
|
1239
|
-
ASSERT_OK(PutWithTTL("large_key1", value_1k, 60, &data));
|
|
1240
|
-
ASSERT_EQ(0, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
|
|
1241
|
-
VerifyDB(data);
|
|
1242
|
-
// large_key2 evicts large_key1
|
|
1243
|
-
ASSERT_OK(PutWithTTL("large_key2", value_1k, 60, &data));
|
|
1244
|
-
ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
|
|
1245
|
-
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
1246
|
-
data.erase("large_key1");
|
|
1247
|
-
VerifyDB(data);
|
|
1248
|
-
// large_key3 get no enough space even after evicting large_key2, so it
|
|
1249
|
-
// instead return no space error.
|
|
1250
|
-
std::string value_2k(2000, 'v');
|
|
1251
|
-
ASSERT_TRUE(PutWithTTL("large_key3", value_2k, 60).IsNoSpace());
|
|
1252
|
-
ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
|
|
1253
|
-
// Verify large_key2 still exists.
|
|
1254
|
-
VerifyDB(data);
|
|
1255
|
-
|
|
1256
|
-
SyncPoint::GetInstance()->DisableProcessing();
|
|
1257
|
-
}
|
|
1258
|
-
|
|
1259
|
-
// Test flush or compaction will trigger FIFO eviction since they update
|
|
1260
|
-
// total SST file size.
|
|
1261
|
-
TEST_F(BlobDBTest, FIFOEviction_TriggerOnSSTSizeChange) {
|
|
1262
|
-
BlobDBOptions bdb_options;
|
|
1263
|
-
bdb_options.max_db_size = 1000;
|
|
1264
|
-
bdb_options.is_fifo = true;
|
|
1265
|
-
bdb_options.min_blob_size = 100;
|
|
1266
|
-
bdb_options.disable_background_tasks = true;
|
|
1267
|
-
Options options;
|
|
1268
|
-
// Use mock env to stop wall clock.
|
|
1269
|
-
options.env = mock_env_.get();
|
|
1270
|
-
auto statistics = CreateDBStatistics();
|
|
1271
|
-
options.statistics = statistics;
|
|
1272
|
-
options.compression = kNoCompression;
|
|
1273
|
-
Open(bdb_options, options);
|
|
1274
|
-
|
|
1275
|
-
SyncPoint::GetInstance()->LoadDependency(
|
|
1276
|
-
{{"DBImpl::NotifyOnFlushCompleted::PostAllOnFlushCompleted",
|
|
1277
|
-
"BlobDBTest.FIFOEviction_TriggerOnSSTSizeChange:AfterFlush"}});
|
|
1278
|
-
|
|
1279
|
-
SyncPoint::GetInstance()->EnableProcessing();
|
|
1280
|
-
|
|
1281
|
-
std::string value(800, 'v');
|
|
1282
|
-
ASSERT_OK(PutWithTTL("large_key", value, 60));
|
|
1283
|
-
ASSERT_EQ(1, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
1284
|
-
ASSERT_EQ(0, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
|
|
1285
|
-
VerifyDB({{"large_key", value}});
|
|
1286
|
-
|
|
1287
|
-
// Insert some small keys and flush to bring DB out of space.
|
|
1288
|
-
std::map<std::string, std::string> data;
|
|
1289
|
-
for (int i = 0; i < 10; i++) {
|
|
1290
|
-
ASSERT_OK(Put("key" + std::to_string(i), "v", &data));
|
|
1291
|
-
}
|
|
1292
|
-
ASSERT_OK(blob_db_->Flush(FlushOptions()));
|
|
1293
|
-
|
|
1294
|
-
TEST_SYNC_POINT("BlobDBTest.FIFOEviction_TriggerOnSSTSizeChange:AfterFlush");
|
|
1295
|
-
|
|
1296
|
-
// Verify large_key is deleted by FIFO eviction.
|
|
1297
|
-
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
1298
|
-
ASSERT_EQ(0, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
1299
|
-
ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
|
|
1300
|
-
VerifyDB(data);
|
|
1301
|
-
|
|
1302
|
-
SyncPoint::GetInstance()->DisableProcessing();
|
|
1303
|
-
}
|
|
1304
|
-
|
|
1305
|
-
TEST_F(BlobDBTest, InlineSmallValues) {
|
|
1306
|
-
constexpr uint64_t kMaxExpiration = 1000;
|
|
1307
|
-
Random rnd(301);
|
|
1308
|
-
BlobDBOptions bdb_options;
|
|
1309
|
-
bdb_options.ttl_range_secs = kMaxExpiration;
|
|
1310
|
-
bdb_options.min_blob_size = 100;
|
|
1311
|
-
bdb_options.blob_file_size = 256 * 1000 * 1000;
|
|
1312
|
-
bdb_options.disable_background_tasks = true;
|
|
1313
|
-
Options options;
|
|
1314
|
-
options.env = mock_env_.get();
|
|
1315
|
-
mock_clock_->SetCurrentTime(0);
|
|
1316
|
-
Open(bdb_options, options);
|
|
1317
|
-
std::map<std::string, std::string> data;
|
|
1318
|
-
std::map<std::string, KeyVersion> versions;
|
|
1319
|
-
for (size_t i = 0; i < 1000; i++) {
|
|
1320
|
-
bool is_small_value = rnd.Next() % 2;
|
|
1321
|
-
bool has_ttl = rnd.Next() % 2;
|
|
1322
|
-
uint64_t expiration = rnd.Next() % kMaxExpiration;
|
|
1323
|
-
int len = is_small_value ? 50 : 200;
|
|
1324
|
-
std::string key = "key" + std::to_string(i);
|
|
1325
|
-
std::string value = rnd.HumanReadableString(len);
|
|
1326
|
-
std::string blob_index;
|
|
1327
|
-
data[key] = value;
|
|
1328
|
-
SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
|
|
1329
|
-
if (!has_ttl) {
|
|
1330
|
-
ASSERT_OK(blob_db_->Put(WriteOptions(), key, value));
|
|
1331
|
-
} else {
|
|
1332
|
-
ASSERT_OK(blob_db_->PutUntil(WriteOptions(), key, value, expiration));
|
|
1333
|
-
}
|
|
1334
|
-
ASSERT_EQ(blob_db_->GetLatestSequenceNumber(), sequence);
|
|
1335
|
-
versions[key] =
|
|
1336
|
-
KeyVersion(key, value, sequence,
|
|
1337
|
-
(is_small_value && !has_ttl) ? kTypeValue : kTypeBlobIndex);
|
|
1338
|
-
}
|
|
1339
|
-
VerifyDB(data);
|
|
1340
|
-
VerifyBaseDB(versions);
|
|
1341
|
-
auto *bdb_impl = static_cast<BlobDBImpl *>(blob_db_);
|
|
1342
|
-
auto blob_files = bdb_impl->TEST_GetBlobFiles();
|
|
1343
|
-
ASSERT_EQ(2, blob_files.size());
|
|
1344
|
-
std::shared_ptr<BlobFile> non_ttl_file;
|
|
1345
|
-
std::shared_ptr<BlobFile> ttl_file;
|
|
1346
|
-
if (blob_files[0]->HasTTL()) {
|
|
1347
|
-
ttl_file = blob_files[0];
|
|
1348
|
-
non_ttl_file = blob_files[1];
|
|
1349
|
-
} else {
|
|
1350
|
-
non_ttl_file = blob_files[0];
|
|
1351
|
-
ttl_file = blob_files[1];
|
|
1352
|
-
}
|
|
1353
|
-
ASSERT_FALSE(non_ttl_file->HasTTL());
|
|
1354
|
-
ASSERT_TRUE(ttl_file->HasTTL());
|
|
1355
|
-
}
|
|
1356
|
-
|
|
1357
895
|
TEST_F(BlobDBTest, UserCompactionFilter) {
|
|
1358
896
|
class CustomerFilter : public CompactionFilter {
|
|
1359
897
|
public:
|
|
1360
|
-
bool Filter(int /*level*/, const Slice
|
|
1361
|
-
std::string
|
|
898
|
+
bool Filter(int /*level*/, const Slice& /*key*/, const Slice& value,
|
|
899
|
+
std::string* new_value, bool* value_changed) const override {
|
|
1362
900
|
*value_changed = false;
|
|
1363
|
-
//
|
|
1364
|
-
// and stored-in-blob data
|
|
901
|
+
// Test compaction filter modifying blob values
|
|
1365
902
|
if (value.size() % 4 == 1) {
|
|
1366
903
|
*new_value = value.ToString();
|
|
1367
904
|
// double size by duplicating value
|
|
@@ -1380,31 +917,23 @@ TEST_F(BlobDBTest, UserCompactionFilter) {
|
|
|
1380
917
|
return false;
|
|
1381
918
|
}
|
|
1382
919
|
bool IgnoreSnapshots() const override { return true; }
|
|
1383
|
-
const char
|
|
920
|
+
const char* Name() const override { return "CustomerFilter"; }
|
|
1384
921
|
};
|
|
1385
922
|
class CustomerFilterFactory : public CompactionFilterFactory {
|
|
1386
|
-
const char
|
|
923
|
+
const char* Name() const override { return "CustomerFilterFactory"; }
|
|
1387
924
|
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
|
1388
|
-
const CompactionFilter::Context
|
|
925
|
+
const CompactionFilter::Context& /*context*/) override {
|
|
1389
926
|
return std::unique_ptr<CompactionFilter>(new CustomerFilter());
|
|
1390
927
|
}
|
|
1391
928
|
};
|
|
1392
929
|
|
|
1393
930
|
constexpr size_t kNumPuts = 1 << 10;
|
|
1394
|
-
// Generate both inlined and blob value
|
|
1395
931
|
constexpr uint64_t kMinValueSize = 1 << 6;
|
|
1396
932
|
constexpr uint64_t kMaxValueSize = 1 << 8;
|
|
1397
|
-
constexpr uint64_t kMinBlobSize = 1 << 7;
|
|
1398
|
-
static_assert(kMinValueSize < kMinBlobSize);
|
|
1399
|
-
static_assert(kMaxValueSize > kMinBlobSize);
|
|
1400
933
|
|
|
1401
934
|
BlobDBOptions bdb_options;
|
|
1402
|
-
bdb_options.min_blob_size = kMinBlobSize;
|
|
1403
935
|
bdb_options.blob_file_size = kMaxValueSize * 10;
|
|
1404
936
|
bdb_options.disable_background_tasks = true;
|
|
1405
|
-
if (Snappy_Supported()) {
|
|
1406
|
-
bdb_options.compression = CompressionType::kSnappyCompression;
|
|
1407
|
-
}
|
|
1408
937
|
// case_num == 0: Test user defined compaction filter
|
|
1409
938
|
// case_num == 1: Test user defined compaction filter factory
|
|
1410
939
|
for (int case_num = 0; case_num < 2; case_num++) {
|
|
@@ -1467,24 +996,22 @@ TEST_F(BlobDBTest, UserCompactionFilter) {
|
|
|
1467
996
|
TEST_F(BlobDBTest, UserCompactionFilter_BlobIOError) {
|
|
1468
997
|
class CustomerFilter : public CompactionFilter {
|
|
1469
998
|
public:
|
|
1470
|
-
bool Filter(int /*level*/, const Slice
|
|
1471
|
-
std::string
|
|
999
|
+
bool Filter(int /*level*/, const Slice& /*key*/, const Slice& value,
|
|
1000
|
+
std::string* new_value, bool* value_changed) const override {
|
|
1472
1001
|
*new_value = value.ToString() + "_new";
|
|
1473
1002
|
*value_changed = true;
|
|
1474
1003
|
return false;
|
|
1475
1004
|
}
|
|
1476
1005
|
bool IgnoreSnapshots() const override { return true; }
|
|
1477
|
-
const char
|
|
1006
|
+
const char* Name() const override { return "CustomerFilter"; }
|
|
1478
1007
|
};
|
|
1479
1008
|
|
|
1480
1009
|
constexpr size_t kNumPuts = 100;
|
|
1481
1010
|
constexpr int kValueSize = 100;
|
|
1482
1011
|
|
|
1483
1012
|
BlobDBOptions bdb_options;
|
|
1484
|
-
bdb_options.min_blob_size = 0;
|
|
1485
1013
|
bdb_options.blob_file_size = kValueSize * 10;
|
|
1486
1014
|
bdb_options.disable_background_tasks = true;
|
|
1487
|
-
bdb_options.compression = CompressionType::kNoCompression;
|
|
1488
1015
|
|
|
1489
1016
|
std::vector<std::string> io_failure_cases = {
|
|
1490
1017
|
"BlobDBImpl::CreateBlobFileAndWriter",
|
|
@@ -1518,7 +1045,7 @@ TEST_F(BlobDBTest, UserCompactionFilter_BlobIOError) {
|
|
|
1518
1045
|
VerifyDB(data);
|
|
1519
1046
|
|
|
1520
1047
|
SyncPoint::GetInstance()->SetCallBack(
|
|
1521
|
-
io_failure_cases[case_num], [&](void
|
|
1048
|
+
io_failure_cases[case_num], [&](void* /*arg*/) {
|
|
1522
1049
|
fault_injection_env_->SetFilesystemActive(false, Status::IOError());
|
|
1523
1050
|
});
|
|
1524
1051
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
@@ -1542,13 +1069,11 @@ TEST_F(BlobDBTest, UserCompactionFilter_BlobIOError) {
|
|
|
1542
1069
|
TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
|
|
1543
1070
|
constexpr size_t kNumKeys = 100;
|
|
1544
1071
|
constexpr size_t kNumPuts = 1000;
|
|
1545
|
-
constexpr uint64_t
|
|
1072
|
+
constexpr uint64_t kMaxTTL = 1000;
|
|
1546
1073
|
constexpr uint64_t kCompactTime = 500;
|
|
1547
|
-
constexpr uint64_t kMinBlobSize = 100;
|
|
1548
1074
|
Random rnd(301);
|
|
1549
1075
|
mock_clock_->SetCurrentTime(0);
|
|
1550
1076
|
BlobDBOptions bdb_options;
|
|
1551
|
-
bdb_options.min_blob_size = kMinBlobSize;
|
|
1552
1077
|
bdb_options.disable_background_tasks = true;
|
|
1553
1078
|
Options options;
|
|
1554
1079
|
options.env = mock_env_.get();
|
|
@@ -1557,25 +1082,18 @@ TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
|
|
|
1557
1082
|
std::map<std::string, std::string> data;
|
|
1558
1083
|
std::map<std::string, std::string> data_after_compact;
|
|
1559
1084
|
for (size_t i = 0; i < kNumPuts; i++) {
|
|
1560
|
-
bool is_small_value = rnd.Next() % 2;
|
|
1561
1085
|
bool has_ttl = rnd.Next() % 2;
|
|
1562
|
-
|
|
1563
|
-
|
|
1086
|
+
// At time 0, stored expiration equals TTL
|
|
1087
|
+
uint64_t ttl = rnd.Next() % kMaxTTL;
|
|
1088
|
+
int len = rnd.Next() % 200 + 10;
|
|
1564
1089
|
std::string key = "key" + std::to_string(rnd.Next() % kNumKeys);
|
|
1565
1090
|
std::string value = rnd.HumanReadableString(len);
|
|
1566
1091
|
if (!has_ttl) {
|
|
1567
|
-
if (is_small_value) {
|
|
1568
|
-
std::string blob_entry;
|
|
1569
|
-
BlobIndex::EncodeInlinedTTL(&blob_entry, expiration, value);
|
|
1570
|
-
// Fake blob index with TTL. See what it will do.
|
|
1571
|
-
ASSERT_GT(kMinBlobSize, blob_entry.size());
|
|
1572
|
-
value = blob_entry;
|
|
1573
|
-
}
|
|
1574
1092
|
ASSERT_OK(Put(key, value));
|
|
1575
1093
|
data_after_compact[key] = value;
|
|
1576
1094
|
} else {
|
|
1577
|
-
ASSERT_OK(
|
|
1578
|
-
if (
|
|
1095
|
+
ASSERT_OK(blob_db_->PutWithTTL(WriteOptions(), key, value, ttl));
|
|
1096
|
+
if (ttl <= kCompactTime) {
|
|
1579
1097
|
data_after_compact.erase(key);
|
|
1580
1098
|
} else {
|
|
1581
1099
|
data_after_compact[key] = value;
|
|
@@ -1588,7 +1106,7 @@ TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
|
|
|
1588
1106
|
mock_clock_->SetCurrentTime(kCompactTime);
|
|
1589
1107
|
// Take a snapshot before compaction. Make sure expired blob indexes is
|
|
1590
1108
|
// filtered regardless of snapshot.
|
|
1591
|
-
const Snapshot
|
|
1109
|
+
const Snapshot* snapshot = blob_db_->GetSnapshot();
|
|
1592
1110
|
// Issue manual compaction to trigger compaction filter.
|
|
1593
1111
|
ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
1594
1112
|
blob_db_->ReleaseSnapshot(snapshot);
|
|
@@ -1597,7 +1115,7 @@ TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
|
|
|
1597
1115
|
const size_t kMaxKeys = 10000;
|
|
1598
1116
|
ASSERT_OK(GetAllKeyVersions(blob_db_, {}, {}, kMaxKeys, &versions));
|
|
1599
1117
|
ASSERT_EQ(data_after_compact.size(), versions.size());
|
|
1600
|
-
for (auto
|
|
1118
|
+
for (auto& version : versions) {
|
|
1601
1119
|
ASSERT_TRUE(data_after_compact.count(version.user_key) > 0);
|
|
1602
1120
|
}
|
|
1603
1121
|
VerifyDB(data_after_compact);
|
|
@@ -1607,7 +1125,6 @@ TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
|
|
|
1607
1125
|
// blob file has been removed.
|
|
1608
1126
|
TEST_F(BlobDBTest, FilterFileNotAvailable) {
|
|
1609
1127
|
BlobDBOptions bdb_options;
|
|
1610
|
-
bdb_options.min_blob_size = 0;
|
|
1611
1128
|
bdb_options.disable_background_tasks = true;
|
|
1612
1129
|
Options options;
|
|
1613
1130
|
options.disable_auto_compactions = true;
|
|
@@ -1627,7 +1144,7 @@ TEST_F(BlobDBTest, FilterFileNotAvailable) {
|
|
|
1627
1144
|
|
|
1628
1145
|
const size_t kMaxKeys = 10000;
|
|
1629
1146
|
|
|
1630
|
-
DB
|
|
1147
|
+
DB* base_db = blob_db_->GetRootDB();
|
|
1631
1148
|
std::vector<KeyVersion> versions;
|
|
1632
1149
|
ASSERT_OK(GetAllKeyVersions(base_db, {}, {}, kMaxKeys, &versions));
|
|
1633
1150
|
ASSERT_EQ(2, versions.size());
|
|
@@ -1660,127 +1177,25 @@ TEST_F(BlobDBTest, FilterFileNotAvailable) {
|
|
|
1660
1177
|
VerifyDB({});
|
|
1661
1178
|
}
|
|
1662
1179
|
|
|
1663
|
-
// Test compaction filter should filter any inlined TTL keys that would have
|
|
1664
|
-
// been dropped by last FIFO eviction if they are store out-of-line.
|
|
1665
|
-
TEST_F(BlobDBTest, FilterForFIFOEviction) {
|
|
1666
|
-
Random rnd(215);
|
|
1667
|
-
BlobDBOptions bdb_options;
|
|
1668
|
-
bdb_options.min_blob_size = 100;
|
|
1669
|
-
bdb_options.ttl_range_secs = 60;
|
|
1670
|
-
bdb_options.max_db_size = 0;
|
|
1671
|
-
bdb_options.disable_background_tasks = true;
|
|
1672
|
-
Options options;
|
|
1673
|
-
// Use mock env to stop wall clock.
|
|
1674
|
-
mock_clock_->SetCurrentTime(0);
|
|
1675
|
-
options.env = mock_env_.get();
|
|
1676
|
-
auto statistics = CreateDBStatistics();
|
|
1677
|
-
options.statistics = statistics;
|
|
1678
|
-
options.disable_auto_compactions = true;
|
|
1679
|
-
Open(bdb_options, options);
|
|
1680
|
-
|
|
1681
|
-
SyncPoint::GetInstance()->LoadDependency(
|
|
1682
|
-
{{"DBImpl::NotifyOnFlushCompleted::PostAllOnFlushCompleted",
|
|
1683
|
-
"BlobDBTest.FilterForFIFOEviction:AfterFlush"}});
|
|
1684
|
-
|
|
1685
|
-
SyncPoint::GetInstance()->EnableProcessing();
|
|
1686
|
-
|
|
1687
|
-
std::map<std::string, std::string> data;
|
|
1688
|
-
std::map<std::string, std::string> data_after_compact;
|
|
1689
|
-
// Insert some small values that will be inlined.
|
|
1690
|
-
for (int i = 0; i < 1000; i++) {
|
|
1691
|
-
std::string key = "key" + std::to_string(i);
|
|
1692
|
-
std::string value = rnd.HumanReadableString(50);
|
|
1693
|
-
uint64_t ttl = rnd.Next() % 120 + 1;
|
|
1694
|
-
ASSERT_OK(PutWithTTL(key, value, ttl, &data));
|
|
1695
|
-
if (ttl >= 60) {
|
|
1696
|
-
data_after_compact[key] = value;
|
|
1697
|
-
}
|
|
1698
|
-
}
|
|
1699
|
-
uint64_t num_keys_to_evict = data.size() - data_after_compact.size();
|
|
1700
|
-
ASSERT_OK(blob_db_->Flush(FlushOptions()));
|
|
1701
|
-
|
|
1702
|
-
TEST_SYNC_POINT("BlobDBTest.FilterForFIFOEviction:AfterFlush");
|
|
1703
|
-
|
|
1704
|
-
uint64_t live_sst_size = blob_db_impl()->TEST_live_sst_size();
|
|
1705
|
-
ASSERT_GT(live_sst_size, 0);
|
|
1706
|
-
VerifyDB(data);
|
|
1707
|
-
|
|
1708
|
-
bdb_options.max_db_size = live_sst_size + 30000;
|
|
1709
|
-
bdb_options.is_fifo = true;
|
|
1710
|
-
Reopen(bdb_options, options);
|
|
1711
|
-
VerifyDB(data);
|
|
1712
|
-
|
|
1713
|
-
// Put two large values, each on a different blob file.
|
|
1714
|
-
std::string large_value(10000, 'v');
|
|
1715
|
-
ASSERT_OK(PutWithTTL("large_key1", large_value, 90));
|
|
1716
|
-
ASSERT_OK(PutWithTTL("large_key2", large_value, 150));
|
|
1717
|
-
ASSERT_EQ(2, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
1718
|
-
ASSERT_EQ(0, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
|
|
1719
|
-
data["large_key1"] = large_value;
|
|
1720
|
-
data["large_key2"] = large_value;
|
|
1721
|
-
VerifyDB(data);
|
|
1722
|
-
|
|
1723
|
-
// Put a third large value which will bring the DB out of space.
|
|
1724
|
-
// FIFO eviction will evict the file of large_key1.
|
|
1725
|
-
ASSERT_OK(PutWithTTL("large_key3", large_value, 150));
|
|
1726
|
-
ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
|
|
1727
|
-
ASSERT_EQ(2, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
1728
|
-
blob_db_impl()->TEST_DeleteObsoleteFiles();
|
|
1729
|
-
ASSERT_EQ(1, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
1730
|
-
data.erase("large_key1");
|
|
1731
|
-
data["large_key3"] = large_value;
|
|
1732
|
-
VerifyDB(data);
|
|
1733
|
-
|
|
1734
|
-
// Putting some more small values. These values shouldn't be evicted by
|
|
1735
|
-
// compaction filter since they are inserted after FIFO eviction.
|
|
1736
|
-
ASSERT_OK(PutWithTTL("foo", "v", 30, &data_after_compact));
|
|
1737
|
-
ASSERT_OK(PutWithTTL("bar", "v", 30, &data_after_compact));
|
|
1738
|
-
|
|
1739
|
-
// FIFO eviction doesn't trigger again since there enough room for the flush.
|
|
1740
|
-
ASSERT_OK(blob_db_->Flush(FlushOptions()));
|
|
1741
|
-
ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
|
|
1742
|
-
|
|
1743
|
-
// Manual compact and check if compaction filter evict those keys with
|
|
1744
|
-
// expiration < 60.
|
|
1745
|
-
ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
1746
|
-
// All keys with expiration < 60, plus large_key1 is filtered by
|
|
1747
|
-
// compaction filter.
|
|
1748
|
-
ASSERT_EQ(num_keys_to_evict + 1,
|
|
1749
|
-
statistics->getTickerCount(BLOB_DB_BLOB_INDEX_EVICTED_COUNT));
|
|
1750
|
-
ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
|
|
1751
|
-
ASSERT_EQ(1, blob_db_impl()->TEST_GetBlobFiles().size());
|
|
1752
|
-
data_after_compact["large_key2"] = large_value;
|
|
1753
|
-
data_after_compact["large_key3"] = large_value;
|
|
1754
|
-
VerifyDB(data_after_compact);
|
|
1755
|
-
|
|
1756
|
-
SyncPoint::GetInstance()->DisableProcessing();
|
|
1757
|
-
}
|
|
1758
|
-
|
|
1759
1180
|
TEST_F(BlobDBTest, GarbageCollection) {
|
|
1760
1181
|
constexpr size_t kNumPuts = 1 << 10;
|
|
1761
1182
|
|
|
1762
|
-
|
|
1183
|
+
// At time 0, stored expiration equals TTL
|
|
1184
|
+
constexpr uint64_t kTTL = 1000;
|
|
1763
1185
|
constexpr uint64_t kCompactTime = 500;
|
|
1764
1186
|
|
|
1765
1187
|
constexpr uint64_t kKeySize = 7; // "key" + 4 digits
|
|
1766
|
-
|
|
1767
|
-
constexpr uint64_t kSmallValueSize = 1 << 6;
|
|
1768
|
-
constexpr uint64_t kLargeValueSize = 1 << 8;
|
|
1769
|
-
constexpr uint64_t kMinBlobSize = 1 << 7;
|
|
1770
|
-
static_assert(kSmallValueSize < kMinBlobSize);
|
|
1771
|
-
static_assert(kLargeValueSize > kMinBlobSize);
|
|
1188
|
+
constexpr uint64_t kValueSize = 1 << 8;
|
|
1772
1189
|
|
|
1773
1190
|
constexpr size_t kBlobsPerFile = 8;
|
|
1774
1191
|
constexpr size_t kNumBlobFiles = kNumPuts / kBlobsPerFile;
|
|
1775
1192
|
constexpr uint64_t kBlobFileSize =
|
|
1776
1193
|
BlobLogHeader::kSize +
|
|
1777
|
-
(BlobLogRecord::kHeaderSize + kKeySize +
|
|
1194
|
+
(BlobLogRecord::kHeaderSize + kKeySize + kValueSize) * kBlobsPerFile;
|
|
1778
1195
|
|
|
1779
1196
|
BlobDBOptions bdb_options;
|
|
1780
|
-
bdb_options.min_blob_size = kMinBlobSize;
|
|
1781
1197
|
bdb_options.blob_file_size = kBlobFileSize;
|
|
1782
1198
|
bdb_options.enable_garbage_collection = true;
|
|
1783
|
-
bdb_options.garbage_collection_cutoff = 0.25;
|
|
1784
1199
|
bdb_options.disable_background_tasks = true;
|
|
1785
1200
|
|
|
1786
1201
|
Options options;
|
|
@@ -1795,14 +1210,14 @@ TEST_F(BlobDBTest, GarbageCollection) {
|
|
|
1795
1210
|
|
|
1796
1211
|
Random rnd(301);
|
|
1797
1212
|
|
|
1798
|
-
// Add a bunch of
|
|
1213
|
+
// Add a bunch of non-TTL values. These will be written to non-TTL
|
|
1799
1214
|
// blob files and will be subject to GC.
|
|
1800
1215
|
for (size_t i = 0; i < kNumPuts; ++i) {
|
|
1801
1216
|
std::ostringstream oss;
|
|
1802
1217
|
oss << "key" << std::setw(4) << std::setfill('0') << i;
|
|
1803
1218
|
|
|
1804
1219
|
const std::string key(oss.str());
|
|
1805
|
-
const std::string value = rnd.HumanReadableString(
|
|
1220
|
+
const std::string value = rnd.HumanReadableString(kValueSize);
|
|
1806
1221
|
const SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
|
|
1807
1222
|
|
|
1808
1223
|
ASSERT_OK(Put(key, value));
|
|
@@ -1815,54 +1230,23 @@ TEST_F(BlobDBTest, GarbageCollection) {
|
|
|
1815
1230
|
sequence, kTypeBlobIndex);
|
|
1816
1231
|
}
|
|
1817
1232
|
|
|
1818
|
-
// Add
|
|
1819
|
-
//
|
|
1233
|
+
// Add a TTL value that will be written to its own TTL blob file (ignored
|
|
1234
|
+
// during GC).
|
|
1820
1235
|
{
|
|
1821
1236
|
const std::string key("key2000");
|
|
1822
|
-
const std::string value = rnd.HumanReadableString(
|
|
1237
|
+
const std::string value = rnd.HumanReadableString(kValueSize);
|
|
1823
1238
|
const SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
|
|
1824
1239
|
|
|
1825
|
-
ASSERT_OK(
|
|
1240
|
+
ASSERT_OK(blob_db_->PutWithTTL(WriteOptions(), key, value, kTTL));
|
|
1826
1241
|
ASSERT_EQ(blob_db_->GetLatestSequenceNumber(), sequence);
|
|
1827
1242
|
|
|
1828
1243
|
data[key] = value;
|
|
1829
1244
|
blob_value_versions[key] = KeyVersion(key, value, sequence, kTypeBlobIndex);
|
|
1830
1245
|
blob_index_versions[key] =
|
|
1831
|
-
BlobIndexVersion(key, /* file_number */ kNumBlobFiles + 1,
|
|
1246
|
+
BlobIndexVersion(key, /* file_number */ kNumBlobFiles + 1, kTTL,
|
|
1832
1247
|
sequence, kTypeBlobIndex);
|
|
1833
1248
|
}
|
|
1834
1249
|
|
|
1835
|
-
// Now add a small TTL value (which will be inlined).
|
|
1836
|
-
{
|
|
1837
|
-
const std::string key("key3000");
|
|
1838
|
-
const std::string value = rnd.HumanReadableString(kSmallValueSize);
|
|
1839
|
-
const SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
|
|
1840
|
-
|
|
1841
|
-
ASSERT_OK(PutUntil(key, value, kExpiration));
|
|
1842
|
-
ASSERT_EQ(blob_db_->GetLatestSequenceNumber(), sequence);
|
|
1843
|
-
|
|
1844
|
-
data[key] = value;
|
|
1845
|
-
blob_value_versions[key] = KeyVersion(key, value, sequence, kTypeBlobIndex);
|
|
1846
|
-
blob_index_versions[key] = BlobIndexVersion(
|
|
1847
|
-
key, kInvalidBlobFileNumber, kExpiration, sequence, kTypeBlobIndex);
|
|
1848
|
-
}
|
|
1849
|
-
|
|
1850
|
-
// Finally, add a small non-TTL value (which will be stored as a regular
|
|
1851
|
-
// value).
|
|
1852
|
-
{
|
|
1853
|
-
const std::string key("key4000");
|
|
1854
|
-
const std::string value = rnd.HumanReadableString(kSmallValueSize);
|
|
1855
|
-
const SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
|
|
1856
|
-
|
|
1857
|
-
ASSERT_OK(Put(key, value));
|
|
1858
|
-
ASSERT_EQ(blob_db_->GetLatestSequenceNumber(), sequence);
|
|
1859
|
-
|
|
1860
|
-
data[key] = value;
|
|
1861
|
-
blob_value_versions[key] = KeyVersion(key, value, sequence, kTypeValue);
|
|
1862
|
-
blob_index_versions[key] = BlobIndexVersion(
|
|
1863
|
-
key, kInvalidBlobFileNumber, kNoExpiration, sequence, kTypeValue);
|
|
1864
|
-
}
|
|
1865
|
-
|
|
1866
1250
|
VerifyDB(data);
|
|
1867
1251
|
VerifyBaseDB(blob_value_versions);
|
|
1868
1252
|
VerifyBaseDBBlobIndex(blob_index_versions);
|
|
@@ -1888,17 +1272,18 @@ TEST_F(BlobDBTest, GarbageCollection) {
|
|
|
1888
1272
|
// compaction.
|
|
1889
1273
|
VerifyDB(data);
|
|
1890
1274
|
|
|
1891
|
-
for (auto
|
|
1892
|
-
KeyVersion
|
|
1275
|
+
for (auto& pair : blob_value_versions) {
|
|
1276
|
+
KeyVersion& version = pair.second;
|
|
1893
1277
|
version.sequence = 0;
|
|
1894
1278
|
}
|
|
1895
1279
|
|
|
1896
1280
|
VerifyBaseDB(blob_value_versions);
|
|
1897
1281
|
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1282
|
+
// GC cutoff is fixed at 0.25
|
|
1283
|
+
constexpr double kGCCutoff = 0.25;
|
|
1284
|
+
const uint64_t cutoff = static_cast<uint64_t>(kGCCutoff * kNumBlobFiles);
|
|
1285
|
+
for (auto& pair : blob_index_versions) {
|
|
1286
|
+
BlobIndexVersion& version = pair.second;
|
|
1902
1287
|
|
|
1903
1288
|
version.sequence = 0;
|
|
1904
1289
|
|
|
@@ -1915,7 +1300,7 @@ TEST_F(BlobDBTest, GarbageCollection) {
|
|
|
1915
1300
|
|
|
1916
1301
|
VerifyBaseDBBlobIndex(blob_index_versions);
|
|
1917
1302
|
|
|
1918
|
-
const Statistics
|
|
1303
|
+
const Statistics* const statistics = options.statistics.get();
|
|
1919
1304
|
assert(statistics);
|
|
1920
1305
|
|
|
1921
1306
|
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_NUM_FILES), cutoff);
|
|
@@ -1924,7 +1309,7 @@ TEST_F(BlobDBTest, GarbageCollection) {
|
|
|
1924
1309
|
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_NUM_KEYS_RELOCATED),
|
|
1925
1310
|
cutoff * kBlobsPerFile);
|
|
1926
1311
|
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_BYTES_RELOCATED),
|
|
1927
|
-
cutoff * kBlobsPerFile *
|
|
1312
|
+
cutoff * kBlobsPerFile * kValueSize);
|
|
1928
1313
|
|
|
1929
1314
|
// At this point, we should have 128 immutable non-TTL files with file numbers
|
|
1930
1315
|
// 33..128 and 130..161. (129 was taken by the TTL blob file.)
|
|
@@ -1946,9 +1331,7 @@ TEST_F(BlobDBTest, GarbageCollection) {
|
|
|
1946
1331
|
|
|
1947
1332
|
TEST_F(BlobDBTest, GarbageCollectionFailure) {
|
|
1948
1333
|
BlobDBOptions bdb_options;
|
|
1949
|
-
bdb_options.min_blob_size = 0;
|
|
1950
1334
|
bdb_options.enable_garbage_collection = true;
|
|
1951
|
-
bdb_options.garbage_collection_cutoff = 1.0;
|
|
1952
1335
|
bdb_options.disable_background_tasks = true;
|
|
1953
1336
|
|
|
1954
1337
|
Options db_options;
|
|
@@ -1956,14 +1339,31 @@ TEST_F(BlobDBTest, GarbageCollectionFailure) {
|
|
|
1956
1339
|
|
|
1957
1340
|
Open(bdb_options, db_options);
|
|
1958
1341
|
|
|
1959
|
-
//
|
|
1342
|
+
// Create 4 blob files. With fixed GC cutoff of 0.25, the oldest file
|
|
1343
|
+
// (floor(0.25 * 4) = 1) will be in the GC zone.
|
|
1344
|
+
// The first file contains valid blobs for "foo" and "dead".
|
|
1960
1345
|
ASSERT_OK(Put("foo", "bar"));
|
|
1961
1346
|
ASSERT_OK(Put("dead", "beef"));
|
|
1962
1347
|
|
|
1963
|
-
|
|
1964
|
-
|
|
1348
|
+
auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
1349
|
+
ASSERT_EQ(blob_files.size(), 1);
|
|
1350
|
+
auto first_file = blob_files[0];
|
|
1351
|
+
uint64_t first_file_number = first_file->BlobFileNumber();
|
|
1352
|
+
ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(first_file));
|
|
1353
|
+
|
|
1354
|
+
// Create 3 more blob files (files 2-4, outside GC zone).
|
|
1355
|
+
for (int i = 1; i < 4; i++) {
|
|
1356
|
+
ASSERT_OK(Put("key" + std::to_string(i), "value"));
|
|
1357
|
+
blob_files = blob_db_impl()->TEST_GetBlobFiles();
|
|
1358
|
+
ASSERT_EQ(static_cast<size_t>(i + 1), blob_files.size());
|
|
1359
|
+
ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_files[i]));
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
// Write a fake blob index that points to the first file (in GC zone)
|
|
1363
|
+
// but with an invalid offset beyond the file size. This will cause
|
|
1364
|
+
// GC to fail when it tries to read this blob.
|
|
1965
1365
|
std::string blob_index;
|
|
1966
|
-
BlobIndex::EncodeBlob(&blob_index,
|
|
1366
|
+
BlobIndex::EncodeBlob(&blob_index, first_file_number, /* offset */ 999999,
|
|
1967
1367
|
/* size */ 5678, kNoCompression);
|
|
1968
1368
|
|
|
1969
1369
|
WriteBatch batch;
|
|
@@ -1971,17 +1371,17 @@ TEST_F(BlobDBTest, GarbageCollectionFailure) {
|
|
|
1971
1371
|
&batch, blob_db_->DefaultColumnFamily()->GetID(), "key", blob_index));
|
|
1972
1372
|
ASSERT_OK(blob_db_->GetRootDB()->Write(WriteOptions(), &batch));
|
|
1973
1373
|
|
|
1974
|
-
|
|
1975
|
-
ASSERT_EQ(blob_files.size(), 1);
|
|
1976
|
-
auto blob_file = blob_files[0];
|
|
1977
|
-
ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_file));
|
|
1978
|
-
|
|
1374
|
+
// Verify compaction fails with IO error due to invalid blob offset.
|
|
1979
1375
|
ASSERT_TRUE(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)
|
|
1980
1376
|
.IsIOError());
|
|
1981
1377
|
|
|
1982
|
-
const Statistics
|
|
1378
|
+
const Statistics* const statistics = db_options.statistics.get();
|
|
1983
1379
|
assert(statistics);
|
|
1984
1380
|
|
|
1381
|
+
// Verify GC statistics:
|
|
1382
|
+
// - Relocated 2 keys ("foo" and "dead") with 7 bytes ("bar" + "beef")
|
|
1383
|
+
// - Failed on "key" which has invalid blob offset
|
|
1384
|
+
// - Created 1 new GC output file before failing
|
|
1985
1385
|
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_NUM_FILES), 0);
|
|
1986
1386
|
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_NUM_NEW_FILES), 1);
|
|
1987
1387
|
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_FAILURES), 1);
|
|
@@ -1993,7 +1393,6 @@ TEST_F(BlobDBTest, GarbageCollectionFailure) {
|
|
|
1993
1393
|
TEST_F(BlobDBTest, EvictExpiredFile) {
|
|
1994
1394
|
BlobDBOptions bdb_options;
|
|
1995
1395
|
bdb_options.ttl_range_secs = 100;
|
|
1996
|
-
bdb_options.min_blob_size = 0;
|
|
1997
1396
|
bdb_options.disable_background_tasks = true;
|
|
1998
1397
|
Options options;
|
|
1999
1398
|
options.env = mock_env_.get();
|
|
@@ -2116,7 +1515,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
|
|
|
2116
1515
|
const std::vector<bool> expected_obsolete{false, false, false, false,
|
|
2117
1516
|
false};
|
|
2118
1517
|
for (size_t i = 0; i < 5; ++i) {
|
|
2119
|
-
const auto
|
|
1518
|
+
const auto& blob_file = blob_files[i];
|
|
2120
1519
|
ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
|
|
2121
1520
|
ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
|
|
2122
1521
|
}
|
|
@@ -2144,7 +1543,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
|
|
|
2144
1543
|
const std::vector<bool> expected_obsolete{false, false, false, false,
|
|
2145
1544
|
false};
|
|
2146
1545
|
for (size_t i = 0; i < 5; ++i) {
|
|
2147
|
-
const auto
|
|
1546
|
+
const auto& blob_file = blob_files[i];
|
|
2148
1547
|
ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
|
|
2149
1548
|
ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
|
|
2150
1549
|
}
|
|
@@ -2173,7 +1572,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
|
|
|
2173
1572
|
const std::vector<bool> expected_obsolete{false, false, false, false,
|
|
2174
1573
|
false};
|
|
2175
1574
|
for (size_t i = 0; i < 5; ++i) {
|
|
2176
|
-
const auto
|
|
1575
|
+
const auto& blob_file = blob_files[i];
|
|
2177
1576
|
ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
|
|
2178
1577
|
ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
|
|
2179
1578
|
}
|
|
@@ -2211,7 +1610,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
|
|
|
2211
1610
|
{}, {7}, {3, 8, 23}, {4, 9}, {5, 10, 22}};
|
|
2212
1611
|
const std::vector<bool> expected_obsolete{true, false, false, false, false};
|
|
2213
1612
|
for (size_t i = 0; i < 5; ++i) {
|
|
2214
|
-
const auto
|
|
1613
|
+
const auto& blob_file = blob_files[i];
|
|
2215
1614
|
ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
|
|
2216
1615
|
ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
|
|
2217
1616
|
}
|
|
@@ -2241,7 +1640,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
|
|
|
2241
1640
|
{}, {7}, {3, 8, 23}, {4, 9}, {5, 10, 22}};
|
|
2242
1641
|
const std::vector<bool> expected_obsolete{true, false, false, false, false};
|
|
2243
1642
|
for (size_t i = 0; i < 5; ++i) {
|
|
2244
|
-
const auto
|
|
1643
|
+
const auto& blob_file = blob_files[i];
|
|
2245
1644
|
ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
|
|
2246
1645
|
ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
|
|
2247
1646
|
}
|
|
@@ -2272,7 +1671,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
|
|
|
2272
1671
|
{}, {}, {3, 8, 23, 25}, {4, 9}, {5, 10}};
|
|
2273
1672
|
const std::vector<bool> expected_obsolete{true, false, false, false, false};
|
|
2274
1673
|
for (size_t i = 0; i < 5; ++i) {
|
|
2275
|
-
const auto
|
|
1674
|
+
const auto& blob_file = blob_files[i];
|
|
2276
1675
|
ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
|
|
2277
1676
|
ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
|
|
2278
1677
|
}
|
|
@@ -2302,7 +1701,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
|
|
|
2302
1701
|
{}, {}, {3, 8, 23, 25}, {4, 9}, {5, 10}};
|
|
2303
1702
|
const std::vector<bool> expected_obsolete{true, true, false, false, false};
|
|
2304
1703
|
for (size_t i = 0; i < 5; ++i) {
|
|
2305
|
-
const auto
|
|
1704
|
+
const auto& blob_file = blob_files[i];
|
|
2306
1705
|
ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
|
|
2307
1706
|
ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
|
|
2308
1707
|
}
|
|
@@ -2323,7 +1722,6 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
|
|
|
2323
1722
|
TEST_F(BlobDBTest, ShutdownWait) {
|
|
2324
1723
|
BlobDBOptions bdb_options;
|
|
2325
1724
|
bdb_options.ttl_range_secs = 100;
|
|
2326
|
-
bdb_options.min_blob_size = 0;
|
|
2327
1725
|
bdb_options.disable_background_tasks = false;
|
|
2328
1726
|
Options options;
|
|
2329
1727
|
options.env = mock_env_.get();
|
|
@@ -2336,15 +1734,15 @@ TEST_F(BlobDBTest, ShutdownWait) {
|
|
|
2336
1734
|
});
|
|
2337
1735
|
// Force all tasks to be scheduled immediately.
|
|
2338
1736
|
SyncPoint::GetInstance()->SetCallBack(
|
|
2339
|
-
"TimeQueue::Add:item.end", [&](void
|
|
2340
|
-
std::chrono::steady_clock::time_point
|
|
2341
|
-
static_cast<std::chrono::steady_clock::time_point
|
|
1737
|
+
"TimeQueue::Add:item.end", [&](void* arg) {
|
|
1738
|
+
std::chrono::steady_clock::time_point* tp =
|
|
1739
|
+
static_cast<std::chrono::steady_clock::time_point*>(arg);
|
|
2342
1740
|
*tp =
|
|
2343
1741
|
std::chrono::steady_clock::now() - std::chrono::milliseconds(10000);
|
|
2344
1742
|
});
|
|
2345
1743
|
|
|
2346
1744
|
SyncPoint::GetInstance()->SetCallBack(
|
|
2347
|
-
"BlobDBImpl::EvictExpiredFiles:cb", [&](void
|
|
1745
|
+
"BlobDBImpl::EvictExpiredFiles:cb", [&](void* /*arg*/) {
|
|
2348
1746
|
// Sleep 3 ms to increase the chance of data race.
|
|
2349
1747
|
// We've synced up the code so that EvictExpiredFiles()
|
|
2350
1748
|
// is called concurrently with ~BlobDBImpl().
|
|
@@ -2387,8 +1785,6 @@ TEST_F(BlobDBTest, SyncBlobFileBeforeClose) {
|
|
|
2387
1785
|
options.statistics = CreateDBStatistics();
|
|
2388
1786
|
|
|
2389
1787
|
BlobDBOptions blob_options;
|
|
2390
|
-
blob_options.min_blob_size = 0;
|
|
2391
|
-
blob_options.bytes_per_sync = 1 << 20;
|
|
2392
1788
|
blob_options.disable_background_tasks = true;
|
|
2393
1789
|
|
|
2394
1790
|
Open(blob_options, options);
|
|
@@ -2407,8 +1803,6 @@ TEST_F(BlobDBTest, SyncBlobFileBeforeCloseIOError) {
|
|
|
2407
1803
|
options.env = fault_injection_env_.get();
|
|
2408
1804
|
|
|
2409
1805
|
BlobDBOptions blob_options;
|
|
2410
|
-
blob_options.min_blob_size = 0;
|
|
2411
|
-
blob_options.bytes_per_sync = 1 << 20;
|
|
2412
1806
|
blob_options.disable_background_tasks = true;
|
|
2413
1807
|
|
|
2414
1808
|
Open(blob_options, options);
|
|
@@ -2419,7 +1813,7 @@ TEST_F(BlobDBTest, SyncBlobFileBeforeCloseIOError) {
|
|
|
2419
1813
|
ASSERT_EQ(blob_files.size(), 1);
|
|
2420
1814
|
|
|
2421
1815
|
SyncPoint::GetInstance()->SetCallBack(
|
|
2422
|
-
"BlobLogWriter::Sync", [this](void
|
|
1816
|
+
"BlobLogWriter::Sync", [this](void* /* arg */) {
|
|
2423
1817
|
fault_injection_env_->SetFilesystemActive(false, Status::IOError());
|
|
2424
1818
|
});
|
|
2425
1819
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
@@ -2436,7 +1830,7 @@ TEST_F(BlobDBTest, SyncBlobFileBeforeCloseIOError) {
|
|
|
2436
1830
|
} // namespace ROCKSDB_NAMESPACE::blob_db
|
|
2437
1831
|
|
|
2438
1832
|
// A black-box test for the ttl wrapper around rocksdb
|
|
2439
|
-
int main(int argc, char
|
|
1833
|
+
int main(int argc, char** argv) {
|
|
2440
1834
|
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
|
|
2441
1835
|
::testing::InitGoogleTest(&argc, argv);
|
|
2442
1836
|
return RUN_ALL_TESTS();
|