@nxtedition/rocksdb 6.0.1 → 6.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BUILDING.md +2 -2
- package/binding.cc +14 -17
- package/deps/rocksdb/build_version.cc +4 -10
- package/deps/rocksdb/rocksdb/CMakeLists.txt +17 -3
- package/deps/rocksdb/rocksdb/Makefile +57 -86
- package/deps/rocksdb/rocksdb/TARGETS +4 -0
- package/deps/rocksdb/rocksdb/cache/cache_test.cc +29 -17
- package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc +511 -0
- package/deps/rocksdb/rocksdb/cache/fast_lru_cache.h +299 -0
- package/deps/rocksdb/rocksdb/cache/lru_cache.cc +3 -0
- package/deps/rocksdb/rocksdb/cache/lru_cache.h +7 -0
- package/deps/rocksdb/rocksdb/common.mk +30 -0
- package/deps/rocksdb/rocksdb/crash_test.mk +3 -3
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +1 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_index.h +3 -3
- package/deps/rocksdb/rocksdb/db/blob/db_blob_index_test.cc +7 -7
- package/deps/rocksdb/rocksdb/db/builder.cc +11 -4
- package/deps/rocksdb/rocksdb/db/c.cc +45 -0
- package/deps/rocksdb/rocksdb/db/c_test.c +25 -2
- package/deps/rocksdb/rocksdb/db/column_family.cc +4 -3
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +23 -22
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +11 -11
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +2 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +36 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +4 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +3 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +25 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +6 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +69 -14
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +7 -7
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +1 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +21 -21
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +1 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +135 -32
- package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/convenience.cc +8 -6
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +209 -38
- package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +28 -25
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +85 -138
- package/deps/rocksdb/rocksdb/db/db_compaction_filter_test.cc +68 -3
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +38 -13
- package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +9 -8
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +5 -4
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +5 -4
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +9 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +6 -3
- package/deps/rocksdb/rocksdb/db/db_info_dumper.cc +26 -18
- package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +8 -7
- package/deps/rocksdb/rocksdb/db/db_iter_test.cc +8 -8
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +6 -3
- package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_options_test.cc +12 -12
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +16 -15
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +6 -4
- package/deps/rocksdb/rocksdb/db/db_rate_limiter_test.cc +15 -14
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +68 -7
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +6 -5
- package/deps/rocksdb/rocksdb/db/db_test.cc +44 -42
- package/deps/rocksdb/rocksdb/db/db_test2.cc +23 -19
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +95 -17
- package/deps/rocksdb/rocksdb/db/db_test_util.h +48 -0
- package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +7 -7
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/dbformat.h +2 -1
- package/deps/rocksdb/rocksdb/db/deletefile_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +8 -8
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +34 -12
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +3 -0
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +7 -7
- package/deps/rocksdb/rocksdb/db/file_indexer.h +1 -4
- package/deps/rocksdb/rocksdb/db/flush_job.cc +18 -4
- package/deps/rocksdb/rocksdb/db/flush_job.h +4 -0
- package/deps/rocksdb/rocksdb/db/flush_job_test.cc +98 -30
- package/deps/rocksdb/rocksdb/db/forward_iterator.cc +1 -1
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +3 -0
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +12 -12
- package/deps/rocksdb/rocksdb/db/listener_test.cc +4 -3
- package/deps/rocksdb/rocksdb/db/memtable.cc +2 -2
- package/deps/rocksdb/rocksdb/db/memtable_list.h +1 -1
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +37 -25
- package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +18 -18
- package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/prefix_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/repair.cc +1 -1
- package/deps/rocksdb/rocksdb/db/repair_test.cc +5 -5
- package/deps/rocksdb/rocksdb/db/snapshot_checker.h +1 -2
- package/deps/rocksdb/rocksdb/db/table_cache.cc +2 -4
- package/deps/rocksdb/rocksdb/db/version_builder.cc +1 -1
- package/deps/rocksdb/rocksdb/db/version_builder_test.cc +3 -5
- package/deps/rocksdb/rocksdb/db/version_edit.cc +2 -2
- package/deps/rocksdb/rocksdb/db/version_set.cc +2 -2
- package/deps/rocksdb/rocksdb/db/version_set.h +3 -3
- package/deps/rocksdb/rocksdb/db/wal_edit.h +2 -1
- package/deps/rocksdb/rocksdb/db/wal_manager.cc +2 -3
- package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/write_batch.cc +178 -30
- package/deps/rocksdb/rocksdb/db/write_batch_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/write_controller.h +1 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +0 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +7 -6
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compaction_filter.h +2 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +4 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +31 -6
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.cc +4 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.cc +0 -10
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +45 -42
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +359 -275
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +53 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +0 -12
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +13 -11
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +276 -109
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.h +63 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +45 -54
- package/deps/rocksdb/rocksdb/env/composite_env.cc +87 -14
- package/deps/rocksdb/rocksdb/env/env.cc +0 -60
- package/deps/rocksdb/rocksdb/env/env_encryption.cc +9 -0
- package/deps/rocksdb/rocksdb/env/env_encryption_ctr.h +1 -1
- package/deps/rocksdb/rocksdb/env/env_posix.cc +6 -5
- package/deps/rocksdb/rocksdb/env/env_test.cc +18 -5
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +17 -12
- package/deps/rocksdb/rocksdb/env/io_posix.cc +39 -37
- package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +9 -9
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +1 -1
- package/deps/rocksdb/rocksdb/file/file_util.h +2 -0
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.h +5 -2
- package/deps/rocksdb/rocksdb/file/sequence_file_reader.cc +7 -0
- package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +69 -45
- package/deps/rocksdb/rocksdb/file/writable_file_writer.h +3 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +35 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +5 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/cleanable.h +59 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/convenience.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +46 -44
- package/deps/rocksdb/rocksdb/include/rocksdb/env.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +2 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +3 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +28 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +91 -40
- package/deps/rocksdb/rocksdb/include/rocksdb/thread_status.h +1 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/unique_id.h +22 -13
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/customizable_util.h +9 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +4 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/object_registry.h +25 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/options_type.h +189 -103
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +14 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +18 -4
- package/deps/rocksdb/rocksdb/memory/arena.h +1 -1
- package/deps/rocksdb/rocksdb/memory/concurrent_arena.cc +1 -5
- package/deps/rocksdb/rocksdb/memory/concurrent_arena.h +1 -5
- package/deps/rocksdb/rocksdb/memory/jemalloc_nodump_allocator.cc +6 -8
- package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +1 -1
- package/deps/rocksdb/rocksdb/memtable/write_buffer_manager.cc +1 -1
- package/deps/rocksdb/rocksdb/memtable/write_buffer_manager_test.cc +5 -3
- package/deps/rocksdb/rocksdb/microbench/db_basic_bench.cc +266 -45
- package/deps/rocksdb/rocksdb/monitoring/histogram.cc +2 -1
- package/deps/rocksdb/rocksdb/monitoring/iostats_context.cc +1 -4
- package/deps/rocksdb/rocksdb/monitoring/iostats_context_imp.h +4 -4
- package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +2 -8
- package/deps/rocksdb/rocksdb/monitoring/perf_context_imp.h +2 -2
- package/deps/rocksdb/rocksdb/monitoring/perf_level.cc +1 -5
- package/deps/rocksdb/rocksdb/monitoring/perf_level_imp.h +1 -5
- package/deps/rocksdb/rocksdb/monitoring/persistent_stats_history.cc +2 -2
- package/deps/rocksdb/rocksdb/monitoring/thread_status_updater.cc +2 -1
- package/deps/rocksdb/rocksdb/monitoring/thread_status_updater.h +1 -1
- package/deps/rocksdb/rocksdb/monitoring/thread_status_util.cc +3 -3
- package/deps/rocksdb/rocksdb/monitoring/thread_status_util.h +2 -2
- package/deps/rocksdb/rocksdb/options/cf_options.cc +34 -38
- package/deps/rocksdb/rocksdb/options/configurable.cc +9 -27
- package/deps/rocksdb/rocksdb/options/configurable_test.cc +1 -1
- package/deps/rocksdb/rocksdb/options/customizable.cc +3 -1
- package/deps/rocksdb/rocksdb/options/customizable_test.cc +379 -318
- package/deps/rocksdb/rocksdb/options/db_options.cc +38 -17
- package/deps/rocksdb/rocksdb/options/db_options.h +1 -0
- package/deps/rocksdb/rocksdb/options/options_helper.cc +82 -39
- package/deps/rocksdb/rocksdb/options/options_parser.cc +10 -10
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +6 -3
- package/deps/rocksdb/rocksdb/options/options_test.cc +115 -59
- package/deps/rocksdb/rocksdb/port/port_posix.h +0 -15
- package/deps/rocksdb/rocksdb/port/win/env_win.cc +5 -4
- package/deps/rocksdb/rocksdb/port/win/env_win.h +2 -2
- package/deps/rocksdb/rocksdb/port/win/port_win.h +0 -31
- package/deps/rocksdb/rocksdb/rocksdb.pc.in +11 -0
- package/deps/rocksdb/rocksdb/src.mk +2 -0
- package/deps/rocksdb/rocksdb/table/block_based/binary_search_index_reader.cc +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/block.cc +4 -2
- package/deps/rocksdb/rocksdb/table/block_based/block.h +21 -25
- package/deps/rocksdb/rocksdb/table/block_based/block_based_filter_block.cc +3 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +14 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +52 -15
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +85 -41
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +42 -72
- package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +7 -5
- package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.h +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +33 -17
- package/deps/rocksdb/rocksdb/table/block_based/hash_index_reader.cc +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +2 -2
- package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.h +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_iterator.cc +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +6 -3
- package/deps/rocksdb/rocksdb/table/block_based/reader_common.cc +4 -4
- package/deps/rocksdb/rocksdb/table/block_fetcher.cc +5 -5
- package/deps/rocksdb/rocksdb/table/cleanable_test.cc +113 -0
- package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder.cc +1 -1
- package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder.h +1 -1
- package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_reader_test.cc +1 -1
- package/deps/rocksdb/rocksdb/table/format.cc +22 -20
- package/deps/rocksdb/rocksdb/table/iterator.cc +1 -81
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +2 -2
- package/deps/rocksdb/rocksdb/table/multiget_context.h +14 -11
- package/deps/rocksdb/rocksdb/table/persistent_cache_options.h +0 -3
- package/deps/rocksdb/rocksdb/table/plain/plain_table_factory.cc +12 -1
- package/deps/rocksdb/rocksdb/table/plain/plain_table_reader.cc +4 -4
- package/deps/rocksdb/rocksdb/table/sst_file_writer_collectors.h +1 -1
- package/deps/rocksdb/rocksdb/table/table_properties.cc +3 -5
- package/deps/rocksdb/rocksdb/table/table_test.cc +202 -78
- package/deps/rocksdb/rocksdb/table/unique_id.cc +57 -25
- package/deps/rocksdb/rocksdb/table/unique_id_impl.h +34 -4
- package/deps/rocksdb/rocksdb/test_util/testutil.cc +1 -1
- package/deps/rocksdb/rocksdb/test_util/testutil.h +11 -8
- package/deps/rocksdb/rocksdb/test_util/transaction_test_util.cc +8 -4
- package/deps/rocksdb/rocksdb/test_util/transaction_test_util.h +17 -0
- package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer.cc +11 -9
- package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +3 -3
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +254 -98
- package/deps/rocksdb/rocksdb/tools/db_sanity_test.cc +4 -4
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +186 -42
- package/deps/rocksdb/rocksdb/tools/ldb_cmd_impl.h +75 -49
- package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +9 -8
- package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +4 -1
- package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +2 -2
- package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +17 -3
- package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.cc +1 -1
- package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.h +1 -1
- package/deps/rocksdb/rocksdb/util/autovector_test.cc +4 -4
- package/deps/rocksdb/rocksdb/util/bloom_test.cc +14 -8
- package/deps/rocksdb/rocksdb/util/build_version.cc.in +5 -6
- package/deps/rocksdb/rocksdb/util/cleanable.cc +180 -0
- package/deps/rocksdb/rocksdb/util/comparator.cc +5 -3
- package/deps/rocksdb/rocksdb/util/compression.h +7 -7
- package/deps/rocksdb/rocksdb/util/file_reader_writer_test.cc +148 -0
- package/deps/rocksdb/rocksdb/util/filelock_test.cc +2 -2
- package/deps/rocksdb/rocksdb/util/filter_bench.cc +12 -4
- package/deps/rocksdb/rocksdb/util/heap.h +5 -3
- package/deps/rocksdb/rocksdb/util/random.cc +1 -5
- package/deps/rocksdb/rocksdb/util/rate_limiter.cc +12 -9
- package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc +1 -1
- package/deps/rocksdb/rocksdb/util/ribbon_alg.h +1 -1
- package/deps/rocksdb/rocksdb/util/ribbon_test.cc +2 -4
- package/deps/rocksdb/rocksdb/util/slice.cc +8 -9
- package/deps/rocksdb/rocksdb/util/string_util.cc +3 -2
- package/deps/rocksdb/rocksdb/util/string_util.h +0 -13
- package/deps/rocksdb/rocksdb/util/thread_local.cc +4 -23
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +99 -22
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_impl.h +7 -0
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +102 -59
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +38 -36
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +28 -0
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +3 -0
- package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/object_registry.cc +71 -0
- package/deps/rocksdb/rocksdb/utilities/object_registry_test.cc +71 -0
- package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache_test.cc +5 -5
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc +3 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_tracker.cc +0 -13
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_locking_test.cc +40 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc +10 -8
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.h +4 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h +17 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc +7 -7
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +8 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/snapshot_checker.cc +5 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +21 -15
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +69 -11
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.cc +22 -9
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.cc +26 -5
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.h +17 -4
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_transaction_test.cc +19 -16
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +7 -3
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +3 -2
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +2 -2
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +6 -6
- package/deps/rocksdb/rocksdb.gyp +8 -2
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
- package/deps/rocksdb/rocksdb/python.mk +0 -9
- package/prebuilds/darwin-x64/node.napi.node +0 -0
- package/prebuilds/linux-arm64/node.napi.node +0 -0
|
@@ -236,8 +236,8 @@ class CompactionJobTestBase : public testing::Test {
|
|
|
236
236
|
for (int i = 0; i < 2; ++i) {
|
|
237
237
|
auto contents = mock::MakeMockFile();
|
|
238
238
|
for (int k = 0; k < kKeysPerFile; ++k) {
|
|
239
|
-
auto key =
|
|
240
|
-
auto value =
|
|
239
|
+
auto key = std::to_string(i * kMatchingKeys + k);
|
|
240
|
+
auto value = std::to_string(i * kKeysPerFile + k);
|
|
241
241
|
InternalKey internal_key(key, ++sequence_number, kTypeValue);
|
|
242
242
|
|
|
243
243
|
// This is how the key will look like once it's written in bottommost
|
|
@@ -321,7 +321,8 @@ class CompactionJobTestBase : public testing::Test {
|
|
|
321
321
|
const std::vector<SequenceNumber>& snapshots = {},
|
|
322
322
|
SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber,
|
|
323
323
|
int output_level = 1, bool verify = true,
|
|
324
|
-
uint64_t expected_oldest_blob_file_number = kInvalidBlobFileNumber
|
|
324
|
+
uint64_t expected_oldest_blob_file_number = kInvalidBlobFileNumber,
|
|
325
|
+
bool check_get_priority = false) {
|
|
325
326
|
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
|
|
326
327
|
|
|
327
328
|
size_t num_input_files = 0;
|
|
@@ -390,6 +391,32 @@ class CompactionJobTestBase : public testing::Test {
|
|
|
390
391
|
expected_oldest_blob_file_number);
|
|
391
392
|
}
|
|
392
393
|
}
|
|
394
|
+
|
|
395
|
+
if (check_get_priority) {
|
|
396
|
+
CheckGetRateLimiterPriority(compaction_job);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
void CheckGetRateLimiterPriority(CompactionJob& compaction_job) {
|
|
401
|
+
// When the state from WriteController is normal.
|
|
402
|
+
ASSERT_EQ(compaction_job.GetRateLimiterPriority(), Env::IO_LOW);
|
|
403
|
+
|
|
404
|
+
WriteController* write_controller =
|
|
405
|
+
compaction_job.versions_->GetColumnFamilySet()->write_controller();
|
|
406
|
+
|
|
407
|
+
{
|
|
408
|
+
// When the state from WriteController is Delayed.
|
|
409
|
+
std::unique_ptr<WriteControllerToken> delay_token =
|
|
410
|
+
write_controller->GetDelayToken(1000000);
|
|
411
|
+
ASSERT_EQ(compaction_job.GetRateLimiterPriority(), Env::IO_USER);
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
{
|
|
415
|
+
// When the state from WriteController is Stopped.
|
|
416
|
+
std::unique_ptr<WriteControllerToken> stop_token =
|
|
417
|
+
write_controller->GetStopToken();
|
|
418
|
+
ASSERT_EQ(compaction_job.GetRateLimiterPriority(), Env::IO_USER);
|
|
419
|
+
}
|
|
393
420
|
}
|
|
394
421
|
|
|
395
422
|
std::shared_ptr<Env> env_guard_;
|
|
@@ -890,10 +917,10 @@ TEST_F(CompactionJobTest, MultiSingleDelete) {
|
|
|
890
917
|
// -> Snapshot Put
|
|
891
918
|
// K: SDel SDel Put SDel Put Put Snapshot SDel Put SDel SDel Put SDel
|
|
892
919
|
// -> Snapshot Put Snapshot SDel
|
|
893
|
-
// L: SDel Put
|
|
894
|
-
// -> Snapshot SDel
|
|
895
|
-
// M: (Put) SDel Put
|
|
896
|
-
// -> SDel Snapshot
|
|
920
|
+
// L: SDel Put SDel Put SDel Snapshot SDel Put SDel SDel Put SDel
|
|
921
|
+
// -> Snapshot SDel Put SDel
|
|
922
|
+
// M: (Put) SDel Put SDel Put SDel Snapshot Put SDel SDel Put SDel SDel
|
|
923
|
+
// -> SDel Snapshot Put SDel
|
|
897
924
|
NewDB();
|
|
898
925
|
|
|
899
926
|
auto file1 = mock::MakeMockFile({
|
|
@@ -924,14 +951,14 @@ TEST_F(CompactionJobTest, MultiSingleDelete) {
|
|
|
924
951
|
{KeyStr("L", 16U, kTypeSingleDeletion), ""},
|
|
925
952
|
{KeyStr("L", 15U, kTypeValue), "val"},
|
|
926
953
|
{KeyStr("L", 14U, kTypeSingleDeletion), ""},
|
|
927
|
-
{KeyStr("L", 13U,
|
|
954
|
+
{KeyStr("L", 13U, kTypeSingleDeletion), ""},
|
|
928
955
|
{KeyStr("L", 12U, kTypeValue), "val"},
|
|
929
|
-
{KeyStr("L", 11U,
|
|
930
|
-
{KeyStr("M", 16U,
|
|
956
|
+
{KeyStr("L", 11U, kTypeSingleDeletion), ""},
|
|
957
|
+
{KeyStr("M", 16U, kTypeSingleDeletion), ""},
|
|
931
958
|
{KeyStr("M", 15U, kTypeSingleDeletion), ""},
|
|
932
959
|
{KeyStr("M", 14U, kTypeValue), "val"},
|
|
933
960
|
{KeyStr("M", 13U, kTypeSingleDeletion), ""},
|
|
934
|
-
{KeyStr("M", 12U,
|
|
961
|
+
{KeyStr("M", 12U, kTypeSingleDeletion), ""},
|
|
935
962
|
{KeyStr("M", 11U, kTypeValue), "val"},
|
|
936
963
|
});
|
|
937
964
|
AddMockFile(file1);
|
|
@@ -972,12 +999,12 @@ TEST_F(CompactionJobTest, MultiSingleDelete) {
|
|
|
972
999
|
{KeyStr("K", 1U, kTypeSingleDeletion), ""},
|
|
973
1000
|
{KeyStr("L", 5U, kTypeSingleDeletion), ""},
|
|
974
1001
|
{KeyStr("L", 4U, kTypeValue), "val"},
|
|
975
|
-
{KeyStr("L", 3U,
|
|
1002
|
+
{KeyStr("L", 3U, kTypeSingleDeletion), ""},
|
|
976
1003
|
{KeyStr("L", 2U, kTypeValue), "val"},
|
|
977
1004
|
{KeyStr("L", 1U, kTypeSingleDeletion), ""},
|
|
978
1005
|
{KeyStr("M", 10U, kTypeSingleDeletion), ""},
|
|
979
1006
|
{KeyStr("M", 7U, kTypeValue), "val"},
|
|
980
|
-
{KeyStr("M", 5U,
|
|
1007
|
+
{KeyStr("M", 5U, kTypeSingleDeletion), ""},
|
|
981
1008
|
{KeyStr("M", 4U, kTypeValue), "val"},
|
|
982
1009
|
{KeyStr("M", 3U, kTypeSingleDeletion), ""},
|
|
983
1010
|
});
|
|
@@ -1019,7 +1046,9 @@ TEST_F(CompactionJobTest, MultiSingleDelete) {
|
|
|
1019
1046
|
{KeyStr("K", 8U, kTypeValue), "val3"},
|
|
1020
1047
|
{KeyStr("L", 16U, kTypeSingleDeletion), ""},
|
|
1021
1048
|
{KeyStr("L", 15U, kTypeValue), ""},
|
|
1022
|
-
{KeyStr("
|
|
1049
|
+
{KeyStr("L", 11U, kTypeSingleDeletion), ""},
|
|
1050
|
+
{KeyStr("M", 15U, kTypeSingleDeletion), ""},
|
|
1051
|
+
{KeyStr("M", 14U, kTypeValue), ""},
|
|
1023
1052
|
{KeyStr("M", 3U, kTypeSingleDeletion), ""}});
|
|
1024
1053
|
|
|
1025
1054
|
SetLastSequence(22U);
|
|
@@ -1105,6 +1134,21 @@ TEST_F(CompactionJobTest, OldestBlobFileNumber) {
|
|
|
1105
1134
|
/* expected_oldest_blob_file_number */ 19);
|
|
1106
1135
|
}
|
|
1107
1136
|
|
|
1137
|
+
TEST_F(CompactionJobTest, NoEnforceSingleDeleteContract) {
|
|
1138
|
+
db_options_.enforce_single_del_contracts = false;
|
|
1139
|
+
NewDB();
|
|
1140
|
+
|
|
1141
|
+
auto file =
|
|
1142
|
+
mock::MakeMockFile({{KeyStr("a", 4U, kTypeSingleDeletion), ""},
|
|
1143
|
+
{KeyStr("a", 3U, kTypeDeletion), "dontcare"}});
|
|
1144
|
+
AddMockFile(file);
|
|
1145
|
+
SetLastSequence(4U);
|
|
1146
|
+
|
|
1147
|
+
auto expected_results = mock::MakeMockFile();
|
|
1148
|
+
auto files = cfd_->current()->storage_info()->LevelFiles(0);
|
|
1149
|
+
RunCompaction({files}, expected_results);
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1108
1152
|
TEST_F(CompactionJobTest, InputSerialization) {
|
|
1109
1153
|
// Setup a random CompactionServiceInput
|
|
1110
1154
|
CompactionServiceInput input;
|
|
@@ -1286,6 +1330,17 @@ TEST_F(CompactionJobTest, ResultSerialization) {
|
|
|
1286
1330
|
}
|
|
1287
1331
|
}
|
|
1288
1332
|
|
|
1333
|
+
TEST_F(CompactionJobTest, GetRateLimiterPriority) {
|
|
1334
|
+
NewDB();
|
|
1335
|
+
|
|
1336
|
+
auto expected_results = CreateTwoFiles(false);
|
|
1337
|
+
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
|
|
1338
|
+
auto files = cfd->current()->storage_info()->LevelFiles(0);
|
|
1339
|
+
ASSERT_EQ(2U, files.size());
|
|
1340
|
+
RunCompaction({files}, expected_results, {}, kMaxSequenceNumber, 1, true,
|
|
1341
|
+
kInvalidBlobFileNumber, true);
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1289
1344
|
class CompactionJobTimestampTest : public CompactionJobTestBase {
|
|
1290
1345
|
public:
|
|
1291
1346
|
CompactionJobTimestampTest()
|
|
@@ -65,7 +65,7 @@ bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
|
|
|
65
65
|
size_t compact_bytes = static_cast<size_t>(level_files[start]->fd.file_size);
|
|
66
66
|
uint64_t compensated_compact_bytes =
|
|
67
67
|
level_files[start]->compensated_file_size;
|
|
68
|
-
size_t compact_bytes_per_del_file =
|
|
68
|
+
size_t compact_bytes_per_del_file = std::numeric_limits<size_t>::max();
|
|
69
69
|
// Compaction range will be [start, limit).
|
|
70
70
|
size_t limit;
|
|
71
71
|
// Pull in files until the amount of compaction work per deleted file begins
|
|
@@ -401,7 +401,7 @@ Status CompactionPicker::GetCompactionInputsFromFileNumbers(
|
|
|
401
401
|
"Cannot find matched SST files for the following file numbers:");
|
|
402
402
|
for (auto fn : *input_set) {
|
|
403
403
|
message += " ";
|
|
404
|
-
message +=
|
|
404
|
+
message += std::to_string(fn);
|
|
405
405
|
}
|
|
406
406
|
return Status::InvalidArgument(message);
|
|
407
407
|
}
|
|
@@ -717,7 +717,7 @@ Compaction* CompactionPicker::CompactRange(
|
|
|
717
717
|
// files that are created during the current compaction.
|
|
718
718
|
if (compact_range_options.bottommost_level_compaction ==
|
|
719
719
|
BottommostLevelCompaction::kForceOptimized &&
|
|
720
|
-
max_file_num_to_ignore !=
|
|
720
|
+
max_file_num_to_ignore != std::numeric_limits<uint64_t>::max()) {
|
|
721
721
|
assert(input_level == output_level);
|
|
722
722
|
// inputs_shrunk holds a continuous subset of input files which were all
|
|
723
723
|
// created before the current manual compaction
|
|
@@ -1004,14 +1004,14 @@ Status CompactionPicker::SanitizeCompactionInputFiles(
|
|
|
1004
1004
|
return Status::InvalidArgument(
|
|
1005
1005
|
"Output level for column family " + cf_meta.name +
|
|
1006
1006
|
" must between [0, " +
|
|
1007
|
-
|
|
1007
|
+
std::to_string(cf_meta.levels[cf_meta.levels.size() - 1].level) + "].");
|
|
1008
1008
|
}
|
|
1009
1009
|
|
|
1010
1010
|
if (output_level > MaxOutputLevel()) {
|
|
1011
1011
|
return Status::InvalidArgument(
|
|
1012
1012
|
"Exceed the maximum output level defined by "
|
|
1013
1013
|
"the current compaction algorithm --- " +
|
|
1014
|
-
|
|
1014
|
+
std::to_string(MaxOutputLevel()));
|
|
1015
1015
|
}
|
|
1016
1016
|
|
|
1017
1017
|
if (output_level < 0) {
|
|
@@ -1061,8 +1061,8 @@ Status CompactionPicker::SanitizeCompactionInputFiles(
|
|
|
1061
1061
|
return Status::InvalidArgument(
|
|
1062
1062
|
"Cannot compact file to up level, input file: " +
|
|
1063
1063
|
MakeTableFileName("", file_num) + " level " +
|
|
1064
|
-
|
|
1065
|
-
|
|
1064
|
+
std::to_string(input_file_level) + " > output level " +
|
|
1065
|
+
std::to_string(output_level));
|
|
1066
1066
|
}
|
|
1067
1067
|
}
|
|
1068
1068
|
|
|
@@ -504,7 +504,7 @@ bool LevelCompactionBuilder::PickIntraL0Compaction() {
|
|
|
504
504
|
return false;
|
|
505
505
|
}
|
|
506
506
|
return FindIntraL0Compaction(level_files, kMinFilesForIntraL0Compaction,
|
|
507
|
-
|
|
507
|
+
std::numeric_limits<uint64_t>::max(),
|
|
508
508
|
mutable_cf_options_.max_compaction_bytes,
|
|
509
509
|
&start_level_inputs_, earliest_mem_seqno_);
|
|
510
510
|
}
|
|
@@ -273,9 +273,9 @@ TEST_F(CompactionPickerTest, NeedsCompactionLevel) {
|
|
|
273
273
|
// start a brand new version in each test.
|
|
274
274
|
NewVersionStorage(kLevels, kCompactionStyleLevel);
|
|
275
275
|
for (int i = 0; i < file_count; ++i) {
|
|
276
|
-
Add(level, i,
|
|
277
|
-
|
|
278
|
-
|
|
276
|
+
Add(level, i, std::to_string((i + 100) * 1000).c_str(),
|
|
277
|
+
std::to_string((i + 100) * 1000 + 999).c_str(), file_size, 0,
|
|
278
|
+
i * 100, i * 100 + 99);
|
|
279
279
|
}
|
|
280
280
|
UpdateVersionStorageInfo();
|
|
281
281
|
ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level);
|
|
@@ -439,8 +439,8 @@ TEST_F(CompactionPickerTest, NeedsCompactionUniversal) {
|
|
|
439
439
|
for (int i = 1;
|
|
440
440
|
i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) {
|
|
441
441
|
NewVersionStorage(1, kCompactionStyleUniversal);
|
|
442
|
-
Add(0, i,
|
|
443
|
-
|
|
442
|
+
Add(0, i, std::to_string((i + 100) * 1000).c_str(),
|
|
443
|
+
std::to_string((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
|
|
444
444
|
i * 100 + 99);
|
|
445
445
|
UpdateVersionStorageInfo();
|
|
446
446
|
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
|
|
@@ -852,17 +852,17 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace4) {
|
|
|
852
852
|
// L3: (1101, 1180) (1201, 1280) ... (7901, 7908)
|
|
853
853
|
// L4: (1130, 1150) (1160, 1210) (1230, 1250) (1260 1310) ... (7960, 8010)
|
|
854
854
|
for (int i = 11; i < 79; i++) {
|
|
855
|
-
Add(3, 100 + i * 3,
|
|
856
|
-
|
|
855
|
+
Add(3, 100 + i * 3, std::to_string(i * 100).c_str(),
|
|
856
|
+
std::to_string(i * 100 + 80).c_str(), kFileSize, 0, 200, 251);
|
|
857
857
|
// Add a tie breaker
|
|
858
858
|
if (i == 66) {
|
|
859
859
|
Add(3, 10000U, "6690", "6699", kFileSize, 0, 200, 251);
|
|
860
860
|
}
|
|
861
861
|
|
|
862
|
-
Add(4, 100 + i * 3 + 1,
|
|
863
|
-
|
|
864
|
-
Add(4, 100 + i * 3 + 2,
|
|
865
|
-
|
|
862
|
+
Add(4, 100 + i * 3 + 1, std::to_string(i * 100 + 30).c_str(),
|
|
863
|
+
std::to_string(i * 100 + 50).c_str(), kFileSize, 0, 200, 251);
|
|
864
|
+
Add(4, 100 + i * 3 + 2, std::to_string(i * 100 + 60).c_str(),
|
|
865
|
+
std::to_string(i * 100 + 110).c_str(), kFileSize, 0, 200, 251);
|
|
866
866
|
}
|
|
867
867
|
UpdateVersionStorageInfo();
|
|
868
868
|
|
|
@@ -899,14 +899,14 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace5) {
|
|
|
899
899
|
// L3: (1101, 1180) (1201, 1280) ... (7901, 7908)
|
|
900
900
|
// L4: (1130, 1150) (1160, 1210) (1230, 1250) (1260 1310) ... (7960, 8010)
|
|
901
901
|
for (int i = 11; i < 70; i++) {
|
|
902
|
-
Add(3, 100 + i * 3,
|
|
903
|
-
|
|
902
|
+
Add(3, 100 + i * 3, std::to_string(i * 100).c_str(),
|
|
903
|
+
std::to_string(i * 100 + 80).c_str(),
|
|
904
904
|
i % 10 == 9 ? kFileSize * 100 : kFileSize, 0, 200, 251);
|
|
905
905
|
|
|
906
|
-
Add(4, 100 + i * 3 + 1,
|
|
907
|
-
|
|
908
|
-
Add(4, 100 + i * 3 + 2,
|
|
909
|
-
|
|
906
|
+
Add(4, 100 + i * 3 + 1, std::to_string(i * 100 + 30).c_str(),
|
|
907
|
+
std::to_string(i * 100 + 50).c_str(), kFileSize, 0, 200, 251);
|
|
908
|
+
Add(4, 100 + i * 3 + 2, std::to_string(i * 100 + 60).c_str(),
|
|
909
|
+
std::to_string(i * 100 + 110).c_str(), kFileSize, 0, 200, 251);
|
|
910
910
|
}
|
|
911
911
|
UpdateVersionStorageInfo();
|
|
912
912
|
|
|
@@ -941,8 +941,8 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
|
|
|
941
941
|
// size of L0 files.
|
|
942
942
|
for (int i = 1; i <= kFileCount; ++i) {
|
|
943
943
|
NewVersionStorage(1, kCompactionStyleFIFO);
|
|
944
|
-
Add(0, i,
|
|
945
|
-
|
|
944
|
+
Add(0, i, std::to_string((i + 100) * 1000).c_str(),
|
|
945
|
+
std::to_string((i + 100) * 1000 + 999).c_str(), kFileSize, 0, i * 100,
|
|
946
946
|
i * 100 + 99);
|
|
947
947
|
UpdateVersionStorageInfo();
|
|
948
948
|
ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()),
|
|
@@ -2653,8 +2653,8 @@ TEST_F(CompactionPickerTest, UniversalMarkedManualCompaction) {
|
|
|
2653
2653
|
universal_compaction_picker.CompactRange(
|
|
2654
2654
|
cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
|
|
2655
2655
|
ColumnFamilyData::kCompactAllLevels, 6, CompactRangeOptions(),
|
|
2656
|
-
nullptr, nullptr, &manual_end, &manual_conflict,
|
|
2657
|
-
""));
|
|
2656
|
+
nullptr, nullptr, &manual_end, &manual_conflict,
|
|
2657
|
+
std::numeric_limits<uint64_t>::max(), ""));
|
|
2658
2658
|
|
|
2659
2659
|
ASSERT_TRUE(compaction);
|
|
2660
2660
|
|
|
@@ -1371,7 +1371,7 @@ Compaction* UniversalCompactionBuilder::PickPeriodicCompaction() {
|
|
|
1371
1371
|
|
|
1372
1372
|
uint64_t UniversalCompactionBuilder::GetMaxOverlappingBytes() const {
|
|
1373
1373
|
if (!mutable_cf_options_.compaction_options_universal.incremental) {
|
|
1374
|
-
return
|
|
1374
|
+
return std::numeric_limits<uint64_t>::max();
|
|
1375
1375
|
} else {
|
|
1376
1376
|
// Try to align cutting boundary with files at the next level if the
|
|
1377
1377
|
// file isn't end up with 1/2 of target size, or it would overlap
|
|
@@ -15,13 +15,17 @@ class MyTestCompactionService : public CompactionService {
|
|
|
15
15
|
MyTestCompactionService(
|
|
16
16
|
std::string db_path, Options& options,
|
|
17
17
|
std::shared_ptr<Statistics>& statistics,
|
|
18
|
-
std::vector<std::shared_ptr<EventListener>>& listeners
|
|
18
|
+
std::vector<std::shared_ptr<EventListener>>& listeners,
|
|
19
|
+
std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
|
|
20
|
+
table_properties_collector_factories)
|
|
19
21
|
: db_path_(std::move(db_path)),
|
|
20
22
|
options_(options),
|
|
21
23
|
statistics_(statistics),
|
|
22
24
|
start_info_("na", "na", "na", 0, Env::TOTAL),
|
|
23
25
|
wait_info_("na", "na", "na", 0, Env::TOTAL),
|
|
24
|
-
listeners_(listeners)
|
|
26
|
+
listeners_(listeners),
|
|
27
|
+
table_properties_collector_factories_(
|
|
28
|
+
std::move(table_properties_collector_factories)) {}
|
|
25
29
|
|
|
26
30
|
static const char* kClassName() { return "MyTestCompactionService"; }
|
|
27
31
|
|
|
@@ -78,12 +82,16 @@ class MyTestCompactionService : public CompactionService {
|
|
|
78
82
|
options_override.listeners = listeners_;
|
|
79
83
|
}
|
|
80
84
|
|
|
85
|
+
if (!table_properties_collector_factories_.empty()) {
|
|
86
|
+
options_override.table_properties_collector_factories =
|
|
87
|
+
table_properties_collector_factories_;
|
|
88
|
+
}
|
|
89
|
+
|
|
81
90
|
OpenAndCompactOptions options;
|
|
82
91
|
options.canceled = &canceled_;
|
|
83
92
|
|
|
84
93
|
Status s = DB::OpenAndCompact(
|
|
85
|
-
options, db_path_,
|
|
86
|
-
db_path_ + "/" + ROCKSDB_NAMESPACE::ToString(info.job_id),
|
|
94
|
+
options, db_path_, db_path_ + "/" + std::to_string(info.job_id),
|
|
87
95
|
compaction_input, compaction_service_result, options_override);
|
|
88
96
|
if (is_override_wait_result_) {
|
|
89
97
|
*compaction_service_result = override_wait_result_;
|
|
@@ -142,6 +150,8 @@ class MyTestCompactionService : public CompactionService {
|
|
|
142
150
|
bool is_override_wait_result_ = false;
|
|
143
151
|
std::string override_wait_result_;
|
|
144
152
|
std::vector<std::shared_ptr<EventListener>> listeners_;
|
|
153
|
+
std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
|
|
154
|
+
table_properties_collector_factories_;
|
|
145
155
|
std::atomic_bool canceled_{false};
|
|
146
156
|
};
|
|
147
157
|
|
|
@@ -158,7 +168,8 @@ class CompactionServiceTest : public DBTestBase {
|
|
|
158
168
|
compactor_statistics_ = CreateDBStatistics();
|
|
159
169
|
|
|
160
170
|
compaction_service_ = std::make_shared<MyTestCompactionService>(
|
|
161
|
-
dbname_, *options, compactor_statistics_, remote_listeners
|
|
171
|
+
dbname_, *options, compactor_statistics_, remote_listeners,
|
|
172
|
+
remote_table_properties_collector_factories);
|
|
162
173
|
options->compaction_service = compaction_service_;
|
|
163
174
|
DestroyAndReopen(*options);
|
|
164
175
|
}
|
|
@@ -177,7 +188,7 @@ class CompactionServiceTest : public DBTestBase {
|
|
|
177
188
|
for (int i = 0; i < 20; i++) {
|
|
178
189
|
for (int j = 0; j < 10; j++) {
|
|
179
190
|
int key_id = i * 10 + j;
|
|
180
|
-
ASSERT_OK(Put(Key(key_id), "value" +
|
|
191
|
+
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
|
181
192
|
}
|
|
182
193
|
ASSERT_OK(Flush());
|
|
183
194
|
}
|
|
@@ -187,7 +198,7 @@ class CompactionServiceTest : public DBTestBase {
|
|
|
187
198
|
for (int i = 0; i < 10; i++) {
|
|
188
199
|
for (int j = 0; j < 10; j++) {
|
|
189
200
|
int key_id = i * 20 + j * 2;
|
|
190
|
-
ASSERT_OK(Put(Key(key_id), "value_new" +
|
|
201
|
+
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
|
191
202
|
}
|
|
192
203
|
ASSERT_OK(Flush());
|
|
193
204
|
}
|
|
@@ -199,14 +210,16 @@ class CompactionServiceTest : public DBTestBase {
|
|
|
199
210
|
for (int i = 0; i < 200; i++) {
|
|
200
211
|
auto result = Get(Key(i));
|
|
201
212
|
if (i % 2) {
|
|
202
|
-
ASSERT_EQ(result, "value" +
|
|
213
|
+
ASSERT_EQ(result, "value" + std::to_string(i));
|
|
203
214
|
} else {
|
|
204
|
-
ASSERT_EQ(result, "value_new" +
|
|
215
|
+
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
|
205
216
|
}
|
|
206
217
|
}
|
|
207
218
|
}
|
|
208
219
|
|
|
209
220
|
std::vector<std::shared_ptr<EventListener>> remote_listeners;
|
|
221
|
+
std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
|
|
222
|
+
remote_table_properties_collector_factories;
|
|
210
223
|
|
|
211
224
|
private:
|
|
212
225
|
std::shared_ptr<Statistics> compactor_statistics_;
|
|
@@ -224,7 +237,7 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
|
|
|
224
237
|
for (int i = 0; i < 20; i++) {
|
|
225
238
|
for (int j = 0; j < 10; j++) {
|
|
226
239
|
int key_id = i * 10 + j;
|
|
227
|
-
ASSERT_OK(Put(Key(key_id), "value" +
|
|
240
|
+
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
|
228
241
|
}
|
|
229
242
|
ASSERT_OK(Flush());
|
|
230
243
|
}
|
|
@@ -232,7 +245,7 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
|
|
|
232
245
|
for (int i = 0; i < 10; i++) {
|
|
233
246
|
for (int j = 0; j < 10; j++) {
|
|
234
247
|
int key_id = i * 20 + j * 2;
|
|
235
|
-
ASSERT_OK(Put(Key(key_id), "value_new" +
|
|
248
|
+
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
|
236
249
|
}
|
|
237
250
|
ASSERT_OK(Flush());
|
|
238
251
|
}
|
|
@@ -242,9 +255,9 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
|
|
|
242
255
|
for (int i = 0; i < 200; i++) {
|
|
243
256
|
auto result = Get(Key(i));
|
|
244
257
|
if (i % 2) {
|
|
245
|
-
ASSERT_EQ(result, "value" +
|
|
258
|
+
ASSERT_EQ(result, "value" + std::to_string(i));
|
|
246
259
|
} else {
|
|
247
|
-
ASSERT_EQ(result, "value_new" +
|
|
260
|
+
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
|
248
261
|
}
|
|
249
262
|
}
|
|
250
263
|
auto my_cs = GetCompactionService();
|
|
@@ -281,7 +294,7 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
|
|
|
281
294
|
for (int i = 0; i < 10; i++) {
|
|
282
295
|
for (int j = 0; j < 10; j++) {
|
|
283
296
|
int key_id = i * 20 + j * 2;
|
|
284
|
-
s = Put(Key(key_id), "value_new" +
|
|
297
|
+
s = Put(Key(key_id), "value_new" + std::to_string(key_id));
|
|
285
298
|
if (s.IsAborted()) {
|
|
286
299
|
break;
|
|
287
300
|
}
|
|
@@ -468,7 +481,7 @@ TEST_F(CompactionServiceTest, CompactionFilter) {
|
|
|
468
481
|
for (int i = 0; i < 20; i++) {
|
|
469
482
|
for (int j = 0; j < 10; j++) {
|
|
470
483
|
int key_id = i * 10 + j;
|
|
471
|
-
ASSERT_OK(Put(Key(key_id), "value" +
|
|
484
|
+
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
|
472
485
|
}
|
|
473
486
|
ASSERT_OK(Flush());
|
|
474
487
|
}
|
|
@@ -476,7 +489,7 @@ TEST_F(CompactionServiceTest, CompactionFilter) {
|
|
|
476
489
|
for (int i = 0; i < 10; i++) {
|
|
477
490
|
for (int j = 0; j < 10; j++) {
|
|
478
491
|
int key_id = i * 20 + j * 2;
|
|
479
|
-
ASSERT_OK(Put(Key(key_id), "value_new" +
|
|
492
|
+
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
|
480
493
|
}
|
|
481
494
|
ASSERT_OK(Flush());
|
|
482
495
|
}
|
|
@@ -490,9 +503,9 @@ TEST_F(CompactionServiceTest, CompactionFilter) {
|
|
|
490
503
|
if (i > 5 && i <= 105) {
|
|
491
504
|
ASSERT_EQ(result, "NOT_FOUND");
|
|
492
505
|
} else if (i % 2) {
|
|
493
|
-
ASSERT_EQ(result, "value" +
|
|
506
|
+
ASSERT_EQ(result, "value" + std::to_string(i));
|
|
494
507
|
} else {
|
|
495
|
-
ASSERT_EQ(result, "value_new" +
|
|
508
|
+
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
|
496
509
|
}
|
|
497
510
|
}
|
|
498
511
|
auto my_cs = GetCompactionService();
|
|
@@ -547,9 +560,9 @@ TEST_F(CompactionServiceTest, ConcurrentCompaction) {
|
|
|
547
560
|
for (int i = 0; i < 200; i++) {
|
|
548
561
|
auto result = Get(Key(i));
|
|
549
562
|
if (i % 2) {
|
|
550
|
-
ASSERT_EQ(result, "value" +
|
|
563
|
+
ASSERT_EQ(result, "value" + std::to_string(i));
|
|
551
564
|
} else {
|
|
552
|
-
ASSERT_EQ(result, "value_new" +
|
|
565
|
+
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
|
553
566
|
}
|
|
554
567
|
}
|
|
555
568
|
auto my_cs = GetCompactionService();
|
|
@@ -564,7 +577,7 @@ TEST_F(CompactionServiceTest, CompactionInfo) {
|
|
|
564
577
|
for (int i = 0; i < 20; i++) {
|
|
565
578
|
for (int j = 0; j < 10; j++) {
|
|
566
579
|
int key_id = i * 10 + j;
|
|
567
|
-
ASSERT_OK(Put(Key(key_id), "value" +
|
|
580
|
+
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
|
568
581
|
}
|
|
569
582
|
ASSERT_OK(Flush());
|
|
570
583
|
}
|
|
@@ -572,7 +585,7 @@ TEST_F(CompactionServiceTest, CompactionInfo) {
|
|
|
572
585
|
for (int i = 0; i < 10; i++) {
|
|
573
586
|
for (int j = 0; j < 10; j++) {
|
|
574
587
|
int key_id = i * 20 + j * 2;
|
|
575
|
-
ASSERT_OK(Put(Key(key_id), "value_new" +
|
|
588
|
+
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
|
576
589
|
}
|
|
577
590
|
ASSERT_OK(Flush());
|
|
578
591
|
}
|
|
@@ -617,7 +630,7 @@ TEST_F(CompactionServiceTest, CompactionInfo) {
|
|
|
617
630
|
for (int i = 0; i < 20; i++) {
|
|
618
631
|
for (int j = 0; j < 10; j++) {
|
|
619
632
|
int key_id = i * 10 + j;
|
|
620
|
-
ASSERT_OK(Put(Key(key_id), "value" +
|
|
633
|
+
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
|
621
634
|
}
|
|
622
635
|
ASSERT_OK(Flush());
|
|
623
636
|
}
|
|
@@ -625,7 +638,7 @@ TEST_F(CompactionServiceTest, CompactionInfo) {
|
|
|
625
638
|
for (int i = 0; i < 4; i++) {
|
|
626
639
|
for (int j = 0; j < 10; j++) {
|
|
627
640
|
int key_id = i * 20 + j * 2;
|
|
628
|
-
ASSERT_OK(Put(Key(key_id), "value_new" +
|
|
641
|
+
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
|
629
642
|
}
|
|
630
643
|
ASSERT_OK(Flush());
|
|
631
644
|
}
|
|
@@ -653,7 +666,7 @@ TEST_F(CompactionServiceTest, FallbackLocalAuto) {
|
|
|
653
666
|
for (int i = 0; i < 20; i++) {
|
|
654
667
|
for (int j = 0; j < 10; j++) {
|
|
655
668
|
int key_id = i * 10 + j;
|
|
656
|
-
ASSERT_OK(Put(Key(key_id), "value" +
|
|
669
|
+
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
|
657
670
|
}
|
|
658
671
|
ASSERT_OK(Flush());
|
|
659
672
|
}
|
|
@@ -661,7 +674,7 @@ TEST_F(CompactionServiceTest, FallbackLocalAuto) {
|
|
|
661
674
|
for (int i = 0; i < 10; i++) {
|
|
662
675
|
for (int j = 0; j < 10; j++) {
|
|
663
676
|
int key_id = i * 20 + j * 2;
|
|
664
|
-
ASSERT_OK(Put(Key(key_id), "value_new" +
|
|
677
|
+
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
|
665
678
|
}
|
|
666
679
|
ASSERT_OK(Flush());
|
|
667
680
|
}
|
|
@@ -671,9 +684,9 @@ TEST_F(CompactionServiceTest, FallbackLocalAuto) {
|
|
|
671
684
|
for (int i = 0; i < 200; i++) {
|
|
672
685
|
auto result = Get(Key(i));
|
|
673
686
|
if (i % 2) {
|
|
674
|
-
ASSERT_EQ(result, "value" +
|
|
687
|
+
ASSERT_EQ(result, "value" + std::to_string(i));
|
|
675
688
|
} else {
|
|
676
|
-
ASSERT_EQ(result, "value_new" +
|
|
689
|
+
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
|
677
690
|
}
|
|
678
691
|
}
|
|
679
692
|
|
|
@@ -796,7 +809,7 @@ TEST_F(CompactionServiceTest, RemoteEventListener) {
|
|
|
796
809
|
for (int i = 0; i < 20; i++) {
|
|
797
810
|
for (int j = 0; j < 10; j++) {
|
|
798
811
|
int key_id = i * 10 + j;
|
|
799
|
-
ASSERT_OK(Put(Key(key_id), "value" +
|
|
812
|
+
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
|
800
813
|
}
|
|
801
814
|
ASSERT_OK(Flush());
|
|
802
815
|
}
|
|
@@ -804,7 +817,7 @@ TEST_F(CompactionServiceTest, RemoteEventListener) {
|
|
|
804
817
|
for (int i = 0; i < 10; i++) {
|
|
805
818
|
for (int j = 0; j < 10; j++) {
|
|
806
819
|
int key_id = i * 20 + j * 2;
|
|
807
|
-
ASSERT_OK(Put(Key(key_id), "value_new" +
|
|
820
|
+
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
|
808
821
|
}
|
|
809
822
|
ASSERT_OK(Flush());
|
|
810
823
|
}
|
|
@@ -821,11 +834,101 @@ TEST_F(CompactionServiceTest, RemoteEventListener) {
|
|
|
821
834
|
for (int i = 0; i < 200; i++) {
|
|
822
835
|
auto result = Get(Key(i));
|
|
823
836
|
if (i % 2) {
|
|
824
|
-
ASSERT_EQ(result, "value" +
|
|
837
|
+
ASSERT_EQ(result, "value" + std::to_string(i));
|
|
825
838
|
} else {
|
|
826
|
-
ASSERT_EQ(result, "value_new" +
|
|
839
|
+
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
TEST_F(CompactionServiceTest, TablePropertiesCollector) {
|
|
845
|
+
const static std::string kUserPropertyName = "TestCount";
|
|
846
|
+
|
|
847
|
+
class TablePropertiesCollectorTest : public TablePropertiesCollector {
|
|
848
|
+
public:
|
|
849
|
+
Status Finish(UserCollectedProperties* properties) override {
|
|
850
|
+
*properties = UserCollectedProperties{
|
|
851
|
+
{kUserPropertyName, std::to_string(count_)},
|
|
852
|
+
};
|
|
853
|
+
return Status::OK();
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
UserCollectedProperties GetReadableProperties() const override {
|
|
857
|
+
return UserCollectedProperties();
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
const char* Name() const override { return "TablePropertiesCollectorTest"; }
|
|
861
|
+
|
|
862
|
+
Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/,
|
|
863
|
+
EntryType /*type*/, SequenceNumber /*seq*/,
|
|
864
|
+
uint64_t /*file_size*/) override {
|
|
865
|
+
count_++;
|
|
866
|
+
return Status::OK();
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
private:
|
|
870
|
+
uint32_t count_ = 0;
|
|
871
|
+
};
|
|
872
|
+
|
|
873
|
+
class TablePropertiesCollectorFactoryTest
|
|
874
|
+
: public TablePropertiesCollectorFactory {
|
|
875
|
+
public:
|
|
876
|
+
TablePropertiesCollector* CreateTablePropertiesCollector(
|
|
877
|
+
TablePropertiesCollectorFactory::Context /*context*/) override {
|
|
878
|
+
return new TablePropertiesCollectorTest();
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
const char* Name() const override {
|
|
882
|
+
return "TablePropertiesCollectorFactoryTest";
|
|
883
|
+
}
|
|
884
|
+
};
|
|
885
|
+
|
|
886
|
+
auto factory = new TablePropertiesCollectorFactoryTest();
|
|
887
|
+
remote_table_properties_collector_factories.emplace_back(factory);
|
|
888
|
+
|
|
889
|
+
const int kNumSst = 3;
|
|
890
|
+
const int kLevel0Trigger = 4;
|
|
891
|
+
Options options = CurrentOptions();
|
|
892
|
+
options.level0_file_num_compaction_trigger = kLevel0Trigger;
|
|
893
|
+
ReopenWithCompactionService(&options);
|
|
894
|
+
|
|
895
|
+
// generate a few SSTs locally which should not have user property
|
|
896
|
+
for (int i = 0; i < kNumSst; i++) {
|
|
897
|
+
for (int j = 0; j < 100; j++) {
|
|
898
|
+
ASSERT_OK(Put(Key(i * 10 + j), "value"));
|
|
899
|
+
}
|
|
900
|
+
ASSERT_OK(Flush());
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
TablePropertiesCollection fname_to_props;
|
|
904
|
+
ASSERT_OK(db_->GetPropertiesOfAllTables(&fname_to_props));
|
|
905
|
+
for (const auto& file_props : fname_to_props) {
|
|
906
|
+
auto properties = file_props.second->user_collected_properties;
|
|
907
|
+
auto it = properties.find(kUserPropertyName);
|
|
908
|
+
ASSERT_EQ(it, properties.end());
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
// trigger compaction
|
|
912
|
+
for (int i = kNumSst; i < kLevel0Trigger; i++) {
|
|
913
|
+
for (int j = 0; j < 100; j++) {
|
|
914
|
+
ASSERT_OK(Put(Key(i * 10 + j), "value"));
|
|
915
|
+
}
|
|
916
|
+
ASSERT_OK(Flush());
|
|
917
|
+
}
|
|
918
|
+
ASSERT_OK(dbfull()->TEST_WaitForCompact(true));
|
|
919
|
+
|
|
920
|
+
ASSERT_OK(db_->GetPropertiesOfAllTables(&fname_to_props));
|
|
921
|
+
|
|
922
|
+
bool has_user_property = false;
|
|
923
|
+
for (const auto& file_props : fname_to_props) {
|
|
924
|
+
auto properties = file_props.second->user_collected_properties;
|
|
925
|
+
auto it = properties.find(kUserPropertyName);
|
|
926
|
+
if (it != properties.end()) {
|
|
927
|
+
has_user_property = true;
|
|
928
|
+
ASSERT_GT(std::stoi(it->second), 0);
|
|
827
929
|
}
|
|
828
930
|
}
|
|
931
|
+
ASSERT_TRUE(has_user_property);
|
|
829
932
|
}
|
|
830
933
|
|
|
831
934
|
} // namespace ROCKSDB_NAMESPACE
|