@nxtedition/rocksdb 13.5.8 → 13.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +209 -2
- package/deps/rocksdb/rocksdb/BUCK +12 -0
- package/deps/rocksdb/rocksdb/CMakeLists.txt +7 -0
- package/deps/rocksdb/rocksdb/Makefile +28 -23
- package/deps/rocksdb/rocksdb/cache/cache.cc +0 -1
- package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +1 -2
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +43 -39
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h +2 -0
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +0 -1
- package/deps/rocksdb/rocksdb/cache/lru_cache.cc +2 -3
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +2 -2
- package/deps/rocksdb/rocksdb/cache/secondary_cache.cc +1 -3
- package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.cc +11 -1
- package/deps/rocksdb/rocksdb/cache/tiered_secondary_cache_test.cc +13 -5
- package/deps/rocksdb/rocksdb/crash_test.mk +61 -15
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +136 -45
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +34 -16
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +10 -7
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +1 -2
- package/deps/rocksdb/rocksdb/db/blob/blob_file_meta.h +1 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +12 -9
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/blob/blob_source.cc +2 -2
- package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/builder.cc +22 -8
- package/deps/rocksdb/rocksdb/db/builder.h +5 -4
- package/deps/rocksdb/rocksdb/db/c.cc +556 -15
- package/deps/rocksdb/rocksdb/db/c_test.c +133 -12
- package/deps/rocksdb/rocksdb/db/column_family.cc +114 -50
- package/deps/rocksdb/rocksdb/db/column_family.h +53 -36
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +95 -70
- package/deps/rocksdb/rocksdb/db/compaction/compaction.h +71 -51
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +7 -86
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +26 -68
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +0 -122
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +453 -258
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +117 -92
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +38 -38
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +24 -17
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +34 -45
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +32 -31
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +12 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +1 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +10 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +82 -34
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +267 -179
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +4 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +273 -89
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +300 -14
- package/deps/rocksdb/rocksdb/db/compaction/compaction_state.cc +4 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_state.h +2 -2
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +28 -23
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +69 -51
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +522 -245
- package/deps/rocksdb/rocksdb/db/convenience.cc +15 -4
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +1 -3
- package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +0 -2
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +196 -17
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +74 -62
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +48 -0
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +682 -250
- package/deps/rocksdb/rocksdb/db/db_dynamic_level_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_encryption_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +11 -16
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +57 -0
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +540 -490
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +347 -188
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +584 -217
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +13 -9
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +5 -7
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +40 -36
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +751 -372
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +35 -32
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +24 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +125 -63
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +2 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +311 -196
- package/deps/rocksdb/rocksdb/db/db_io_failure_test.cc +15 -5
- package/deps/rocksdb/rocksdb/db/db_iter.cc +42 -29
- package/deps/rocksdb/rocksdb/db/db_iter.h +96 -31
- package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/db_iter_test.cc +168 -228
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +454 -0
- package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +8 -8
- package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +90 -0
- package/deps/rocksdb/rocksdb/db/db_merge_operand_test.cc +60 -2
- package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +7 -3
- package/deps/rocksdb/rocksdb/db/db_options_test.cc +85 -27
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +3 -1
- package/deps/rocksdb/rocksdb/db/db_rate_limiter_test.cc +0 -2
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +114 -2
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +51 -3
- package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_test.cc +325 -18
- package/deps/rocksdb/rocksdb/db/db_test2.cc +644 -20
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +14 -6
- package/deps/rocksdb/rocksdb/db/db_test_util.h +9 -0
- package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +64 -45
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +203 -14
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +259 -30
- package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +75 -1
- package/deps/rocksdb/rocksdb/db/dbformat.h +70 -6
- package/deps/rocksdb/rocksdb/db/deletefile_test.cc +0 -190
- package/deps/rocksdb/rocksdb/db/error_handler.cc +22 -7
- package/deps/rocksdb/rocksdb/db/error_handler.h +16 -1
- package/deps/rocksdb/rocksdb/db/event_helpers.cc +41 -26
- package/deps/rocksdb/rocksdb/db/experimental.cc +4 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +464 -78
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +166 -69
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +54 -25
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +1 -3
- package/deps/rocksdb/rocksdb/db/flush_job.cc +98 -81
- package/deps/rocksdb/rocksdb/db/flush_job.h +4 -9
- package/deps/rocksdb/rocksdb/db/flush_job_test.cc +80 -84
- package/deps/rocksdb/rocksdb/db/forward_iterator.cc +1 -1
- package/deps/rocksdb/rocksdb/db/forward_iterator.h +2 -2
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +12 -19
- package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +0 -2
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +41 -15
- package/deps/rocksdb/rocksdb/db/internal_stats.h +63 -52
- package/deps/rocksdb/rocksdb/db/job_context.h +59 -24
- package/deps/rocksdb/rocksdb/db/listener_test.cc +69 -10
- package/deps/rocksdb/rocksdb/db/log_format.h +11 -2
- package/deps/rocksdb/rocksdb/db/log_reader.cc +147 -34
- package/deps/rocksdb/rocksdb/db/log_reader.h +40 -11
- package/deps/rocksdb/rocksdb/db/log_test.cc +16 -3
- package/deps/rocksdb/rocksdb/db/log_writer.cc +102 -55
- package/deps/rocksdb/rocksdb/db/log_writer.h +21 -2
- package/deps/rocksdb/rocksdb/db/malloc_stats.h +0 -2
- package/deps/rocksdb/rocksdb/db/memtable.cc +16 -47
- package/deps/rocksdb/rocksdb/db/memtable.h +76 -12
- package/deps/rocksdb/rocksdb/db/memtable_list.cc +23 -20
- package/deps/rocksdb/rocksdb/db/memtable_list.h +9 -11
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +18 -37
- package/deps/rocksdb/rocksdb/db/merge_context.h +2 -1
- package/deps/rocksdb/rocksdb/db/merge_test.cc +8 -0
- package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +3 -5
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +15 -7
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.h +6 -3
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +22 -4
- package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +41 -1
- package/deps/rocksdb/rocksdb/db/prefix_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/repair.cc +29 -34
- package/deps/rocksdb/rocksdb/db/repair_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +14 -15
- package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc +1 -3
- package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h +47 -1
- package/deps/rocksdb/rocksdb/db/table_cache.cc +3 -3
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +1 -3
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +2 -1
- package/deps/rocksdb/rocksdb/db/version_builder.cc +2 -2
- package/deps/rocksdb/rocksdb/db/version_edit.cc +8 -37
- package/deps/rocksdb/rocksdb/db/version_edit.h +32 -1
- package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +26 -18
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -5
- package/deps/rocksdb/rocksdb/db/version_set.cc +282 -197
- package/deps/rocksdb/rocksdb/db/version_set.h +54 -57
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +28 -35
- package/deps/rocksdb/rocksdb/db/version_util.h +2 -3
- package/deps/rocksdb/rocksdb/db/wal_manager.cc +3 -2
- package/deps/rocksdb/rocksdb/db/wal_manager.h +0 -1
- package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/wide/wide_columns.cc +1 -0
- package/deps/rocksdb/rocksdb/db/write_batch.cc +22 -8
- package/deps/rocksdb/rocksdb/db/write_batch_internal.h +5 -4
- package/deps/rocksdb/rocksdb/db/write_batch_test.cc +7 -6
- package/deps/rocksdb/rocksdb/db/write_callback_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/write_thread.h +3 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +13 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +9 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compaction_service.h +39 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compression_manager.h +65 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +45 -22
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +7 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +22 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_table_properties_collector.h +28 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +143 -38
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +4 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +80 -32
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.h +51 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +23 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +305 -15
- package/deps/rocksdb/rocksdb/env/env.cc +32 -2
- package/deps/rocksdb/rocksdb/env/env_encryption.cc +0 -2
- package/deps/rocksdb/rocksdb/env/env_encryption_ctr.h +2 -4
- package/deps/rocksdb/rocksdb/env/env_posix.cc +4 -2
- package/deps/rocksdb/rocksdb/env/env_test.cc +0 -1
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +20 -11
- package/deps/rocksdb/rocksdb/env/fs_readonly.h +0 -2
- package/deps/rocksdb/rocksdb/env/fs_remap.cc +0 -2
- package/deps/rocksdb/rocksdb/env/fs_remap.h +0 -2
- package/deps/rocksdb/rocksdb/env/io_posix.cc +6 -4
- package/deps/rocksdb/rocksdb/env/io_posix.h +3 -2
- package/deps/rocksdb/rocksdb/env/mock_env.cc +0 -1
- package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +2 -2
- package/deps/rocksdb/rocksdb/file/delete_scheduler.h +0 -2
- package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +0 -2
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +30 -21
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +16 -0
- package/deps/rocksdb/rocksdb/file/file_util.cc +32 -14
- package/deps/rocksdb/rocksdb/file/file_util.h +22 -5
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +229 -76
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +21 -12
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.h +10 -7
- package/deps/rocksdb/rocksdb/file/random_access_file_reader_test.cc +12 -8
- package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.cc +1 -2
- package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.h +0 -2
- package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +3 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +598 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_iterator.h +36 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +70 -11
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +232 -11
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/compaction_job_stats.h +3 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/compression_type.h +149 -15
- package/deps/rocksdb/rocksdb/include/rocksdb/convenience.h +17 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/data_structure.h +132 -34
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +158 -79
- package/deps/rocksdb/rocksdb/include/rocksdb/db_bench_tool.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/env.h +4 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/env_encryption.h +1 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/external_table.h +275 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +50 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +10 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/ldb_tool.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +5 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/multi_scan.h +237 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +230 -39
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +15 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_level.h +31 -11
- package/deps/rocksdb/rocksdb/include/rocksdb/slice.h +41 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_dump_tool.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h +5 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +18 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/status.h +2 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +20 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +19 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/thread_status.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/tool_hooks.h +124 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/trace_record.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +26 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h +55 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/debug.h +3 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/env_mirror.h +0 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +1 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/memory_util.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/object_registry.h +1 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/options_util.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/secondary_index.h +96 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/secondary_index_faiss.h +117 -0
- package/deps/rocksdb/rocksdb/{utilities/secondary_index/faiss_ivf_index.h → include/rocksdb/utilities/secondary_index_simple.h} +11 -14
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +26 -11
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/table_properties_collectors.h +16 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +0 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +63 -7
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db_mutex.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/write_batch_with_index.h +28 -12
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +3 -3
- package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +0 -2
- package/deps/rocksdb/rocksdb/logging/event_logger_test.cc +1 -2
- package/deps/rocksdb/rocksdb/memory/memory_allocator_impl.h +1 -1
- package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc +0 -1
- package/deps/rocksdb/rocksdb/memtable/hash_linklist_rep.cc +0 -1
- package/deps/rocksdb/rocksdb/memtable/memtablerep_bench.cc +3 -1
- package/deps/rocksdb/rocksdb/memtable/skiplist.h +2 -2
- package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +2 -4
- package/deps/rocksdb/rocksdb/memtable/vectorrep.cc +69 -8
- package/deps/rocksdb/rocksdb/memtable/wbwi_memtable.cc +32 -9
- package/deps/rocksdb/rocksdb/memtable/wbwi_memtable.h +58 -45
- package/deps/rocksdb/rocksdb/monitoring/histogram.h +1 -1
- package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +5 -3
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +5 -0
- package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +1 -1
- package/deps/rocksdb/rocksdb/monitoring/thread_status_util_debug.cc +3 -2
- package/deps/rocksdb/rocksdb/options/cf_options.cc +44 -13
- package/deps/rocksdb/rocksdb/options/cf_options.h +21 -7
- package/deps/rocksdb/rocksdb/options/configurable.cc +5 -5
- package/deps/rocksdb/rocksdb/options/configurable_test.h +1 -2
- package/deps/rocksdb/rocksdb/options/customizable.cc +0 -1
- package/deps/rocksdb/rocksdb/options/customizable_test.cc +4 -11
- package/deps/rocksdb/rocksdb/options/db_options.cc +18 -15
- package/deps/rocksdb/rocksdb/options/db_options.h +2 -2
- package/deps/rocksdb/rocksdb/options/options.cc +296 -305
- package/deps/rocksdb/rocksdb/options/options_helper.cc +188 -62
- package/deps/rocksdb/rocksdb/options/options_helper.h +3 -3
- package/deps/rocksdb/rocksdb/options/options_parser.cc +2 -4
- package/deps/rocksdb/rocksdb/options/options_parser.h +0 -1
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +17 -4
- package/deps/rocksdb/rocksdb/options/options_test.cc +101 -76
- package/deps/rocksdb/rocksdb/port/lang.h +2 -1
- package/deps/rocksdb/rocksdb/port/port_posix.cc +2 -1
- package/deps/rocksdb/rocksdb/port/stack_trace.cc +5 -4
- package/deps/rocksdb/rocksdb/port/win/env_win.cc +3 -2
- package/deps/rocksdb/rocksdb/port/win/xpress_win.cc +99 -1
- package/deps/rocksdb/rocksdb/port/win/xpress_win.h +6 -0
- package/deps/rocksdb/rocksdb/src.mk +17 -11
- package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.h +0 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +1094 -929
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +6 -19
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +76 -22
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h +2 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +221 -131
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +12 -9
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +23 -24
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +38 -38
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +7 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +5 -5
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +10 -12
- package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +6 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +35 -43
- package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index_test.cc +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc +1 -2
- package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +0 -4
- package/deps/rocksdb/rocksdb/table/block_based/filter_policy_internal.h +0 -1
- package/deps/rocksdb/rocksdb/table/block_based/hash_index_reader.cc +3 -3
- package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc +3 -3
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +4 -4
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc +4 -5
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +4 -4
- package/deps/rocksdb/rocksdb/table/block_fetcher.cc +37 -35
- package/deps/rocksdb/rocksdb/table/block_fetcher.h +11 -7
- package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +4 -3
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +31 -5
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +2 -1
- package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder.h +0 -1
- package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder_test.cc +0 -1
- package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_reader_test.cc +0 -1
- package/deps/rocksdb/rocksdb/table/external_table.cc +483 -0
- package/deps/rocksdb/rocksdb/table/format.cc +62 -44
- package/deps/rocksdb/rocksdb/table/format.h +35 -12
- package/deps/rocksdb/rocksdb/table/internal_iterator.h +3 -13
- package/deps/rocksdb/rocksdb/table/iterator_wrapper.h +8 -0
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +6 -0
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +150 -141
- package/deps/rocksdb/rocksdb/table/meta_blocks.h +5 -0
- package/deps/rocksdb/rocksdb/table/multiget_context.h +3 -2
- package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +8 -0
- package/deps/rocksdb/rocksdb/table/plain/plain_table_index.cc +0 -1
- package/deps/rocksdb/rocksdb/table/plain/plain_table_index.h +0 -2
- package/deps/rocksdb/rocksdb/table/plain/plain_table_key_coding.h +0 -2
- package/deps/rocksdb/rocksdb/table/plain/plain_table_reader.cc +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +6 -6
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +86 -7
- package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +88 -2
- package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +0 -1
- package/deps/rocksdb/rocksdb/table/table_builder.h +10 -1
- package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +3 -2
- package/deps/rocksdb/rocksdb/table/table_test.cc +899 -22
- package/deps/rocksdb/rocksdb/test_util/testutil.cc +3 -4
- package/deps/rocksdb/rocksdb/test_util/testutil.h +132 -1
- package/deps/rocksdb/rocksdb/test_util/transaction_test_util.cc +0 -1
- package/deps/rocksdb/rocksdb/test_util/transaction_test_util.h +0 -2
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +163 -77
- package/deps/rocksdb/rocksdb/tools/db_bench_tool_test.cc +0 -2
- package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +0 -1
- package/deps/rocksdb/rocksdb/tools/dump/db_dump_tool.cc +0 -1
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +120 -52
- package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +1 -0
- package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +1 -1
- package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +0 -2
- package/deps/rocksdb/rocksdb/tools/simulated_hybrid_file_system.cc +2 -2
- package/deps/rocksdb/rocksdb/tools/simulated_hybrid_file_system.h +0 -2
- package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +2 -1
- package/deps/rocksdb/rocksdb/tools/tool_hooks.cc +94 -0
- package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.cc +0 -1
- package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.h +0 -1
- package/deps/rocksdb/rocksdb/trace_replay/io_tracer.cc +1 -1
- package/deps/rocksdb/rocksdb/trace_replay/io_tracer_test.cc +2 -1
- package/deps/rocksdb/rocksdb/trace_replay/trace_replay.cc +3 -5
- package/deps/rocksdb/rocksdb/util/async_file_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/util/async_file_reader.h +15 -8
- package/deps/rocksdb/rocksdb/util/auto_skip_compressor.cc +131 -0
- package/deps/rocksdb/rocksdb/util/auto_skip_compressor.h +90 -0
- package/deps/rocksdb/rocksdb/util/autovector.h +1 -1
- package/deps/rocksdb/rocksdb/util/autovector_test.cc +2 -2
- package/deps/rocksdb/rocksdb/util/compaction_job_stats_impl.cc +0 -2
- package/deps/rocksdb/rocksdb/util/compression.cc +936 -4
- package/deps/rocksdb/rocksdb/util/compression.h +348 -232
- package/deps/rocksdb/rocksdb/util/compression_test.cc +229 -0
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +10 -10
- package/deps/rocksdb/rocksdb/util/crc32c_ppc.c +1 -0
- package/deps/rocksdb/rocksdb/util/data_structure.cc +2 -0
- package/deps/rocksdb/rocksdb/util/file_reader_writer_test.cc +1 -3
- package/deps/rocksdb/rocksdb/util/ppc-opcode.h +5 -5
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +108 -0
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +67 -0
- package/deps/rocksdb/rocksdb/util/slice_test.cc +83 -0
- package/deps/rocksdb/rocksdb/util/string_util.cc +0 -2
- package/deps/rocksdb/rocksdb/util/string_util.h +10 -0
- package/deps/rocksdb/rocksdb/util/thread_operation.h +2 -1
- package/deps/rocksdb/rocksdb/util/udt_util.cc +18 -5
- package/deps/rocksdb/rocksdb/util/udt_util.h +10 -7
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +650 -154
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +438 -144
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_gc_stats.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +16 -17
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +2 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +7 -8
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +4 -3
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/cache_dump_load.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +0 -48
- package/deps/rocksdb/rocksdb/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/compaction_filters/remove_emptyvalue_compactionfilter.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/debug.cc +7 -14
- package/deps/rocksdb/rocksdb/utilities/env_mirror.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/env_mirror_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/env_timed.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/env_timed_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +5 -3
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +10 -9
- package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/memory/memory_util.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/memory_allocators.h +1 -0
- package/deps/rocksdb/rocksdb/utilities/object_registry_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/options/options_util.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_metadata.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_metadata.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table_evictable.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/lrulist.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_test.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_tier.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_tier.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/secondary_index/faiss_ivf_index.cc +183 -32
- package/deps/rocksdb/rocksdb/utilities/secondary_index/faiss_ivf_index_test.cc +258 -12
- package/deps/rocksdb/rocksdb/utilities/secondary_index/secondary_index_helper.h +33 -0
- package/deps/rocksdb/rocksdb/utilities/secondary_index/secondary_index_iterator.cc +99 -0
- package/deps/rocksdb/rocksdb/utilities/secondary_index/secondary_index_mixin.h +280 -120
- package/deps/rocksdb/rocksdb/utilities/secondary_index/simple_secondary_index.cc +79 -0
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc +52 -16
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.h +10 -6
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc +55 -0
- package/deps/rocksdb/rocksdb/utilities/trace/replayer_impl.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/lock_manager.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/lock_manager.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.cc +37 -12
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.h +2 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_tracker.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_locking_test.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/db.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/util/dbt.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.cc +1 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +36 -10
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +5 -7
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +4 -5
- package/deps/rocksdb/rocksdb/utilities/transactions/snapshot_checker.cc +1 -4
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.h +1 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_db_mutex_impl.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_db_mutex_impl.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +1118 -37
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +4 -7
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +3 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_transaction_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +1 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.h +1 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn_db.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +0 -3
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +125 -127
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +45 -23
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +54 -22
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +477 -58
- package/deps/rocksdb/rocksdb.gyp +5 -0
- package/index.js +47 -2
- package/package.json +8 -1
- package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
- package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
|
@@ -91,13 +91,13 @@ CacheAllocationPtr CopyBufferToHeap(MemoryAllocator* allocator, Slice& buf) {
|
|
|
91
91
|
#define INSTANTIATE_BLOCKLIKE_TEMPLATES(T) \
|
|
92
92
|
template Status BlockBasedTable::RetrieveBlock<T>( \
|
|
93
93
|
FilePrefetchBuffer * prefetch_buffer, const ReadOptions& ro, \
|
|
94
|
-
const BlockHandle& handle,
|
|
94
|
+
const BlockHandle& handle, UnownedPtr<Decompressor> decomp, \
|
|
95
95
|
CachableEntry<T>* out_parsed_block, GetContext* get_context, \
|
|
96
96
|
BlockCacheLookupContext* lookup_context, bool for_compaction, \
|
|
97
97
|
bool use_cache, bool async_read, bool use_block_cache_for_lookup) const; \
|
|
98
98
|
template Status BlockBasedTable::MaybeReadBlockAndLoadToCache<T>( \
|
|
99
99
|
FilePrefetchBuffer * prefetch_buffer, const ReadOptions& ro, \
|
|
100
|
-
const BlockHandle& handle,
|
|
100
|
+
const BlockHandle& handle, UnownedPtr<Decompressor> decomp, \
|
|
101
101
|
bool for_compaction, CachableEntry<T>* block_entry, \
|
|
102
102
|
GetContext* get_context, BlockCacheLookupContext* lookup_context, \
|
|
103
103
|
BlockContents* contents, bool async_read, \
|
|
@@ -107,7 +107,7 @@ CacheAllocationPtr CopyBufferToHeap(MemoryAllocator* allocator, Slice& buf) {
|
|
|
107
107
|
CachableEntry<T>* out_parsed_block) const;
|
|
108
108
|
|
|
109
109
|
INSTANTIATE_BLOCKLIKE_TEMPLATES(ParsedFullFilterBlock);
|
|
110
|
-
INSTANTIATE_BLOCKLIKE_TEMPLATES(
|
|
110
|
+
INSTANTIATE_BLOCKLIKE_TEMPLATES(DecompressorDict);
|
|
111
111
|
INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kData);
|
|
112
112
|
INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kIndex);
|
|
113
113
|
INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kFilterPartitionIndex);
|
|
@@ -195,7 +195,7 @@ Status ReadAndParseBlockFromFile(
|
|
|
195
195
|
const Footer& footer, const ReadOptions& options, const BlockHandle& handle,
|
|
196
196
|
std::unique_ptr<TBlocklike>* result, const ImmutableOptions& ioptions,
|
|
197
197
|
BlockCreateContext& create_context, bool maybe_compressed,
|
|
198
|
-
|
|
198
|
+
UnownedPtr<Decompressor> decomp,
|
|
199
199
|
const PersistentCacheOptions& cache_options,
|
|
200
200
|
MemoryAllocator* memory_allocator, bool for_compaction, bool async_read) {
|
|
201
201
|
assert(result);
|
|
@@ -204,8 +204,8 @@ Status ReadAndParseBlockFromFile(
|
|
|
204
204
|
BlockFetcher block_fetcher(
|
|
205
205
|
file, prefetch_buffer, footer, options, handle, &contents, ioptions,
|
|
206
206
|
/*do_uncompress*/ maybe_compressed, maybe_compressed,
|
|
207
|
-
TBlocklike::kBlockType,
|
|
208
|
-
|
|
207
|
+
TBlocklike::kBlockType, decomp, cache_options, memory_allocator, nullptr,
|
|
208
|
+
for_compaction);
|
|
209
209
|
Status s;
|
|
210
210
|
// If prefetch_buffer is not allocated, it will fallback to synchronous
|
|
211
211
|
// reading of block contents.
|
|
@@ -562,6 +562,112 @@ Status GetGlobalSequenceNumber(const TableProperties& table_properties,
|
|
|
562
562
|
|
|
563
563
|
return Status::OK();
|
|
564
564
|
}
|
|
565
|
+
|
|
566
|
+
Status GetDecompressor(const std::string& compression_name,
|
|
567
|
+
UnownedPtr<CompressionManager> compression_manager,
|
|
568
|
+
uint32_t table_format_version,
|
|
569
|
+
std::shared_ptr<Decompressor>* out_decompressor) {
|
|
570
|
+
if (compression_name.empty()) {
|
|
571
|
+
// Very old file (before RocksDB 4.9.0) that might contain compressed
|
|
572
|
+
// blocks. Get a general decompressor for the format version.
|
|
573
|
+
auto mgr_to_use = GetBuiltinCompressionManager(
|
|
574
|
+
GetCompressFormatForVersion(table_format_version));
|
|
575
|
+
*out_decompressor = mgr_to_use->GetDecompressor();
|
|
576
|
+
return Status::OK();
|
|
577
|
+
}
|
|
578
|
+
if (FormatVersionUsesCompressionManagerName(table_format_version)) {
|
|
579
|
+
constexpr char kFieldSep = ';';
|
|
580
|
+
size_t separator_pos = compression_name.find_first_of(kFieldSep);
|
|
581
|
+
if (separator_pos == std::string::npos) {
|
|
582
|
+
return Status::Corruption(
|
|
583
|
+
"Missing separator in compression_name property");
|
|
584
|
+
}
|
|
585
|
+
// Built with explicit CompressionManager and schema support for
|
|
586
|
+
// identifying its compatibility name, which is the first field here.
|
|
587
|
+
Slice compatibility_name(compression_name.data(), separator_pos);
|
|
588
|
+
std::shared_ptr<CompressionManager> mgr_to_use;
|
|
589
|
+
if (compression_manager) {
|
|
590
|
+
// First attempt to go through the compression manager configured for
|
|
591
|
+
// writing new files, for efficiency (usually correct) and not forcing
|
|
592
|
+
// use of ObjectLibrary registration (dependency injection).
|
|
593
|
+
mgr_to_use = compression_manager->FindCompatibleCompressionManager(
|
|
594
|
+
compatibility_name);
|
|
595
|
+
}
|
|
596
|
+
if (mgr_to_use == nullptr) {
|
|
597
|
+
ConfigOptions strict;
|
|
598
|
+
strict.ignore_unknown_options = false;
|
|
599
|
+
strict.ignore_unsupported_options = false;
|
|
600
|
+
Status s = CompressionManager::CreateFromString(
|
|
601
|
+
strict, compatibility_name.ToString(), &mgr_to_use);
|
|
602
|
+
// Even though we might be able to recover from "not found" if only
|
|
603
|
+
// built-in compression types are used (would be checked below), it
|
|
604
|
+
// would provide misleading or unreliable success to allow that to
|
|
605
|
+
// succeed.
|
|
606
|
+
if (!s.ok()) {
|
|
607
|
+
return s;
|
|
608
|
+
}
|
|
609
|
+
assert(mgr_to_use || compatibility_name == kNullptrString ||
|
|
610
|
+
compatibility_name.empty());
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
// Second field is set of compression types actually used in the file
|
|
614
|
+
size_t start_pos = separator_pos + 1;
|
|
615
|
+
separator_pos = compression_name.find_first_of(kFieldSep, start_pos);
|
|
616
|
+
if (UNLIKELY(separator_pos == std::string::npos)) {
|
|
617
|
+
return Status::Corruption("Missing second field from compression_name");
|
|
618
|
+
}
|
|
619
|
+
if (UNLIKELY((separator_pos - start_pos) & 1)) {
|
|
620
|
+
return Status::Corruption(
|
|
621
|
+
"Second field of compression_name has odd size");
|
|
622
|
+
}
|
|
623
|
+
size_t count = (separator_pos - start_pos) / 2;
|
|
624
|
+
auto ctypes = std::make_unique<CompressionType[]>(count);
|
|
625
|
+
const char* ptr = compression_name.data() + start_pos;
|
|
626
|
+
for (size_t i = 0; i < count; ++i) {
|
|
627
|
+
uint64_t val = 0;
|
|
628
|
+
bool success = ParseBaseChars<16>(&ptr, 2, &val);
|
|
629
|
+
if (UNLIKELY(!success || val == kNoCompression ||
|
|
630
|
+
val >= kDisableCompressionOption)) {
|
|
631
|
+
return Status::Corruption(
|
|
632
|
+
"Error parsing second field of compression_name");
|
|
633
|
+
}
|
|
634
|
+
ctypes[i] = static_cast<CompressionType>(val);
|
|
635
|
+
}
|
|
636
|
+
if (mgr_to_use) {
|
|
637
|
+
*out_decompressor = mgr_to_use->GetDecompressorForTypes(
|
|
638
|
+
ctypes.get(), ctypes.get() + count);
|
|
639
|
+
assert(*out_decompressor || count == 0);
|
|
640
|
+
} else {
|
|
641
|
+
// Compression/decompression disabled
|
|
642
|
+
*out_decompressor = nullptr;
|
|
643
|
+
assert(count == 0);
|
|
644
|
+
}
|
|
645
|
+
// Can ignore possible additional future fields
|
|
646
|
+
} else {
|
|
647
|
+
// No explicit CompressionManager, e.g. legacy file support where
|
|
648
|
+
// decompressing with built-in CompressionManager works.
|
|
649
|
+
CompressionType saved_comp_type =
|
|
650
|
+
CompressionTypeFromString(compression_name);
|
|
651
|
+
if (saved_comp_type == kDisableCompressionOption) {
|
|
652
|
+
// Unrecognized. For RocksDB versions able to read format_version=7,
|
|
653
|
+
// this is considered an error so that we can continue to evolve the
|
|
654
|
+
// schema of the compression_name property and report good error
|
|
655
|
+
// messages.
|
|
656
|
+
return Status::Corruption("Unrecognized compression_name: " +
|
|
657
|
+
compression_name);
|
|
658
|
+
} else if (saved_comp_type != kNoCompression) {
|
|
659
|
+
// Use built-in compression manager
|
|
660
|
+
auto mgr_to_use = GetBuiltinCompressionManager(
|
|
661
|
+
GetCompressFormatForVersion(table_format_version));
|
|
662
|
+
*out_decompressor =
|
|
663
|
+
mgr_to_use->GetDecompressorOptimizeFor(saved_comp_type);
|
|
664
|
+
} else {
|
|
665
|
+
// No compression -> decompressor not needed
|
|
666
|
+
*out_decompressor = nullptr;
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
return Status::OK();
|
|
670
|
+
}
|
|
565
671
|
} // namespace
|
|
566
672
|
|
|
567
673
|
void BlockBasedTable::SetupBaseCacheKey(const TableProperties* properties,
|
|
@@ -629,6 +735,7 @@ Status BlockBasedTable::Open(
|
|
|
629
735
|
std::unique_ptr<TableReader>* table_reader, uint64_t tail_size,
|
|
630
736
|
std::shared_ptr<CacheReservationManager> table_reader_cache_res_mgr,
|
|
631
737
|
const std::shared_ptr<const SliceTransform>& prefix_extractor,
|
|
738
|
+
UnownedPtr<CompressionManager> compression_manager,
|
|
632
739
|
const bool prefetch_index_and_filter_in_cache, const bool skip_filters,
|
|
633
740
|
const int level, const bool immortal_table,
|
|
634
741
|
const SequenceNumber largest_seqno, const bool force_direct_prefetch,
|
|
@@ -683,7 +790,8 @@ Status BlockBasedTable::Open(
|
|
|
683
790
|
// 6. [meta block: index]
|
|
684
791
|
// 7. [meta block: filter]
|
|
685
792
|
IOOptions opts;
|
|
686
|
-
|
|
793
|
+
IODebugContext dbg;
|
|
794
|
+
s = file->PrepareIOOptions(ro, opts, &dbg);
|
|
687
795
|
if (s.ok()) {
|
|
688
796
|
s = ReadFooterFromFile(opts, file.get(), *ioptions.fs,
|
|
689
797
|
prefetch_buffer.get(), file_size, &footer,
|
|
@@ -695,7 +803,8 @@ Status BlockBasedTable::Open(
|
|
|
695
803
|
}
|
|
696
804
|
return s;
|
|
697
805
|
}
|
|
698
|
-
if (!IsSupportedFormatVersion(footer.format_version())
|
|
806
|
+
if (!IsSupportedFormatVersion(footer.format_version()) &&
|
|
807
|
+
!TEST_AllowUnsupportedFormatVersion()) {
|
|
699
808
|
return Status::Corruption(
|
|
700
809
|
"Unknown Footer version. Maybe this file was created with newer "
|
|
701
810
|
"version of RocksDB?");
|
|
@@ -708,6 +817,13 @@ Status BlockBasedTable::Open(
|
|
|
708
817
|
rep->file = std::move(file);
|
|
709
818
|
rep->footer = footer;
|
|
710
819
|
|
|
820
|
+
// Some ancient versions (~2.5 - 2.7, format_version=1) could compress the
|
|
821
|
+
// metaindex block, so we need to allow for that
|
|
822
|
+
if (footer.format_version() < 2) {
|
|
823
|
+
auto mgr = GetBuiltinCompressionManager(/*compression_format_version=*/1);
|
|
824
|
+
rep->decompressor = mgr->GetDecompressor();
|
|
825
|
+
}
|
|
826
|
+
|
|
711
827
|
// For fully portable/stable cache keys, we need to read the properties
|
|
712
828
|
// block before setting up cache keys. TODO: consider setting up a bootstrap
|
|
713
829
|
// cache key for PersistentCache to use for metaindex and properties blocks.
|
|
@@ -738,16 +854,19 @@ Status BlockBasedTable::Open(
|
|
|
738
854
|
return s;
|
|
739
855
|
}
|
|
740
856
|
|
|
857
|
+
// Read compression metadata and configure decompressor
|
|
858
|
+
s = GetDecompressor(
|
|
859
|
+
rep->table_properties ? rep->table_properties->compression_name
|
|
860
|
+
: std::string{},
|
|
861
|
+
compression_manager, footer.format_version(), &rep->decompressor);
|
|
862
|
+
if (!s.ok()) {
|
|
863
|
+
return s;
|
|
864
|
+
}
|
|
865
|
+
|
|
741
866
|
// Populate BlockCreateContext
|
|
742
|
-
bool blocks_definitely_zstd_compressed =
|
|
743
|
-
rep->table_properties &&
|
|
744
|
-
(rep->table_properties->compression_name ==
|
|
745
|
-
CompressionTypeToString(kZSTD) ||
|
|
746
|
-
rep->table_properties->compression_name ==
|
|
747
|
-
CompressionTypeToString(kZSTDNotFinalCompression));
|
|
748
867
|
rep->create_context = BlockCreateContext(
|
|
749
868
|
&rep->table_options, &rep->ioptions, rep->ioptions.stats,
|
|
750
|
-
|
|
869
|
+
rep->decompressor.get(), block_protection_bytes_per_key,
|
|
751
870
|
rep->internal_comparator.user_comparator(), rep->index_value_is_full,
|
|
752
871
|
rep->index_has_first_key);
|
|
753
872
|
|
|
@@ -917,6 +1036,7 @@ Status BlockBasedTable::PrefetchTail(
|
|
|
917
1036
|
"TailPrefetchStats.",
|
|
918
1037
|
file->file_name().c_str(), tail_prefetch_size);
|
|
919
1038
|
}
|
|
1039
|
+
TEST_SYNC_POINT("BlockBasedTable::PrefetchTail::TaiSizeNotRecorded");
|
|
920
1040
|
}
|
|
921
1041
|
size_t prefetch_off;
|
|
922
1042
|
size_t prefetch_len;
|
|
@@ -936,7 +1056,8 @@ Status BlockBasedTable::PrefetchTail(
|
|
|
936
1056
|
#endif // NDEBUG
|
|
937
1057
|
|
|
938
1058
|
IOOptions opts;
|
|
939
|
-
|
|
1059
|
+
IODebugContext dbg;
|
|
1060
|
+
Status s = file->PrepareIOOptions(ro, opts, &dbg);
|
|
940
1061
|
// Try file system prefetch
|
|
941
1062
|
if (s.ok() && !file->use_direct_io() && !force_direct_prefetch) {
|
|
942
1063
|
if (!file->Prefetch(opts, prefetch_off, prefetch_len).IsNotSupported()) {
|
|
@@ -999,9 +1120,6 @@ Status BlockBasedTable::ReadPropertiesBlock(
|
|
|
999
1120
|
"Problem reading or processing seqno-to-time mapping: %s",
|
|
1000
1121
|
s.ToString().c_str());
|
|
1001
1122
|
}
|
|
1002
|
-
rep_->blocks_maybe_compressed =
|
|
1003
|
-
rep_->table_properties->compression_name !=
|
|
1004
|
-
CompressionTypeToString(kNoCompression);
|
|
1005
1123
|
}
|
|
1006
1124
|
} else {
|
|
1007
1125
|
ROCKS_LOG_ERROR(rep_->ioptions.logger,
|
|
@@ -1107,71 +1225,24 @@ Status BlockBasedTable::PrefetchIndexAndFilterBlocks(
|
|
|
1107
1225
|
// Find filter handle and filter type
|
|
1108
1226
|
if (rep_->filter_policy) {
|
|
1109
1227
|
auto name = rep_->filter_policy->CompatibilityName();
|
|
1110
|
-
bool builtin_compatible =
|
|
1111
|
-
strcmp(name, BuiltinFilterPolicy::kCompatibilityName()) == 0;
|
|
1112
|
-
|
|
1113
1228
|
for (const auto& [filter_type, prefix] :
|
|
1114
1229
|
{std::make_pair(Rep::FilterType::kFullFilter, kFullFilterBlockPrefix),
|
|
1115
1230
|
std::make_pair(Rep::FilterType::kPartitionedFilter,
|
|
1116
1231
|
kPartitionedFilterBlockPrefix),
|
|
1117
1232
|
std::make_pair(Rep::FilterType::kNoFilter,
|
|
1118
1233
|
kObsoleteFilterBlockPrefix)}) {
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
test::FastLocalBloomFilterPolicy::kClassName(),
|
|
1130
|
-
test::Standard128RibbonFilterPolicy::kClassName(),
|
|
1131
|
-
"rocksdb.internal.DeprecatedBlockBasedBloomFilter",
|
|
1132
|
-
BloomFilterPolicy::kClassName(),
|
|
1133
|
-
RibbonFilterPolicy::kClassName(),
|
|
1134
|
-
};
|
|
1135
|
-
|
|
1136
|
-
// For efficiency, do a prefix seek and see if the first match is
|
|
1137
|
-
// good.
|
|
1138
|
-
meta_iter->Seek(prefix);
|
|
1139
|
-
if (meta_iter->status().ok() && meta_iter->Valid()) {
|
|
1140
|
-
Slice key = meta_iter->key();
|
|
1141
|
-
if (key.starts_with(prefix)) {
|
|
1142
|
-
key.remove_prefix(prefix.size());
|
|
1143
|
-
if (kBuiltinNameAndAliases.find(key.ToString()) !=
|
|
1144
|
-
kBuiltinNameAndAliases.end()) {
|
|
1145
|
-
Slice v = meta_iter->value();
|
|
1146
|
-
Status s = rep_->filter_handle.DecodeFrom(&v);
|
|
1147
|
-
if (s.ok()) {
|
|
1148
|
-
rep_->filter_type = filter_type;
|
|
1149
|
-
if (filter_type == Rep::FilterType::kNoFilter) {
|
|
1150
|
-
ROCKS_LOG_WARN(rep_->ioptions.logger,
|
|
1151
|
-
"Detected obsolete filter type in %s. Read "
|
|
1152
|
-
"performance might suffer until DB is fully "
|
|
1153
|
-
"re-compacted.",
|
|
1154
|
-
rep_->file->file_name().c_str());
|
|
1155
|
-
}
|
|
1156
|
-
break;
|
|
1157
|
-
}
|
|
1158
|
-
}
|
|
1159
|
-
}
|
|
1160
|
-
}
|
|
1161
|
-
} else {
|
|
1162
|
-
std::string filter_block_key = prefix + name;
|
|
1163
|
-
if (FindMetaBlock(meta_iter, filter_block_key, &rep_->filter_handle)
|
|
1164
|
-
.ok()) {
|
|
1165
|
-
rep_->filter_type = filter_type;
|
|
1166
|
-
if (filter_type == Rep::FilterType::kNoFilter) {
|
|
1167
|
-
ROCKS_LOG_WARN(
|
|
1168
|
-
rep_->ioptions.logger,
|
|
1169
|
-
"Detected obsolete filter type in %s. Read performance might "
|
|
1170
|
-
"suffer until DB is fully re-compacted.",
|
|
1171
|
-
rep_->file->file_name().c_str());
|
|
1172
|
-
}
|
|
1173
|
-
break;
|
|
1234
|
+
std::string filter_block_key = prefix + name;
|
|
1235
|
+
if (FindMetaBlock(meta_iter, filter_block_key, &rep_->filter_handle)
|
|
1236
|
+
.ok()) {
|
|
1237
|
+
rep_->filter_type = filter_type;
|
|
1238
|
+
if (filter_type == Rep::FilterType::kNoFilter) {
|
|
1239
|
+
ROCKS_LOG_WARN(
|
|
1240
|
+
rep_->ioptions.logger,
|
|
1241
|
+
"Detected obsolete filter type in %s. Read performance might "
|
|
1242
|
+
"suffer until DB is fully re-compacted.",
|
|
1243
|
+
rep_->file->file_name().c_str());
|
|
1174
1244
|
}
|
|
1245
|
+
break;
|
|
1175
1246
|
}
|
|
1176
1247
|
}
|
|
1177
1248
|
}
|
|
@@ -1288,7 +1359,10 @@ Status BlockBasedTable::PrefetchIndexAndFilterBlocks(
|
|
|
1288
1359
|
}
|
|
1289
1360
|
}
|
|
1290
1361
|
|
|
1291
|
-
|
|
1362
|
+
// NOTE: before the fix to https://github.com/facebook/rocksdb/issues/12409, a
|
|
1363
|
+
// file could have a (de)compression dictionary block without a configured
|
|
1364
|
+
// compression, so we need to ignore the dictionary in that case.
|
|
1365
|
+
if (!rep_->compression_dict_handle.IsNull() && rep_->decompressor) {
|
|
1292
1366
|
std::unique_ptr<UncompressionDictReader> uncompression_dict_reader;
|
|
1293
1367
|
s = UncompressionDictReader::Create(
|
|
1294
1368
|
this, ro, prefetch_buffer, use_cache, prefetch_all || pin_unpartitioned,
|
|
@@ -1350,10 +1424,9 @@ Status BlockBasedTable::ReadMetaIndexBlock(
|
|
|
1350
1424
|
Status s = ReadAndParseBlockFromFile(
|
|
1351
1425
|
rep_->file.get(), prefetch_buffer, rep_->footer, ro,
|
|
1352
1426
|
rep_->footer.metaindex_handle(), &metaindex, rep_->ioptions,
|
|
1353
|
-
rep_->create_context, true /*maybe_compressed*/,
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
false /* async_read */);
|
|
1427
|
+
rep_->create_context, true /*maybe_compressed*/, rep_->decompressor.get(),
|
|
1428
|
+
rep_->persistent_cache_options, GetMemoryAllocator(rep_->table_options),
|
|
1429
|
+
false /* for_compaction */, false /* async_read */);
|
|
1357
1430
|
|
|
1358
1431
|
if (!s.ok()) {
|
|
1359
1432
|
ROCKS_LOG_ERROR(rep_->ioptions.logger,
|
|
@@ -1392,7 +1465,7 @@ template <typename TBlocklike>
|
|
|
1392
1465
|
WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::GetDataBlockFromCache(
|
|
1393
1466
|
const Slice& cache_key, BlockCacheInterface<TBlocklike> block_cache,
|
|
1394
1467
|
CachableEntry<TBlocklike>* out_parsed_block, GetContext* get_context,
|
|
1395
|
-
|
|
1468
|
+
UnownedPtr<Decompressor> decomp) const {
|
|
1396
1469
|
assert(out_parsed_block);
|
|
1397
1470
|
assert(out_parsed_block->IsEmpty());
|
|
1398
1471
|
|
|
@@ -1401,12 +1474,24 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::GetDataBlockFromCache(
|
|
|
1401
1474
|
|
|
1402
1475
|
// Lookup uncompressed cache first
|
|
1403
1476
|
if (block_cache) {
|
|
1404
|
-
BlockCreateContext create_ctx = rep_->create_context;
|
|
1405
|
-
create_ctx.dict = dict;
|
|
1406
1477
|
assert(!cache_key.empty());
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1478
|
+
typename BlockCacheInterface<TBlocklike>::TypedHandle* cache_handle;
|
|
1479
|
+
if (decomp.get() != rep_->decompressor.get() && decomp) {
|
|
1480
|
+
// `decomp` must be a dictionary-aware decompressor, which is only
|
|
1481
|
+
// available in the block cache (so that dictionaries can be evicted
|
|
1482
|
+
// from memory) and can't live in the table reader.
|
|
1483
|
+
// NOTE: inefficient BlockCreateContext copy for dict-aware decompressor
|
|
1484
|
+
// (see TODO in block_cache.h)
|
|
1485
|
+
BlockCreateContext create_ctx = rep_->create_context;
|
|
1486
|
+
create_ctx.decompressor = decomp.get();
|
|
1487
|
+
cache_handle = block_cache.LookupFull(
|
|
1488
|
+
cache_key, &create_ctx, GetCachePriority<TBlocklike>(), statistics,
|
|
1489
|
+
rep_->ioptions.lowest_used_cache_tier);
|
|
1490
|
+
} else {
|
|
1491
|
+
cache_handle = block_cache.LookupFull(
|
|
1492
|
+
cache_key, &rep_->create_context, GetCachePriority<TBlocklike>(),
|
|
1493
|
+
statistics, rep_->ioptions.lowest_used_cache_tier);
|
|
1494
|
+
}
|
|
1410
1495
|
|
|
1411
1496
|
// Avoid updating metrics here if the handle is not complete yet. This
|
|
1412
1497
|
// happens with MultiGet and secondary cache. So update the metrics only
|
|
@@ -1436,10 +1521,9 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::PutDataBlockToCache(
|
|
|
1436
1521
|
CachableEntry<TBlocklike>* out_parsed_block,
|
|
1437
1522
|
BlockContents&& uncompressed_block_contents,
|
|
1438
1523
|
BlockContents&& compressed_block_contents, CompressionType block_comp_type,
|
|
1439
|
-
|
|
1440
|
-
|
|
1524
|
+
UnownedPtr<Decompressor> decomp, MemoryAllocator* memory_allocator,
|
|
1525
|
+
GetContext* get_context) const {
|
|
1441
1526
|
const ImmutableOptions& ioptions = rep_->ioptions;
|
|
1442
|
-
const uint32_t format_version = rep_->table_options.format_version;
|
|
1443
1527
|
assert(out_parsed_block);
|
|
1444
1528
|
assert(out_parsed_block->IsEmpty());
|
|
1445
1529
|
|
|
@@ -1451,12 +1535,10 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::PutDataBlockToCache(
|
|
|
1451
1535
|
uncompressed_block_contents.data.empty()) {
|
|
1452
1536
|
assert(compressed_block_contents.data.data());
|
|
1453
1537
|
// Retrieve the uncompressed contents into a new buffer
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
&uncompressed_block_contents, format_version,
|
|
1459
|
-
ioptions, memory_allocator);
|
|
1538
|
+
s = DecompressBlockData(
|
|
1539
|
+
compressed_block_contents.data.data(),
|
|
1540
|
+
compressed_block_contents.data.size(), block_comp_type, *decomp,
|
|
1541
|
+
&uncompressed_block_contents, ioptions, memory_allocator);
|
|
1460
1542
|
if (!s.ok()) {
|
|
1461
1543
|
return s;
|
|
1462
1544
|
}
|
|
@@ -1569,15 +1651,18 @@ Status BlockBasedTable::LookupAndPinBlocksInCache(
|
|
|
1569
1651
|
assert(block_cache);
|
|
1570
1652
|
|
|
1571
1653
|
Status s;
|
|
1572
|
-
CachableEntry<
|
|
1654
|
+
CachableEntry<DecompressorDict> cached_dict;
|
|
1573
1655
|
if (rep_->uncompression_dict_reader) {
|
|
1574
1656
|
s = rep_->uncompression_dict_reader->GetOrReadUncompressionDictionary(
|
|
1575
1657
|
/* prefetch_buffer= */ nullptr, ro,
|
|
1576
1658
|
/* get_context= */ nullptr, /* lookup_context= */ nullptr,
|
|
1577
|
-
&
|
|
1659
|
+
&cached_dict);
|
|
1578
1660
|
if (!s.ok()) {
|
|
1579
1661
|
return s;
|
|
1580
1662
|
}
|
|
1663
|
+
if (!cached_dict.GetValue()) {
|
|
1664
|
+
return Status::Corruption("Success but no dictionary read");
|
|
1665
|
+
}
|
|
1581
1666
|
}
|
|
1582
1667
|
|
|
1583
1668
|
// Do the lookup.
|
|
@@ -1586,14 +1671,20 @@ Status BlockBasedTable::LookupAndPinBlocksInCache(
|
|
|
1586
1671
|
|
|
1587
1672
|
Statistics* statistics = rep_->ioptions.statistics.get();
|
|
1588
1673
|
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1674
|
+
typename BlockCacheInterface<TBlocklike>::TypedHandle* cache_handle;
|
|
1675
|
+
if (cached_dict.GetValue()) {
|
|
1676
|
+
// NOTE: inefficient BlockCreateContext copy for dict-aware decompressor
|
|
1677
|
+
// (see TODO in block_cache.h)
|
|
1678
|
+
BlockCreateContext create_ctx = rep_->create_context;
|
|
1679
|
+
create_ctx.decompressor = cached_dict.GetValue()->decompressor_.get();
|
|
1680
|
+
cache_handle = block_cache.LookupFull(
|
|
1681
|
+
key, &create_ctx, GetCachePriority<TBlocklike>(), statistics,
|
|
1682
|
+
rep_->ioptions.lowest_used_cache_tier);
|
|
1683
|
+
} else {
|
|
1684
|
+
cache_handle = block_cache.LookupFull(
|
|
1685
|
+
key, &rep_->create_context, GetCachePriority<TBlocklike>(), statistics,
|
|
1686
|
+
rep_->ioptions.lowest_used_cache_tier);
|
|
1687
|
+
}
|
|
1597
1688
|
|
|
1598
1689
|
if (!cache_handle) {
|
|
1599
1690
|
UpdateCacheMissMetrics(TBlocklike::kBlockType, /* get_context = */ nullptr);
|
|
@@ -1622,7 +1713,7 @@ template <typename TBlocklike>
|
|
|
1622
1713
|
WithBlocklikeCheck<Status, TBlocklike>
|
|
1623
1714
|
BlockBasedTable::MaybeReadBlockAndLoadToCache(
|
|
1624
1715
|
FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro,
|
|
1625
|
-
const BlockHandle& handle,
|
|
1716
|
+
const BlockHandle& handle, UnownedPtr<Decompressor> decomp,
|
|
1626
1717
|
bool for_compaction, CachableEntry<TBlocklike>* out_parsed_block,
|
|
1627
1718
|
GetContext* get_context, BlockCacheLookupContext* lookup_context,
|
|
1628
1719
|
BlockContents* contents, bool async_read,
|
|
@@ -1646,7 +1737,7 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache(
|
|
|
1646
1737
|
if (!contents) {
|
|
1647
1738
|
if (use_block_cache_for_lookup) {
|
|
1648
1739
|
s = GetDataBlockFromCache(key, block_cache, out_parsed_block,
|
|
1649
|
-
get_context,
|
|
1740
|
+
get_context, decomp);
|
|
1650
1741
|
// Value could still be null at this point, so check the cache handle
|
|
1651
1742
|
// and update the read pattern for prefetching
|
|
1652
1743
|
if (out_parsed_block->GetValue() ||
|
|
@@ -1676,7 +1767,7 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache(
|
|
|
1676
1767
|
const bool maybe_compressed =
|
|
1677
1768
|
TBlocklike::kBlockType != BlockType::kFilter &&
|
|
1678
1769
|
TBlocklike::kBlockType != BlockType::kCompressionDictionary &&
|
|
1679
|
-
rep_->
|
|
1770
|
+
rep_->decompressor;
|
|
1680
1771
|
// This flag, if true, tells BlockFetcher to return the uncompressed
|
|
1681
1772
|
// block when ReadBlockContents() is called.
|
|
1682
1773
|
const bool do_uncompress = maybe_compressed;
|
|
@@ -1700,8 +1791,7 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache(
|
|
|
1700
1791
|
BlockFetcher block_fetcher(
|
|
1701
1792
|
rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle,
|
|
1702
1793
|
&tmp_contents, rep_->ioptions, do_uncompress, maybe_compressed,
|
|
1703
|
-
TBlocklike::kBlockType,
|
|
1704
|
-
rep_->persistent_cache_options,
|
|
1794
|
+
TBlocklike::kBlockType, decomp, rep_->persistent_cache_options,
|
|
1705
1795
|
GetMemoryAllocator(rep_->table_options),
|
|
1706
1796
|
/*allocator=*/nullptr);
|
|
1707
1797
|
|
|
@@ -1716,7 +1806,7 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache(
|
|
|
1716
1806
|
s = block_fetcher.ReadBlockContents();
|
|
1717
1807
|
}
|
|
1718
1808
|
|
|
1719
|
-
contents_comp_type = block_fetcher.
|
|
1809
|
+
contents_comp_type = block_fetcher.compression_type();
|
|
1720
1810
|
if (get_context) {
|
|
1721
1811
|
switch (TBlocklike::kBlockType) {
|
|
1722
1812
|
case BlockType::kIndex:
|
|
@@ -1748,7 +1838,7 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache(
|
|
|
1748
1838
|
// block in block_fetcher
|
|
1749
1839
|
s = PutDataBlockToCache(
|
|
1750
1840
|
key, block_cache, out_parsed_block, std::move(uncomp_contents),
|
|
1751
|
-
std::move(comp_contents), contents_comp_type,
|
|
1841
|
+
std::move(comp_contents), contents_comp_type, decomp,
|
|
1752
1842
|
GetMemoryAllocator(rep_->table_options), get_context);
|
|
1753
1843
|
}
|
|
1754
1844
|
} else {
|
|
@@ -1764,7 +1854,7 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache(
|
|
|
1764
1854
|
// the block to the cache.
|
|
1765
1855
|
s = PutDataBlockToCache(
|
|
1766
1856
|
key, block_cache, out_parsed_block, std::move(uncomp_contents),
|
|
1767
|
-
std::move(comp_contents), contents_comp_type,
|
|
1857
|
+
std::move(comp_contents), contents_comp_type, decomp,
|
|
1768
1858
|
GetMemoryAllocator(rep_->table_options), get_context);
|
|
1769
1859
|
}
|
|
1770
1860
|
}
|
|
@@ -1879,7 +1969,7 @@ void BlockBasedTable::FinishTraceRecord(
|
|
|
1879
1969
|
template <typename TBlocklike /*, auto*/>
|
|
1880
1970
|
WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::RetrieveBlock(
|
|
1881
1971
|
FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro,
|
|
1882
|
-
const BlockHandle& handle,
|
|
1972
|
+
const BlockHandle& handle, UnownedPtr<Decompressor> decomp,
|
|
1883
1973
|
CachableEntry<TBlocklike>* out_parsed_block, GetContext* get_context,
|
|
1884
1974
|
BlockCacheLookupContext* lookup_context, bool for_compaction,
|
|
1885
1975
|
bool use_cache, bool async_read, bool use_block_cache_for_lookup) const {
|
|
@@ -1889,8 +1979,8 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::RetrieveBlock(
|
|
|
1889
1979
|
Status s;
|
|
1890
1980
|
if (use_cache) {
|
|
1891
1981
|
s = MaybeReadBlockAndLoadToCache(
|
|
1892
|
-
prefetch_buffer, ro, handle,
|
|
1893
|
-
|
|
1982
|
+
prefetch_buffer, ro, handle, decomp, for_compaction, out_parsed_block,
|
|
1983
|
+
get_context, lookup_context,
|
|
1894
1984
|
/*contents=*/nullptr, async_read, use_block_cache_for_lookup);
|
|
1895
1985
|
|
|
1896
1986
|
if (!s.ok()) {
|
|
@@ -1914,7 +2004,7 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::RetrieveBlock(
|
|
|
1914
2004
|
const bool maybe_compressed =
|
|
1915
2005
|
TBlocklike::kBlockType != BlockType::kFilter &&
|
|
1916
2006
|
TBlocklike::kBlockType != BlockType::kCompressionDictionary &&
|
|
1917
|
-
rep_->
|
|
2007
|
+
rep_->decompressor;
|
|
1918
2008
|
std::unique_ptr<TBlocklike> block;
|
|
1919
2009
|
|
|
1920
2010
|
{
|
|
@@ -1923,9 +2013,9 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::RetrieveBlock(
|
|
|
1923
2013
|
StopWatch sw(rep_->ioptions.clock, rep_->ioptions.stats, histogram);
|
|
1924
2014
|
s = ReadAndParseBlockFromFile(
|
|
1925
2015
|
rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle, &block,
|
|
1926
|
-
rep_->ioptions, rep_->create_context, maybe_compressed,
|
|
1927
|
-
|
|
1928
|
-
|
|
2016
|
+
rep_->ioptions, rep_->create_context, maybe_compressed, decomp,
|
|
2017
|
+
rep_->persistent_cache_options, GetMemoryAllocator(rep_->table_options),
|
|
2018
|
+
for_compaction, async_read);
|
|
1929
2019
|
|
|
1930
2020
|
if (get_context) {
|
|
1931
2021
|
switch (TBlocklike::kBlockType) {
|
|
@@ -2607,8 +2697,8 @@ Status BlockBasedTable::VerifyChecksumInBlocks(
|
|
|
2607
2697
|
BlockFetcher block_fetcher(
|
|
2608
2698
|
rep_->file.get(), &prefetch_buffer, rep_->footer, read_options, handle,
|
|
2609
2699
|
&contents, rep_->ioptions, false /* decompress */,
|
|
2610
|
-
false /*maybe_compressed*/, BlockType::kData,
|
|
2611
|
-
|
|
2700
|
+
false /*maybe_compressed*/, BlockType::kData, nullptr /*decompressor*/,
|
|
2701
|
+
rep_->persistent_cache_options);
|
|
2612
2702
|
s = block_fetcher.ReadBlockContents();
|
|
2613
2703
|
if (!s.ok()) {
|
|
2614
2704
|
break;
|
|
@@ -2697,12 +2787,12 @@ Status BlockBasedTable::VerifyChecksumInMetaBlocks(
|
|
|
2697
2787
|
// if it was checked on open.
|
|
2698
2788
|
} else {
|
|
2699
2789
|
// FIXME? Need to verify checksums of index and filter partitions?
|
|
2700
|
-
s = BlockFetcher(
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2790
|
+
s = BlockFetcher(rep_->file.get(), nullptr /* prefetch buffer */,
|
|
2791
|
+
rep_->footer, read_options, handle, &contents,
|
|
2792
|
+
rep_->ioptions, false /* decompress */,
|
|
2793
|
+
false /*maybe_compressed*/,
|
|
2794
|
+
GetBlockTypeForMetaBlockByName(meta_block_name),
|
|
2795
|
+
nullptr /*decompressor*/, rep_->persistent_cache_options)
|
|
2706
2796
|
.ReadBlockContents();
|
|
2707
2797
|
}
|
|
2708
2798
|
if (!s.ok()) {
|
|
@@ -3082,7 +3172,7 @@ Status BlockBasedTable::DumpTable(WritableFile* out_file) {
|
|
|
3082
3172
|
|
|
3083
3173
|
// Output compression dictionary
|
|
3084
3174
|
if (rep_->uncompression_dict_reader) {
|
|
3085
|
-
CachableEntry<
|
|
3175
|
+
CachableEntry<DecompressorDict> uncompression_dict;
|
|
3086
3176
|
s = rep_->uncompression_dict_reader->GetOrReadUncompressionDictionary(
|
|
3087
3177
|
nullptr /* prefetch_buffer */, ro, nullptr /* get_context */,
|
|
3088
3178
|
nullptr /* lookup_context */, &uncompression_dict);
|