@nxtedition/rocksdb 13.5.7 → 13.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +248 -70
- package/binding.gyp +2 -2
- package/deps/rocksdb/rocksdb/BUCK +12 -0
- package/deps/rocksdb/rocksdb/CMakeLists.txt +7 -0
- package/deps/rocksdb/rocksdb/Makefile +28 -23
- package/deps/rocksdb/rocksdb/cache/cache.cc +0 -1
- package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +1 -2
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +43 -39
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h +2 -0
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +0 -1
- package/deps/rocksdb/rocksdb/cache/lru_cache.cc +2 -3
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +2 -2
- package/deps/rocksdb/rocksdb/cache/secondary_cache.cc +1 -3
- package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.cc +11 -1
- package/deps/rocksdb/rocksdb/cache/tiered_secondary_cache_test.cc +13 -5
- package/deps/rocksdb/rocksdb/crash_test.mk +61 -15
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +136 -45
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +34 -16
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +10 -7
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +1 -2
- package/deps/rocksdb/rocksdb/db/blob/blob_file_meta.h +1 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +12 -9
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/blob/blob_source.cc +2 -2
- package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/builder.cc +22 -8
- package/deps/rocksdb/rocksdb/db/builder.h +5 -4
- package/deps/rocksdb/rocksdb/db/c.cc +556 -15
- package/deps/rocksdb/rocksdb/db/c_test.c +133 -12
- package/deps/rocksdb/rocksdb/db/column_family.cc +114 -50
- package/deps/rocksdb/rocksdb/db/column_family.h +53 -36
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +95 -70
- package/deps/rocksdb/rocksdb/db/compaction/compaction.h +71 -51
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +7 -86
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +26 -68
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +0 -122
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +453 -258
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +117 -92
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +38 -38
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +24 -17
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +34 -45
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +32 -31
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +12 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +1 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +10 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +82 -34
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +267 -179
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +4 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +273 -89
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +300 -14
- package/deps/rocksdb/rocksdb/db/compaction/compaction_state.cc +4 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_state.h +2 -2
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +28 -23
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +69 -51
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +522 -245
- package/deps/rocksdb/rocksdb/db/convenience.cc +15 -4
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +1 -3
- package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +0 -2
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +196 -17
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +74 -62
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +48 -0
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +682 -250
- package/deps/rocksdb/rocksdb/db/db_dynamic_level_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_encryption_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +11 -16
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +57 -0
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +540 -490
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +347 -188
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +584 -217
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +13 -9
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +5 -7
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +40 -36
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +751 -372
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +35 -32
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +24 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +125 -63
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +2 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +311 -196
- package/deps/rocksdb/rocksdb/db/db_io_failure_test.cc +15 -5
- package/deps/rocksdb/rocksdb/db/db_iter.cc +42 -29
- package/deps/rocksdb/rocksdb/db/db_iter.h +96 -31
- package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/db_iter_test.cc +168 -228
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +454 -0
- package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +8 -8
- package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +90 -0
- package/deps/rocksdb/rocksdb/db/db_merge_operand_test.cc +60 -2
- package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +7 -3
- package/deps/rocksdb/rocksdb/db/db_options_test.cc +85 -27
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +3 -1
- package/deps/rocksdb/rocksdb/db/db_rate_limiter_test.cc +0 -2
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +114 -2
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +51 -3
- package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_test.cc +325 -18
- package/deps/rocksdb/rocksdb/db/db_test2.cc +644 -20
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +14 -6
- package/deps/rocksdb/rocksdb/db/db_test_util.h +9 -0
- package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +64 -45
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +203 -14
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +259 -30
- package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +75 -1
- package/deps/rocksdb/rocksdb/db/dbformat.h +70 -6
- package/deps/rocksdb/rocksdb/db/deletefile_test.cc +0 -190
- package/deps/rocksdb/rocksdb/db/error_handler.cc +22 -7
- package/deps/rocksdb/rocksdb/db/error_handler.h +16 -1
- package/deps/rocksdb/rocksdb/db/event_helpers.cc +41 -26
- package/deps/rocksdb/rocksdb/db/experimental.cc +4 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +464 -78
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +166 -69
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +54 -25
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +1 -3
- package/deps/rocksdb/rocksdb/db/flush_job.cc +98 -81
- package/deps/rocksdb/rocksdb/db/flush_job.h +4 -9
- package/deps/rocksdb/rocksdb/db/flush_job_test.cc +80 -84
- package/deps/rocksdb/rocksdb/db/forward_iterator.cc +1 -1
- package/deps/rocksdb/rocksdb/db/forward_iterator.h +2 -2
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +12 -19
- package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +0 -2
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +41 -15
- package/deps/rocksdb/rocksdb/db/internal_stats.h +63 -52
- package/deps/rocksdb/rocksdb/db/job_context.h +59 -24
- package/deps/rocksdb/rocksdb/db/listener_test.cc +69 -10
- package/deps/rocksdb/rocksdb/db/log_format.h +11 -2
- package/deps/rocksdb/rocksdb/db/log_reader.cc +147 -34
- package/deps/rocksdb/rocksdb/db/log_reader.h +40 -11
- package/deps/rocksdb/rocksdb/db/log_test.cc +16 -3
- package/deps/rocksdb/rocksdb/db/log_writer.cc +102 -55
- package/deps/rocksdb/rocksdb/db/log_writer.h +21 -2
- package/deps/rocksdb/rocksdb/db/malloc_stats.h +0 -2
- package/deps/rocksdb/rocksdb/db/memtable.cc +16 -47
- package/deps/rocksdb/rocksdb/db/memtable.h +76 -12
- package/deps/rocksdb/rocksdb/db/memtable_list.cc +23 -20
- package/deps/rocksdb/rocksdb/db/memtable_list.h +9 -11
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +18 -37
- package/deps/rocksdb/rocksdb/db/merge_context.h +2 -1
- package/deps/rocksdb/rocksdb/db/merge_test.cc +8 -0
- package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +3 -5
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +15 -7
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.h +6 -3
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +22 -4
- package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +41 -1
- package/deps/rocksdb/rocksdb/db/prefix_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/repair.cc +29 -34
- package/deps/rocksdb/rocksdb/db/repair_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +14 -15
- package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc +1 -3
- package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h +47 -1
- package/deps/rocksdb/rocksdb/db/table_cache.cc +3 -3
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +1 -3
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +2 -1
- package/deps/rocksdb/rocksdb/db/version_builder.cc +2 -2
- package/deps/rocksdb/rocksdb/db/version_edit.cc +8 -37
- package/deps/rocksdb/rocksdb/db/version_edit.h +32 -1
- package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +26 -18
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -5
- package/deps/rocksdb/rocksdb/db/version_set.cc +282 -197
- package/deps/rocksdb/rocksdb/db/version_set.h +54 -57
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +28 -35
- package/deps/rocksdb/rocksdb/db/version_util.h +2 -3
- package/deps/rocksdb/rocksdb/db/wal_manager.cc +3 -2
- package/deps/rocksdb/rocksdb/db/wal_manager.h +0 -1
- package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/wide/wide_columns.cc +1 -0
- package/deps/rocksdb/rocksdb/db/write_batch.cc +22 -8
- package/deps/rocksdb/rocksdb/db/write_batch_internal.h +5 -4
- package/deps/rocksdb/rocksdb/db/write_batch_test.cc +7 -6
- package/deps/rocksdb/rocksdb/db/write_callback_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/write_thread.h +3 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +13 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +9 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compaction_service.h +39 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compression_manager.h +65 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +45 -22
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +7 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +22 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_table_properties_collector.h +28 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +143 -38
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +4 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +80 -32
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.h +51 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +23 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +305 -15
- package/deps/rocksdb/rocksdb/env/env.cc +32 -2
- package/deps/rocksdb/rocksdb/env/env_encryption.cc +0 -2
- package/deps/rocksdb/rocksdb/env/env_encryption_ctr.h +2 -4
- package/deps/rocksdb/rocksdb/env/env_posix.cc +4 -2
- package/deps/rocksdb/rocksdb/env/env_test.cc +0 -1
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +20 -11
- package/deps/rocksdb/rocksdb/env/fs_readonly.h +0 -2
- package/deps/rocksdb/rocksdb/env/fs_remap.cc +0 -2
- package/deps/rocksdb/rocksdb/env/fs_remap.h +0 -2
- package/deps/rocksdb/rocksdb/env/io_posix.cc +6 -4
- package/deps/rocksdb/rocksdb/env/io_posix.h +3 -2
- package/deps/rocksdb/rocksdb/env/mock_env.cc +0 -1
- package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +2 -2
- package/deps/rocksdb/rocksdb/file/delete_scheduler.h +0 -2
- package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +0 -2
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +30 -21
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +16 -0
- package/deps/rocksdb/rocksdb/file/file_util.cc +32 -14
- package/deps/rocksdb/rocksdb/file/file_util.h +22 -5
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +229 -76
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +21 -12
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.h +10 -7
- package/deps/rocksdb/rocksdb/file/random_access_file_reader_test.cc +12 -8
- package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.cc +1 -2
- package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.h +0 -2
- package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +3 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +598 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_iterator.h +36 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +70 -11
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +232 -11
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/compaction_job_stats.h +3 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/compression_type.h +149 -15
- package/deps/rocksdb/rocksdb/include/rocksdb/convenience.h +17 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/data_structure.h +132 -34
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +158 -79
- package/deps/rocksdb/rocksdb/include/rocksdb/db_bench_tool.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/env.h +4 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/env_encryption.h +1 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/external_table.h +275 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +50 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +10 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/ldb_tool.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +5 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/multi_scan.h +237 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +230 -39
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +15 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_level.h +31 -11
- package/deps/rocksdb/rocksdb/include/rocksdb/slice.h +41 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_dump_tool.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h +5 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +18 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/status.h +2 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +20 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +19 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/thread_status.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/tool_hooks.h +124 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/trace_record.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +26 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h +55 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/debug.h +3 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/env_mirror.h +0 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +1 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/memory_util.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/object_registry.h +1 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/options_util.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/secondary_index.h +96 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/secondary_index_faiss.h +117 -0
- package/deps/rocksdb/rocksdb/{utilities/secondary_index/faiss_ivf_index.h → include/rocksdb/utilities/secondary_index_simple.h} +11 -14
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +26 -11
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/table_properties_collectors.h +16 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +0 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +63 -7
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db_mutex.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/write_batch_with_index.h +28 -12
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +3 -3
- package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +0 -2
- package/deps/rocksdb/rocksdb/logging/event_logger_test.cc +1 -2
- package/deps/rocksdb/rocksdb/memory/memory_allocator_impl.h +1 -1
- package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc +0 -1
- package/deps/rocksdb/rocksdb/memtable/hash_linklist_rep.cc +0 -1
- package/deps/rocksdb/rocksdb/memtable/memtablerep_bench.cc +3 -1
- package/deps/rocksdb/rocksdb/memtable/skiplist.h +2 -2
- package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +2 -4
- package/deps/rocksdb/rocksdb/memtable/vectorrep.cc +69 -8
- package/deps/rocksdb/rocksdb/memtable/wbwi_memtable.cc +32 -9
- package/deps/rocksdb/rocksdb/memtable/wbwi_memtable.h +58 -45
- package/deps/rocksdb/rocksdb/monitoring/histogram.h +1 -1
- package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +5 -3
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +5 -0
- package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +1 -1
- package/deps/rocksdb/rocksdb/monitoring/thread_status_util_debug.cc +3 -2
- package/deps/rocksdb/rocksdb/options/cf_options.cc +44 -13
- package/deps/rocksdb/rocksdb/options/cf_options.h +21 -7
- package/deps/rocksdb/rocksdb/options/configurable.cc +5 -5
- package/deps/rocksdb/rocksdb/options/configurable_test.h +1 -2
- package/deps/rocksdb/rocksdb/options/customizable.cc +0 -1
- package/deps/rocksdb/rocksdb/options/customizable_test.cc +4 -11
- package/deps/rocksdb/rocksdb/options/db_options.cc +18 -15
- package/deps/rocksdb/rocksdb/options/db_options.h +2 -2
- package/deps/rocksdb/rocksdb/options/options.cc +296 -305
- package/deps/rocksdb/rocksdb/options/options_helper.cc +188 -62
- package/deps/rocksdb/rocksdb/options/options_helper.h +3 -3
- package/deps/rocksdb/rocksdb/options/options_parser.cc +2 -4
- package/deps/rocksdb/rocksdb/options/options_parser.h +0 -1
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +17 -4
- package/deps/rocksdb/rocksdb/options/options_test.cc +101 -76
- package/deps/rocksdb/rocksdb/port/lang.h +2 -1
- package/deps/rocksdb/rocksdb/port/port_posix.cc +2 -1
- package/deps/rocksdb/rocksdb/port/stack_trace.cc +5 -4
- package/deps/rocksdb/rocksdb/port/win/env_win.cc +3 -2
- package/deps/rocksdb/rocksdb/port/win/xpress_win.cc +99 -1
- package/deps/rocksdb/rocksdb/port/win/xpress_win.h +6 -0
- package/deps/rocksdb/rocksdb/src.mk +17 -11
- package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.h +0 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +1094 -929
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +6 -19
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +76 -22
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h +2 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +221 -131
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +12 -9
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +23 -24
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +38 -38
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +7 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +5 -5
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +10 -12
- package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +6 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +35 -43
- package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index_test.cc +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc +1 -2
- package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +0 -4
- package/deps/rocksdb/rocksdb/table/block_based/filter_policy_internal.h +0 -1
- package/deps/rocksdb/rocksdb/table/block_based/hash_index_reader.cc +3 -3
- package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc +3 -3
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +4 -4
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc +4 -5
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +4 -4
- package/deps/rocksdb/rocksdb/table/block_fetcher.cc +37 -35
- package/deps/rocksdb/rocksdb/table/block_fetcher.h +11 -7
- package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +4 -3
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +31 -5
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +2 -1
- package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder.h +0 -1
- package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder_test.cc +0 -1
- package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_reader_test.cc +0 -1
- package/deps/rocksdb/rocksdb/table/external_table.cc +483 -0
- package/deps/rocksdb/rocksdb/table/format.cc +62 -44
- package/deps/rocksdb/rocksdb/table/format.h +35 -12
- package/deps/rocksdb/rocksdb/table/internal_iterator.h +3 -13
- package/deps/rocksdb/rocksdb/table/iterator_wrapper.h +8 -0
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +6 -0
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +150 -141
- package/deps/rocksdb/rocksdb/table/meta_blocks.h +5 -0
- package/deps/rocksdb/rocksdb/table/multiget_context.h +3 -2
- package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +8 -0
- package/deps/rocksdb/rocksdb/table/plain/plain_table_index.cc +0 -1
- package/deps/rocksdb/rocksdb/table/plain/plain_table_index.h +0 -2
- package/deps/rocksdb/rocksdb/table/plain/plain_table_key_coding.h +0 -2
- package/deps/rocksdb/rocksdb/table/plain/plain_table_reader.cc +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +6 -6
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +0 -1
- package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +86 -7
- package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +88 -2
- package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +0 -1
- package/deps/rocksdb/rocksdb/table/table_builder.h +10 -1
- package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +3 -2
- package/deps/rocksdb/rocksdb/table/table_test.cc +899 -22
- package/deps/rocksdb/rocksdb/test_util/testutil.cc +3 -4
- package/deps/rocksdb/rocksdb/test_util/testutil.h +132 -1
- package/deps/rocksdb/rocksdb/test_util/transaction_test_util.cc +0 -1
- package/deps/rocksdb/rocksdb/test_util/transaction_test_util.h +0 -2
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +163 -77
- package/deps/rocksdb/rocksdb/tools/db_bench_tool_test.cc +0 -2
- package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +0 -1
- package/deps/rocksdb/rocksdb/tools/dump/db_dump_tool.cc +0 -1
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +120 -52
- package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +1 -0
- package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +1 -1
- package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +0 -2
- package/deps/rocksdb/rocksdb/tools/simulated_hybrid_file_system.cc +2 -2
- package/deps/rocksdb/rocksdb/tools/simulated_hybrid_file_system.h +0 -2
- package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +2 -1
- package/deps/rocksdb/rocksdb/tools/tool_hooks.cc +94 -0
- package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.cc +0 -1
- package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.h +0 -1
- package/deps/rocksdb/rocksdb/trace_replay/io_tracer.cc +1 -1
- package/deps/rocksdb/rocksdb/trace_replay/io_tracer_test.cc +2 -1
- package/deps/rocksdb/rocksdb/trace_replay/trace_replay.cc +3 -5
- package/deps/rocksdb/rocksdb/util/async_file_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/util/async_file_reader.h +15 -8
- package/deps/rocksdb/rocksdb/util/auto_skip_compressor.cc +131 -0
- package/deps/rocksdb/rocksdb/util/auto_skip_compressor.h +90 -0
- package/deps/rocksdb/rocksdb/util/autovector.h +1 -1
- package/deps/rocksdb/rocksdb/util/autovector_test.cc +2 -2
- package/deps/rocksdb/rocksdb/util/compaction_job_stats_impl.cc +0 -2
- package/deps/rocksdb/rocksdb/util/compression.cc +936 -4
- package/deps/rocksdb/rocksdb/util/compression.h +348 -232
- package/deps/rocksdb/rocksdb/util/compression_test.cc +229 -0
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +10 -10
- package/deps/rocksdb/rocksdb/util/crc32c_ppc.c +1 -0
- package/deps/rocksdb/rocksdb/util/data_structure.cc +2 -0
- package/deps/rocksdb/rocksdb/util/file_reader_writer_test.cc +1 -3
- package/deps/rocksdb/rocksdb/util/ppc-opcode.h +5 -5
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +108 -0
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +67 -0
- package/deps/rocksdb/rocksdb/util/slice_test.cc +83 -0
- package/deps/rocksdb/rocksdb/util/string_util.cc +0 -2
- package/deps/rocksdb/rocksdb/util/string_util.h +10 -0
- package/deps/rocksdb/rocksdb/util/thread_operation.h +2 -1
- package/deps/rocksdb/rocksdb/util/udt_util.cc +18 -5
- package/deps/rocksdb/rocksdb/util/udt_util.h +10 -7
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +650 -154
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +438 -144
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_gc_stats.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +16 -17
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +2 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +7 -8
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +4 -3
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/cache_dump_load.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +0 -48
- package/deps/rocksdb/rocksdb/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/compaction_filters/remove_emptyvalue_compactionfilter.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/debug.cc +7 -14
- package/deps/rocksdb/rocksdb/utilities/env_mirror.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/env_mirror_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/env_timed.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/env_timed_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +5 -3
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +10 -9
- package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/memory/memory_util.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/memory_allocators.h +1 -0
- package/deps/rocksdb/rocksdb/utilities/object_registry_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/options/options_util.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_metadata.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_metadata.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table_evictable.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/lrulist.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_test.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_tier.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_tier.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/secondary_index/faiss_ivf_index.cc +183 -32
- package/deps/rocksdb/rocksdb/utilities/secondary_index/faiss_ivf_index_test.cc +258 -12
- package/deps/rocksdb/rocksdb/utilities/secondary_index/secondary_index_helper.h +33 -0
- package/deps/rocksdb/rocksdb/utilities/secondary_index/secondary_index_iterator.cc +99 -0
- package/deps/rocksdb/rocksdb/utilities/secondary_index/secondary_index_mixin.h +280 -120
- package/deps/rocksdb/rocksdb/utilities/secondary_index/simple_secondary_index.cc +79 -0
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc +52 -16
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.h +10 -6
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc +55 -0
- package/deps/rocksdb/rocksdb/utilities/trace/replayer_impl.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/lock_manager.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/lock_manager.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.cc +37 -12
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.h +2 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_tracker.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_locking_test.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/db.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/util/dbt.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.cc +1 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +36 -10
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +5 -7
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +4 -5
- package/deps/rocksdb/rocksdb/utilities/transactions/snapshot_checker.cc +1 -4
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.h +1 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_db_mutex_impl.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_db_mutex_impl.h +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +1118 -37
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +4 -7
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.h +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +3 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_transaction_test.cc +0 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +1 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.h +1 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn_db.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +0 -3
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +125 -127
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +45 -23
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +54 -22
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +477 -58
- package/deps/rocksdb/rocksdb.gyp +9 -4
- package/index.js +50 -9
- package/package.json +8 -1
- package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
- package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
|
@@ -24,7 +24,10 @@ Reader::Reporter::~Reporter() = default;
|
|
|
24
24
|
|
|
25
25
|
Reader::Reader(std::shared_ptr<Logger> info_log,
|
|
26
26
|
std::unique_ptr<SequentialFileReader>&& _file,
|
|
27
|
-
Reporter* reporter, bool checksum, uint64_t log_num
|
|
27
|
+
Reporter* reporter, bool checksum, uint64_t log_num,
|
|
28
|
+
bool track_and_verify_wals, bool stop_replay_for_corruption,
|
|
29
|
+
uint64_t min_wal_number_to_keep,
|
|
30
|
+
const PredecessorWALInfo& observed_predecessor_wal_info)
|
|
28
31
|
: info_log_(info_log),
|
|
29
32
|
file_(std::move(_file)),
|
|
30
33
|
reporter_(reporter),
|
|
@@ -37,13 +40,17 @@ Reader::Reader(std::shared_ptr<Logger> info_log,
|
|
|
37
40
|
last_record_offset_(0),
|
|
38
41
|
end_of_buffer_offset_(0),
|
|
39
42
|
log_number_(log_num),
|
|
43
|
+
track_and_verify_wals_(track_and_verify_wals),
|
|
44
|
+
stop_replay_for_corruption_(stop_replay_for_corruption),
|
|
45
|
+
min_wal_number_to_keep_(min_wal_number_to_keep),
|
|
46
|
+
observed_predecessor_wal_info_(observed_predecessor_wal_info),
|
|
40
47
|
recycled_(false),
|
|
41
48
|
first_record_read_(false),
|
|
42
49
|
compression_type_(kNoCompression),
|
|
43
50
|
compression_type_record_read_(false),
|
|
44
51
|
uncompress_(nullptr),
|
|
45
52
|
hash_state_(nullptr),
|
|
46
|
-
uncompress_hash_state_(nullptr){}
|
|
53
|
+
uncompress_hash_state_(nullptr) {}
|
|
47
54
|
|
|
48
55
|
Reader::~Reader() {
|
|
49
56
|
delete[] backing_store_;
|
|
@@ -65,6 +72,9 @@ Reader::~Reader() {
|
|
|
65
72
|
//
|
|
66
73
|
// TODO krad: Evaluate if we need to move to a more strict mode where we
|
|
67
74
|
// restrict the inconsistency to only the last log
|
|
75
|
+
// TODO (hx235): move `wal_recovery_mode` to be a member data like other
|
|
76
|
+
// information (e.g, `stop_replay_for_corruption`) to decide whether to
|
|
77
|
+
// check for and surface corruption in `ReadRecord()`
|
|
68
78
|
bool Reader::ReadRecord(Slice* record, std::string* scratch,
|
|
69
79
|
WALRecoveryMode wal_recovery_mode,
|
|
70
80
|
uint64_t* record_checksum) {
|
|
@@ -88,7 +98,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
|
|
|
88
98
|
while (true) {
|
|
89
99
|
uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
|
|
90
100
|
size_t drop_size = 0;
|
|
91
|
-
const
|
|
101
|
+
const uint8_t record_type =
|
|
92
102
|
ReadPhysicalRecord(&fragment, &drop_size, record_checksum);
|
|
93
103
|
switch (record_type) {
|
|
94
104
|
case kFullType:
|
|
@@ -185,6 +195,23 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
|
|
|
185
195
|
}
|
|
186
196
|
break;
|
|
187
197
|
}
|
|
198
|
+
case kPredecessorWALInfoType:
|
|
199
|
+
case kRecyclePredecessorWALInfoType: {
|
|
200
|
+
prospective_record_offset = physical_record_offset;
|
|
201
|
+
scratch->clear();
|
|
202
|
+
last_record_offset_ = prospective_record_offset;
|
|
203
|
+
|
|
204
|
+
PredecessorWALInfo recorded_predecessor_wal_info;
|
|
205
|
+
Status s = recorded_predecessor_wal_info.DecodeFrom(&fragment);
|
|
206
|
+
if (!s.ok()) {
|
|
207
|
+
ReportCorruption(fragment.size(),
|
|
208
|
+
"could not decode PredecessorWALInfoType record");
|
|
209
|
+
} else {
|
|
210
|
+
MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
|
|
211
|
+
recorded_predecessor_wal_info);
|
|
212
|
+
}
|
|
213
|
+
break;
|
|
214
|
+
}
|
|
188
215
|
case kUserDefinedTimestampSizeType:
|
|
189
216
|
case kRecyclableUserDefinedTimestampSizeType: {
|
|
190
217
|
if (in_fragmented_record && !scratch->empty()) {
|
|
@@ -313,11 +340,13 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
|
|
|
313
340
|
break;
|
|
314
341
|
|
|
315
342
|
default: {
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
343
|
+
if ((record_type & kRecordTypeSafeIgnoreMask) == 0) {
|
|
344
|
+
std::string reason =
|
|
345
|
+
"unknown record type " + std::to_string(record_type);
|
|
346
|
+
ReportCorruption(
|
|
347
|
+
(fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
|
|
348
|
+
reason.c_str());
|
|
349
|
+
}
|
|
321
350
|
in_fragmented_record = false;
|
|
322
351
|
scratch->clear();
|
|
323
352
|
break;
|
|
@@ -327,6 +356,61 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
|
|
|
327
356
|
return false;
|
|
328
357
|
}
|
|
329
358
|
|
|
359
|
+
void Reader::MaybeVerifyPredecessorWALInfo(
|
|
360
|
+
WALRecoveryMode wal_recovery_mode, Slice fragment,
|
|
361
|
+
const PredecessorWALInfo& recorded_predecessor_wal_info) {
|
|
362
|
+
if (!track_and_verify_wals_ ||
|
|
363
|
+
wal_recovery_mode == WALRecoveryMode::kSkipAnyCorruptedRecords ||
|
|
364
|
+
stop_replay_for_corruption_) {
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
assert(recorded_predecessor_wal_info.IsInitialized());
|
|
368
|
+
uint64_t recorded_predecessor_log_number =
|
|
369
|
+
recorded_predecessor_wal_info.GetLogNumber();
|
|
370
|
+
|
|
371
|
+
// This is the first WAL recovered thus with no predecessor WAL info has been
|
|
372
|
+
// initialized
|
|
373
|
+
if (!observed_predecessor_wal_info_.IsInitialized()) {
|
|
374
|
+
if (recorded_predecessor_log_number >= min_wal_number_to_keep_) {
|
|
375
|
+
std::string reason = "Missing WAL of log number " +
|
|
376
|
+
std::to_string(recorded_predecessor_log_number);
|
|
377
|
+
ReportCorruption(fragment.size(), reason.c_str(),
|
|
378
|
+
recorded_predecessor_log_number);
|
|
379
|
+
}
|
|
380
|
+
} else {
|
|
381
|
+
if (observed_predecessor_wal_info_.GetLogNumber() !=
|
|
382
|
+
recorded_predecessor_log_number) {
|
|
383
|
+
std::string reason = "Missing WAL of log number " +
|
|
384
|
+
std::to_string(recorded_predecessor_log_number);
|
|
385
|
+
ReportCorruption(fragment.size(), reason.c_str(),
|
|
386
|
+
recorded_predecessor_log_number);
|
|
387
|
+
} else if (observed_predecessor_wal_info_.GetLastSeqnoRecorded() !=
|
|
388
|
+
recorded_predecessor_wal_info.GetLastSeqnoRecorded()) {
|
|
389
|
+
std::string reason =
|
|
390
|
+
"Mismatched last sequence number recorded in the WAL of log number " +
|
|
391
|
+
std::to_string(recorded_predecessor_log_number) + ". Recorded " +
|
|
392
|
+
std::to_string(recorded_predecessor_wal_info.GetLastSeqnoRecorded()) +
|
|
393
|
+
". Observed " +
|
|
394
|
+
std::to_string(
|
|
395
|
+
observed_predecessor_wal_info_.GetLastSeqnoRecorded()) +
|
|
396
|
+
". (Last sequence number equal to 0 indicates no WAL records)";
|
|
397
|
+
ReportCorruption(fragment.size(), reason.c_str(),
|
|
398
|
+
recorded_predecessor_log_number);
|
|
399
|
+
} else if (observed_predecessor_wal_info_.GetSizeBytes() !=
|
|
400
|
+
recorded_predecessor_wal_info.GetSizeBytes()) {
|
|
401
|
+
std::string reason =
|
|
402
|
+
"Mismatched size of the WAL of log number " +
|
|
403
|
+
std::to_string(recorded_predecessor_log_number) + ". Recorded " +
|
|
404
|
+
std::to_string(recorded_predecessor_wal_info.GetSizeBytes()) +
|
|
405
|
+
" bytes. Observed " +
|
|
406
|
+
std::to_string(observed_predecessor_wal_info_.GetSizeBytes()) +
|
|
407
|
+
" bytes.";
|
|
408
|
+
ReportCorruption(fragment.size(), reason.c_str(),
|
|
409
|
+
recorded_predecessor_log_number);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
|
|
330
414
|
uint64_t Reader::LastRecordOffset() { return last_record_offset_; }
|
|
331
415
|
|
|
332
416
|
uint64_t Reader::LastRecordEnd() {
|
|
@@ -403,13 +487,15 @@ void Reader::UnmarkEOFInternal() {
|
|
|
403
487
|
}
|
|
404
488
|
}
|
|
405
489
|
|
|
406
|
-
void Reader::ReportCorruption(size_t bytes, const char* reason
|
|
407
|
-
|
|
490
|
+
void Reader::ReportCorruption(size_t bytes, const char* reason,
|
|
491
|
+
uint64_t log_number) {
|
|
492
|
+
ReportDrop(bytes, Status::Corruption(reason), log_number);
|
|
408
493
|
}
|
|
409
494
|
|
|
410
|
-
void Reader::ReportDrop(size_t bytes, const Status& reason
|
|
495
|
+
void Reader::ReportDrop(size_t bytes, const Status& reason,
|
|
496
|
+
uint64_t log_number) {
|
|
411
497
|
if (reporter_ != nullptr) {
|
|
412
|
-
reporter_->Corruption(bytes, reason);
|
|
498
|
+
reporter_->Corruption(bytes, reason, log_number);
|
|
413
499
|
}
|
|
414
500
|
}
|
|
415
501
|
|
|
@@ -419,7 +505,7 @@ void Reader::ReportOldLogRecord(size_t bytes) {
|
|
|
419
505
|
}
|
|
420
506
|
}
|
|
421
507
|
|
|
422
|
-
bool Reader::ReadMore(size_t* drop_size,
|
|
508
|
+
bool Reader::ReadMore(size_t* drop_size, uint8_t* error) {
|
|
423
509
|
if (!eof_ && !read_error_) {
|
|
424
510
|
// Last read was a full read, so this is a trailer to skip
|
|
425
511
|
buffer_.clear();
|
|
@@ -460,15 +546,15 @@ bool Reader::ReadMore(size_t* drop_size, int* error) {
|
|
|
460
546
|
}
|
|
461
547
|
}
|
|
462
548
|
|
|
463
|
-
|
|
464
|
-
|
|
549
|
+
uint8_t Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
|
|
550
|
+
uint64_t* fragment_checksum) {
|
|
465
551
|
while (true) {
|
|
466
552
|
// We need at least the minimum header size
|
|
467
553
|
if (buffer_.size() < static_cast<size_t>(kHeaderSize)) {
|
|
468
554
|
// the default value of r is meaningless because ReadMore will overwrite
|
|
469
555
|
// it if it returns false; in case it returns true, the return value will
|
|
470
556
|
// not be used anyway
|
|
471
|
-
|
|
557
|
+
uint8_t r = kEof;
|
|
472
558
|
if (!ReadMore(drop_size, &r)) {
|
|
473
559
|
return r;
|
|
474
560
|
}
|
|
@@ -479,12 +565,13 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
|
|
|
479
565
|
const char* header = buffer_.data();
|
|
480
566
|
const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
|
|
481
567
|
const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
|
|
482
|
-
const
|
|
568
|
+
const uint8_t type = static_cast<uint8_t>(header[6]);
|
|
483
569
|
const uint32_t length = a | (b << 8);
|
|
484
570
|
int header_size = kHeaderSize;
|
|
485
571
|
const bool is_recyclable_type =
|
|
486
572
|
((type >= kRecyclableFullType && type <= kRecyclableLastType) ||
|
|
487
|
-
type == kRecyclableUserDefinedTimestampSizeType
|
|
573
|
+
type == kRecyclableUserDefinedTimestampSizeType ||
|
|
574
|
+
type == kRecyclePredecessorWALInfoType);
|
|
488
575
|
if (is_recyclable_type) {
|
|
489
576
|
header_size = kRecyclableHeaderSize;
|
|
490
577
|
if (first_record_read_ && !recycled_) {
|
|
@@ -494,7 +581,7 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
|
|
|
494
581
|
recycled_ = true;
|
|
495
582
|
// We need enough for the larger header
|
|
496
583
|
if (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
|
|
497
|
-
|
|
584
|
+
uint8_t r = kEof;
|
|
498
585
|
if (!ReadMore(drop_size, &r)) {
|
|
499
586
|
return r;
|
|
500
587
|
}
|
|
@@ -549,6 +636,8 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
|
|
|
549
636
|
buffer_.remove_prefix(header_size + length);
|
|
550
637
|
|
|
551
638
|
if (!uncompress_ || type == kSetCompressionType ||
|
|
639
|
+
type == kPredecessorWALInfoType ||
|
|
640
|
+
type == kRecyclePredecessorWALInfoType ||
|
|
552
641
|
type == kUserDefinedTimestampSizeType ||
|
|
553
642
|
type == kRecyclableUserDefinedTimestampSizeType) {
|
|
554
643
|
*result = Slice(header + header_size, length);
|
|
@@ -638,7 +727,9 @@ Status Reader::UpdateRecordedTimestampSize(
|
|
|
638
727
|
}
|
|
639
728
|
|
|
640
729
|
bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
|
|
641
|
-
WALRecoveryMode
|
|
730
|
+
WALRecoveryMode wal_recovery_mode
|
|
731
|
+
|
|
732
|
+
,
|
|
642
733
|
uint64_t* /* checksum */) {
|
|
643
734
|
assert(record != nullptr);
|
|
644
735
|
assert(scratch != nullptr);
|
|
@@ -651,7 +742,7 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
|
|
|
651
742
|
uint64_t prospective_record_offset = 0;
|
|
652
743
|
uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
|
|
653
744
|
size_t drop_size = 0;
|
|
654
|
-
|
|
745
|
+
uint8_t fragment_type_or_err = 0; // Initialize to make compiler happy
|
|
655
746
|
Slice fragment;
|
|
656
747
|
while (TryReadFragment(&fragment, &drop_size, &fragment_type_or_err)) {
|
|
657
748
|
switch (fragment_type_or_err) {
|
|
@@ -728,7 +819,24 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
|
|
|
728
819
|
}
|
|
729
820
|
break;
|
|
730
821
|
}
|
|
822
|
+
case kPredecessorWALInfoType:
|
|
823
|
+
case kRecyclePredecessorWALInfoType: {
|
|
824
|
+
fragments_.clear();
|
|
825
|
+
prospective_record_offset = physical_record_offset;
|
|
826
|
+
last_record_offset_ = prospective_record_offset;
|
|
827
|
+
in_fragmented_record_ = false;
|
|
731
828
|
|
|
829
|
+
PredecessorWALInfo recorded_predecessor_wal_info;
|
|
830
|
+
Status s = recorded_predecessor_wal_info.DecodeFrom(&fragment);
|
|
831
|
+
if (!s.ok()) {
|
|
832
|
+
ReportCorruption(fragment.size(),
|
|
833
|
+
"could not decode PredecessorWALInfoType record");
|
|
834
|
+
} else {
|
|
835
|
+
MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
|
|
836
|
+
recorded_predecessor_wal_info);
|
|
837
|
+
}
|
|
838
|
+
break;
|
|
839
|
+
}
|
|
732
840
|
case kUserDefinedTimestampSizeType:
|
|
733
841
|
case kRecyclableUserDefinedTimestampSizeType: {
|
|
734
842
|
if (in_fragmented_record_ && !scratch->empty()) {
|
|
@@ -781,11 +889,13 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
|
|
|
781
889
|
break;
|
|
782
890
|
|
|
783
891
|
default: {
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
892
|
+
if ((fragment_type_or_err & kRecordTypeSafeIgnoreMask) == 0) {
|
|
893
|
+
std::string reason =
|
|
894
|
+
"unknown record type " + std::to_string(fragment_type_or_err);
|
|
895
|
+
ReportCorruption(
|
|
896
|
+
fragment.size() + (in_fragmented_record_ ? fragments_.size() : 0),
|
|
897
|
+
reason.c_str());
|
|
898
|
+
}
|
|
789
899
|
in_fragmented_record_ = false;
|
|
790
900
|
fragments_.clear();
|
|
791
901
|
break;
|
|
@@ -803,7 +913,7 @@ void FragmentBufferedReader::UnmarkEOF() {
|
|
|
803
913
|
UnmarkEOFInternal();
|
|
804
914
|
}
|
|
805
915
|
|
|
806
|
-
bool FragmentBufferedReader::TryReadMore(size_t* drop_size,
|
|
916
|
+
bool FragmentBufferedReader::TryReadMore(size_t* drop_size, uint8_t* error) {
|
|
807
917
|
if (!eof_ && !read_error_) {
|
|
808
918
|
// Last read was a full read, so this is a trailer to skip
|
|
809
919
|
buffer_.clear();
|
|
@@ -844,15 +954,15 @@ bool FragmentBufferedReader::TryReadMore(size_t* drop_size, int* error) {
|
|
|
844
954
|
}
|
|
845
955
|
|
|
846
956
|
// return true if the caller should process the fragment_type_or_err.
|
|
847
|
-
bool FragmentBufferedReader::TryReadFragment(
|
|
848
|
-
|
|
957
|
+
bool FragmentBufferedReader::TryReadFragment(Slice* fragment, size_t* drop_size,
|
|
958
|
+
uint8_t* fragment_type_or_err) {
|
|
849
959
|
assert(fragment != nullptr);
|
|
850
960
|
assert(drop_size != nullptr);
|
|
851
961
|
assert(fragment_type_or_err != nullptr);
|
|
852
962
|
|
|
853
963
|
while (buffer_.size() < static_cast<size_t>(kHeaderSize)) {
|
|
854
964
|
size_t old_size = buffer_.size();
|
|
855
|
-
|
|
965
|
+
uint8_t error = kEof;
|
|
856
966
|
if (!TryReadMore(drop_size, &error)) {
|
|
857
967
|
*fragment_type_or_err = error;
|
|
858
968
|
return false;
|
|
@@ -863,11 +973,12 @@ bool FragmentBufferedReader::TryReadFragment(
|
|
|
863
973
|
const char* header = buffer_.data();
|
|
864
974
|
const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
|
|
865
975
|
const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
|
|
866
|
-
const
|
|
976
|
+
const uint8_t type = static_cast<uint8_t>(header[6]);
|
|
867
977
|
const uint32_t length = a | (b << 8);
|
|
868
978
|
int header_size = kHeaderSize;
|
|
869
979
|
if ((type >= kRecyclableFullType && type <= kRecyclableLastType) ||
|
|
870
|
-
type == kRecyclableUserDefinedTimestampSizeType
|
|
980
|
+
type == kRecyclableUserDefinedTimestampSizeType ||
|
|
981
|
+
type == kRecyclePredecessorWALInfoType) {
|
|
871
982
|
if (first_record_read_ && !recycled_) {
|
|
872
983
|
// A recycled log should have started with a recycled record
|
|
873
984
|
*fragment_type_or_err = kBadRecord;
|
|
@@ -877,7 +988,7 @@ bool FragmentBufferedReader::TryReadFragment(
|
|
|
877
988
|
header_size = kRecyclableHeaderSize;
|
|
878
989
|
while (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
|
|
879
990
|
size_t old_size = buffer_.size();
|
|
880
|
-
|
|
991
|
+
uint8_t error = kEof;
|
|
881
992
|
if (!TryReadMore(drop_size, &error)) {
|
|
882
993
|
*fragment_type_or_err = error;
|
|
883
994
|
return false;
|
|
@@ -894,7 +1005,7 @@ bool FragmentBufferedReader::TryReadFragment(
|
|
|
894
1005
|
|
|
895
1006
|
while (header_size + length > buffer_.size()) {
|
|
896
1007
|
size_t old_size = buffer_.size();
|
|
897
|
-
|
|
1008
|
+
uint8_t error = kEof;
|
|
898
1009
|
if (!TryReadMore(drop_size, &error)) {
|
|
899
1010
|
*fragment_type_or_err = error;
|
|
900
1011
|
return false;
|
|
@@ -923,6 +1034,8 @@ bool FragmentBufferedReader::TryReadFragment(
|
|
|
923
1034
|
buffer_.remove_prefix(header_size + length);
|
|
924
1035
|
|
|
925
1036
|
if (!uncompress_ || type == kSetCompressionType ||
|
|
1037
|
+
type == kPredecessorWALInfoType ||
|
|
1038
|
+
type == kRecyclePredecessorWALInfoType ||
|
|
926
1039
|
type == kUserDefinedTimestampSizeType ||
|
|
927
1040
|
type == kRecyclableUserDefinedTimestampSizeType) {
|
|
928
1041
|
*fragment = Slice(header + header_size, length);
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
#pragma once
|
|
11
11
|
#include <stdint.h>
|
|
12
12
|
|
|
13
|
+
#include <cstdint>
|
|
13
14
|
#include <memory>
|
|
14
15
|
#include <unordered_map>
|
|
15
16
|
#include <vector>
|
|
@@ -44,7 +45,8 @@ class Reader {
|
|
|
44
45
|
|
|
45
46
|
// Some corruption was detected. "size" is the approximate number
|
|
46
47
|
// of bytes dropped due to the corruption.
|
|
47
|
-
virtual void Corruption(size_t bytes, const Status& status
|
|
48
|
+
virtual void Corruption(size_t bytes, const Status& status,
|
|
49
|
+
uint64_t log_number = kMaxSequenceNumber) = 0;
|
|
48
50
|
|
|
49
51
|
virtual void OldLogRecord(size_t /*bytes*/) {}
|
|
50
52
|
};
|
|
@@ -57,9 +59,15 @@ class Reader {
|
|
|
57
59
|
// live while this Reader is in use.
|
|
58
60
|
//
|
|
59
61
|
// If "checksum" is true, verify checksums if available.
|
|
62
|
+
// TODO(hx235): seperate WAL related parameters from general `Reader`
|
|
63
|
+
// parameters
|
|
60
64
|
Reader(std::shared_ptr<Logger> info_log,
|
|
61
65
|
std::unique_ptr<SequentialFileReader>&& file, Reporter* reporter,
|
|
62
|
-
bool checksum, uint64_t log_num
|
|
66
|
+
bool checksum, uint64_t log_num, bool track_and_verify_wals = false,
|
|
67
|
+
bool stop_replay_for_corruption = false,
|
|
68
|
+
uint64_t min_wal_number_to_keep = std::numeric_limits<uint64_t>::max(),
|
|
69
|
+
const PredecessorWALInfo& observed_predecessor_wal_info =
|
|
70
|
+
PredecessorWALInfo());
|
|
63
71
|
// No copying allowed
|
|
64
72
|
Reader(const Reader&) = delete;
|
|
65
73
|
void operator=(const Reader&) = delete;
|
|
@@ -147,6 +155,17 @@ class Reader {
|
|
|
147
155
|
// which log number this is
|
|
148
156
|
uint64_t const log_number_;
|
|
149
157
|
|
|
158
|
+
// See `Optinos::track_and_verify_wals`
|
|
159
|
+
bool track_and_verify_wals_;
|
|
160
|
+
// Below variables are used for WAL verification
|
|
161
|
+
// TODO(hx235): To revise `stop_replay_for_corruption_` inside `LogReader`
|
|
162
|
+
// since we have `observed_predecessor_wal_info_` to verify against the
|
|
163
|
+
// `recorded_predecessor_wal_info_` recorded in current WAL. If there is no
|
|
164
|
+
// WAL hole, we can revise `stop_replay_for_corruption_` to be false.
|
|
165
|
+
bool stop_replay_for_corruption_;
|
|
166
|
+
uint64_t min_wal_number_to_keep_;
|
|
167
|
+
PredecessorWALInfo observed_predecessor_wal_info_;
|
|
168
|
+
|
|
150
169
|
// Whether this is a recycled log file
|
|
151
170
|
bool recycled_;
|
|
152
171
|
|
|
@@ -171,7 +190,7 @@ class Reader {
|
|
|
171
190
|
UnorderedMap<uint32_t, size_t> recorded_cf_to_ts_sz_;
|
|
172
191
|
|
|
173
192
|
// Extend record types with the following special values
|
|
174
|
-
enum {
|
|
193
|
+
enum : uint8_t {
|
|
175
194
|
kEof = kMaxRecordType + 1,
|
|
176
195
|
// Returned whenever we find an invalid physical record.
|
|
177
196
|
// Currently there are three situations in which this happens:
|
|
@@ -192,24 +211,30 @@ class Reader {
|
|
|
192
211
|
// If WAL compressioned is enabled, fragment_checksum is the checksum of the
|
|
193
212
|
// fragment computed from the orginal buffer containinng uncompressed
|
|
194
213
|
// fragment.
|
|
195
|
-
|
|
196
|
-
|
|
214
|
+
uint8_t ReadPhysicalRecord(Slice* result, size_t* drop_size,
|
|
215
|
+
uint64_t* fragment_checksum = nullptr);
|
|
197
216
|
|
|
198
217
|
// Read some more
|
|
199
|
-
bool ReadMore(size_t* drop_size,
|
|
218
|
+
bool ReadMore(size_t* drop_size, uint8_t* error);
|
|
200
219
|
|
|
201
220
|
void UnmarkEOFInternal();
|
|
202
221
|
|
|
203
222
|
// Reports dropped bytes to the reporter.
|
|
204
223
|
// buffer_ must be updated to remove the dropped bytes prior to invocation.
|
|
205
|
-
void ReportCorruption(size_t bytes, const char* reason
|
|
206
|
-
|
|
224
|
+
void ReportCorruption(size_t bytes, const char* reason,
|
|
225
|
+
uint64_t log_number = kMaxSequenceNumber);
|
|
226
|
+
void ReportDrop(size_t bytes, const Status& reason,
|
|
227
|
+
uint64_t log_number = kMaxSequenceNumber);
|
|
207
228
|
void ReportOldLogRecord(size_t bytes);
|
|
208
229
|
|
|
209
230
|
void InitCompression(const CompressionTypeRecord& compression_record);
|
|
210
231
|
|
|
211
232
|
Status UpdateRecordedTimestampSize(
|
|
212
233
|
const std::vector<std::pair<uint32_t, size_t>>& cf_to_ts_sz);
|
|
234
|
+
|
|
235
|
+
void MaybeVerifyPredecessorWALInfo(
|
|
236
|
+
WALRecoveryMode wal_recovery_mode, Slice fragment,
|
|
237
|
+
const PredecessorWALInfo& recorded_predecessor_wal_info);
|
|
213
238
|
};
|
|
214
239
|
|
|
215
240
|
class FragmentBufferedReader : public Reader {
|
|
@@ -217,7 +242,11 @@ class FragmentBufferedReader : public Reader {
|
|
|
217
242
|
FragmentBufferedReader(std::shared_ptr<Logger> info_log,
|
|
218
243
|
std::unique_ptr<SequentialFileReader>&& _file,
|
|
219
244
|
Reporter* reporter, bool checksum, uint64_t log_num)
|
|
220
|
-
: Reader(info_log, std::move(_file), reporter, checksum, log_num
|
|
245
|
+
: Reader(info_log, std::move(_file), reporter, checksum, log_num,
|
|
246
|
+
false /*verify_and_track_wals*/,
|
|
247
|
+
false /*stop_replay_for_corruption*/,
|
|
248
|
+
std::numeric_limits<uint64_t>::max() /*min_wal_number_to_keep*/,
|
|
249
|
+
PredecessorWALInfo() /*observed_predecessor_wal_info*/),
|
|
221
250
|
fragments_(),
|
|
222
251
|
in_fragmented_record_(false) {}
|
|
223
252
|
~FragmentBufferedReader() override {}
|
|
@@ -232,9 +261,9 @@ class FragmentBufferedReader : public Reader {
|
|
|
232
261
|
bool in_fragmented_record_;
|
|
233
262
|
|
|
234
263
|
bool TryReadFragment(Slice* result, size_t* drop_size,
|
|
235
|
-
|
|
264
|
+
uint8_t* fragment_type_or_err);
|
|
236
265
|
|
|
237
|
-
bool TryReadMore(size_t* drop_size,
|
|
266
|
+
bool TryReadMore(size_t* drop_size, uint8_t* error);
|
|
238
267
|
|
|
239
268
|
// No copy allowed
|
|
240
269
|
FragmentBufferedReader(const FragmentBufferedReader&);
|
|
@@ -129,7 +129,8 @@ class LogTest
|
|
|
129
129
|
std::string message_;
|
|
130
130
|
|
|
131
131
|
ReportCollector() : dropped_bytes_(0) {}
|
|
132
|
-
void Corruption(size_t bytes, const Status& status
|
|
132
|
+
void Corruption(size_t bytes, const Status& status,
|
|
133
|
+
uint64_t /*log_number*/ = kMaxSequenceNumber) override {
|
|
133
134
|
dropped_bytes_ += bytes;
|
|
134
135
|
message_.append(status.ToString());
|
|
135
136
|
}
|
|
@@ -436,6 +437,17 @@ TEST_P(LogTest, BadRecordType) {
|
|
|
436
437
|
ASSERT_EQ("OK", MatchError("unknown record type"));
|
|
437
438
|
}
|
|
438
439
|
|
|
440
|
+
TEST_P(LogTest, IgnorableRecordType) {
|
|
441
|
+
Write("foo");
|
|
442
|
+
// Type is stored in header[6]
|
|
443
|
+
SetByte(6, static_cast<char>(kRecordTypeSafeIgnoreMask + 100));
|
|
444
|
+
FixChecksum(0, 3, false);
|
|
445
|
+
ASSERT_EQ("EOF", Read());
|
|
446
|
+
// The new type has value 129 and masked to be ignorable if unknown
|
|
447
|
+
ASSERT_EQ(0U, DroppedBytes());
|
|
448
|
+
ASSERT_EQ("", ReportMessage());
|
|
449
|
+
}
|
|
450
|
+
|
|
439
451
|
TEST_P(LogTest, TruncatedTrailingRecordIsIgnored) {
|
|
440
452
|
Write("foo");
|
|
441
453
|
ShrinkSize(4); // Drop all payload as well as a header byte
|
|
@@ -454,7 +466,7 @@ TEST_P(LogTest, TruncatedTrailingRecordIsNotIgnored) {
|
|
|
454
466
|
Write("foo");
|
|
455
467
|
ShrinkSize(4); // Drop all payload as well as a header byte
|
|
456
468
|
ASSERT_EQ("EOF", Read(WALRecoveryMode::kAbsoluteConsistency));
|
|
457
|
-
// Truncated last record is ignored,
|
|
469
|
+
// Truncated last record is not ignored, treated as an error
|
|
458
470
|
ASSERT_GT(DroppedBytes(), 0U);
|
|
459
471
|
ASSERT_EQ("OK", MatchError("Corruption: truncated header"));
|
|
460
472
|
}
|
|
@@ -814,7 +826,8 @@ class RetriableLogTest : public ::testing::TestWithParam<int> {
|
|
|
814
826
|
std::string message_;
|
|
815
827
|
|
|
816
828
|
ReportCollector() : dropped_bytes_(0) {}
|
|
817
|
-
void Corruption(size_t bytes, const Status& status
|
|
829
|
+
void Corruption(size_t bytes, const Status& status,
|
|
830
|
+
uint64_t /*log_number*/ = kMaxSequenceNumber) override {
|
|
818
831
|
dropped_bytes_ += bytes;
|
|
819
832
|
message_.append(status.ToString());
|
|
820
833
|
}
|