@nxtedition/rocksdb 13.5.13 → 15.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +55 -180
- package/binding.gyp +2 -2
- package/chained-batch.js +9 -16
- package/deps/rocksdb/rocksdb/BUCK +18 -1
- package/deps/rocksdb/rocksdb/CMakeLists.txt +10 -3
- package/deps/rocksdb/rocksdb/Makefile +20 -9
- package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +90 -13
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +88 -75
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +44 -36
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +184 -148
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h +5 -11
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +116 -47
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +1 -1
- package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.cc +3 -6
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +1 -1
- package/deps/rocksdb/rocksdb/db/builder.cc +4 -2
- package/deps/rocksdb/rocksdb/db/c.cc +207 -0
- package/deps/rocksdb/rocksdb/db/c_test.c +72 -0
- package/deps/rocksdb/rocksdb/db/column_family.cc +3 -2
- package/deps/rocksdb/rocksdb/db/column_family.h +5 -0
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +4 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +2 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +51 -38
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +29 -12
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +5 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +566 -366
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +131 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +1 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +7 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +4 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +13 -14
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +12 -7
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +8 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +97 -76
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +11 -14
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +1 -1
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +8 -0
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +16 -3
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +1 -0
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +448 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +22 -20
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +4 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +5 -5
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +7 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_iter.cc +104 -0
- package/deps/rocksdb/rocksdb/db/db_iter.h +4 -11
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +331 -58
- package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +129 -0
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +64 -0
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +40 -0
- package/deps/rocksdb/rocksdb/db/db_test2.cc +25 -15
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +42 -24
- package/deps/rocksdb/rocksdb/db/db_test_util.h +29 -14
- package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +69 -36
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/event_helpers.cc +1 -0
- package/deps/rocksdb/rocksdb/db/experimental.cc +5 -4
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +8 -1
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +275 -79
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +23 -5
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +591 -175
- package/deps/rocksdb/rocksdb/db/flush_job.cc +3 -4
- package/deps/rocksdb/rocksdb/db/log_reader.cc +5 -2
- package/deps/rocksdb/rocksdb/db/memtable.cc +84 -35
- package/deps/rocksdb/rocksdb/db/memtable.h +39 -34
- package/deps/rocksdb/rocksdb/db/merge_helper.cc +1 -0
- package/deps/rocksdb/rocksdb/db/merge_operator.cc +1 -1
- package/deps/rocksdb/rocksdb/db/multi_scan.cc +11 -5
- package/deps/rocksdb/rocksdb/db/version_edit.cc +1 -1
- package/deps/rocksdb/rocksdb/db/version_edit.h +1 -1
- package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +34 -14
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +28 -5
- package/deps/rocksdb/rocksdb/db/version_set.cc +159 -14
- package/deps/rocksdb/rocksdb/db/version_set.h +2 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/CMakeLists.txt +1 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +60 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +16 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compaction_service.h +75 -10
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compression_manager.cc +28 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compression_manager.h +2 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +31 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +50 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +57 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_stat.h +0 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +266 -35
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +5 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +0 -6
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +18 -2
- package/deps/rocksdb/rocksdb/env/env.cc +12 -0
- package/deps/rocksdb/rocksdb/env/env_test.cc +18 -0
- package/deps/rocksdb/rocksdb/env/file_system_tracer.cc +2 -0
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +9 -5
- package/deps/rocksdb/rocksdb/env/io_posix.cc +4 -2
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +19 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +33 -31
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +42 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +93 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +43 -49
- package/deps/rocksdb/rocksdb/include/rocksdb/compaction_job_stats.h +4 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/compression_type.h +8 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/data_structure.h +487 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +11 -12
- package/deps/rocksdb/rocksdb/include/rocksdb/env.h +135 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +12 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/ldb_tool.h +8 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +12 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/metadata.h +3 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/multi_scan.h +19 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +219 -24
- package/deps/rocksdb/rocksdb/include/rocksdb/point_lock_bench_tool.h +14 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/secondary_cache.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/slice.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +7 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/status.h +16 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +16 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +13 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/types.h +4 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +0 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/user_defined_index.h +45 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/cache_dump_load.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +6 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +21 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/memory/memory_allocator_impl.h +3 -3
- package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +77 -51
- package/deps/rocksdb/rocksdb/memtable/skiplist.h +10 -13
- package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +16 -7
- package/deps/rocksdb/rocksdb/memtable/vectorrep.cc +9 -4
- package/deps/rocksdb/rocksdb/monitoring/iostats_context.cc +2 -0
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +6 -0
- package/deps/rocksdb/rocksdb/options/cf_options.cc +13 -1
- package/deps/rocksdb/rocksdb/options/cf_options.h +6 -2
- package/deps/rocksdb/rocksdb/options/options.cc +2 -0
- package/deps/rocksdb/rocksdb/options/options_helper.cc +9 -8
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +9 -5
- package/deps/rocksdb/rocksdb/port/mmap.cc +1 -1
- package/deps/rocksdb/rocksdb/port/win/xpress_win.cc +51 -0
- package/deps/rocksdb/rocksdb/port/win/xpress_win.h +4 -0
- package/deps/rocksdb/rocksdb/src.mk +8 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +1125 -765
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +35 -24
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +29 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +732 -256
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +225 -16
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +102 -26
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +2 -75
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +433 -141
- package/deps/rocksdb/rocksdb/table/block_based/block_builder.h +2 -0
- package/deps/rocksdb/rocksdb/table/block_based/flush_block_policy.cc +17 -10
- package/deps/rocksdb/rocksdb/table/block_based/flush_block_policy_impl.h +20 -0
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +112 -85
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +191 -36
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +2 -2
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/user_defined_index_wrapper.h +108 -31
- package/deps/rocksdb/rocksdb/table/external_table.cc +7 -3
- package/deps/rocksdb/rocksdb/table/format.cc +6 -12
- package/deps/rocksdb/rocksdb/table/format.h +10 -0
- package/deps/rocksdb/rocksdb/table/internal_iterator.h +1 -1
- package/deps/rocksdb/rocksdb/table/iterator_wrapper.h +1 -1
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +1 -1
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +5 -0
- package/deps/rocksdb/rocksdb/table/multiget_context.h +3 -1
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +118 -46
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +9 -8
- package/deps/rocksdb/rocksdb/table/table_builder.h +5 -0
- package/deps/rocksdb/rocksdb/table/table_properties.cc +16 -0
- package/deps/rocksdb/rocksdb/table/table_test.cc +1540 -155
- package/deps/rocksdb/rocksdb/test_util/testutil.h +21 -5
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +26 -5
- package/deps/rocksdb/rocksdb/tools/ldb.cc +1 -2
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +2 -0
- package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +9 -3
- package/deps/rocksdb/rocksdb/tools/sst_dump_test.cc +133 -165
- package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +173 -64
- package/deps/rocksdb/rocksdb/util/aligned_buffer.h +69 -0
- package/deps/rocksdb/rocksdb/util/atomic.h +6 -0
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.cc +29 -20
- package/deps/rocksdb/rocksdb/util/auto_tune_compressor.h +10 -6
- package/deps/rocksdb/rocksdb/util/bit_fields.h +338 -0
- package/deps/rocksdb/rocksdb/util/coding.h +3 -3
- package/deps/rocksdb/rocksdb/util/compaction_job_stats_impl.cc +2 -2
- package/deps/rocksdb/rocksdb/util/compression.cc +777 -82
- package/deps/rocksdb/rocksdb/util/compression.h +5 -0
- package/deps/rocksdb/rocksdb/util/compression_test.cc +5 -3
- package/deps/rocksdb/rocksdb/util/dynamic_bloom.cc +2 -2
- package/deps/rocksdb/rocksdb/util/dynamic_bloom.h +15 -14
- package/deps/rocksdb/rocksdb/util/interval_test.cc +102 -0
- package/deps/rocksdb/rocksdb/util/semaphore.h +164 -0
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +10 -6
- package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +4 -2
- package/deps/rocksdb/rocksdb/util/slice_test.cc +136 -0
- package/deps/rocksdb/rocksdb/util/status.cc +1 -0
- package/deps/rocksdb/rocksdb/util/string_util.cc +2 -16
- package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.cc +1 -1
- package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +7 -4
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +35 -14
- package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table_test.cc +2 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/lock_manager.cc +5 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/any_lock_manager_test.h +244 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_bench.cc +18 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_bench_tool.cc +159 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.cc +1244 -161
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.h +66 -12
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_stress_test.cc +103 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_test.cc +1275 -8
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_test.h +40 -262
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_test_common.h +78 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_validation_test_runner.h +469 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_locking_test.cc +2 -6
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +4 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +9 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/timestamped_snapshot_test.cc +18 -9
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.h +2 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_db_mutex_impl.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +72 -44
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +92 -15
- package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +6 -20
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +143 -112
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_transaction_test.cc +23 -16
- package/index.js +18 -42
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
- package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
- package/util.h +38 -12
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_stat.cc +0 -17
|
@@ -34,6 +34,7 @@
|
|
|
34
34
|
#include "util/string_util.h"
|
|
35
35
|
|
|
36
36
|
#ifdef SNAPPY
|
|
37
|
+
#include <snappy-sinksource.h>
|
|
37
38
|
#include <snappy.h>
|
|
38
39
|
#endif
|
|
39
40
|
|
|
@@ -48,10 +49,14 @@
|
|
|
48
49
|
#if defined(LZ4)
|
|
49
50
|
#include <lz4.h>
|
|
50
51
|
#include <lz4hc.h>
|
|
52
|
+
#if LZ4_VERSION_NUMBER < 10700 // < r129
|
|
53
|
+
#error "LZ4 support requires version >= 1.7.0 (lz4-devel)"
|
|
54
|
+
#endif
|
|
51
55
|
#endif
|
|
52
56
|
|
|
53
57
|
#ifdef ZSTD
|
|
54
58
|
#include <zstd.h>
|
|
59
|
+
#include <zstd_errors.h>
|
|
55
60
|
// ZSTD_Compress2(), ZSTD_compressStream2() and frame parameters all belong to
|
|
56
61
|
// advanced APIs and require v1.4.0+, which is from April 2019.
|
|
57
62
|
// https://github.com/facebook/zstd/blob/eb9f881eb810f2242f1ef36b3f3e7014eecb8fa6/lib/zstd.h#L297C40-L297C45
|
|
@@ -1102,8 +1102,8 @@ TEST_F(DBCompressionTest, CompressionManagerWrapper) {
|
|
|
1102
1102
|
using CompressorWrapper::CompressorWrapper;
|
|
1103
1103
|
const char* Name() const override { return "MyCompressor"; }
|
|
1104
1104
|
|
|
1105
|
-
Status CompressBlock(Slice uncompressed_data,
|
|
1106
|
-
|
|
1105
|
+
Status CompressBlock(Slice uncompressed_data, char* compressed_output,
|
|
1106
|
+
size_t* compressed_output_size,
|
|
1107
1107
|
CompressionType* out_compression_type,
|
|
1108
1108
|
ManagedWorkingArea* working_area) override {
|
|
1109
1109
|
auto begin = uncompressed_data.data();
|
|
@@ -1111,16 +1111,18 @@ TEST_F(DBCompressionTest, CompressionManagerWrapper) {
|
|
|
1111
1111
|
if (std::search(begin, end, kDoNotCompress.begin(),
|
|
1112
1112
|
kDoNotCompress.end()) != end) {
|
|
1113
1113
|
// Do not attempt compression
|
|
1114
|
+
*compressed_output_size = 0;
|
|
1114
1115
|
EXPECT_EQ(*out_compression_type, kNoCompression);
|
|
1115
1116
|
return Status::OK();
|
|
1116
1117
|
} else if (std::search(begin, end, kRejectCompression.begin(),
|
|
1117
1118
|
kRejectCompression.end()) != end) {
|
|
1118
1119
|
// Simulate attempted & rejected compression
|
|
1119
|
-
*
|
|
1120
|
+
*compressed_output_size = 1;
|
|
1120
1121
|
EXPECT_EQ(*out_compression_type, kNoCompression);
|
|
1121
1122
|
return Status::OK();
|
|
1122
1123
|
} else {
|
|
1123
1124
|
return wrapped_->CompressBlock(uncompressed_data, compressed_output,
|
|
1125
|
+
compressed_output_size,
|
|
1124
1126
|
out_compression_type, working_area);
|
|
1125
1127
|
}
|
|
1126
1128
|
}
|
|
@@ -62,9 +62,9 @@ DynamicBloom::DynamicBloom(Allocator* allocator, uint32_t total_bits,
|
|
|
62
62
|
// Align on block_bytes boundary
|
|
63
63
|
raw += block_bytes - block_offset;
|
|
64
64
|
}
|
|
65
|
-
static_assert(sizeof(
|
|
65
|
+
static_assert(sizeof(RelaxedAtomic<uint64_t>) == sizeof(uint64_t),
|
|
66
66
|
"Expecting zero-space-overhead atomic");
|
|
67
|
-
data_ = reinterpret_cast<
|
|
67
|
+
data_ = reinterpret_cast<RelaxedAtomic<uint64_t>*>(raw);
|
|
68
68
|
}
|
|
69
69
|
|
|
70
70
|
} // namespace ROCKSDB_NAMESPACE
|
|
@@ -7,12 +7,10 @@
|
|
|
7
7
|
|
|
8
8
|
#include <array>
|
|
9
9
|
#include <atomic>
|
|
10
|
-
#include <memory>
|
|
11
|
-
#include <string>
|
|
12
10
|
|
|
13
|
-
#include "port/port.h"
|
|
14
11
|
#include "rocksdb/slice.h"
|
|
15
12
|
#include "table/multiget_context.h"
|
|
13
|
+
#include "util/atomic.h"
|
|
16
14
|
#include "util/hash.h"
|
|
17
15
|
|
|
18
16
|
namespace ROCKSDB_NAMESPACE {
|
|
@@ -50,16 +48,20 @@ class DynamicBloom {
|
|
|
50
48
|
|
|
51
49
|
~DynamicBloom() {}
|
|
52
50
|
|
|
53
|
-
// Assuming single
|
|
51
|
+
// Assuming single thread adding to the DynamicBloom
|
|
54
52
|
void Add(const Slice& key);
|
|
55
53
|
|
|
56
|
-
// Like Add, but may be called
|
|
54
|
+
// Like Add, but may be called concurrently with other functions. Does not
|
|
55
|
+
// establish happens-before relationship with other functions so requires some
|
|
56
|
+
// external mechanism to ensure other threads can see the change.
|
|
57
57
|
void AddConcurrently(const Slice& key);
|
|
58
58
|
|
|
59
59
|
// Assuming single threaded access to this function.
|
|
60
60
|
void AddHash(uint32_t hash);
|
|
61
61
|
|
|
62
|
-
// Like AddHash, but may be called
|
|
62
|
+
// Like AddHash, but may be called concurrently with other functions. Does not
|
|
63
|
+
// establish happens-before relationship with other functions so requires some
|
|
64
|
+
// external mechanism to ensure other threads can see the change.
|
|
63
65
|
void AddHashConcurrently(uint32_t hash);
|
|
64
66
|
|
|
65
67
|
// Multithreaded access to this function is OK
|
|
@@ -80,7 +82,7 @@ class DynamicBloom {
|
|
|
80
82
|
// this stores k/2, the number of words to double-probe.
|
|
81
83
|
const uint32_t kNumDoubleProbes;
|
|
82
84
|
|
|
83
|
-
|
|
85
|
+
RelaxedAtomic<uint64_t>* data_;
|
|
84
86
|
|
|
85
87
|
// or_func(ptr, mask) should effect *ptr |= mask with the appropriate
|
|
86
88
|
// concurrency safety, working with bytes.
|
|
@@ -97,21 +99,20 @@ inline void DynamicBloom::AddConcurrently(const Slice& key) {
|
|
|
97
99
|
}
|
|
98
100
|
|
|
99
101
|
inline void DynamicBloom::AddHash(uint32_t hash) {
|
|
100
|
-
AddHash(hash, [](
|
|
101
|
-
ptr->
|
|
102
|
-
std::memory_order_relaxed);
|
|
102
|
+
AddHash(hash, [](RelaxedAtomic<uint64_t>* ptr, uint64_t mask) {
|
|
103
|
+
ptr->StoreRelaxed(ptr->LoadRelaxed() | mask);
|
|
103
104
|
});
|
|
104
105
|
}
|
|
105
106
|
|
|
106
107
|
inline void DynamicBloom::AddHashConcurrently(uint32_t hash) {
|
|
107
|
-
AddHash(hash, [](
|
|
108
|
+
AddHash(hash, [](RelaxedAtomic<uint64_t>* ptr, uint64_t mask) {
|
|
108
109
|
// Happens-before between AddHash and MaybeContains is handled by
|
|
109
110
|
// access to versions_->LastSequence(), so all we have to do here is
|
|
110
111
|
// avoid races (so we don't give the compiler a license to mess up
|
|
111
112
|
// our code) and not lose bits. std::memory_order_relaxed is enough
|
|
112
113
|
// for that.
|
|
113
|
-
if ((mask & ptr->
|
|
114
|
-
ptr->
|
|
114
|
+
if ((mask & ptr->LoadRelaxed()) != mask) {
|
|
115
|
+
ptr->FetchOrRelaxed(mask);
|
|
115
116
|
}
|
|
116
117
|
});
|
|
117
118
|
}
|
|
@@ -183,7 +184,7 @@ inline bool DynamicBloom::DoubleProbe(uint32_t h32, size_t byte_offset) const {
|
|
|
183
184
|
// Two bit probes per uint64_t probe
|
|
184
185
|
uint64_t mask =
|
|
185
186
|
((uint64_t)1 << (h & 63)) | ((uint64_t)1 << ((h >> 6) & 63));
|
|
186
|
-
uint64_t val = data_[byte_offset ^ i].
|
|
187
|
+
uint64_t val = data_[byte_offset ^ i].LoadRelaxed();
|
|
187
188
|
if (i + 1 >= kNumDoubleProbes) {
|
|
188
189
|
return (val & mask) == mask;
|
|
189
190
|
} else if ((val & mask) != mask) {
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
2
|
+
// This source code is licensed under both the GPLv2 (found in the
|
|
3
|
+
// COPYING file in the root directory) and Apache 2.0 License
|
|
4
|
+
// (found in the LICENSE.Apache file in the root directory).
|
|
5
|
+
|
|
6
|
+
#include "db/db_test_util.h"
|
|
7
|
+
#include "rocksdb/data_structure.h"
|
|
8
|
+
#include "rocksdb/slice.h"
|
|
9
|
+
|
|
10
|
+
namespace ROCKSDB_NAMESPACE {
|
|
11
|
+
|
|
12
|
+
class IntervalSetTest : public testing::Test {};
|
|
13
|
+
|
|
14
|
+
TEST_F(IntervalSetTest, BasicTest) {
|
|
15
|
+
IntervalSet<int> set;
|
|
16
|
+
set.insert({2, 15});
|
|
17
|
+
EXPECT_EQ(set.size(), 1);
|
|
18
|
+
set.insert({5, 9});
|
|
19
|
+
EXPECT_EQ(set.size(), 1);
|
|
20
|
+
set.insert({0, 10});
|
|
21
|
+
EXPECT_EQ(set.size(), 1);
|
|
22
|
+
set.insert({25, 30});
|
|
23
|
+
EXPECT_EQ(set.size(), 2);
|
|
24
|
+
set.insert({16, 25});
|
|
25
|
+
EXPECT_EQ(set.size(), 2);
|
|
26
|
+
set.insert({45, 85});
|
|
27
|
+
ASSERT_EQ(set.size(), 3);
|
|
28
|
+
auto iter = set.begin();
|
|
29
|
+
ASSERT_EQ(*iter, Interval<int>(0, 15));
|
|
30
|
+
iter++;
|
|
31
|
+
ASSERT_EQ(*iter, Interval<int>(16, 30));
|
|
32
|
+
iter++;
|
|
33
|
+
ASSERT_EQ(*iter, Interval<int>(45, 85));
|
|
34
|
+
set.insert({31});
|
|
35
|
+
iter = set.begin();
|
|
36
|
+
ASSERT_EQ(*iter, Interval<int>(0, 15));
|
|
37
|
+
iter++;
|
|
38
|
+
ASSERT_EQ(*iter, Interval<int>(16, 30));
|
|
39
|
+
iter++;
|
|
40
|
+
ASSERT_EQ(*iter, Interval<int>(31));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
TEST_F(IntervalSetTest, SliceTest) {
|
|
44
|
+
IntervalSet<Slice, Comparator> set(BytewiseComparator());
|
|
45
|
+
EXPECT_TRUE(set.insert("k00", "k10"));
|
|
46
|
+
// Should do nothing
|
|
47
|
+
EXPECT_TRUE(set.insert("k02", "k08"));
|
|
48
|
+
auto iter = set.begin();
|
|
49
|
+
ASSERT_EQ(iter->start().ToString(), "k00");
|
|
50
|
+
ASSERT_EQ(iter->end().ToString(), "k10");
|
|
51
|
+
ASSERT_EQ(set.size(), 1);
|
|
52
|
+
iter++;
|
|
53
|
+
ASSERT_EQ(iter, set.end());
|
|
54
|
+
EXPECT_TRUE(set.insert("k15", "k20"));
|
|
55
|
+
EXPECT_TRUE(set.insert("k16"));
|
|
56
|
+
ASSERT_EQ(set.size(), 2);
|
|
57
|
+
iter = set.begin();
|
|
58
|
+
ASSERT_EQ(iter->start().ToString(), "k00");
|
|
59
|
+
ASSERT_EQ(iter->end().ToString(), "k10");
|
|
60
|
+
iter++;
|
|
61
|
+
ASSERT_EQ(iter->start().ToString(), "k15");
|
|
62
|
+
ASSERT_EQ(iter->has_end(), false);
|
|
63
|
+
//
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
TEST_F(IntervalSetTest, PropModeTest) {
|
|
67
|
+
IntervalSet<Slice, Comparator> set(BytewiseComparator(), true);
|
|
68
|
+
EXPECT_TRUE(set.insert("k00", "k10"));
|
|
69
|
+
// Should do nothing
|
|
70
|
+
EXPECT_FALSE(set.insert("k02", "k08"));
|
|
71
|
+
EXPECT_EQ(set.size(), 1);
|
|
72
|
+
EXPECT_TRUE(set.insert("k15", "k20"));
|
|
73
|
+
EXPECT_EQ(set.size(), 2);
|
|
74
|
+
EXPECT_FALSE(set.insert("k16"));
|
|
75
|
+
ASSERT_EQ(set.size(), 2);
|
|
76
|
+
auto iter = set.begin();
|
|
77
|
+
ASSERT_EQ(iter->start().ToString(), "k00");
|
|
78
|
+
ASSERT_EQ(iter->end().ToString(), "k10");
|
|
79
|
+
iter++;
|
|
80
|
+
ASSERT_EQ(iter->start().ToString(), "k15");
|
|
81
|
+
ASSERT_EQ(iter->end().ToString(), "k20");
|
|
82
|
+
EXPECT_TRUE(set.insert("k12", "k14"));
|
|
83
|
+
iter = set.begin();
|
|
84
|
+
ASSERT_EQ(set.size(), 3);
|
|
85
|
+
ASSERT_EQ(iter->start().ToString(), "k00");
|
|
86
|
+
ASSERT_EQ(iter->end().ToString(), "k10");
|
|
87
|
+
iter++;
|
|
88
|
+
ASSERT_EQ(iter->start().ToString(), "k12");
|
|
89
|
+
ASSERT_EQ(iter->end().ToString(), "k14");
|
|
90
|
+
iter++;
|
|
91
|
+
ASSERT_EQ(iter->start().ToString(), "k15");
|
|
92
|
+
ASSERT_EQ(iter->end().ToString(), "k20");
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
} // namespace ROCKSDB_NAMESPACE
|
|
96
|
+
|
|
97
|
+
int main(int argc, char** argv) {
|
|
98
|
+
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
|
|
99
|
+
::testing::InitGoogleTest(&argc, argv);
|
|
100
|
+
|
|
101
|
+
return RUN_ALL_TESTS();
|
|
102
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
2
|
+
// This source code is licensed under both the GPLv2 (found in the
|
|
3
|
+
// COPYING file in the root directory) and Apache 2.0 License
|
|
4
|
+
// (found in the LICENSE.Apache file in the root directory).
|
|
5
|
+
|
|
6
|
+
#pragma once
|
|
7
|
+
|
|
8
|
+
#include <cassert>
|
|
9
|
+
#include <condition_variable>
|
|
10
|
+
#include <mutex>
|
|
11
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
12
|
+
#include <semaphore>
|
|
13
|
+
#endif
|
|
14
|
+
|
|
15
|
+
#include "port/port.h"
|
|
16
|
+
#include "rocksdb/rocksdb_namespace.h"
|
|
17
|
+
|
|
18
|
+
namespace ROCKSDB_NAMESPACE {
|
|
19
|
+
|
|
20
|
+
// Wrapper providing a chosen counting semaphore implementation. The default
|
|
21
|
+
// implementation based on a mutex and condvar unfortunately can result in
|
|
22
|
+
// Release() temporarily waiting on another thread to make progress (if that
|
|
23
|
+
// other thread is preempted while holding the mutex), but that should be rare.
|
|
24
|
+
// However, alternative implementations may have correctness issues or even
|
|
25
|
+
// worse performance. See std::counting_semaphore for general contract.
|
|
26
|
+
//
|
|
27
|
+
// NOTE1: std::counting_semaphore is known to be buggy on many std library
|
|
28
|
+
// implementations, so be cautious about enabling it. Reportedly, an acquire()
|
|
29
|
+
// can falsely block indefinitely. And we can't easily work around that with
|
|
30
|
+
// try_acquire_for because another common bug has that function consistently
|
|
31
|
+
// sleeping for the entire timeout duration even if a release() happens earlier.
|
|
32
|
+
// Therefore, using std::counting_semaphore/binary_semaphore is strictly opt-in
|
|
33
|
+
// for now.
|
|
34
|
+
//
|
|
35
|
+
// NOTE2: Also tried wrapping folly::fibers::Semaphore here but it was not as
|
|
36
|
+
// efficient (for parallel compression) as even the mutex+condvar version.
|
|
37
|
+
class ALIGN_AS(CACHE_LINE_SIZE) CountingSemaphore {
|
|
38
|
+
public:
|
|
39
|
+
explicit CountingSemaphore(std::ptrdiff_t starting_count)
|
|
40
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
41
|
+
: sem_(starting_count)
|
|
42
|
+
#else
|
|
43
|
+
: count_(static_cast<int32_t>(starting_count))
|
|
44
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
45
|
+
{
|
|
46
|
+
assert(starting_count >= 0);
|
|
47
|
+
assert(starting_count <= INT32_MAX);
|
|
48
|
+
}
|
|
49
|
+
void Acquire() {
|
|
50
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
51
|
+
sem_.acquire();
|
|
52
|
+
#else
|
|
53
|
+
std::unique_lock<std::mutex> lock(mutex_);
|
|
54
|
+
assert(count_ >= 0);
|
|
55
|
+
cv_.wait(lock, [this] { return count_ > 0; });
|
|
56
|
+
--count_;
|
|
57
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
58
|
+
}
|
|
59
|
+
bool TryAcquire() {
|
|
60
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
61
|
+
return sem_.try_acquire();
|
|
62
|
+
#else
|
|
63
|
+
std::unique_lock<std::mutex> lock(mutex_);
|
|
64
|
+
assert(count_ >= 0);
|
|
65
|
+
if (count_ == 0) {
|
|
66
|
+
return false;
|
|
67
|
+
} else {
|
|
68
|
+
--count_;
|
|
69
|
+
return true;
|
|
70
|
+
}
|
|
71
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
72
|
+
}
|
|
73
|
+
void Release(std::ptrdiff_t n = 1) {
|
|
74
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
75
|
+
sem_.release(n);
|
|
76
|
+
#else
|
|
77
|
+
assert(n >= 0);
|
|
78
|
+
assert(n <= INT32_MAX);
|
|
79
|
+
if (n > 0) {
|
|
80
|
+
std::unique_lock<std::mutex> lock(mutex_);
|
|
81
|
+
assert(count_ >= 0);
|
|
82
|
+
count_ += static_cast<int32_t>(n);
|
|
83
|
+
assert(count_ >= 0); // no overflow
|
|
84
|
+
if (n == 1) {
|
|
85
|
+
cv_.notify_one();
|
|
86
|
+
} else {
|
|
87
|
+
cv_.notify_all();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
private:
|
|
94
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
95
|
+
std::counting_semaphore<INT32_MAX> sem_;
|
|
96
|
+
#else
|
|
97
|
+
int32_t count_;
|
|
98
|
+
std::mutex mutex_;
|
|
99
|
+
std::condition_variable cv_;
|
|
100
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
101
|
+
}; // namespace ROCKSDB_NAMESPACE
|
|
102
|
+
|
|
103
|
+
// Wrapper providing a chosen binary semaphore implementation. See notes on
|
|
104
|
+
// CountingSemaphore above, and on Release() below.
|
|
105
|
+
class BinarySemaphore {
|
|
106
|
+
public:
|
|
107
|
+
explicit BinarySemaphore(std::ptrdiff_t starting_count)
|
|
108
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
109
|
+
: sem_(starting_count)
|
|
110
|
+
#else
|
|
111
|
+
: state_(starting_count > 0)
|
|
112
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
113
|
+
{
|
|
114
|
+
assert(starting_count >= 0);
|
|
115
|
+
}
|
|
116
|
+
void Acquire() {
|
|
117
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
118
|
+
sem_.acquire();
|
|
119
|
+
#else
|
|
120
|
+
std::unique_lock<std::mutex> lock(mutex_);
|
|
121
|
+
cv_.wait(lock, [this] { return state_; });
|
|
122
|
+
state_ = false;
|
|
123
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
124
|
+
}
|
|
125
|
+
bool TryAcquire() {
|
|
126
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
127
|
+
return sem_.try_acquire();
|
|
128
|
+
#else
|
|
129
|
+
std::unique_lock<std::mutex> lock(mutex_);
|
|
130
|
+
if (state_) {
|
|
131
|
+
state_ = false;
|
|
132
|
+
return true;
|
|
133
|
+
} else {
|
|
134
|
+
return false;
|
|
135
|
+
}
|
|
136
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
137
|
+
}
|
|
138
|
+
void Release() {
|
|
139
|
+
// NOTE: implementations of std::binary_semaphore::release() tend to behave
|
|
140
|
+
// like counting semaphores in the case of multiple Release() calls without
|
|
141
|
+
// Acquire() in between, though it is undefined behavior. It is also OK to
|
|
142
|
+
// cap the count at 1.
|
|
143
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
144
|
+
sem_.release();
|
|
145
|
+
#else
|
|
146
|
+
std::unique_lock<std::mutex> lock(mutex_);
|
|
147
|
+
// check precondition to avoid UB in std implementation
|
|
148
|
+
assert(state_ == false);
|
|
149
|
+
state_ = true;
|
|
150
|
+
cv_.notify_one();
|
|
151
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
private:
|
|
155
|
+
#ifdef ROCKSDB_USE_STD_SEMAPHORES
|
|
156
|
+
std::binary_semaphore sem_;
|
|
157
|
+
#else
|
|
158
|
+
bool state_;
|
|
159
|
+
std::mutex mutex_;
|
|
160
|
+
std::condition_variable cv_;
|
|
161
|
+
#endif // ROCKSDB_USE_STD_SEMAPHORES
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
} // namespace ROCKSDB_NAMESPACE
|
|
@@ -60,13 +60,15 @@ const char* RandomMixedCompressor::Name() const {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
Status RandomMixedCompressor::CompressBlock(
|
|
63
|
-
Slice uncompressed_data,
|
|
64
|
-
|
|
63
|
+
Slice uncompressed_data, char* compressed_output,
|
|
64
|
+
size_t* compressed_output_size, CompressionType* out_compression_type,
|
|
65
|
+
ManagedWorkingArea* wa) {
|
|
65
66
|
auto selected =
|
|
66
67
|
Random::GetTLSInstance()->Uniform(static_cast<int>(compressors_.size()));
|
|
67
68
|
auto& compressor = compressors_[selected];
|
|
68
69
|
return compressor->CompressBlock(uncompressed_data, compressed_output,
|
|
69
|
-
out_compression_type,
|
|
70
|
+
compressed_output_size, out_compression_type,
|
|
71
|
+
wa);
|
|
70
72
|
}
|
|
71
73
|
|
|
72
74
|
const char* RandomMixedCompressionManager::Name() const {
|
|
@@ -85,13 +87,15 @@ const char* RoundRobinCompressor::Name() const {
|
|
|
85
87
|
}
|
|
86
88
|
|
|
87
89
|
Status RoundRobinCompressor::CompressBlock(
|
|
88
|
-
Slice uncompressed_data,
|
|
89
|
-
|
|
90
|
+
Slice uncompressed_data, char* compressed_output,
|
|
91
|
+
size_t* compressed_output_size, CompressionType* out_compression_type,
|
|
92
|
+
ManagedWorkingArea* wa) {
|
|
90
93
|
auto counter = block_counter.FetchAddRelaxed(1);
|
|
91
94
|
auto sel_idx = counter % (compressors_.size());
|
|
92
95
|
auto& compressor = compressors_[sel_idx];
|
|
93
96
|
return compressor->CompressBlock(uncompressed_data, compressed_output,
|
|
94
|
-
out_compression_type,
|
|
97
|
+
compressed_output_size, out_compression_type,
|
|
98
|
+
wa);
|
|
95
99
|
}
|
|
96
100
|
|
|
97
101
|
RelaxedAtomic<uint64_t> RoundRobinCompressor::block_counter{0};
|
|
@@ -34,7 +34,8 @@ class MultiCompressorWrapper : public Compressor {
|
|
|
34
34
|
struct RandomMixedCompressor : public MultiCompressorWrapper {
|
|
35
35
|
using MultiCompressorWrapper::MultiCompressorWrapper;
|
|
36
36
|
const char* Name() const override;
|
|
37
|
-
Status CompressBlock(Slice uncompressed_data,
|
|
37
|
+
Status CompressBlock(Slice uncompressed_data, char* compressed_output,
|
|
38
|
+
size_t* compressed_output_size,
|
|
38
39
|
CompressionType* out_compression_type,
|
|
39
40
|
ManagedWorkingArea* wa) override;
|
|
40
41
|
};
|
|
@@ -50,7 +51,8 @@ class RandomMixedCompressionManager : public CompressionManagerWrapper {
|
|
|
50
51
|
struct RoundRobinCompressor : public MultiCompressorWrapper {
|
|
51
52
|
using MultiCompressorWrapper::MultiCompressorWrapper;
|
|
52
53
|
const char* Name() const override;
|
|
53
|
-
Status CompressBlock(Slice uncompressed_data,
|
|
54
|
+
Status CompressBlock(Slice uncompressed_data, char* compressed_output,
|
|
55
|
+
size_t* compressed_output_size,
|
|
54
56
|
CompressionType* out_compression_type,
|
|
55
57
|
ManagedWorkingArea* wa) override;
|
|
56
58
|
static RelaxedAtomic<uint64_t> block_counter;
|
|
@@ -3,17 +3,25 @@
|
|
|
3
3
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
4
4
|
// (found in the LICENSE.Apache file in the root directory).
|
|
5
5
|
|
|
6
|
+
// Because there are a small set of tests for Slice and there's a cost in having
|
|
7
|
+
// extra test binaries for each component, this test file has evolved into a
|
|
8
|
+
// "grab bag" of small tests for various reusable components, mostly in util/.
|
|
9
|
+
|
|
6
10
|
#include "rocksdb/slice.h"
|
|
7
11
|
|
|
8
12
|
#include <gtest/gtest.h>
|
|
9
13
|
|
|
14
|
+
#include <semaphore>
|
|
15
|
+
|
|
10
16
|
#include "port/port.h"
|
|
11
17
|
#include "port/stack_trace.h"
|
|
12
18
|
#include "rocksdb/data_structure.h"
|
|
13
19
|
#include "rocksdb/types.h"
|
|
14
20
|
#include "test_util/testharness.h"
|
|
15
21
|
#include "test_util/testutil.h"
|
|
22
|
+
#include "util/bit_fields.h"
|
|
16
23
|
#include "util/cast_util.h"
|
|
24
|
+
#include "util/semaphore.h"
|
|
17
25
|
#include "util/string_util.h"
|
|
18
26
|
|
|
19
27
|
namespace ROCKSDB_NAMESPACE {
|
|
@@ -422,6 +430,134 @@ TEST(ToBaseCharsStringTest, Tests) {
|
|
|
422
430
|
ASSERT_EQ(ToBaseCharsString<32>(2, 255, false), "7v");
|
|
423
431
|
}
|
|
424
432
|
|
|
433
|
+
TEST(SemaphoreTest, CountingSemaphore) {
|
|
434
|
+
CountingSemaphore sem{0};
|
|
435
|
+
int kCount = 5;
|
|
436
|
+
std::vector<std::thread> threads;
|
|
437
|
+
for (int i = 0; i < kCount; ++i) {
|
|
438
|
+
threads.emplace_back([&sem] { sem.Release(); });
|
|
439
|
+
}
|
|
440
|
+
for (int i = 0; i < kCount; ++i) {
|
|
441
|
+
threads.emplace_back([&sem] { sem.Acquire(); });
|
|
442
|
+
}
|
|
443
|
+
for (auto& t : threads) {
|
|
444
|
+
t.join();
|
|
445
|
+
}
|
|
446
|
+
// Nothing left on the semaphore
|
|
447
|
+
ASSERT_FALSE(sem.TryAcquire());
|
|
448
|
+
// Keep testing
|
|
449
|
+
sem.Release(2);
|
|
450
|
+
ASSERT_TRUE(sem.TryAcquire());
|
|
451
|
+
sem.Acquire();
|
|
452
|
+
ASSERT_FALSE(sem.TryAcquire());
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
TEST(SemaphoreTest, BinarySemaphore) {
|
|
456
|
+
BinarySemaphore sem{0};
|
|
457
|
+
int kCount = 5;
|
|
458
|
+
std::vector<std::thread> threads;
|
|
459
|
+
for (int i = 0; i < kCount; ++i) {
|
|
460
|
+
threads.emplace_back([&sem] {
|
|
461
|
+
sem.Acquire();
|
|
462
|
+
sem.Release();
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
threads.emplace_back([&sem] { sem.Release(); });
|
|
466
|
+
for (auto& t : threads) {
|
|
467
|
+
t.join();
|
|
468
|
+
}
|
|
469
|
+
// Only able to acquire one excess release
|
|
470
|
+
ASSERT_TRUE(sem.TryAcquire());
|
|
471
|
+
ASSERT_FALSE(sem.TryAcquire());
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
TEST(BitFieldsTest, BitFields) {
|
|
475
|
+
// Start by verifying example from BitFields comment
|
|
476
|
+
struct MyState : public BitFields<uint32_t, MyState> {
|
|
477
|
+
// Extra helper declarations and/or field type declarations
|
|
478
|
+
};
|
|
479
|
+
|
|
480
|
+
using Field1 = UnsignedBitField<MyState, 16, NoPrevBitField>;
|
|
481
|
+
using Field2 = BoolBitField<MyState, Field1>;
|
|
482
|
+
using Field3 = BoolBitField<MyState, Field2>;
|
|
483
|
+
using Field4 = UnsignedBitField<MyState, 5, Field3>;
|
|
484
|
+
|
|
485
|
+
// MyState{} is zero-initialized
|
|
486
|
+
auto state = MyState{}.With<Field1>(42U).With<Field2>(true);
|
|
487
|
+
state.Set<Field4>(3U);
|
|
488
|
+
state.Ref<Field1>() += state.Get<Field4>();
|
|
489
|
+
|
|
490
|
+
ASSERT_EQ(state.Get<Field1>(), 45U);
|
|
491
|
+
ASSERT_EQ(state.Get<Field2>(), true);
|
|
492
|
+
ASSERT_EQ(state.Get<Field3>(), false);
|
|
493
|
+
ASSERT_EQ(state.Get<Field4>(), 3U);
|
|
494
|
+
|
|
495
|
+
// Misc operators
|
|
496
|
+
auto ref = state.Ref<Field3>();
|
|
497
|
+
auto ref2 = std::move(ref);
|
|
498
|
+
ref2 = true;
|
|
499
|
+
ASSERT_EQ(state.Get<Field3>(), true);
|
|
500
|
+
|
|
501
|
+
MyState state2;
|
|
502
|
+
// Basic non-concurrent tests for atomic wrappers
|
|
503
|
+
{
|
|
504
|
+
RelaxedBitFieldsAtomic<MyState> relaxed{state};
|
|
505
|
+
ASSERT_EQ(state, relaxed.LoadRelaxed());
|
|
506
|
+
relaxed.StoreRelaxed(state2);
|
|
507
|
+
ASSERT_EQ(state2, relaxed.LoadRelaxed());
|
|
508
|
+
MyState state3 = relaxed.ExchangeRelaxed(state);
|
|
509
|
+
ASSERT_EQ(state2, state3);
|
|
510
|
+
ASSERT_TRUE(relaxed.CasStrongRelaxed(state, state2));
|
|
511
|
+
while (!relaxed.CasWeakRelaxed(state2, state)) {
|
|
512
|
+
}
|
|
513
|
+
ASSERT_EQ(state2, state3);
|
|
514
|
+
ASSERT_EQ(state, relaxed.LoadRelaxed());
|
|
515
|
+
|
|
516
|
+
auto transform1 = Field2::ClearTransform() + Field3::ClearTransform();
|
|
517
|
+
MyState before, after;
|
|
518
|
+
relaxed.ApplyRelaxed(transform1, &before, &after);
|
|
519
|
+
ASSERT_EQ(before, state);
|
|
520
|
+
ASSERT_NE(after, state);
|
|
521
|
+
ASSERT_EQ(after.Get<Field2>(), false);
|
|
522
|
+
ASSERT_EQ(after.Get<Field3>(), false);
|
|
523
|
+
|
|
524
|
+
auto transform2 = Field2::SetTransform() + Field3::SetTransform();
|
|
525
|
+
relaxed.ApplyRelaxed(transform2, &before, &after);
|
|
526
|
+
ASSERT_NE(before, state);
|
|
527
|
+
ASSERT_EQ(before.Get<Field2>(), false);
|
|
528
|
+
ASSERT_EQ(before.Get<Field3>(), false);
|
|
529
|
+
ASSERT_EQ(after, state);
|
|
530
|
+
}
|
|
531
|
+
{
|
|
532
|
+
AcqRelBitFieldsAtomic<MyState> acqrel{state};
|
|
533
|
+
ASSERT_EQ(state, acqrel.Load());
|
|
534
|
+
acqrel.Store(state2);
|
|
535
|
+
ASSERT_EQ(state2, acqrel.Load());
|
|
536
|
+
MyState state3 = acqrel.Exchange(state);
|
|
537
|
+
ASSERT_EQ(state2, state3);
|
|
538
|
+
ASSERT_TRUE(acqrel.CasStrong(state, state2));
|
|
539
|
+
while (!acqrel.CasWeak(state2, state)) {
|
|
540
|
+
}
|
|
541
|
+
ASSERT_EQ(state2, state3);
|
|
542
|
+
ASSERT_EQ(state, acqrel.Load());
|
|
543
|
+
|
|
544
|
+
auto transform1 = Field2::ClearTransform() + Field3::ClearTransform();
|
|
545
|
+
MyState before, after;
|
|
546
|
+
acqrel.Apply(transform1, &before, &after);
|
|
547
|
+
ASSERT_EQ(before, state);
|
|
548
|
+
ASSERT_NE(after, state);
|
|
549
|
+
ASSERT_EQ(after.Get<Field2>(), false);
|
|
550
|
+
ASSERT_EQ(after.Get<Field3>(), false);
|
|
551
|
+
|
|
552
|
+
auto transform2 = Field2::SetTransform() + Field3::SetTransform();
|
|
553
|
+
acqrel.Apply(transform2, &before, &after);
|
|
554
|
+
ASSERT_NE(before, state);
|
|
555
|
+
ASSERT_EQ(before.Get<Field2>(), false);
|
|
556
|
+
ASSERT_EQ(before.Get<Field3>(), false);
|
|
557
|
+
ASSERT_EQ(after, state);
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
425
561
|
} // namespace ROCKSDB_NAMESPACE
|
|
426
562
|
|
|
427
563
|
int main(int argc, char** argv) {
|
|
@@ -46,6 +46,7 @@ static const char* msgs[static_cast<int>(Status::kMaxSubCode)] = {
|
|
|
46
46
|
"IO fenced off", // kIOFenced
|
|
47
47
|
"Merge operator failed", // kMergeOperatorFailed
|
|
48
48
|
"Number of operands merged exceeded threshold", // kMergeOperandThresholdExceeded
|
|
49
|
+
"MultiScan reached file prefetch limit", // kMultiScanPrefetchLimit
|
|
49
50
|
};
|
|
50
51
|
|
|
51
52
|
Status::Status(Code _code, SubCode _subcode, const Slice& msg,
|
|
@@ -20,20 +20,6 @@
|
|
|
20
20
|
#include "port/sys_time.h"
|
|
21
21
|
#include "rocksdb/slice.h"
|
|
22
22
|
|
|
23
|
-
#ifndef __has_cpp_attribute
|
|
24
|
-
#define ROCKSDB_HAS_CPP_ATTRIBUTE(x) 0
|
|
25
|
-
#else
|
|
26
|
-
#define ROCKSDB_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
|
|
27
|
-
#endif
|
|
28
|
-
|
|
29
|
-
#if ROCKSDB_HAS_CPP_ATTRIBUTE(maybe_unused) && __cplusplus >= 201703L
|
|
30
|
-
#define ROCKSDB_MAYBE_UNUSED [[maybe_unused]]
|
|
31
|
-
#elif ROCKSDB_HAS_CPP_ATTRIBUTE(gnu::unused) || __GNUC__
|
|
32
|
-
#define ROCKSDB_MAYBE_UNUSED [[gnu::unused]]
|
|
33
|
-
#else
|
|
34
|
-
#define ROCKSDB_MAYBE_UNUSED
|
|
35
|
-
#endif
|
|
36
|
-
|
|
37
23
|
namespace ROCKSDB_NAMESPACE {
|
|
38
24
|
|
|
39
25
|
const std::string kNullptrString = "nullptr";
|
|
@@ -501,7 +487,7 @@ bool TryParseTimeRangeString(const std::string& value, int& start_time,
|
|
|
501
487
|
// selects proper function.
|
|
502
488
|
|
|
503
489
|
#if !(defined(_WIN32) && (defined(__MINGW32__) || defined(_MSC_VER)))
|
|
504
|
-
|
|
490
|
+
[[maybe_unused]]
|
|
505
491
|
static std::string invoke_strerror_r(int (*strerror_r)(int, char*, size_t),
|
|
506
492
|
int err, char* buf, size_t buflen) {
|
|
507
493
|
// Using XSI-compatible strerror_r
|
|
@@ -515,7 +501,7 @@ static std::string invoke_strerror_r(int (*strerror_r)(int, char*, size_t),
|
|
|
515
501
|
return buf;
|
|
516
502
|
}
|
|
517
503
|
|
|
518
|
-
|
|
504
|
+
[[maybe_unused]]
|
|
519
505
|
static std::string invoke_strerror_r(char* (*strerror_r)(int, char*, size_t),
|
|
520
506
|
int err, char* buf, size_t buflen) {
|
|
521
507
|
// Using GNU strerror_r
|