@nxtedition/rocksdb 10.1.4 → 10.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +16 -12
- package/deps/rocksdb/rocksdb/CMakeLists.txt +16 -5
- package/deps/rocksdb/rocksdb/Makefile +38 -15
- package/deps/rocksdb/rocksdb/TARGETS +10 -0
- package/deps/rocksdb/rocksdb/cache/cache_test.cc +58 -0
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +4 -4
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +4 -2
- package/deps/rocksdb/rocksdb/db/builder.cc +2 -2
- package/deps/rocksdb/rocksdb/db/builder.h +1 -1
- package/deps/rocksdb/rocksdb/db/c.cc +205 -6
- package/deps/rocksdb/rocksdb/db/c_test.c +189 -1
- package/deps/rocksdb/rocksdb/db/column_family.cc +28 -0
- package/deps/rocksdb/rocksdb/db/column_family.h +17 -0
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +234 -60
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +8 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction.h +11 -9
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +4 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +2 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +1 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +22 -25
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +2 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +112 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +72 -21
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +2 -0
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +77 -0
- package/deps/rocksdb/rocksdb/db/convenience.cc +3 -0
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +269 -112
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +107 -43
- package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +93 -24
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +5 -5
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +157 -68
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +56 -15
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +78 -105
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +39 -9
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +21 -14
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +107 -63
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +43 -2
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +4 -0
- package/deps/rocksdb/rocksdb/db/db_rate_limiter_test.cc +6 -0
- package/deps/rocksdb/rocksdb/db/db_test.cc +10 -2
- package/deps/rocksdb/rocksdb/db/db_test2.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +5 -0
- package/deps/rocksdb/rocksdb/db/db_test_util.h +7 -6
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +92 -2
- package/deps/rocksdb/rocksdb/db/error_handler.cc +34 -39
- package/deps/rocksdb/rocksdb/db/error_handler.h +3 -4
- package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +8 -4
- package/deps/rocksdb/rocksdb/db/event_helpers.cc +6 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +71 -15
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +11 -0
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +383 -4
- package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +88 -72
- package/deps/rocksdb/rocksdb/db/flush_job.cc +30 -3
- package/deps/rocksdb/rocksdb/db/flush_job.h +14 -0
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +60 -1
- package/deps/rocksdb/rocksdb/db/internal_stats.h +20 -1
- package/deps/rocksdb/rocksdb/db/log_writer.cc +24 -0
- package/deps/rocksdb/rocksdb/db/log_writer.h +5 -0
- package/deps/rocksdb/rocksdb/db/memtable.cc +6 -4
- package/deps/rocksdb/rocksdb/db/memtable.h +10 -10
- package/deps/rocksdb/rocksdb/db/memtable_list.cc +4 -4
- package/deps/rocksdb/rocksdb/db/multi_cf_iterator_impl.h +10 -3
- package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.h +8 -10
- package/deps/rocksdb/rocksdb/db/repair.cc +4 -3
- package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc +30 -0
- package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h +9 -0
- package/deps/rocksdb/rocksdb/db/table_cache.cc +17 -2
- package/deps/rocksdb/rocksdb/db/table_cache.h +9 -1
- package/deps/rocksdb/rocksdb/db/table_properties_collector.h +9 -2
- package/deps/rocksdb/rocksdb/db/table_properties_collector_test.cc +3 -1
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +3 -3
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +7 -7
- package/deps/rocksdb/rocksdb/db/version_edit.cc +0 -1
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -6
- package/deps/rocksdb/rocksdb/db/version_set.cc +54 -31
- package/deps/rocksdb/rocksdb/db/version_set.h +14 -7
- package/deps/rocksdb/rocksdb/db/wal_manager.cc +37 -29
- package/deps/rocksdb/rocksdb/db/wal_manager.h +6 -5
- package/deps/rocksdb/rocksdb/db/wide/wide_columns_helper.cc +6 -0
- package/deps/rocksdb/rocksdb/db/write_batch.cc +54 -23
- package/deps/rocksdb/rocksdb/db/write_callback_test.cc +46 -5
- package/deps/rocksdb/rocksdb/db/write_thread.cc +53 -5
- package/deps/rocksdb/rocksdb/db/write_thread.h +36 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/CMakeLists.txt +1 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/batched_ops_stress.cc +5 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +57 -17
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +11 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +8 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +10 -25
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +25 -88
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_filters.cc +93 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_filters.h +16 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +43 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +109 -21
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +8 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +666 -205
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +55 -10
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +18 -16
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +19 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.h +5 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +782 -494
- package/deps/rocksdb/rocksdb/env/composite_env_wrapper.h +21 -0
- package/deps/rocksdb/rocksdb/env/env.cc +6 -0
- package/deps/rocksdb/rocksdb/env/io_posix.cc +0 -1
- package/deps/rocksdb/rocksdb/file/file_util.cc +8 -2
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +34 -19
- package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +29 -32
- package/deps/rocksdb/rocksdb/file/writable_file_writer.h +41 -15
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +4 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +63 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +16 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/env.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +0 -16
- package/deps/rocksdb/rocksdb/include/rocksdb/iterator_base.h +16 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +21 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +76 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +17 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/transaction_log.h +12 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +31 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/user_write_callback.h +29 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/checkpoint.h +4 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/customizable_util.h +0 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +17 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/table_properties_collectors.h +46 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +7 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/options/cf_options.cc +13 -2
- package/deps/rocksdb/rocksdb/options/cf_options.h +6 -2
- package/deps/rocksdb/rocksdb/options/db_options.cc +8 -0
- package/deps/rocksdb/rocksdb/options/db_options.h +9 -5
- package/deps/rocksdb/rocksdb/options/options.cc +3 -0
- package/deps/rocksdb/rocksdb/options/options_helper.cc +1 -0
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +3 -1
- package/deps/rocksdb/rocksdb/port/jemalloc_helper.h +2 -2
- package/deps/rocksdb/rocksdb/port/stack_trace.cc +1 -0
- package/deps/rocksdb/rocksdb/port/win/port_win.cc +3 -2
- package/deps/rocksdb/rocksdb/src.mk +4 -0
- package/deps/rocksdb/rocksdb/table/block_based/binary_search_index_reader.cc +1 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +4 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +15 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +102 -41
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +15 -7
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +1 -3
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +5 -6
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +31 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +6 -0
- package/deps/rocksdb/rocksdb/table/block_based/cachable_entry.h +10 -5
- package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +11 -15
- package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc +17 -11
- package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.h +5 -2
- package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.cc +28 -21
- package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.h +9 -11
- package/deps/rocksdb/rocksdb/table/block_based/full_filter_block_test.cc +16 -16
- package/deps/rocksdb/rocksdb/table/block_based/hash_index_reader.cc +1 -2
- package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc +14 -9
- package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.h +4 -1
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +82 -41
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.h +13 -14
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +18 -22
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +51 -13
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.h +2 -0
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc +3 -11
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +2 -3
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +9 -10
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +3 -2
- package/deps/rocksdb/rocksdb/table/format.cc +1 -2
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +18 -13
- package/deps/rocksdb/rocksdb/table/merging_iterator.h +5 -3
- package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +2 -2
- package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/table/sst_file_writer_collectors.h +3 -1
- package/deps/rocksdb/rocksdb/table/table_builder.h +8 -7
- package/deps/rocksdb/rocksdb/table/table_reader.h +9 -0
- package/deps/rocksdb/rocksdb/test_util/testutil.cc +1 -0
- package/deps/rocksdb/rocksdb/test_util/testutil.h +6 -0
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +19 -0
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +434 -110
- package/deps/rocksdb/rocksdb/tools/ldb_cmd_impl.h +3 -1
- package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +3 -0
- package/deps/rocksdb/rocksdb/util/aligned_storage.h +24 -0
- package/deps/rocksdb/rocksdb/util/filter_bench.cc +1 -1
- package/deps/rocksdb/rocksdb/util/random.cc +2 -1
- package/deps/rocksdb/rocksdb/util/stderr_logger.h +1 -1
- package/deps/rocksdb/rocksdb/util/udt_util.cc +33 -0
- package/deps/rocksdb/rocksdb/util/udt_util.h +7 -0
- package/deps/rocksdb/rocksdb/util/udt_util_test.cc +33 -0
- package/deps/rocksdb/rocksdb/util/write_batch_util.h +5 -0
- package/deps/rocksdb/rocksdb/util/xxhash.h +10 -3
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +13 -13
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +104 -48
- package/deps/rocksdb/rocksdb/utilities/debug.cc +16 -4
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +647 -235
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +274 -157
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_for_tiering_collector.cc +144 -0
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_for_tiering_collector.h +45 -0
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_for_tiering_collector_test.cc +139 -0
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc +12 -0
- package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc +3 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_test.cc +105 -6
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +64 -8
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +5 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.cc +43 -5
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.h +5 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +154 -6
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +158 -2
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.cc +16 -11
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.cc +4 -4
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +9 -8
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn_db.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +43 -7
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +2 -0
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +1 -1
- package/index.js +1 -2
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
- package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
- package/util.h +25 -2
- package/.tap/test-results/node_modules/abstract-level/test/chained-batch-test.js.tap +0 -0
- package/.tap/test-results/node_modules/abstract-level/test/get-test.js.tap +0 -0
- package/.tap/test-results/test/abstract-level-test.js.tap +0 -1077
- package/.tap/test-results/test/batch-test.js.tap +0 -12
- package/.tap/test-results/test/chained-batch-gc-test.js.tap +0 -11
- package/.tap/test-results/test/cleanup-hanging-iterators-test.js.tap +0 -135
- package/.tap/test-results/test/clear-gc-test.js.tap +0 -13
- package/.tap/test-results/test/column-test.js.tap +0 -55
- package/.tap/test-results/test/common.js.tap +0 -0
- package/.tap/test-results/test/compression-test.js.tap +0 -30
- package/.tap/test-results/test/db-identity.js.tap +0 -12
- package/.tap/test-results/test/electron.js.tap +0 -0
- package/.tap/test-results/test/env-cleanup-hook-test.js.tap +0 -40
- package/.tap/test-results/test/env-cleanup-hook.js.tap +0 -0
- package/.tap/test-results/test/gc.js.tap +0 -0
- package/.tap/test-results/test/getproperty-test.js.tap +0 -29
- package/.tap/test-results/test/iterator-gc-test.js.tap +0 -15
- package/.tap/test-results/test/iterator-hwm-test.js.tap +0 -131
- package/.tap/test-results/test/iterator-recursion-test.js.tap +0 -12
- package/.tap/test-results/test/iterator-starvation-test.js.tap +0 -73
- package/.tap/test-results/test/iterator-test.js.tap +0 -6
- package/.tap/test-results/test/leak-tester-batch.js.tap +0 -0
- package/.tap/test-results/test/leak-tester-iterator.js.tap +0 -0
- package/.tap/test-results/test/leak-tester.js.tap +0 -0
- package/.tap/test-results/test/lock-test.js.tap +0 -18
- package/.tap/test-results/test/lock.js.tap +0 -0
- package/.tap/test-results/test/make.js.tap +0 -0
- package/.tap/test-results/test/max-rev-merge.js.tap +0 -0
- package/.tap/test-results/test/merge-operator-test.js.tap +0 -12
- package/.tap/test-results/test/mkdir-test.js.tap +0 -15
- package/.tap/test-results/test/segfault-test.js.tap +0 -76
- package/.tap/test-results/test/stack-blower.js.tap +0 -0
- package/deps/rocksdb/rocksdb/README.md +0 -29
- package/deps/rocksdb/rocksdb/microbench/README.md +0 -60
- package/deps/rocksdb/rocksdb/plugin/README.md +0 -43
- package/deps/rocksdb/rocksdb/port/README +0 -10
- package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/README +0 -13
- package/tmp/000099.sst +0 -0
- package/tmp/000102.sst +0 -0
- package/tmp/000103.log +0 -0
- package/tmp/CURRENT +0 -1
- package/tmp/IDENTITY +0 -1
- package/tmp/LOCK +0 -0
- package/tmp/MANIFEST-000104 +0 -0
- package/tmp/OPTIONS-000098 +0 -207
- package/tmp/OPTIONS-000106 +0 -207
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
#include "rocksdb/table_properties.h"
|
|
27
27
|
#include "table/block_based/block_based_table_reader.h"
|
|
28
28
|
#include "table/unique_id_impl.h"
|
|
29
|
+
#include "test_util/secondary_cache_test_util.h"
|
|
29
30
|
#include "util/compression.h"
|
|
30
31
|
#include "util/defer.h"
|
|
31
32
|
#include "util/hash.h"
|
|
@@ -740,118 +741,6 @@ class LookupLiarCache : public CacheWrapper {
|
|
|
740
741
|
|
|
741
742
|
} // anonymous namespace
|
|
742
743
|
|
|
743
|
-
TEST_F(DBBlockCacheTest, AddRedundantStats) {
|
|
744
|
-
const size_t capacity = size_t{1} << 25;
|
|
745
|
-
const int num_shard_bits = 0; // 1 shard
|
|
746
|
-
int iterations_tested = 0;
|
|
747
|
-
for (const std::shared_ptr<Cache>& base_cache :
|
|
748
|
-
{NewLRUCache(capacity, num_shard_bits),
|
|
749
|
-
// FixedHyperClockCache
|
|
750
|
-
HyperClockCacheOptions(
|
|
751
|
-
capacity,
|
|
752
|
-
BlockBasedTableOptions().block_size /*estimated_value_size*/,
|
|
753
|
-
num_shard_bits)
|
|
754
|
-
.MakeSharedCache(),
|
|
755
|
-
// AutoHyperClockCache
|
|
756
|
-
HyperClockCacheOptions(capacity, 0 /*estimated_value_size*/,
|
|
757
|
-
num_shard_bits)
|
|
758
|
-
.MakeSharedCache()}) {
|
|
759
|
-
if (!base_cache) {
|
|
760
|
-
// Skip clock cache when not supported
|
|
761
|
-
continue;
|
|
762
|
-
}
|
|
763
|
-
++iterations_tested;
|
|
764
|
-
Options options = CurrentOptions();
|
|
765
|
-
options.create_if_missing = true;
|
|
766
|
-
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
|
767
|
-
|
|
768
|
-
std::shared_ptr<LookupLiarCache> cache =
|
|
769
|
-
std::make_shared<LookupLiarCache>(base_cache);
|
|
770
|
-
|
|
771
|
-
BlockBasedTableOptions table_options;
|
|
772
|
-
table_options.cache_index_and_filter_blocks = true;
|
|
773
|
-
table_options.block_cache = cache;
|
|
774
|
-
table_options.filter_policy.reset(NewBloomFilterPolicy(50));
|
|
775
|
-
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
776
|
-
DestroyAndReopen(options);
|
|
777
|
-
|
|
778
|
-
// Create a new table.
|
|
779
|
-
ASSERT_OK(Put("foo", "value"));
|
|
780
|
-
ASSERT_OK(Put("bar", "value"));
|
|
781
|
-
ASSERT_OK(Flush());
|
|
782
|
-
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
783
|
-
|
|
784
|
-
// Normal access filter+index+data.
|
|
785
|
-
ASSERT_EQ("value", Get("foo"));
|
|
786
|
-
|
|
787
|
-
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
|
|
788
|
-
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
|
|
789
|
-
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
|
|
790
|
-
// --------
|
|
791
|
-
ASSERT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
|
792
|
-
|
|
793
|
-
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
|
|
794
|
-
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
|
|
795
|
-
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
|
|
796
|
-
// --------
|
|
797
|
-
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
|
|
798
|
-
|
|
799
|
-
// Againt access filter+index+data, but force redundant load+insert on index
|
|
800
|
-
cache->SetNthLookupNotFound(2);
|
|
801
|
-
ASSERT_EQ("value", Get("bar"));
|
|
802
|
-
|
|
803
|
-
ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
|
|
804
|
-
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
|
|
805
|
-
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
|
|
806
|
-
// --------
|
|
807
|
-
ASSERT_EQ(4, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
|
808
|
-
|
|
809
|
-
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
|
|
810
|
-
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
|
|
811
|
-
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
|
|
812
|
-
// --------
|
|
813
|
-
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
|
|
814
|
-
|
|
815
|
-
// Access just filter (with high probability), and force redundant
|
|
816
|
-
// load+insert
|
|
817
|
-
cache->SetNthLookupNotFound(1);
|
|
818
|
-
ASSERT_EQ("NOT_FOUND", Get("this key was not added"));
|
|
819
|
-
|
|
820
|
-
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
|
|
821
|
-
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
|
|
822
|
-
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
|
|
823
|
-
// --------
|
|
824
|
-
EXPECT_EQ(5, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
|
825
|
-
|
|
826
|
-
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
|
|
827
|
-
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
|
|
828
|
-
EXPECT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
|
|
829
|
-
// --------
|
|
830
|
-
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
|
|
831
|
-
|
|
832
|
-
// Access just data, forcing redundant load+insert
|
|
833
|
-
ReadOptions read_options;
|
|
834
|
-
std::unique_ptr<Iterator> iter{db_->NewIterator(read_options)};
|
|
835
|
-
cache->SetNthLookupNotFound(1);
|
|
836
|
-
iter->SeekToFirst();
|
|
837
|
-
ASSERT_TRUE(iter->Valid());
|
|
838
|
-
ASSERT_EQ(iter->key(), "bar");
|
|
839
|
-
|
|
840
|
-
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
|
|
841
|
-
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
|
|
842
|
-
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
|
|
843
|
-
// --------
|
|
844
|
-
EXPECT_EQ(6, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
|
845
|
-
|
|
846
|
-
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
|
|
847
|
-
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
|
|
848
|
-
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
|
|
849
|
-
// --------
|
|
850
|
-
EXPECT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
|
|
851
|
-
}
|
|
852
|
-
EXPECT_GE(iterations_tested, 1);
|
|
853
|
-
}
|
|
854
|
-
|
|
855
744
|
TEST_F(DBBlockCacheTest, ParanoidFileChecks) {
|
|
856
745
|
Options options = CurrentOptions();
|
|
857
746
|
options.create_if_missing = true;
|
|
@@ -1347,6 +1236,274 @@ TEST_F(DBBlockCacheTest, HyperClockCacheReportProblems) {
|
|
|
1347
1236
|
EXPECT_EQ(logger->PopCounts(), (std::array<int, 3>{{0, 1, 0}}));
|
|
1348
1237
|
}
|
|
1349
1238
|
|
|
1239
|
+
class DBBlockCacheTypeTest
|
|
1240
|
+
: public DBBlockCacheTest,
|
|
1241
|
+
public secondary_cache_test_util::WithCacheTypeParam {};
|
|
1242
|
+
|
|
1243
|
+
INSTANTIATE_TEST_CASE_P(DBBlockCacheTypeTestInstance, DBBlockCacheTypeTest,
|
|
1244
|
+
secondary_cache_test_util::GetTestingCacheTypes());
|
|
1245
|
+
|
|
1246
|
+
TEST_P(DBBlockCacheTypeTest, AddRedundantStats) {
|
|
1247
|
+
BlockBasedTableOptions table_options;
|
|
1248
|
+
|
|
1249
|
+
const size_t capacity = size_t{1} << 25;
|
|
1250
|
+
const int num_shard_bits = 0; // 1 shard
|
|
1251
|
+
estimated_value_size_ = table_options.block_size;
|
|
1252
|
+
std::shared_ptr<Cache> base_cache =
|
|
1253
|
+
NewCache(capacity, num_shard_bits, /*strict_capacity_limit=*/false);
|
|
1254
|
+
Options options = CurrentOptions();
|
|
1255
|
+
options.create_if_missing = true;
|
|
1256
|
+
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
|
1257
|
+
|
|
1258
|
+
std::shared_ptr<LookupLiarCache> cache =
|
|
1259
|
+
std::make_shared<LookupLiarCache>(base_cache);
|
|
1260
|
+
|
|
1261
|
+
table_options.cache_index_and_filter_blocks = true;
|
|
1262
|
+
table_options.block_cache = cache;
|
|
1263
|
+
table_options.filter_policy.reset(NewBloomFilterPolicy(50));
|
|
1264
|
+
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
1265
|
+
DestroyAndReopen(options);
|
|
1266
|
+
|
|
1267
|
+
// Create a new table.
|
|
1268
|
+
ASSERT_OK(Put("foo", "value"));
|
|
1269
|
+
ASSERT_OK(Put("bar", "value"));
|
|
1270
|
+
ASSERT_OK(Flush());
|
|
1271
|
+
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
1272
|
+
|
|
1273
|
+
// Normal access filter+index+data.
|
|
1274
|
+
ASSERT_EQ("value", Get("foo"));
|
|
1275
|
+
|
|
1276
|
+
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
|
|
1277
|
+
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
|
|
1278
|
+
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
|
|
1279
|
+
// --------
|
|
1280
|
+
ASSERT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
|
1281
|
+
|
|
1282
|
+
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
|
|
1283
|
+
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
|
|
1284
|
+
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
|
|
1285
|
+
// --------
|
|
1286
|
+
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
|
|
1287
|
+
|
|
1288
|
+
// Againt access filter+index+data, but force redundant load+insert on index
|
|
1289
|
+
cache->SetNthLookupNotFound(2);
|
|
1290
|
+
ASSERT_EQ("value", Get("bar"));
|
|
1291
|
+
|
|
1292
|
+
ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
|
|
1293
|
+
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
|
|
1294
|
+
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
|
|
1295
|
+
// --------
|
|
1296
|
+
ASSERT_EQ(4, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
|
1297
|
+
|
|
1298
|
+
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
|
|
1299
|
+
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
|
|
1300
|
+
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
|
|
1301
|
+
// --------
|
|
1302
|
+
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
|
|
1303
|
+
|
|
1304
|
+
// Access just filter (with high probability), and force redundant
|
|
1305
|
+
// load+insert
|
|
1306
|
+
cache->SetNthLookupNotFound(1);
|
|
1307
|
+
ASSERT_EQ("NOT_FOUND", Get("this key was not added"));
|
|
1308
|
+
|
|
1309
|
+
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
|
|
1310
|
+
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
|
|
1311
|
+
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
|
|
1312
|
+
// --------
|
|
1313
|
+
EXPECT_EQ(5, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
|
1314
|
+
|
|
1315
|
+
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
|
|
1316
|
+
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
|
|
1317
|
+
EXPECT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
|
|
1318
|
+
// --------
|
|
1319
|
+
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
|
|
1320
|
+
|
|
1321
|
+
// Access just data, forcing redundant load+insert
|
|
1322
|
+
ReadOptions read_options;
|
|
1323
|
+
std::unique_ptr<Iterator> iter{db_->NewIterator(read_options)};
|
|
1324
|
+
cache->SetNthLookupNotFound(1);
|
|
1325
|
+
iter->SeekToFirst();
|
|
1326
|
+
ASSERT_TRUE(iter->Valid());
|
|
1327
|
+
ASSERT_EQ(iter->key(), "bar");
|
|
1328
|
+
|
|
1329
|
+
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
|
|
1330
|
+
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
|
|
1331
|
+
EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
|
|
1332
|
+
// --------
|
|
1333
|
+
EXPECT_EQ(6, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
|
1334
|
+
|
|
1335
|
+
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
|
|
1336
|
+
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
|
|
1337
|
+
EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
|
|
1338
|
+
// --------
|
|
1339
|
+
EXPECT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
namespace {
|
|
1343
|
+
std::string AltKey(int i) {
|
|
1344
|
+
char buf[100];
|
|
1345
|
+
snprintf(buf, sizeof(buf), "altkey%06d", i);
|
|
1346
|
+
return std::string(buf);
|
|
1347
|
+
}
|
|
1348
|
+
} // namespace
|
|
1349
|
+
|
|
1350
|
+
TEST_P(DBBlockCacheTypeTest, Uncache) {
|
|
1351
|
+
for (bool partitioned : {false, true}) {
|
|
1352
|
+
SCOPED_TRACE("partitioned=" + std::to_string(partitioned));
|
|
1353
|
+
for (uint32_t ua : {0, 1, 2, 10000}) {
|
|
1354
|
+
SCOPED_TRACE("ua=" + std::to_string(ua));
|
|
1355
|
+
|
|
1356
|
+
BlockBasedTableOptions table_options;
|
|
1357
|
+
Options options = CurrentOptions();
|
|
1358
|
+
options.uncache_aggressiveness = ua;
|
|
1359
|
+
options.create_if_missing = true;
|
|
1360
|
+
// Don't allow background operations to keep Versions referenced
|
|
1361
|
+
options.stats_dump_period_sec = 0;
|
|
1362
|
+
options.stats_persist_period_sec = 0;
|
|
1363
|
+
auto stats = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
|
1364
|
+
options.statistics = stats;
|
|
1365
|
+
|
|
1366
|
+
const size_t capacity = size_t{1} << 25;
|
|
1367
|
+
const int num_shard_bits = 0; // 1 shard
|
|
1368
|
+
estimated_value_size_ = table_options.block_size;
|
|
1369
|
+
std::shared_ptr<Cache> cache =
|
|
1370
|
+
NewCache(capacity, num_shard_bits, /*strict_capacity_limit=*/false);
|
|
1371
|
+
|
|
1372
|
+
table_options.cache_index_and_filter_blocks = true;
|
|
1373
|
+
table_options.block_cache = cache;
|
|
1374
|
+
table_options.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
1375
|
+
table_options.partition_filters = partitioned;
|
|
1376
|
+
table_options.index_type =
|
|
1377
|
+
partitioned ? BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch
|
|
1378
|
+
: BlockBasedTableOptions::IndexType::kBinarySearch;
|
|
1379
|
+
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
1380
|
+
DestroyAndReopen(options);
|
|
1381
|
+
|
|
1382
|
+
size_t kBaselineCount = 1; // Because of entry stats collector
|
|
1383
|
+
|
|
1384
|
+
ASSERT_EQ(kBaselineCount, cache->GetOccupancyCount());
|
|
1385
|
+
ASSERT_EQ(0U, cache->GetUsage());
|
|
1386
|
+
|
|
1387
|
+
constexpr uint8_t kNumDataBlocks = 10;
|
|
1388
|
+
constexpr uint8_t kNumFiles = 3;
|
|
1389
|
+
for (int i = 0; i < kNumDataBlocks; i++) {
|
|
1390
|
+
// Force some overlap with ordering
|
|
1391
|
+
ASSERT_OK(Put(Key((i * 7) % kNumDataBlocks),
|
|
1392
|
+
Random::GetTLSInstance()->RandomBinaryString(
|
|
1393
|
+
static_cast<int>(table_options.block_size))));
|
|
1394
|
+
if (i >= kNumDataBlocks - kNumFiles) {
|
|
1395
|
+
ASSERT_OK(Flush());
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
ASSERT_EQ(int{kNumFiles}, NumTableFilesAtLevel(0));
|
|
1399
|
+
|
|
1400
|
+
for (int i = 0; i < kNumDataBlocks; i++) {
|
|
1401
|
+
ASSERT_NE(Get(Key(i)), "NOT_FOUND");
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
size_t meta_blocks_per_file = /*index & filter*/ 2U * (1U + partitioned);
|
|
1405
|
+
ASSERT_EQ(
|
|
1406
|
+
cache->GetOccupancyCount(),
|
|
1407
|
+
kBaselineCount + kNumDataBlocks + meta_blocks_per_file * kNumFiles);
|
|
1408
|
+
ASSERT_GE(cache->GetUsage(), kNumDataBlocks * table_options.block_size);
|
|
1409
|
+
|
|
1410
|
+
// Combine into one file, making the originals obsolete
|
|
1411
|
+
ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
|
|
1412
|
+
ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
|
|
1413
|
+
|
|
1414
|
+
ASSERT_EQ(1, NumTableFilesAtLevel(1));
|
|
1415
|
+
|
|
1416
|
+
for (int i = 0; i < kNumDataBlocks; i++) {
|
|
1417
|
+
ASSERT_NE(Get(Key(i)), "NOT_FOUND");
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
if (ua == 0) {
|
|
1421
|
+
// Expect to see cache entries for new file and obsolete files
|
|
1422
|
+
EXPECT_EQ(cache->GetOccupancyCount(),
|
|
1423
|
+
kBaselineCount + kNumDataBlocks * 2U +
|
|
1424
|
+
meta_blocks_per_file * (kNumFiles + 1));
|
|
1425
|
+
EXPECT_GE(cache->GetUsage(),
|
|
1426
|
+
kNumDataBlocks * table_options.block_size * 2U);
|
|
1427
|
+
} else {
|
|
1428
|
+
// Expect only to see cache entries for new file
|
|
1429
|
+
EXPECT_EQ(cache->GetOccupancyCount(),
|
|
1430
|
+
kBaselineCount + kNumDataBlocks + meta_blocks_per_file);
|
|
1431
|
+
EXPECT_GE(cache->GetUsage(), kNumDataBlocks * table_options.block_size);
|
|
1432
|
+
EXPECT_LT(cache->GetUsage(),
|
|
1433
|
+
kNumDataBlocks * table_options.block_size * 2U);
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
size_t alt_baseline_count = cache->GetOccupancyCount();
|
|
1437
|
+
size_t alt_baseline_usage = cache->GetUsage();
|
|
1438
|
+
ASSERT_OK(stats->Reset());
|
|
1439
|
+
// We aren't generally cleaning up cache entries on DB::Close, especially
|
|
1440
|
+
// because someone might just re-open the same DB.
|
|
1441
|
+
Reopen(options);
|
|
1442
|
+
for (int i = 0; i < kNumDataBlocks; i++) {
|
|
1443
|
+
ASSERT_NE(Get(Key(i)), "NOT_FOUND");
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
EXPECT_EQ(cache->GetOccupancyCount(), alt_baseline_count);
|
|
1447
|
+
EXPECT_EQ(cache->GetUsage(), alt_baseline_usage);
|
|
1448
|
+
|
|
1449
|
+
// Check for unnecessary unncessary cache churn
|
|
1450
|
+
ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_ADD), 0U);
|
|
1451
|
+
ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_MISS), 0U);
|
|
1452
|
+
ASSERT_GT(stats->getTickerCount(BLOCK_CACHE_HIT), 0U);
|
|
1453
|
+
|
|
1454
|
+
// And now do a similar test as above except with trivial moves, making
|
|
1455
|
+
// sure that we aren't falsely uncaching in that case, which would cause
|
|
1456
|
+
// unnecessary cache misses. Using AltKey instead of Key to avoid
|
|
1457
|
+
// interference.
|
|
1458
|
+
for (int i = 0; i < kNumDataBlocks; i++) {
|
|
1459
|
+
// No overlap
|
|
1460
|
+
ASSERT_OK(
|
|
1461
|
+
Put(AltKey(i), Random::GetTLSInstance()->RandomBinaryString(
|
|
1462
|
+
static_cast<int>(table_options.block_size))));
|
|
1463
|
+
if (i >= kNumDataBlocks - kNumFiles) {
|
|
1464
|
+
ASSERT_OK(Flush());
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
ASSERT_EQ(int{kNumFiles}, NumTableFilesAtLevel(0));
|
|
1468
|
+
|
|
1469
|
+
for (int i = 0; i < kNumDataBlocks; i++) {
|
|
1470
|
+
ASSERT_NE(Get(AltKey(i)), "NOT_FOUND");
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
ASSERT_EQ(cache->GetOccupancyCount(),
|
|
1474
|
+
alt_baseline_count + kNumDataBlocks +
|
|
1475
|
+
meta_blocks_per_file * kNumFiles);
|
|
1476
|
+
ASSERT_GE(cache->GetUsage(),
|
|
1477
|
+
alt_baseline_usage + kNumDataBlocks * table_options.block_size);
|
|
1478
|
+
|
|
1479
|
+
ASSERT_OK(stats->Reset());
|
|
1480
|
+
|
|
1481
|
+
// Make trivial move
|
|
1482
|
+
{
|
|
1483
|
+
auto a = AltKey(0);
|
|
1484
|
+
auto b = AltKey(kNumDataBlocks);
|
|
1485
|
+
Slice slice_a{a};
|
|
1486
|
+
Slice slice_b{b};
|
|
1487
|
+
ASSERT_OK(db_->CompactRange({}, &slice_a, &slice_b));
|
|
1488
|
+
}
|
|
1489
|
+
ASSERT_EQ(/*old*/ 1 + /*new*/ int{kNumFiles}, NumTableFilesAtLevel(1));
|
|
1490
|
+
|
|
1491
|
+
for (int i = 0; i < kNumDataBlocks; i++) {
|
|
1492
|
+
ASSERT_NE(Get(AltKey(i)), "NOT_FOUND");
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
// Should be the same if trivial move
|
|
1496
|
+
ASSERT_EQ(cache->GetOccupancyCount(),
|
|
1497
|
+
alt_baseline_count + kNumDataBlocks +
|
|
1498
|
+
meta_blocks_per_file * kNumFiles);
|
|
1499
|
+
|
|
1500
|
+
// Check for unnecessary unncessary cache churn
|
|
1501
|
+
ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_ADD), 0U);
|
|
1502
|
+
ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_MISS), 0U);
|
|
1503
|
+
ASSERT_GT(stats->getTickerCount(BLOCK_CACHE_HIT), 0U);
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1350
1507
|
|
|
1351
1508
|
class DBBlockCacheKeyTest
|
|
1352
1509
|
: public DBTestBase,
|
|
@@ -322,6 +322,67 @@ TEST_F(DBBloomFilterTest, GetFilterByPrefixBloom) {
|
|
|
322
322
|
}
|
|
323
323
|
}
|
|
324
324
|
|
|
325
|
+
TEST_F(DBBloomFilterTest, FilterNumEntriesCoalesce) {
|
|
326
|
+
for (bool partition_filters : {true, false}) {
|
|
327
|
+
SCOPED_TRACE("partition_filters=" + std::to_string(partition_filters));
|
|
328
|
+
for (bool prefix : {true, false}) {
|
|
329
|
+
SCOPED_TRACE("prefix=" + std::to_string(prefix));
|
|
330
|
+
for (bool whole : {true, false}) {
|
|
331
|
+
SCOPED_TRACE("whole=" + std::to_string(whole));
|
|
332
|
+
Options options = last_options_;
|
|
333
|
+
options.prefix_extractor.reset();
|
|
334
|
+
if (prefix) {
|
|
335
|
+
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
|
|
336
|
+
}
|
|
337
|
+
BlockBasedTableOptions bbto;
|
|
338
|
+
bbto.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
339
|
+
bbto.whole_key_filtering = whole;
|
|
340
|
+
if (partition_filters) {
|
|
341
|
+
bbto.partition_filters = true;
|
|
342
|
+
bbto.index_type =
|
|
343
|
+
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
|
344
|
+
}
|
|
345
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
346
|
+
DestroyAndReopen(options);
|
|
347
|
+
|
|
348
|
+
// Need a snapshot to allow keeping multiple entries for the same key
|
|
349
|
+
std::vector<const Snapshot*> snapshots;
|
|
350
|
+
for (int i = 1; i <= 3; ++i) {
|
|
351
|
+
std::string val = "val" + std::to_string(i);
|
|
352
|
+
ASSERT_OK(Put("foo1", val));
|
|
353
|
+
ASSERT_OK(Put("foo2", val));
|
|
354
|
+
ASSERT_OK(Put("bar1", val));
|
|
355
|
+
ASSERT_OK(Put("bar2", val));
|
|
356
|
+
ASSERT_OK(Put("bar3", val));
|
|
357
|
+
snapshots.push_back(db_->GetSnapshot());
|
|
358
|
+
}
|
|
359
|
+
ASSERT_OK(Flush());
|
|
360
|
+
|
|
361
|
+
TablePropertiesCollection tpc;
|
|
362
|
+
ASSERT_OK(db_->GetPropertiesOfAllTables(&tpc));
|
|
363
|
+
// sanity checks
|
|
364
|
+
ASSERT_EQ(tpc.size(), 1U);
|
|
365
|
+
auto& tp = *tpc.begin()->second;
|
|
366
|
+
EXPECT_EQ(tp.num_entries, 3U * 5U);
|
|
367
|
+
|
|
368
|
+
// test checks
|
|
369
|
+
unsigned ex_filter_entries = 0;
|
|
370
|
+
if (whole) {
|
|
371
|
+
ex_filter_entries += 5; // unique keys
|
|
372
|
+
}
|
|
373
|
+
if (prefix) {
|
|
374
|
+
ex_filter_entries += 2; // unique prefixes
|
|
375
|
+
}
|
|
376
|
+
EXPECT_EQ(tp.num_filter_entries, ex_filter_entries);
|
|
377
|
+
|
|
378
|
+
for (auto* sn : snapshots) {
|
|
379
|
+
db_->ReleaseSnapshot(sn);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
325
386
|
TEST_F(DBBloomFilterTest, WholeKeyFilterProp) {
|
|
326
387
|
for (bool partition_filters : {true, false}) {
|
|
327
388
|
Options options = last_options_;
|
|
@@ -3143,52 +3204,55 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) {
|
|
|
3143
3204
|
}
|
|
3144
3205
|
|
|
3145
3206
|
TEST_F(DBBloomFilterTest, SeekForPrevWithPartitionedFilters) {
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3207
|
+
for (bool wkf : {true, false}) {
|
|
3208
|
+
SCOPED_TRACE("whole_key_filtering=" + std::to_string(wkf));
|
|
3209
|
+
Options options = CurrentOptions();
|
|
3210
|
+
constexpr size_t kNumKeys = 10000;
|
|
3211
|
+
static_assert(kNumKeys <= 10000, "kNumKeys have to be <= 10000");
|
|
3212
|
+
options.memtable_factory.reset(
|
|
3213
|
+
test::NewSpecialSkipListFactory(kNumKeys + 10));
|
|
3214
|
+
options.create_if_missing = true;
|
|
3215
|
+
constexpr size_t kPrefixLength = 4;
|
|
3216
|
+
options.prefix_extractor.reset(NewFixedPrefixTransform(kPrefixLength));
|
|
3217
|
+
options.compression = kNoCompression;
|
|
3218
|
+
BlockBasedTableOptions bbto;
|
|
3219
|
+
bbto.filter_policy.reset(NewBloomFilterPolicy(50));
|
|
3220
|
+
bbto.index_shortening =
|
|
3221
|
+
BlockBasedTableOptions::IndexShorteningMode::kNoShortening;
|
|
3222
|
+
bbto.block_size = 128;
|
|
3223
|
+
bbto.metadata_block_size = 128;
|
|
3224
|
+
bbto.partition_filters = true;
|
|
3225
|
+
bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
|
3226
|
+
bbto.whole_key_filtering = wkf;
|
|
3227
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
3228
|
+
DestroyAndReopen(options);
|
|
3165
3229
|
|
|
3166
|
-
|
|
3230
|
+
const std::string value(64, '\0');
|
|
3167
3231
|
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3232
|
+
WriteOptions write_opts;
|
|
3233
|
+
write_opts.disableWAL = true;
|
|
3234
|
+
for (size_t i = 0; i < kNumKeys; ++i) {
|
|
3235
|
+
std::ostringstream oss;
|
|
3236
|
+
oss << std::setfill('0') << std::setw(4) << std::fixed << i;
|
|
3237
|
+
ASSERT_OK(db_->Put(write_opts, oss.str(), value));
|
|
3238
|
+
}
|
|
3239
|
+
ASSERT_OK(Flush());
|
|
3176
3240
|
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3241
|
+
ReadOptions read_opts;
|
|
3242
|
+
// Use legacy, implicit prefix seek
|
|
3243
|
+
read_opts.total_order_seek = false;
|
|
3244
|
+
read_opts.auto_prefix_mode = false;
|
|
3245
|
+
std::unique_ptr<Iterator> it(db_->NewIterator(read_opts));
|
|
3246
|
+
for (size_t i = 0; i < kNumKeys; ++i) {
|
|
3247
|
+
// Seek with a key after each one added but with same prefix. One will
|
|
3248
|
+
// surely cross a partition boundary.
|
|
3249
|
+
std::ostringstream oss;
|
|
3250
|
+
oss << std::setfill('0') << std::setw(4) << std::fixed << i << "a";
|
|
3251
|
+
it->SeekForPrev(oss.str());
|
|
3252
|
+
ASSERT_OK(it->status());
|
|
3253
|
+
ASSERT_TRUE(it->Valid());
|
|
3254
|
+
}
|
|
3255
|
+
}
|
|
3192
3256
|
}
|
|
3193
3257
|
|
|
3194
3258
|
namespace {
|