@nxtedition/rocksdb 11.0.2 → 11.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/binding.cc +133 -122
  2. package/deps/rocksdb/rocksdb/db/column_family_test.cc +15 -7
  3. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +4 -2
  4. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +8 -4
  5. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +11 -7
  6. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +17 -11
  7. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +15 -0
  8. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +155 -0
  9. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +564 -461
  10. package/deps/rocksdb/rocksdb/db/db_follower_test.cc +8 -4
  11. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +40 -24
  12. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +8 -1
  13. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +7 -4
  14. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +5 -0
  15. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +3 -1
  16. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +19 -1
  17. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +20 -16
  18. package/deps/rocksdb/rocksdb/db/db_io_failure_test.cc +27 -0
  19. package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +10 -2
  20. package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +85 -0
  21. package/deps/rocksdb/rocksdb/db/db_sst_test.cc +55 -2
  22. package/deps/rocksdb/rocksdb/db/db_test2.cc +231 -0
  23. package/deps/rocksdb/rocksdb/db/db_test_util.cc +5 -0
  24. package/deps/rocksdb/rocksdb/db/db_test_util.h +10 -1
  25. package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +0 -1
  26. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +175 -1
  27. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +64 -0
  28. package/deps/rocksdb/rocksdb/db/dbformat.h +5 -6
  29. package/deps/rocksdb/rocksdb/db/dbformat_test.cc +8 -8
  30. package/deps/rocksdb/rocksdb/db/experimental.cc +3 -2
  31. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +2 -4
  32. package/deps/rocksdb/rocksdb/db/flush_job.cc +7 -2
  33. package/deps/rocksdb/rocksdb/db/flush_job_test.cc +4 -2
  34. package/deps/rocksdb/rocksdb/db/listener_test.cc +5 -5
  35. package/deps/rocksdb/rocksdb/db/log_writer.cc +12 -3
  36. package/deps/rocksdb/rocksdb/db/memtable.cc +83 -23
  37. package/deps/rocksdb/rocksdb/db/memtable.h +11 -3
  38. package/deps/rocksdb/rocksdb/db/memtable_list.cc +7 -5
  39. package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +21 -0
  40. package/deps/rocksdb/rocksdb/db/version_builder.cc +462 -33
  41. package/deps/rocksdb/rocksdb/db/version_builder.h +70 -23
  42. package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +95 -207
  43. package/deps/rocksdb/rocksdb/db/version_edit_handler.h +54 -35
  44. package/deps/rocksdb/rocksdb/db/version_set.cc +13 -11
  45. package/deps/rocksdb/rocksdb/db/version_set_test.cc +313 -59
  46. package/deps/rocksdb/rocksdb/db/write_batch.cc +124 -64
  47. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +2 -3
  48. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compaction_filter.h +1 -1
  49. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +4 -1
  50. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +9 -0
  51. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +4 -32
  52. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +7 -3
  53. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +60 -172
  54. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +57 -2
  55. package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +23 -15
  56. package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.h +2 -3
  57. package/deps/rocksdb/rocksdb/db_stress_tool/expected_value.cc +1 -1
  58. package/deps/rocksdb/rocksdb/db_stress_tool/expected_value.h +4 -1
  59. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +200 -92
  60. package/deps/rocksdb/rocksdb/env/file_system.cc +3 -3
  61. package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +124 -23
  62. package/deps/rocksdb/rocksdb/file/delete_scheduler.h +61 -8
  63. package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +141 -2
  64. package/deps/rocksdb/rocksdb/file/file_util.cc +17 -2
  65. package/deps/rocksdb/rocksdb/file/file_util.h +10 -0
  66. package/deps/rocksdb/rocksdb/file/filename.cc +11 -3
  67. package/deps/rocksdb/rocksdb/file/filename.h +2 -1
  68. package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.cc +18 -0
  69. package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.h +27 -4
  70. package/deps/rocksdb/rocksdb/file/writable_file_writer.h +8 -1
  71. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +8 -13
  72. package/deps/rocksdb/rocksdb/include/rocksdb/env.h +4 -0
  73. package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +5 -0
  74. package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +5 -2
  75. package/deps/rocksdb/rocksdb/include/rocksdb/filter_policy.h +2 -1
  76. package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +34 -0
  77. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +25 -1
  78. package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +5 -0
  79. package/deps/rocksdb/rocksdb/include/rocksdb/table.h +27 -9
  80. package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +2 -0
  81. package/deps/rocksdb/rocksdb/include/rocksdb/types.h +12 -0
  82. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +21 -0
  83. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
  84. package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +29 -1
  85. package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +102 -33
  86. package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +46 -3
  87. package/deps/rocksdb/rocksdb/monitoring/statistics.cc +4 -0
  88. package/deps/rocksdb/rocksdb/options/cf_options.cc +6 -0
  89. package/deps/rocksdb/rocksdb/options/cf_options.h +2 -0
  90. package/deps/rocksdb/rocksdb/options/db_options.cc +15 -1
  91. package/deps/rocksdb/rocksdb/options/db_options.h +2 -0
  92. package/deps/rocksdb/rocksdb/options/options_helper.cc +10 -0
  93. package/deps/rocksdb/rocksdb/options/options_parser.cc +3 -2
  94. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +9 -2
  95. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +75 -35
  96. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +6 -0
  97. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +4 -0
  98. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +8 -1
  99. package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +40 -15
  100. package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +98 -17
  101. package/deps/rocksdb/rocksdb/table/block_based/filter_policy_internal.h +14 -2
  102. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.cc +21 -91
  103. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.h +13 -21
  104. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block_test.cc +14 -5
  105. package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +62 -53
  106. package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +60 -38
  107. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +175 -78
  108. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.h +65 -36
  109. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +25 -15
  110. package/deps/rocksdb/rocksdb/table/block_fetcher.cc +13 -1
  111. package/deps/rocksdb/rocksdb/table/meta_blocks.cc +18 -4
  112. package/deps/rocksdb/rocksdb/table/meta_blocks.h +4 -0
  113. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +11 -0
  114. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +2 -2
  115. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +47 -18
  116. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.h +1 -2
  117. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +95 -0
  118. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +26 -15
  119. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +62 -19
  120. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +73 -34
  121. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +5 -0
  122. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +10 -3
  123. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.h +2 -1
  124. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.cc +8 -5
  125. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.h +7 -4
  126. package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +225 -0
  127. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.cc +2 -1
  128. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.h +17 -0
  129. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +5 -2
  130. package/index.js +5 -17
  131. package/iterator.js +9 -1
  132. package/package.json +1 -1
  133. package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
  134. package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
@@ -45,6 +45,12 @@ const std::string kStandard128Ribbon =
45
45
  const std::string kAutoBloom = BloomFilterPolicy::kClassName();
46
46
  const std::string kAutoRibbon = RibbonFilterPolicy::kClassName();
47
47
 
48
+ enum class FilterPartitioning {
49
+ kUnpartitionedFilter,
50
+ kCoupledPartitionedFilter,
51
+ kDecoupledPartitionedFilter,
52
+ };
53
+
48
54
  template <typename T>
49
55
  T Pop(T& var) {
50
56
  auto rv = var;
@@ -62,32 +68,61 @@ class DBBloomFilterTest : public DBTestBase {
62
68
  public:
63
69
  DBBloomFilterTest()
64
70
  : DBTestBase("db_bloom_filter_test", /*env_do_fsync=*/true) {}
71
+
72
+ bool PartitionFilters() {
73
+ return filter_partitioning_ != FilterPartitioning::kUnpartitionedFilter;
74
+ }
75
+
76
+ void SetInTableOptions(BlockBasedTableOptions* table_options) {
77
+ table_options->partition_filters = PartitionFilters();
78
+ if (PartitionFilters()) {
79
+ table_options->index_type =
80
+ BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
81
+ }
82
+ }
83
+
84
+ protected:
85
+ FilterPartitioning filter_partitioning_ =
86
+ FilterPartitioning::kUnpartitionedFilter;
65
87
  };
66
88
 
67
- class DBBloomFilterTestWithParam
68
- : public DBTestBase,
89
+ class DBBloomFilterTestWithPartitioningParam
90
+ : public DBBloomFilterTest,
91
+ public testing::WithParamInterface<FilterPartitioning> {
92
+ public:
93
+ ~DBBloomFilterTestWithPartitioningParam() override = default;
94
+
95
+ void SetUp() override { filter_partitioning_ = GetParam(); }
96
+ };
97
+
98
+ class DBBloomFilterTestWithFormatParams
99
+ : public DBBloomFilterTest,
69
100
  public testing::WithParamInterface<
70
- std::tuple<std::string, bool, uint32_t>> {
71
- // public testing::WithParamInterface<bool> {
101
+ std::tuple<std::string, FilterPartitioning, uint32_t>> {
72
102
  protected:
73
103
  std::string bfp_impl_;
74
- bool partition_filters_;
104
+ double bits_per_key_;
75
105
  uint32_t format_version_;
76
106
 
77
107
  public:
78
- DBBloomFilterTestWithParam()
79
- : DBTestBase("db_bloom_filter_tests", /*env_do_fsync=*/true) {}
80
-
81
- ~DBBloomFilterTestWithParam() override = default;
108
+ ~DBBloomFilterTestWithFormatParams() override = default;
82
109
 
83
110
  void SetUp() override {
84
111
  bfp_impl_ = std::get<0>(GetParam());
85
- partition_filters_ = std::get<1>(GetParam());
112
+ bits_per_key_ = 10; // default;
113
+ filter_partitioning_ = std::get<1>(GetParam());
86
114
  format_version_ = std::get<2>(GetParam());
87
115
  }
116
+
117
+ void SetInTableOptions(BlockBasedTableOptions* table_options) {
118
+ DBBloomFilterTest::SetInTableOptions(table_options);
119
+ table_options->filter_policy = Create(bits_per_key_, bfp_impl_);
120
+ table_options->format_version = format_version_;
121
+ }
88
122
  };
89
123
 
90
- class DBBloomFilterTestDefFormatVersion : public DBBloomFilterTestWithParam {};
124
+ class DBBloomFilterTestDefFormatVersion
125
+ : public DBBloomFilterTestWithFormatParams {};
91
126
 
92
127
  class SliceTransformLimitedDomainGeneric : public SliceTransform {
93
128
  const char* Name() const override {
@@ -118,11 +153,11 @@ TEST_P(DBBloomFilterTestDefFormatVersion, KeyMayExist) {
118
153
  std::string value;
119
154
  anon::OptionsOverride options_override;
120
155
  options_override.filter_policy = Create(20, bfp_impl_);
121
- options_override.partition_filters = partition_filters_;
156
+ options_override.partition_filters = PartitionFilters();
122
157
  options_override.metadata_block_size = 32;
123
158
  options_override.full_block_cache = true;
124
159
  Options options = CurrentOptions(options_override);
125
- if (partition_filters_) {
160
+ if (PartitionFilters()) {
126
161
  auto* table_options =
127
162
  options.table_factory->GetOptions<BlockBasedTableOptions>();
128
163
  if (table_options != nullptr &&
@@ -190,397 +225,383 @@ TEST_P(DBBloomFilterTestDefFormatVersion, KeyMayExist) {
190
225
  ChangeOptions(kSkipPlainTable | kSkipHashIndex | kSkipFIFOCompaction));
191
226
  }
192
227
 
193
- TEST_F(DBBloomFilterTest, GetFilterByPrefixBloomCustomPrefixExtractor) {
194
- for (bool partition_filters : {true, false}) {
195
- Options options = last_options_;
196
- options.prefix_extractor =
197
- std::make_shared<SliceTransformLimitedDomainGeneric>();
198
- options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
199
- get_perf_context()->EnablePerLevelPerfContext();
200
- BlockBasedTableOptions bbto;
201
- bbto.filter_policy.reset(NewBloomFilterPolicy(10));
202
- if (partition_filters) {
203
- bbto.partition_filters = true;
204
- bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
205
- }
206
- bbto.whole_key_filtering = false;
207
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
208
- DestroyAndReopen(options);
228
+ TEST_P(DBBloomFilterTestWithPartitioningParam,
229
+ GetFilterByPrefixBloomCustomPrefixExtractor) {
230
+ Options options = last_options_;
231
+ options.prefix_extractor =
232
+ std::make_shared<SliceTransformLimitedDomainGeneric>();
233
+ options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
234
+ get_perf_context()->EnablePerLevelPerfContext();
235
+ BlockBasedTableOptions bbto;
236
+ SetInTableOptions(&bbto);
237
+ bbto.filter_policy.reset(NewBloomFilterPolicy(10));
238
+ bbto.whole_key_filtering = false;
239
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
240
+ DestroyAndReopen(options);
209
241
 
210
- WriteOptions wo;
211
- ReadOptions ro;
212
- FlushOptions fo;
213
- fo.wait = true;
214
- std::string value;
242
+ WriteOptions wo;
243
+ ReadOptions ro;
244
+ FlushOptions fo;
245
+ fo.wait = true;
246
+ std::string value;
215
247
 
216
- ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
217
- ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
218
- ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
248
+ ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
249
+ ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
250
+ ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
219
251
 
220
- ASSERT_OK(dbfull()->Flush(fo));
252
+ ASSERT_OK(dbfull()->Flush(fo));
221
253
 
222
- ASSERT_EQ("foo", Get("barbarbar"));
223
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
224
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
254
+ ASSERT_EQ("foo", Get("barbarbar"));
255
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
256
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
225
257
 
226
- ASSERT_EQ("foo2", Get("barbarbar2"));
227
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
228
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
258
+ ASSERT_EQ("foo2", Get("barbarbar2"));
259
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
260
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
229
261
 
230
- ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
231
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
232
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
262
+ ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
263
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
264
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
233
265
 
234
- ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
235
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
236
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
266
+ ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
267
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
268
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
237
269
 
238
- ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
239
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
240
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
270
+ ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
271
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
272
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
241
273
 
242
- ro.total_order_seek = true;
243
- // NOTE: total_order_seek no longer affects Get()
244
- ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
245
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
246
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
274
+ ro.total_order_seek = true;
275
+ // NOTE: total_order_seek no longer affects Get()
276
+ ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
277
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
278
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
247
279
 
248
- // No bloom on extractor changed
249
- ASSERT_OK(db_->SetOptions({{"prefix_extractor", "capped:10"}}));
250
- ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
251
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
252
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
280
+ // No bloom on extractor changed
281
+ ASSERT_OK(db_->SetOptions({{"prefix_extractor", "capped:10"}}));
282
+ ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
283
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
284
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
253
285
 
254
- // No bloom on extractor changed, after re-open
255
- options.prefix_extractor.reset(NewCappedPrefixTransform(10));
256
- Reopen(options);
257
- ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
258
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
259
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
286
+ // No bloom on extractor changed, after re-open
287
+ options.prefix_extractor.reset(NewCappedPrefixTransform(10));
288
+ Reopen(options);
289
+ ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
290
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
291
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
260
292
 
261
- get_perf_context()->Reset();
262
- }
293
+ get_perf_context()->Reset();
263
294
  }
264
295
 
265
- TEST_F(DBBloomFilterTest, GetFilterByPrefixBloom) {
266
- for (bool partition_filters : {true, false}) {
267
- Options options = last_options_;
268
- options.prefix_extractor.reset(NewFixedPrefixTransform(8));
269
- options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
270
- get_perf_context()->EnablePerLevelPerfContext();
271
- BlockBasedTableOptions bbto;
272
- bbto.filter_policy.reset(NewBloomFilterPolicy(10));
273
- if (partition_filters) {
274
- bbto.partition_filters = true;
275
- bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
276
- }
277
- bbto.whole_key_filtering = false;
278
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
279
- DestroyAndReopen(options);
280
-
281
- WriteOptions wo;
282
- ReadOptions ro;
283
- FlushOptions fo;
284
- fo.wait = true;
285
- std::string value;
286
-
287
- ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
288
- ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
289
- ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
290
-
291
- ASSERT_OK(dbfull()->Flush(fo));
292
-
293
- ASSERT_EQ("foo", Get("barbarbar"));
294
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
295
- ASSERT_EQ("foo2", Get("barbarbar2"));
296
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
297
- ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
298
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
299
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
300
-
301
- ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
302
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
303
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
304
-
305
- ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
306
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
307
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
308
-
309
- ro.total_order_seek = true;
310
- // NOTE: total_order_seek no longer affects Get()
311
- ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
312
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
313
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
296
+ TEST_P(DBBloomFilterTestWithPartitioningParam, GetFilterByPrefixBloom) {
297
+ Options options = last_options_;
298
+ options.prefix_extractor.reset(NewFixedPrefixTransform(8));
299
+ options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
300
+ get_perf_context()->EnablePerLevelPerfContext();
301
+ BlockBasedTableOptions bbto;
302
+ SetInTableOptions(&bbto);
303
+ bbto.filter_policy.reset(NewBloomFilterPolicy(10));
304
+ bbto.whole_key_filtering = false;
305
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
306
+ DestroyAndReopen(options);
314
307
 
315
- // No bloom on extractor changed
316
- ASSERT_OK(db_->SetOptions({{"prefix_extractor", "capped:10"}}));
317
- ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
318
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
319
- EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
308
+ WriteOptions wo;
309
+ ReadOptions ro;
310
+ FlushOptions fo;
311
+ fo.wait = true;
312
+ std::string value;
313
+
314
+ ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
315
+ ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
316
+ ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
317
+
318
+ ASSERT_OK(dbfull()->Flush(fo));
319
+
320
+ ASSERT_EQ("foo", Get("barbarbar"));
321
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
322
+ ASSERT_EQ("foo2", Get("barbarbar2"));
323
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
324
+ ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
325
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
326
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
327
+
328
+ ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
329
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
330
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
331
+
332
+ ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
333
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
334
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
335
+
336
+ ro.total_order_seek = true;
337
+ // NOTE: total_order_seek no longer affects Get()
338
+ ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
339
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
340
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
341
+
342
+ // No bloom on extractor changed
343
+ ASSERT_OK(db_->SetOptions({{"prefix_extractor", "capped:10"}}));
344
+ ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
345
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
346
+ EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
320
347
 
321
- get_perf_context()->Reset();
322
- }
348
+ get_perf_context()->Reset();
323
349
  }
324
350
 
325
- TEST_F(DBBloomFilterTest, FilterNumEntriesCoalesce) {
326
- for (bool partition_filters : {true, false}) {
327
- SCOPED_TRACE("partition_filters=" + std::to_string(partition_filters));
328
- for (bool prefix : {true, false}) {
329
- SCOPED_TRACE("prefix=" + std::to_string(prefix));
330
- for (bool whole : {true, false}) {
331
- SCOPED_TRACE("whole=" + std::to_string(whole));
332
- Options options = last_options_;
333
- options.prefix_extractor.reset();
334
- if (prefix) {
335
- options.prefix_extractor.reset(NewFixedPrefixTransform(3));
336
- }
337
- BlockBasedTableOptions bbto;
338
- bbto.filter_policy.reset(NewBloomFilterPolicy(10));
339
- bbto.whole_key_filtering = whole;
340
- if (partition_filters) {
341
- bbto.partition_filters = true;
342
- bbto.index_type =
343
- BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
344
- }
345
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
346
- DestroyAndReopen(options);
347
-
348
- // Need a snapshot to allow keeping multiple entries for the same key
349
- std::vector<const Snapshot*> snapshots;
350
- for (int i = 1; i <= 3; ++i) {
351
- std::string val = "val" + std::to_string(i);
352
- ASSERT_OK(Put("foo1", val));
353
- ASSERT_OK(Put("foo2", val));
354
- ASSERT_OK(Put("bar1", val));
355
- ASSERT_OK(Put("bar2", val));
356
- ASSERT_OK(Put("bar3", val));
357
- snapshots.push_back(db_->GetSnapshot());
358
- }
359
- ASSERT_OK(Flush());
351
+ TEST_P(DBBloomFilterTestWithPartitioningParam, FilterNumEntriesCoalesce) {
352
+ for (bool prefix : {true, false}) {
353
+ SCOPED_TRACE("prefix=" + std::to_string(prefix));
354
+ for (bool whole : {true, false}) {
355
+ SCOPED_TRACE("whole=" + std::to_string(whole));
356
+ Options options = last_options_;
357
+ options.prefix_extractor.reset();
358
+ if (prefix) {
359
+ options.prefix_extractor.reset(NewFixedPrefixTransform(3));
360
+ }
361
+ BlockBasedTableOptions bbto;
362
+ SetInTableOptions(&bbto);
363
+ bbto.filter_policy.reset(NewBloomFilterPolicy(10));
364
+ bbto.whole_key_filtering = whole;
365
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
366
+ DestroyAndReopen(options);
367
+
368
+ // Need a snapshot to allow keeping multiple entries for the same key
369
+ std::vector<const Snapshot*> snapshots;
370
+ for (int i = 1; i <= 3; ++i) {
371
+ std::string val = "val" + std::to_string(i);
372
+ ASSERT_OK(Put("foo1", val));
373
+ ASSERT_OK(Put("foo2", val));
374
+ ASSERT_OK(Put("bar1", val));
375
+ ASSERT_OK(Put("bar2", val));
376
+ ASSERT_OK(Put("bar3", val));
377
+ snapshots.push_back(db_->GetSnapshot());
378
+ }
379
+ ASSERT_OK(Flush());
360
380
 
361
- TablePropertiesCollection tpc;
362
- ASSERT_OK(db_->GetPropertiesOfAllTables(&tpc));
363
- // sanity checks
364
- ASSERT_EQ(tpc.size(), 1U);
365
- auto& tp = *tpc.begin()->second;
366
- EXPECT_EQ(tp.num_entries, 3U * 5U);
367
-
368
- // test checks
369
- unsigned ex_filter_entries = 0;
370
- if (whole) {
371
- ex_filter_entries += 5; // unique keys
372
- }
373
- if (prefix) {
374
- ex_filter_entries += 2; // unique prefixes
375
- }
376
- EXPECT_EQ(tp.num_filter_entries, ex_filter_entries);
381
+ TablePropertiesCollection tpc;
382
+ ASSERT_OK(db_->GetPropertiesOfAllTables(&tpc));
383
+ // sanity checks
384
+ ASSERT_EQ(tpc.size(), 1U);
385
+ auto& tp = *tpc.begin()->second;
386
+ EXPECT_EQ(tp.num_entries, 3U * 5U);
387
+
388
+ // test checks
389
+ unsigned ex_filter_entries = 0;
390
+ if (whole) {
391
+ ex_filter_entries += 5; // unique keys
392
+ }
393
+ if (prefix) {
394
+ ex_filter_entries += 2; // unique prefixes
395
+ }
396
+ EXPECT_EQ(tp.num_filter_entries, ex_filter_entries);
377
397
 
378
- for (auto* sn : snapshots) {
379
- db_->ReleaseSnapshot(sn);
380
- }
398
+ for (auto* sn : snapshots) {
399
+ db_->ReleaseSnapshot(sn);
381
400
  }
382
401
  }
383
402
  }
384
403
  }
385
404
 
386
- TEST_F(DBBloomFilterTest, WholeKeyFilterProp) {
387
- for (bool partition_filters : {true, false}) {
388
- Options options = last_options_;
389
- options.prefix_extractor.reset(NewFixedPrefixTransform(3));
390
- options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
391
- get_perf_context()->EnablePerLevelPerfContext();
405
+ TEST_P(DBBloomFilterTestWithPartitioningParam, WholeKeyFilterProp) {
406
+ Options options = last_options_;
407
+ options.prefix_extractor.reset(NewFixedPrefixTransform(3));
408
+ options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
409
+ get_perf_context()->EnablePerLevelPerfContext();
392
410
 
393
- BlockBasedTableOptions bbto;
394
- bbto.filter_policy.reset(NewBloomFilterPolicy(10));
395
- bbto.whole_key_filtering = false;
396
- if (partition_filters) {
397
- bbto.partition_filters = true;
398
- bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
399
- }
400
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
401
- DestroyAndReopen(options);
411
+ BlockBasedTableOptions bbto;
412
+ SetInTableOptions(&bbto);
413
+ bbto.filter_policy.reset(NewBloomFilterPolicy(10));
414
+ bbto.whole_key_filtering = false;
415
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
416
+ DestroyAndReopen(options);
402
417
 
403
- WriteOptions wo;
404
- ReadOptions ro;
405
- FlushOptions fo;
406
- fo.wait = true;
407
- std::string value;
418
+ WriteOptions wo;
419
+ ReadOptions ro;
420
+ FlushOptions fo;
421
+ fo.wait = true;
422
+ std::string value;
408
423
 
409
- ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
410
- // Needs insert some keys to make sure files are not filtered out by key
411
- // ranges.
412
- ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
413
- ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
414
- ASSERT_OK(dbfull()->Flush(fo));
424
+ ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
425
+ // Needs insert some keys to make sure files are not filtered out by key
426
+ // ranges.
427
+ ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
428
+ ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
429
+ ASSERT_OK(dbfull()->Flush(fo));
415
430
 
416
- Reopen(options);
417
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
418
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
419
- ASSERT_EQ("NOT_FOUND", Get("foo"));
420
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
421
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
422
- ASSERT_EQ("NOT_FOUND", Get("bar"));
423
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
424
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
425
- ASSERT_EQ("foo", Get("foobar"));
426
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
427
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
428
-
429
- // Reopen with whole key filtering enabled and prefix extractor
430
- // NULL. Bloom filter should be off for both of whole key and
431
- // prefix bloom.
432
- bbto.whole_key_filtering = true;
433
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
434
- options.prefix_extractor.reset();
435
- Reopen(options);
431
+ Reopen(options);
432
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
433
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
434
+ ASSERT_EQ("NOT_FOUND", Get("foo"));
435
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
436
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
437
+ ASSERT_EQ("NOT_FOUND", Get("bar"));
438
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
439
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
440
+ ASSERT_EQ("foo", Get("foobar"));
441
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
442
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
443
+
444
+ // Reopen with whole key filtering enabled and prefix extractor
445
+ // NULL. Bloom filter should be off for both of whole key and
446
+ // prefix bloom.
447
+ bbto.whole_key_filtering = true;
448
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
449
+ options.prefix_extractor.reset();
450
+ Reopen(options);
436
451
 
437
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
438
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
439
- ASSERT_EQ("NOT_FOUND", Get("foo"));
440
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
441
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
442
- ASSERT_EQ("NOT_FOUND", Get("bar"));
443
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
444
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
445
- ASSERT_EQ("foo", Get("foobar"));
446
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
447
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
448
- // Write DB with only full key filtering.
449
- ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
450
- // Needs insert some keys to make sure files are not filtered out by key
451
- // ranges.
452
- ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
453
- ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
454
- ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
455
-
456
- // Reopen with both of whole key off and prefix extractor enabled.
457
- // Still no bloom filter should be used.
458
- options.prefix_extractor.reset(NewFixedPrefixTransform(3));
459
- bbto.whole_key_filtering = false;
460
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
461
- Reopen(options);
452
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
453
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
454
+ ASSERT_EQ("NOT_FOUND", Get("foo"));
455
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
456
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
457
+ ASSERT_EQ("NOT_FOUND", Get("bar"));
458
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
459
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
460
+ ASSERT_EQ("foo", Get("foobar"));
461
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
462
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
463
+ // Write DB with only full key filtering.
464
+ ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
465
+ // Needs insert some keys to make sure files are not filtered out by key
466
+ // ranges.
467
+ ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
468
+ ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
469
+ ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
470
+
471
+ // Reopen with both of whole key off and prefix extractor enabled.
472
+ // Still no bloom filter should be used.
473
+ options.prefix_extractor.reset(NewFixedPrefixTransform(3));
474
+ bbto.whole_key_filtering = false;
475
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
476
+ Reopen(options);
462
477
 
463
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
464
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
465
- ASSERT_EQ("NOT_FOUND", Get("foo"));
466
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
467
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
468
- ASSERT_EQ("NOT_FOUND", Get("bar"));
469
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
470
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
471
- ASSERT_EQ("foo", Get("foobar"));
472
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
473
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
474
-
475
- // Try to create a DB with mixed files:
476
- ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
477
- // Needs insert some keys to make sure files are not filtered out by key
478
- // ranges.
479
- ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
480
- ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
481
- ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
482
-
483
- options.prefix_extractor.reset();
484
- bbto.whole_key_filtering = true;
485
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
486
- Reopen(options);
478
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
479
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
480
+ ASSERT_EQ("NOT_FOUND", Get("foo"));
481
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
482
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
483
+ ASSERT_EQ("NOT_FOUND", Get("bar"));
484
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
485
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
486
+ ASSERT_EQ("foo", Get("foobar"));
487
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
488
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
489
+
490
+ // Try to create a DB with mixed files:
491
+ ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
492
+ // Needs insert some keys to make sure files are not filtered out by key
493
+ // ranges.
494
+ ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
495
+ ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
496
+ ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
487
497
 
488
- // Try to create a DB with mixed files.
489
- ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar"));
490
- // In this case needs insert some keys to make sure files are
491
- // not filtered out by key ranges.
492
- ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
493
- ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
494
- ASSERT_OK(Flush());
498
+ options.prefix_extractor.reset();
499
+ bbto.whole_key_filtering = true;
500
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
501
+ Reopen(options);
495
502
 
496
- // Now we have two files:
497
- // File 1: An older file with prefix bloom (disabled)
498
- // File 2: A newer file with whole bloom filter.
499
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
500
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
501
- ASSERT_EQ("NOT_FOUND", Get("foo"));
502
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
503
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
504
- ASSERT_EQ("NOT_FOUND", Get("bar"));
505
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
506
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
507
- ASSERT_EQ("foo", Get("foobar"));
508
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
509
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
510
- ASSERT_EQ("bar", Get("barfoo"));
511
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
512
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
513
-
514
- // Reopen with the same setting: only whole key is used
515
- Reopen(options);
516
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
517
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
518
- ASSERT_EQ("NOT_FOUND", Get("foo"));
519
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
520
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
521
- ASSERT_EQ("NOT_FOUND", Get("bar"));
522
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
523
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
524
- ASSERT_EQ("foo", Get("foobar"));
525
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
526
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
527
- ASSERT_EQ("bar", Get("barfoo"));
528
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
529
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
530
-
531
- // Restart with both filters are allowed
532
- options.prefix_extractor.reset(NewFixedPrefixTransform(3));
533
- bbto.whole_key_filtering = true;
534
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
535
- Reopen(options);
536
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
537
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
538
- // File 1 will has it filtered out.
539
- // File 2 will not, as prefix `foo` exists in the file.
540
- ASSERT_EQ("NOT_FOUND", Get("foo"));
541
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
542
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
543
- ASSERT_EQ("NOT_FOUND", Get("bar"));
544
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
545
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
546
- ASSERT_EQ("foo", Get("foobar"));
547
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
548
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
549
- ASSERT_EQ("bar", Get("barfoo"));
550
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
551
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
552
-
553
- // Restart with only prefix bloom is allowed.
554
- options.prefix_extractor.reset(NewFixedPrefixTransform(3));
555
- bbto.whole_key_filtering = false;
556
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
557
- Reopen(options);
558
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
559
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
560
- ASSERT_EQ("NOT_FOUND", Get("foo"));
561
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
562
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
563
- ASSERT_EQ("NOT_FOUND", Get("bar"));
564
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
565
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
566
- ASSERT_EQ("foo", Get("foobar"));
567
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
568
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
569
- ASSERT_EQ("bar", Get("barfoo"));
570
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
571
- EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
572
- uint64_t bloom_filter_useful_all_levels = 0;
573
- for (auto& kv : (*(get_perf_context()->level_to_perf_context))) {
574
- if (kv.second.bloom_filter_useful > 0) {
575
- bloom_filter_useful_all_levels += kv.second.bloom_filter_useful;
576
- }
503
+ // Try to create a DB with mixed files.
504
+ ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar"));
505
+ // In this case needs insert some keys to make sure files are
506
+ // not filtered out by key ranges.
507
+ ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
508
+ ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
509
+ ASSERT_OK(Flush());
510
+
511
+ // Now we have two files:
512
+ // File 1: An older file with prefix bloom (disabled)
513
+ // File 2: A newer file with whole bloom filter.
514
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
515
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
516
+ ASSERT_EQ("NOT_FOUND", Get("foo"));
517
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
518
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
519
+ ASSERT_EQ("NOT_FOUND", Get("bar"));
520
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
521
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
522
+ ASSERT_EQ("foo", Get("foobar"));
523
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
524
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
525
+ ASSERT_EQ("bar", Get("barfoo"));
526
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
527
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
528
+
529
+ // Reopen with the same setting: only whole key is used
530
+ Reopen(options);
531
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
532
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
533
+ ASSERT_EQ("NOT_FOUND", Get("foo"));
534
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
535
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
536
+ ASSERT_EQ("NOT_FOUND", Get("bar"));
537
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
538
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
539
+ ASSERT_EQ("foo", Get("foobar"));
540
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
541
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
542
+ ASSERT_EQ("bar", Get("barfoo"));
543
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
544
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
545
+
546
+ // Restart with both filters are allowed
547
+ options.prefix_extractor.reset(NewFixedPrefixTransform(3));
548
+ bbto.whole_key_filtering = true;
549
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
550
+ Reopen(options);
551
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
552
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
553
+ // File 1 will has it filtered out.
554
+ // File 2 will not, as prefix `foo` exists in the file.
555
+ ASSERT_EQ("NOT_FOUND", Get("foo"));
556
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
557
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
558
+ ASSERT_EQ("NOT_FOUND", Get("bar"));
559
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
560
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
561
+ ASSERT_EQ("foo", Get("foobar"));
562
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
563
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
564
+ ASSERT_EQ("bar", Get("barfoo"));
565
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
566
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
567
+
568
+ // Restart with only prefix bloom is allowed.
569
+ options.prefix_extractor.reset(NewFixedPrefixTransform(3));
570
+ bbto.whole_key_filtering = false;
571
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
572
+ Reopen(options);
573
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
574
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
575
+ ASSERT_EQ("NOT_FOUND", Get("foo"));
576
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
577
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
578
+ ASSERT_EQ("NOT_FOUND", Get("bar"));
579
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
580
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
581
+ ASSERT_EQ("foo", Get("foobar"));
582
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
583
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
584
+ ASSERT_EQ("bar", Get("barfoo"));
585
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
586
+ EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
587
+ uint64_t bloom_filter_useful_all_levels = 0;
588
+ for (auto& kv : (*(get_perf_context()->level_to_perf_context))) {
589
+ if (kv.second.bloom_filter_useful > 0) {
590
+ bloom_filter_useful_all_levels += kv.second.bloom_filter_useful;
577
591
  }
578
- ASSERT_EQ(12, bloom_filter_useful_all_levels);
579
- get_perf_context()->Reset();
580
592
  }
593
+ ASSERT_EQ(12, bloom_filter_useful_all_levels);
594
+ get_perf_context()->Reset();
581
595
  }
582
596
 
583
- TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
597
+ INSTANTIATE_TEST_CASE_P(
598
+ DBBloomFilterTestWithPartitioningParam,
599
+ DBBloomFilterTestWithPartitioningParam,
600
+ ::testing::Values(FilterPartitioning::kUnpartitionedFilter,
601
+ FilterPartitioning::kCoupledPartitionedFilter,
602
+ FilterPartitioning::kDecoupledPartitionedFilter));
603
+
604
+ TEST_P(DBBloomFilterTestWithFormatParams, BloomFilter) {
584
605
  do {
585
606
  Options options = CurrentOptions();
586
607
  env_->count_random_reads_ = true;
@@ -588,20 +609,23 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
588
609
  // ChangeCompactOptions() only changes compaction style, which does not
589
610
  // trigger reset of table_factory
590
611
  BlockBasedTableOptions table_options;
612
+ // When partitioned filters are coupled to index blocks, they tend to get
613
+ // extra fractional bits per key when rounding up to the next cache line
614
+ // size. Here we correct for that to get similar effective bits per key.
615
+ bits_per_key_ = table_options.decouple_partitioned_filters ? 10.5 : 10;
616
+ SetInTableOptions(&table_options);
591
617
  table_options.no_block_cache = true;
592
- table_options.filter_policy = Create(10, bfp_impl_);
593
618
  table_options.optimize_filters_for_memory = false;
594
- table_options.partition_filters = partition_filters_;
595
- if (partition_filters_) {
596
- table_options.index_type =
597
- BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
598
- }
599
- table_options.format_version = format_version_;
600
619
  if (format_version_ >= 4) {
601
620
  // value delta encoding challenged more with index interval > 1
602
621
  table_options.index_block_restart_interval = 8;
603
622
  }
604
- table_options.metadata_block_size = 32;
623
+ // This test is rather sensitive to the actual filter partition block size,
624
+ // and keeping that consistent between coupled and uncoupled requires a
625
+ // different metadata block size for this example (where it controls index
626
+ // block size).
627
+ table_options.metadata_block_size =
628
+ table_options.decouple_partitioned_filters ? 320 : 32;
605
629
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
606
630
 
607
631
  CreateAndReopenWithCF({"pikachu"}, options);
@@ -628,7 +652,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
628
652
  int reads = env_->random_read_counter_.Read();
629
653
  fprintf(stderr, "%d present => %d reads\n", N, reads);
630
654
  ASSERT_GE(reads, N);
631
- if (partition_filters_) {
655
+ if (PartitionFilters()) {
632
656
  // Without block cache, we read an extra partition filter per each
633
657
  // level*read and a partition index per each read
634
658
  ASSERT_LE(reads, 4 * N + 2 * N / 100);
@@ -643,7 +667,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
643
667
  }
644
668
  reads = env_->random_read_counter_.Read();
645
669
  fprintf(stderr, "%d missing => %d reads\n", N, reads);
646
- if (partition_filters_) {
670
+ if (PartitionFilters()) {
647
671
  // With partitioned filter we read one extra filter per level per each
648
672
  // missed read.
649
673
  ASSERT_LE(reads, 2 * N + 3 * N / 100);
@@ -658,7 +682,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
658
682
  uint64_t nkeys = N + N / 100;
659
683
  uint64_t filter_size = ParseUint64(props["filter_size"]);
660
684
  EXPECT_LE(filter_size,
661
- (partition_filters_ ? 12 : 11) * nkeys / /*bits / byte*/ 8);
685
+ (PartitionFilters() ? 12 : 11) * nkeys / /*bits / byte*/ 8);
662
686
  if (bfp_impl_ == kAutoRibbon) {
663
687
  // Sometimes using Ribbon filter which is more space-efficient
664
688
  EXPECT_GE(filter_size, 7 * nkeys / /*bits / byte*/ 8);
@@ -679,15 +703,20 @@ namespace {
679
703
 
680
704
  class AlwaysTrueBitsBuilder : public FilterBitsBuilder {
681
705
  public:
682
- void AddKey(const Slice&) override {}
683
- size_t EstimateEntriesAdded() override { return 0U; }
706
+ void AddKey(const Slice&) override { ++count_; }
707
+ void AddKeyAndAlt(const Slice&, const Slice&) override { count_ += 2; }
708
+ size_t EstimateEntriesAdded() override { return count_; }
684
709
  Slice Finish(std::unique_ptr<const char[]>* /* buf */) override {
710
+ count_ = 0;
685
711
  // Interpreted as "always true" filter (0 probes over 1 byte of
686
712
  // payload, 5 bytes metadata)
687
713
  return Slice("\0\0\0\0\0\0", 6);
688
714
  }
689
715
  using FilterBitsBuilder::Finish;
690
716
  size_t ApproximateNumEntries(size_t) override { return SIZE_MAX; }
717
+
718
+ private:
719
+ size_t count_ = 0;
691
720
  };
692
721
 
693
722
  class AlwaysTrueFilterPolicy : public ReadOnlyBuiltinFilterPolicy {
@@ -709,7 +738,7 @@ class AlwaysTrueFilterPolicy : public ReadOnlyBuiltinFilterPolicy {
709
738
 
710
739
  } // anonymous namespace
711
740
 
712
- TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
741
+ TEST_P(DBBloomFilterTestWithFormatParams, SkipFilterOnEssentiallyZeroBpk) {
713
742
  constexpr int maxKey = 10;
714
743
  auto PutFn = [&]() {
715
744
  int i;
@@ -740,12 +769,7 @@ TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
740
769
  Options options = CurrentOptions();
741
770
  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
742
771
  BlockBasedTableOptions table_options;
743
- table_options.partition_filters = partition_filters_;
744
- if (partition_filters_) {
745
- table_options.index_type =
746
- BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
747
- }
748
- table_options.format_version = format_version_;
772
+ SetInTableOptions(&table_options);
749
773
 
750
774
  // Test 1: bits per key < 0.5 means skip filters -> no filter
751
775
  // constructed or read.
@@ -819,27 +843,85 @@ TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
819
843
  EXPECT_EQ(props["filter_size"], "0");
820
844
  }
821
845
 
846
+ TEST_P(DBBloomFilterTestWithFormatParams, FilterBitsBuilderDedup) {
847
+ BlockBasedTableOptions table_options;
848
+ SetInTableOptions(&table_options);
849
+ FilterBuildingContext context{table_options};
850
+ std::unique_ptr<FilterBitsBuilder> builder{
851
+ table_options.filter_policy->GetBuilderWithContext(context)};
852
+
853
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 0U);
854
+ // Check for sufficient de-duplication between regular keys and alt keys
855
+ // (prefixes), keeping in mind that the key might equal its prefix.
856
+
857
+ builder->AddKey("abc");
858
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 1U);
859
+ builder->AddKeyAndAlt("abc1", "abc");
860
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 2U);
861
+ builder->AddKeyAndAlt("bcd", "bcd");
862
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 3U);
863
+ builder->AddKeyAndAlt("cde-1", "cde");
864
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 5U);
865
+ builder->AddKeyAndAlt("cde", "cde");
866
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 5U);
867
+ builder->AddKeyAndAlt("cde1", "cde");
868
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 6U);
869
+ builder->AddKeyAndAlt("def-1", "def");
870
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 8U);
871
+ builder->AddKeyAndAlt("def", "def");
872
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 8U);
873
+ builder->AddKey("def$$"); // Like not in extractor domain
874
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 9U);
875
+ builder->AddKey("def$$");
876
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 9U);
877
+ builder->AddKeyAndAlt("efg42", "efg");
878
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 11U);
879
+ builder->AddKeyAndAlt("efg", "efg"); // Like extra "alt" on a partition
880
+ ASSERT_EQ(builder->EstimateEntriesAdded(), 11U);
881
+ }
882
+
822
883
  #if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
823
884
  INSTANTIATE_TEST_CASE_P(
824
885
  FormatDef, DBBloomFilterTestDefFormatVersion,
825
886
  ::testing::Values(
826
- std::make_tuple(kAutoBloom, true, test::kDefaultFormatVersion),
827
- std::make_tuple(kAutoBloom, false, test::kDefaultFormatVersion),
828
- std::make_tuple(kAutoRibbon, false, test::kDefaultFormatVersion)));
887
+ std::make_tuple(kAutoBloom,
888
+ FilterPartitioning::kCoupledPartitionedFilter,
889
+ test::kDefaultFormatVersion),
890
+ std::make_tuple(kAutoBloom,
891
+ FilterPartitioning::kDecoupledPartitionedFilter,
892
+ test::kDefaultFormatVersion),
893
+ std::make_tuple(kAutoBloom, FilterPartitioning::kUnpartitionedFilter,
894
+ test::kDefaultFormatVersion),
895
+ std::make_tuple(kAutoRibbon, FilterPartitioning::kUnpartitionedFilter,
896
+ test::kDefaultFormatVersion)));
829
897
 
830
898
  INSTANTIATE_TEST_CASE_P(
831
- FormatDef, DBBloomFilterTestWithParam,
899
+ FormatDef, DBBloomFilterTestWithFormatParams,
832
900
  ::testing::Values(
833
- std::make_tuple(kAutoBloom, true, test::kDefaultFormatVersion),
834
- std::make_tuple(kAutoBloom, false, test::kDefaultFormatVersion),
835
- std::make_tuple(kAutoRibbon, false, test::kDefaultFormatVersion)));
901
+ std::make_tuple(kAutoBloom,
902
+ FilterPartitioning::kCoupledPartitionedFilter,
903
+ test::kDefaultFormatVersion),
904
+ std::make_tuple(kAutoBloom,
905
+ FilterPartitioning::kDecoupledPartitionedFilter,
906
+ test::kDefaultFormatVersion),
907
+ std::make_tuple(kAutoBloom, FilterPartitioning::kUnpartitionedFilter,
908
+ test::kDefaultFormatVersion),
909
+ std::make_tuple(kAutoRibbon, FilterPartitioning::kUnpartitionedFilter,
910
+ test::kDefaultFormatVersion)));
836
911
 
837
912
  INSTANTIATE_TEST_CASE_P(
838
- FormatLatest, DBBloomFilterTestWithParam,
839
- ::testing::Values(std::make_tuple(kAutoBloom, true, kLatestFormatVersion),
840
- std::make_tuple(kAutoBloom, false, kLatestFormatVersion),
841
- std::make_tuple(kAutoRibbon, false,
842
- kLatestFormatVersion)));
913
+ FormatLatest, DBBloomFilterTestWithFormatParams,
914
+ ::testing::Values(
915
+ std::make_tuple(kAutoBloom,
916
+ FilterPartitioning::kCoupledPartitionedFilter,
917
+ kLatestFormatVersion),
918
+ std::make_tuple(kAutoBloom,
919
+ FilterPartitioning::kDecoupledPartitionedFilter,
920
+ kLatestFormatVersion),
921
+ std::make_tuple(kAutoBloom, FilterPartitioning::kUnpartitionedFilter,
922
+ kLatestFormatVersion),
923
+ std::make_tuple(kAutoRibbon, FilterPartitioning::kUnpartitionedFilter,
924
+ kLatestFormatVersion)));
843
925
  #endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
844
926
 
845
927
  TEST_F(DBBloomFilterTest, BloomFilterRate) {
@@ -957,18 +1039,17 @@ TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
957
1039
  using FilterConstructionReserveMemoryHash = uint64_t;
958
1040
 
959
1041
  class ChargeFilterConstructionTestWithParam
960
- : public DBTestBase,
961
- public testing::WithParamInterface<std::tuple<
962
- CacheEntryRoleOptions::Decision, std::string, bool, bool>> {
1042
+ : public DBBloomFilterTest,
1043
+ public testing::WithParamInterface<
1044
+ std::tuple<CacheEntryRoleOptions::Decision, std::string,
1045
+ FilterPartitioning, bool>> {
963
1046
  public:
964
1047
  ChargeFilterConstructionTestWithParam()
965
- : DBTestBase("db_bloom_filter_tests",
966
- /*env_do_fsync=*/true),
967
- num_key_(0),
1048
+ : num_key_(0),
968
1049
  charge_filter_construction_(std::get<0>(GetParam())),
969
1050
  policy_(std::get<1>(GetParam())),
970
- partition_filters_(std::get<2>(GetParam())),
971
1051
  detect_filter_construct_corruption_(std::get<3>(GetParam())) {
1052
+ filter_partitioning_ = std::get<2>(GetParam());
972
1053
  if (charge_filter_construction_ ==
973
1054
  CacheEntryRoleOptions::Decision::kDisabled ||
974
1055
  policy_ == kLegacyBloom) {
@@ -976,7 +1057,7 @@ class ChargeFilterConstructionTestWithParam
976
1057
  // cache charging happens instead of its accuracy. Therefore we don't
977
1058
  // need many keys.
978
1059
  num_key_ = 5;
979
- } else if (partition_filters_) {
1060
+ } else if (PartitionFilters()) {
980
1061
  // For PartitionFilter case, since we set
981
1062
  // table_options.metadata_block_size big enough such that each partition
982
1063
  // trigger at least 1 dummy entry reservation each for hash entries and
@@ -1013,6 +1094,7 @@ class ChargeFilterConstructionTestWithParam
1013
1094
 
1014
1095
  BlockBasedTableOptions GetBlockBasedTableOptions() {
1015
1096
  BlockBasedTableOptions table_options;
1097
+ SetInTableOptions(&table_options);
1016
1098
 
1017
1099
  // We set cache capacity big enough to prevent cache full for convenience in
1018
1100
  // calculation.
@@ -1022,10 +1104,7 @@ class ChargeFilterConstructionTestWithParam
1022
1104
  {CacheEntryRole::kFilterConstruction,
1023
1105
  {/*.charged = */ charge_filter_construction_}});
1024
1106
  table_options.filter_policy = Create(10, policy_);
1025
- table_options.partition_filters = partition_filters_;
1026
1107
  if (table_options.partition_filters) {
1027
- table_options.index_type =
1028
- BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
1029
1108
  // We set table_options.metadata_block_size big enough so that each
1030
1109
  // partition trigger at least 1 dummy entry insertion each for hash
1031
1110
  // entries and final filter.
@@ -1054,8 +1133,6 @@ class ChargeFilterConstructionTestWithParam
1054
1133
 
1055
1134
  std::string GetFilterPolicy() { return policy_; }
1056
1135
 
1057
- bool PartitionFilters() { return partition_filters_; }
1058
-
1059
1136
  std::shared_ptr<
1060
1137
  TargetCacheChargeTrackingCache<CacheEntryRole::kFilterConstruction>>
1061
1138
  GetCache() {
@@ -1066,7 +1143,6 @@ class ChargeFilterConstructionTestWithParam
1066
1143
  std::size_t num_key_;
1067
1144
  CacheEntryRoleOptions::Decision charge_filter_construction_;
1068
1145
  std::string policy_;
1069
- bool partition_filters_;
1070
1146
  std::shared_ptr<
1071
1147
  TargetCacheChargeTrackingCache<CacheEntryRole::kFilterConstruction>>
1072
1148
  cache_;
@@ -1078,28 +1154,43 @@ INSTANTIATE_TEST_CASE_P(
1078
1154
  ChargeFilterConstructionTestWithParam,
1079
1155
  ::testing::Values(
1080
1156
  std::make_tuple(CacheEntryRoleOptions::Decision::kDisabled,
1081
- kFastLocalBloom, false, false),
1157
+ kFastLocalBloom,
1158
+ FilterPartitioning::kUnpartitionedFilter, false),
1082
1159
 
1083
1160
  std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1084
- kFastLocalBloom, false, false),
1161
+ kFastLocalBloom,
1162
+ FilterPartitioning::kUnpartitionedFilter, false),
1163
+ std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1164
+ kFastLocalBloom,
1165
+ FilterPartitioning::kUnpartitionedFilter, true),
1085
1166
  std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1086
- kFastLocalBloom, false, true),
1167
+ kFastLocalBloom,
1168
+ FilterPartitioning::kCoupledPartitionedFilter, false),
1087
1169
  std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1088
- kFastLocalBloom, true, false),
1170
+ kFastLocalBloom,
1171
+ FilterPartitioning::kCoupledPartitionedFilter, true),
1089
1172
  std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1090
- kFastLocalBloom, true, true),
1173
+ kFastLocalBloom,
1174
+ FilterPartitioning::kDecoupledPartitionedFilter, true),
1091
1175
 
1092
1176
  std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1093
- kStandard128Ribbon, false, false),
1177
+ kStandard128Ribbon,
1178
+ FilterPartitioning::kUnpartitionedFilter, false),
1094
1179
  std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1095
- kStandard128Ribbon, false, true),
1180
+ kStandard128Ribbon,
1181
+ FilterPartitioning::kUnpartitionedFilter, true),
1096
1182
  std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1097
- kStandard128Ribbon, true, false),
1183
+ kStandard128Ribbon,
1184
+ FilterPartitioning::kCoupledPartitionedFilter, false),
1098
1185
  std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1099
- kStandard128Ribbon, true, true),
1186
+ kStandard128Ribbon,
1187
+ FilterPartitioning::kCoupledPartitionedFilter, true),
1188
+ std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
1189
+ kStandard128Ribbon,
1190
+ FilterPartitioning::kDecoupledPartitionedFilter, true),
1100
1191
 
1101
1192
  std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, kLegacyBloom,
1102
- false, false)));
1193
+ FilterPartitioning::kUnpartitionedFilter, false)));
1103
1194
 
1104
1195
  // TODO: Speed up this test, and reduce disk space usage (~700MB)
1105
1196
  // The current test inserts many keys (on the scale of dummy entry size)
@@ -1160,7 +1251,6 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
1160
1251
  bool charge_filter_construction = (ChargeFilterConstructMemory() ==
1161
1252
  CacheEntryRoleOptions::Decision::kEnabled);
1162
1253
  std::string policy = GetFilterPolicy();
1163
- bool partition_filters = PartitionFilters();
1164
1254
  bool detect_filter_construct_corruption =
1165
1255
  table_options.detect_filter_construct_corruption;
1166
1256
 
@@ -1247,7 +1337,7 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
1247
1337
  * last longer since we release hash entries reservation later.
1248
1338
  *
1249
1339
  */
1250
- if (!partition_filters) {
1340
+ if (!PartitionFilters()) {
1251
1341
  EXPECT_EQ(filter_construction_cache_res_peaks.size(), 1)
1252
1342
  << "Filter construction cache charging should have only 1 peak in "
1253
1343
  "case: kFastLocalBloom + FullFilter";
@@ -1369,7 +1459,7 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
1369
1459
  * = hash entries + banding + final filter
1370
1460
  *
1371
1461
  */
1372
- if (!partition_filters) {
1462
+ if (!PartitionFilters()) {
1373
1463
  ASSERT_GE(
1374
1464
  std::floor(
1375
1465
  1.0 * predicted_final_filter_cache_res /
@@ -1444,29 +1534,21 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
1444
1534
  }
1445
1535
 
1446
1536
  class DBFilterConstructionCorruptionTestWithParam
1447
- : public DBTestBase,
1537
+ : public DBBloomFilterTest,
1448
1538
  public testing::WithParamInterface<
1449
1539
  std::tuple<bool /* detect_filter_construct_corruption */, std::string,
1450
- bool /* partition_filters */>> {
1540
+ FilterPartitioning>> {
1451
1541
  public:
1452
- DBFilterConstructionCorruptionTestWithParam()
1453
- : DBTestBase("db_bloom_filter_tests",
1454
- /*env_do_fsync=*/true) {}
1455
-
1456
- BlockBasedTableOptions GetBlockBasedTableOptions() {
1457
- BlockBasedTableOptions table_options;
1458
- table_options.detect_filter_construct_corruption = std::get<0>(GetParam());
1459
- table_options.filter_policy = Create(10, std::get<1>(GetParam()));
1460
- table_options.partition_filters = std::get<2>(GetParam());
1461
- if (table_options.partition_filters) {
1462
- table_options.index_type =
1463
- BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
1542
+ void SetInTableOptions(BlockBasedTableOptions* table_options) {
1543
+ table_options->detect_filter_construct_corruption = std::get<0>(GetParam());
1544
+ table_options->filter_policy = Create(10, std::get<1>(GetParam()));
1545
+ filter_partitioning_ = std::get<2>(GetParam());
1546
+ DBBloomFilterTest::SetInTableOptions(table_options);
1547
+ if (PartitionFilters()) {
1464
1548
  // We set table_options.metadata_block_size small enough so we can
1465
1549
  // trigger filter partitioning with GetNumKey() amount of keys
1466
- table_options.metadata_block_size = 10;
1550
+ table_options->metadata_block_size = 10;
1467
1551
  }
1468
-
1469
- return table_options;
1470
1552
  }
1471
1553
 
1472
1554
  // Return an appropriate amount of keys for testing
@@ -1477,15 +1559,26 @@ class DBFilterConstructionCorruptionTestWithParam
1477
1559
  INSTANTIATE_TEST_CASE_P(
1478
1560
  DBFilterConstructionCorruptionTestWithParam,
1479
1561
  DBFilterConstructionCorruptionTestWithParam,
1480
- ::testing::Values(std::make_tuple(false, kFastLocalBloom, false),
1481
- std::make_tuple(true, kFastLocalBloom, false),
1482
- std::make_tuple(true, kFastLocalBloom, true),
1483
- std::make_tuple(true, kStandard128Ribbon, false),
1484
- std::make_tuple(true, kStandard128Ribbon, true)));
1562
+ ::testing::Values(
1563
+ std::make_tuple(false, kFastLocalBloom,
1564
+ FilterPartitioning::kUnpartitionedFilter),
1565
+ std::make_tuple(true, kFastLocalBloom,
1566
+ FilterPartitioning::kUnpartitionedFilter),
1567
+ std::make_tuple(true, kFastLocalBloom,
1568
+ FilterPartitioning::kCoupledPartitionedFilter),
1569
+ std::make_tuple(true, kFastLocalBloom,
1570
+ FilterPartitioning::kDecoupledPartitionedFilter),
1571
+ std::make_tuple(true, kStandard128Ribbon,
1572
+ FilterPartitioning::kUnpartitionedFilter),
1573
+ std::make_tuple(true, kStandard128Ribbon,
1574
+ FilterPartitioning::kCoupledPartitionedFilter),
1575
+ std::make_tuple(true, kStandard128Ribbon,
1576
+ FilterPartitioning::kDecoupledPartitionedFilter)));
1485
1577
 
1486
1578
  TEST_P(DBFilterConstructionCorruptionTestWithParam, DetectCorruption) {
1487
1579
  Options options = CurrentOptions();
1488
- BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
1580
+ BlockBasedTableOptions table_options;
1581
+ SetInTableOptions(&table_options);
1489
1582
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
1490
1583
  options.create_if_missing = true;
1491
1584
  options.disable_auto_compactions = true;
@@ -1572,7 +1665,8 @@ TEST_P(DBFilterConstructionCorruptionTestWithParam, DetectCorruption) {
1572
1665
  TEST_P(DBFilterConstructionCorruptionTestWithParam,
1573
1666
  DynamicallyTurnOnAndOffDetectConstructCorruption) {
1574
1667
  Options options = CurrentOptions();
1575
- BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
1668
+ BlockBasedTableOptions table_options;
1669
+ SetInTableOptions(&table_options);
1576
1670
  // We intend to turn on
1577
1671
  // table_options.detect_filter_construct_corruption dynamically
1578
1672
  // therefore we override this test parmater's value
@@ -2309,11 +2403,12 @@ static const std::string kPlainTable = "test_PlainTableBloom";
2309
2403
 
2310
2404
  class BloomStatsTestWithParam
2311
2405
  : public DBBloomFilterTest,
2312
- public testing::WithParamInterface<std::tuple<std::string, bool>> {
2406
+ public testing::WithParamInterface<
2407
+ std::tuple<std::string, FilterPartitioning>> {
2313
2408
  public:
2314
2409
  BloomStatsTestWithParam() {
2315
2410
  bfp_impl_ = std::get<0>(GetParam());
2316
- partition_filters_ = std::get<1>(GetParam());
2411
+ filter_partitioning_ = std::get<1>(GetParam());
2317
2412
 
2318
2413
  options_.create_if_missing = true;
2319
2414
  options_.prefix_extractor.reset(
@@ -2321,13 +2416,13 @@ class BloomStatsTestWithParam
2321
2416
  options_.memtable_prefix_bloom_size_ratio =
2322
2417
  8.0 * 1024.0 / static_cast<double>(options_.write_buffer_size);
2323
2418
  if (bfp_impl_ == kPlainTable) {
2324
- assert(!partition_filters_); // not supported in plain table
2419
+ assert(!PartitionFilters()); // not supported in plain table
2325
2420
  PlainTableOptions table_options;
2326
2421
  options_.table_factory.reset(NewPlainTableFactory(table_options));
2327
2422
  } else {
2328
2423
  BlockBasedTableOptions table_options;
2329
- if (partition_filters_) {
2330
- table_options.partition_filters = partition_filters_;
2424
+ if (PartitionFilters()) {
2425
+ table_options.partition_filters = PartitionFilters();
2331
2426
  table_options.index_type =
2332
2427
  BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
2333
2428
  }
@@ -2350,7 +2445,6 @@ class BloomStatsTestWithParam
2350
2445
  static void TearDownTestCase() {}
2351
2446
 
2352
2447
  std::string bfp_impl_;
2353
- bool partition_filters_;
2354
2448
  Options options_;
2355
2449
  };
2356
2450
 
@@ -2463,11 +2557,20 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
2463
2557
 
2464
2558
  INSTANTIATE_TEST_CASE_P(
2465
2559
  BloomStatsTestWithParam, BloomStatsTestWithParam,
2466
- ::testing::Values(std::make_tuple(kLegacyBloom, false),
2467
- std::make_tuple(kLegacyBloom, true),
2468
- std::make_tuple(kFastLocalBloom, false),
2469
- std::make_tuple(kFastLocalBloom, true),
2470
- std::make_tuple(kPlainTable, false)));
2560
+ ::testing::Values(
2561
+ std::make_tuple(kLegacyBloom, FilterPartitioning::kUnpartitionedFilter),
2562
+ std::make_tuple(kLegacyBloom,
2563
+ FilterPartitioning::kCoupledPartitionedFilter),
2564
+ std::make_tuple(kLegacyBloom,
2565
+ FilterPartitioning::kDecoupledPartitionedFilter),
2566
+ std::make_tuple(kFastLocalBloom,
2567
+ FilterPartitioning::kUnpartitionedFilter),
2568
+ std::make_tuple(kFastLocalBloom,
2569
+ FilterPartitioning::kCoupledPartitionedFilter),
2570
+ std::make_tuple(kFastLocalBloom,
2571
+ FilterPartitioning::kDecoupledPartitionedFilter),
2572
+ std::make_tuple(kPlainTable,
2573
+ FilterPartitioning::kUnpartitionedFilter)));
2471
2574
 
2472
2575
  namespace {
2473
2576
  void PrefixScanInit(DBBloomFilterTest* dbtest) {