@nxtedition/rocksdb 11.0.2 → 11.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +133 -122
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +15 -7
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +4 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +8 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +11 -7
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +17 -11
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +15 -0
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +155 -0
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +564 -461
- package/deps/rocksdb/rocksdb/db/db_follower_test.cc +8 -4
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +40 -24
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +8 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +7 -4
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +5 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +3 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +19 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +20 -16
- package/deps/rocksdb/rocksdb/db/db_io_failure_test.cc +27 -0
- package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +10 -2
- package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +85 -0
- package/deps/rocksdb/rocksdb/db/db_sst_test.cc +55 -2
- package/deps/rocksdb/rocksdb/db/db_test2.cc +231 -0
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +5 -0
- package/deps/rocksdb/rocksdb/db/db_test_util.h +10 -1
- package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +175 -1
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +64 -0
- package/deps/rocksdb/rocksdb/db/dbformat.h +5 -6
- package/deps/rocksdb/rocksdb/db/dbformat_test.cc +8 -8
- package/deps/rocksdb/rocksdb/db/experimental.cc +3 -2
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +2 -4
- package/deps/rocksdb/rocksdb/db/flush_job.cc +7 -2
- package/deps/rocksdb/rocksdb/db/flush_job_test.cc +4 -2
- package/deps/rocksdb/rocksdb/db/listener_test.cc +5 -5
- package/deps/rocksdb/rocksdb/db/log_writer.cc +12 -3
- package/deps/rocksdb/rocksdb/db/memtable.cc +83 -23
- package/deps/rocksdb/rocksdb/db/memtable.h +11 -3
- package/deps/rocksdb/rocksdb/db/memtable_list.cc +7 -5
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +21 -0
- package/deps/rocksdb/rocksdb/db/version_builder.cc +462 -33
- package/deps/rocksdb/rocksdb/db/version_builder.h +70 -23
- package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +95 -207
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +54 -35
- package/deps/rocksdb/rocksdb/db/version_set.cc +13 -11
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +313 -59
- package/deps/rocksdb/rocksdb/db/write_batch.cc +124 -64
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +2 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compaction_filter.h +1 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +4 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +9 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +4 -32
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +7 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +60 -172
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +57 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +23 -15
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.h +2 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_value.cc +1 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_value.h +4 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +200 -92
- package/deps/rocksdb/rocksdb/env/file_system.cc +3 -3
- package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +124 -23
- package/deps/rocksdb/rocksdb/file/delete_scheduler.h +61 -8
- package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +141 -2
- package/deps/rocksdb/rocksdb/file/file_util.cc +17 -2
- package/deps/rocksdb/rocksdb/file/file_util.h +10 -0
- package/deps/rocksdb/rocksdb/file/filename.cc +11 -3
- package/deps/rocksdb/rocksdb/file/filename.h +2 -1
- package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.cc +18 -0
- package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.h +27 -4
- package/deps/rocksdb/rocksdb/file/writable_file_writer.h +8 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +8 -13
- package/deps/rocksdb/rocksdb/include/rocksdb/env.h +4 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +5 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/filter_policy.h +2 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +34 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +25 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/table.h +27 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +2 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/types.h +12 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +21 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +29 -1
- package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +102 -33
- package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +46 -3
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +4 -0
- package/deps/rocksdb/rocksdb/options/cf_options.cc +6 -0
- package/deps/rocksdb/rocksdb/options/cf_options.h +2 -0
- package/deps/rocksdb/rocksdb/options/db_options.cc +15 -1
- package/deps/rocksdb/rocksdb/options/db_options.h +2 -0
- package/deps/rocksdb/rocksdb/options/options_helper.cc +10 -0
- package/deps/rocksdb/rocksdb/options/options_parser.cc +3 -2
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +9 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +75 -35
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +6 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +4 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +8 -1
- package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +40 -15
- package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +98 -17
- package/deps/rocksdb/rocksdb/table/block_based/filter_policy_internal.h +14 -2
- package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.cc +21 -91
- package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.h +13 -21
- package/deps/rocksdb/rocksdb/table/block_based/full_filter_block_test.cc +14 -5
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +62 -53
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +60 -38
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +175 -78
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.h +65 -36
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +25 -15
- package/deps/rocksdb/rocksdb/table/block_fetcher.cc +13 -1
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +18 -4
- package/deps/rocksdb/rocksdb/table/meta_blocks.h +4 -0
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +11 -0
- package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +2 -2
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +47 -18
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.h +1 -2
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +95 -0
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +26 -15
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +62 -19
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +73 -34
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +5 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +10 -3
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.h +2 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.cc +8 -5
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.h +7 -4
- package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +225 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.h +17 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +5 -2
- package/index.js +5 -17
- package/iterator.js +9 -1
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
- package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
|
@@ -45,6 +45,12 @@ const std::string kStandard128Ribbon =
|
|
|
45
45
|
const std::string kAutoBloom = BloomFilterPolicy::kClassName();
|
|
46
46
|
const std::string kAutoRibbon = RibbonFilterPolicy::kClassName();
|
|
47
47
|
|
|
48
|
+
enum class FilterPartitioning {
|
|
49
|
+
kUnpartitionedFilter,
|
|
50
|
+
kCoupledPartitionedFilter,
|
|
51
|
+
kDecoupledPartitionedFilter,
|
|
52
|
+
};
|
|
53
|
+
|
|
48
54
|
template <typename T>
|
|
49
55
|
T Pop(T& var) {
|
|
50
56
|
auto rv = var;
|
|
@@ -62,32 +68,61 @@ class DBBloomFilterTest : public DBTestBase {
|
|
|
62
68
|
public:
|
|
63
69
|
DBBloomFilterTest()
|
|
64
70
|
: DBTestBase("db_bloom_filter_test", /*env_do_fsync=*/true) {}
|
|
71
|
+
|
|
72
|
+
bool PartitionFilters() {
|
|
73
|
+
return filter_partitioning_ != FilterPartitioning::kUnpartitionedFilter;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
void SetInTableOptions(BlockBasedTableOptions* table_options) {
|
|
77
|
+
table_options->partition_filters = PartitionFilters();
|
|
78
|
+
if (PartitionFilters()) {
|
|
79
|
+
table_options->index_type =
|
|
80
|
+
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
protected:
|
|
85
|
+
FilterPartitioning filter_partitioning_ =
|
|
86
|
+
FilterPartitioning::kUnpartitionedFilter;
|
|
65
87
|
};
|
|
66
88
|
|
|
67
|
-
class
|
|
68
|
-
: public
|
|
89
|
+
class DBBloomFilterTestWithPartitioningParam
|
|
90
|
+
: public DBBloomFilterTest,
|
|
91
|
+
public testing::WithParamInterface<FilterPartitioning> {
|
|
92
|
+
public:
|
|
93
|
+
~DBBloomFilterTestWithPartitioningParam() override = default;
|
|
94
|
+
|
|
95
|
+
void SetUp() override { filter_partitioning_ = GetParam(); }
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
class DBBloomFilterTestWithFormatParams
|
|
99
|
+
: public DBBloomFilterTest,
|
|
69
100
|
public testing::WithParamInterface<
|
|
70
|
-
std::tuple<std::string,
|
|
71
|
-
// public testing::WithParamInterface<bool> {
|
|
101
|
+
std::tuple<std::string, FilterPartitioning, uint32_t>> {
|
|
72
102
|
protected:
|
|
73
103
|
std::string bfp_impl_;
|
|
74
|
-
|
|
104
|
+
double bits_per_key_;
|
|
75
105
|
uint32_t format_version_;
|
|
76
106
|
|
|
77
107
|
public:
|
|
78
|
-
|
|
79
|
-
: DBTestBase("db_bloom_filter_tests", /*env_do_fsync=*/true) {}
|
|
80
|
-
|
|
81
|
-
~DBBloomFilterTestWithParam() override = default;
|
|
108
|
+
~DBBloomFilterTestWithFormatParams() override = default;
|
|
82
109
|
|
|
83
110
|
void SetUp() override {
|
|
84
111
|
bfp_impl_ = std::get<0>(GetParam());
|
|
85
|
-
|
|
112
|
+
bits_per_key_ = 10; // default;
|
|
113
|
+
filter_partitioning_ = std::get<1>(GetParam());
|
|
86
114
|
format_version_ = std::get<2>(GetParam());
|
|
87
115
|
}
|
|
116
|
+
|
|
117
|
+
void SetInTableOptions(BlockBasedTableOptions* table_options) {
|
|
118
|
+
DBBloomFilterTest::SetInTableOptions(table_options);
|
|
119
|
+
table_options->filter_policy = Create(bits_per_key_, bfp_impl_);
|
|
120
|
+
table_options->format_version = format_version_;
|
|
121
|
+
}
|
|
88
122
|
};
|
|
89
123
|
|
|
90
|
-
class DBBloomFilterTestDefFormatVersion
|
|
124
|
+
class DBBloomFilterTestDefFormatVersion
|
|
125
|
+
: public DBBloomFilterTestWithFormatParams {};
|
|
91
126
|
|
|
92
127
|
class SliceTransformLimitedDomainGeneric : public SliceTransform {
|
|
93
128
|
const char* Name() const override {
|
|
@@ -118,11 +153,11 @@ TEST_P(DBBloomFilterTestDefFormatVersion, KeyMayExist) {
|
|
|
118
153
|
std::string value;
|
|
119
154
|
anon::OptionsOverride options_override;
|
|
120
155
|
options_override.filter_policy = Create(20, bfp_impl_);
|
|
121
|
-
options_override.partition_filters =
|
|
156
|
+
options_override.partition_filters = PartitionFilters();
|
|
122
157
|
options_override.metadata_block_size = 32;
|
|
123
158
|
options_override.full_block_cache = true;
|
|
124
159
|
Options options = CurrentOptions(options_override);
|
|
125
|
-
if (
|
|
160
|
+
if (PartitionFilters()) {
|
|
126
161
|
auto* table_options =
|
|
127
162
|
options.table_factory->GetOptions<BlockBasedTableOptions>();
|
|
128
163
|
if (table_options != nullptr &&
|
|
@@ -190,397 +225,383 @@ TEST_P(DBBloomFilterTestDefFormatVersion, KeyMayExist) {
|
|
|
190
225
|
ChangeOptions(kSkipPlainTable | kSkipHashIndex | kSkipFIFOCompaction));
|
|
191
226
|
}
|
|
192
227
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
bbto.whole_key_filtering = false;
|
|
207
|
-
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
208
|
-
DestroyAndReopen(options);
|
|
228
|
+
TEST_P(DBBloomFilterTestWithPartitioningParam,
|
|
229
|
+
GetFilterByPrefixBloomCustomPrefixExtractor) {
|
|
230
|
+
Options options = last_options_;
|
|
231
|
+
options.prefix_extractor =
|
|
232
|
+
std::make_shared<SliceTransformLimitedDomainGeneric>();
|
|
233
|
+
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
|
234
|
+
get_perf_context()->EnablePerLevelPerfContext();
|
|
235
|
+
BlockBasedTableOptions bbto;
|
|
236
|
+
SetInTableOptions(&bbto);
|
|
237
|
+
bbto.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
238
|
+
bbto.whole_key_filtering = false;
|
|
239
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
240
|
+
DestroyAndReopen(options);
|
|
209
241
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
242
|
+
WriteOptions wo;
|
|
243
|
+
ReadOptions ro;
|
|
244
|
+
FlushOptions fo;
|
|
245
|
+
fo.wait = true;
|
|
246
|
+
std::string value;
|
|
215
247
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
248
|
+
ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
|
|
249
|
+
ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
|
|
250
|
+
ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
|
|
219
251
|
|
|
220
|
-
|
|
252
|
+
ASSERT_OK(dbfull()->Flush(fo));
|
|
221
253
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
254
|
+
ASSERT_EQ("foo", Get("barbarbar"));
|
|
255
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
256
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
|
|
225
257
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
258
|
+
ASSERT_EQ("foo2", Get("barbarbar2"));
|
|
259
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
260
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
|
|
229
261
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
262
|
+
ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
|
|
263
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
264
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
|
|
233
265
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
266
|
+
ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
|
|
267
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
268
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
|
|
237
269
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
270
|
+
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
|
271
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
272
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
|
|
241
273
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
274
|
+
ro.total_order_seek = true;
|
|
275
|
+
// NOTE: total_order_seek no longer affects Get()
|
|
276
|
+
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
|
277
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
278
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
|
|
247
279
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
280
|
+
// No bloom on extractor changed
|
|
281
|
+
ASSERT_OK(db_->SetOptions({{"prefix_extractor", "capped:10"}}));
|
|
282
|
+
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
|
283
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
284
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
|
|
253
285
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
286
|
+
// No bloom on extractor changed, after re-open
|
|
287
|
+
options.prefix_extractor.reset(NewCappedPrefixTransform(10));
|
|
288
|
+
Reopen(options);
|
|
289
|
+
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
|
290
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
291
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
|
|
260
292
|
|
|
261
|
-
|
|
262
|
-
}
|
|
293
|
+
get_perf_context()->Reset();
|
|
263
294
|
}
|
|
264
295
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
}
|
|
277
|
-
bbto.whole_key_filtering = false;
|
|
278
|
-
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
279
|
-
DestroyAndReopen(options);
|
|
280
|
-
|
|
281
|
-
WriteOptions wo;
|
|
282
|
-
ReadOptions ro;
|
|
283
|
-
FlushOptions fo;
|
|
284
|
-
fo.wait = true;
|
|
285
|
-
std::string value;
|
|
286
|
-
|
|
287
|
-
ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
|
|
288
|
-
ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
|
|
289
|
-
ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
|
|
290
|
-
|
|
291
|
-
ASSERT_OK(dbfull()->Flush(fo));
|
|
292
|
-
|
|
293
|
-
ASSERT_EQ("foo", Get("barbarbar"));
|
|
294
|
-
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
295
|
-
ASSERT_EQ("foo2", Get("barbarbar2"));
|
|
296
|
-
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
297
|
-
ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
|
|
298
|
-
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
299
|
-
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
|
|
300
|
-
|
|
301
|
-
ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
|
|
302
|
-
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
303
|
-
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
|
|
304
|
-
|
|
305
|
-
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
|
306
|
-
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
307
|
-
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
|
|
308
|
-
|
|
309
|
-
ro.total_order_seek = true;
|
|
310
|
-
// NOTE: total_order_seek no longer affects Get()
|
|
311
|
-
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
|
312
|
-
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
313
|
-
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
|
|
296
|
+
TEST_P(DBBloomFilterTestWithPartitioningParam, GetFilterByPrefixBloom) {
|
|
297
|
+
Options options = last_options_;
|
|
298
|
+
options.prefix_extractor.reset(NewFixedPrefixTransform(8));
|
|
299
|
+
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
|
300
|
+
get_perf_context()->EnablePerLevelPerfContext();
|
|
301
|
+
BlockBasedTableOptions bbto;
|
|
302
|
+
SetInTableOptions(&bbto);
|
|
303
|
+
bbto.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
304
|
+
bbto.whole_key_filtering = false;
|
|
305
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
306
|
+
DestroyAndReopen(options);
|
|
314
307
|
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
308
|
+
WriteOptions wo;
|
|
309
|
+
ReadOptions ro;
|
|
310
|
+
FlushOptions fo;
|
|
311
|
+
fo.wait = true;
|
|
312
|
+
std::string value;
|
|
313
|
+
|
|
314
|
+
ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
|
|
315
|
+
ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
|
|
316
|
+
ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
|
|
317
|
+
|
|
318
|
+
ASSERT_OK(dbfull()->Flush(fo));
|
|
319
|
+
|
|
320
|
+
ASSERT_EQ("foo", Get("barbarbar"));
|
|
321
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
322
|
+
ASSERT_EQ("foo2", Get("barbarbar2"));
|
|
323
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
324
|
+
ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
|
|
325
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
326
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
|
|
327
|
+
|
|
328
|
+
ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
|
|
329
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
330
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
|
|
331
|
+
|
|
332
|
+
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
|
333
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
334
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
|
|
335
|
+
|
|
336
|
+
ro.total_order_seek = true;
|
|
337
|
+
// NOTE: total_order_seek no longer affects Get()
|
|
338
|
+
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
|
339
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
340
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1);
|
|
341
|
+
|
|
342
|
+
// No bloom on extractor changed
|
|
343
|
+
ASSERT_OK(db_->SetOptions({{"prefix_extractor", "capped:10"}}));
|
|
344
|
+
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
|
345
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
346
|
+
EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0);
|
|
320
347
|
|
|
321
|
-
|
|
322
|
-
}
|
|
348
|
+
get_perf_context()->Reset();
|
|
323
349
|
}
|
|
324
350
|
|
|
325
|
-
|
|
326
|
-
for (bool
|
|
327
|
-
SCOPED_TRACE("
|
|
328
|
-
for (bool
|
|
329
|
-
SCOPED_TRACE("
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
options.prefix_extractor.reset();
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
ASSERT_OK(Put("bar1", val));
|
|
355
|
-
ASSERT_OK(Put("bar2", val));
|
|
356
|
-
ASSERT_OK(Put("bar3", val));
|
|
357
|
-
snapshots.push_back(db_->GetSnapshot());
|
|
358
|
-
}
|
|
359
|
-
ASSERT_OK(Flush());
|
|
351
|
+
TEST_P(DBBloomFilterTestWithPartitioningParam, FilterNumEntriesCoalesce) {
|
|
352
|
+
for (bool prefix : {true, false}) {
|
|
353
|
+
SCOPED_TRACE("prefix=" + std::to_string(prefix));
|
|
354
|
+
for (bool whole : {true, false}) {
|
|
355
|
+
SCOPED_TRACE("whole=" + std::to_string(whole));
|
|
356
|
+
Options options = last_options_;
|
|
357
|
+
options.prefix_extractor.reset();
|
|
358
|
+
if (prefix) {
|
|
359
|
+
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
|
|
360
|
+
}
|
|
361
|
+
BlockBasedTableOptions bbto;
|
|
362
|
+
SetInTableOptions(&bbto);
|
|
363
|
+
bbto.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
364
|
+
bbto.whole_key_filtering = whole;
|
|
365
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
366
|
+
DestroyAndReopen(options);
|
|
367
|
+
|
|
368
|
+
// Need a snapshot to allow keeping multiple entries for the same key
|
|
369
|
+
std::vector<const Snapshot*> snapshots;
|
|
370
|
+
for (int i = 1; i <= 3; ++i) {
|
|
371
|
+
std::string val = "val" + std::to_string(i);
|
|
372
|
+
ASSERT_OK(Put("foo1", val));
|
|
373
|
+
ASSERT_OK(Put("foo2", val));
|
|
374
|
+
ASSERT_OK(Put("bar1", val));
|
|
375
|
+
ASSERT_OK(Put("bar2", val));
|
|
376
|
+
ASSERT_OK(Put("bar3", val));
|
|
377
|
+
snapshots.push_back(db_->GetSnapshot());
|
|
378
|
+
}
|
|
379
|
+
ASSERT_OK(Flush());
|
|
360
380
|
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
381
|
+
TablePropertiesCollection tpc;
|
|
382
|
+
ASSERT_OK(db_->GetPropertiesOfAllTables(&tpc));
|
|
383
|
+
// sanity checks
|
|
384
|
+
ASSERT_EQ(tpc.size(), 1U);
|
|
385
|
+
auto& tp = *tpc.begin()->second;
|
|
386
|
+
EXPECT_EQ(tp.num_entries, 3U * 5U);
|
|
387
|
+
|
|
388
|
+
// test checks
|
|
389
|
+
unsigned ex_filter_entries = 0;
|
|
390
|
+
if (whole) {
|
|
391
|
+
ex_filter_entries += 5; // unique keys
|
|
392
|
+
}
|
|
393
|
+
if (prefix) {
|
|
394
|
+
ex_filter_entries += 2; // unique prefixes
|
|
395
|
+
}
|
|
396
|
+
EXPECT_EQ(tp.num_filter_entries, ex_filter_entries);
|
|
377
397
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
}
|
|
398
|
+
for (auto* sn : snapshots) {
|
|
399
|
+
db_->ReleaseSnapshot(sn);
|
|
381
400
|
}
|
|
382
401
|
}
|
|
383
402
|
}
|
|
384
403
|
}
|
|
385
404
|
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
get_perf_context()->EnablePerLevelPerfContext();
|
|
405
|
+
TEST_P(DBBloomFilterTestWithPartitioningParam, WholeKeyFilterProp) {
|
|
406
|
+
Options options = last_options_;
|
|
407
|
+
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
|
|
408
|
+
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
|
409
|
+
get_perf_context()->EnablePerLevelPerfContext();
|
|
392
410
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
}
|
|
400
|
-
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
401
|
-
DestroyAndReopen(options);
|
|
411
|
+
BlockBasedTableOptions bbto;
|
|
412
|
+
SetInTableOptions(&bbto);
|
|
413
|
+
bbto.filter_policy.reset(NewBloomFilterPolicy(10));
|
|
414
|
+
bbto.whole_key_filtering = false;
|
|
415
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
416
|
+
DestroyAndReopen(options);
|
|
402
417
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
418
|
+
WriteOptions wo;
|
|
419
|
+
ReadOptions ro;
|
|
420
|
+
FlushOptions fo;
|
|
421
|
+
fo.wait = true;
|
|
422
|
+
std::string value;
|
|
408
423
|
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
424
|
+
ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
|
|
425
|
+
// Needs insert some keys to make sure files are not filtered out by key
|
|
426
|
+
// ranges.
|
|
427
|
+
ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
|
|
428
|
+
ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
|
|
429
|
+
ASSERT_OK(dbfull()->Flush(fo));
|
|
415
430
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
431
|
+
Reopen(options);
|
|
432
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
433
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
434
|
+
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
|
435
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
436
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
437
|
+
ASSERT_EQ("NOT_FOUND", Get("bar"));
|
|
438
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
439
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
440
|
+
ASSERT_EQ("foo", Get("foobar"));
|
|
441
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
442
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
443
|
+
|
|
444
|
+
// Reopen with whole key filtering enabled and prefix extractor
|
|
445
|
+
// NULL. Bloom filter should be off for both of whole key and
|
|
446
|
+
// prefix bloom.
|
|
447
|
+
bbto.whole_key_filtering = true;
|
|
448
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
449
|
+
options.prefix_extractor.reset();
|
|
450
|
+
Reopen(options);
|
|
436
451
|
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
452
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
453
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
454
|
+
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
|
455
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
456
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
457
|
+
ASSERT_EQ("NOT_FOUND", Get("bar"));
|
|
458
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
459
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
460
|
+
ASSERT_EQ("foo", Get("foobar"));
|
|
461
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
462
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
463
|
+
// Write DB with only full key filtering.
|
|
464
|
+
ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
|
|
465
|
+
// Needs insert some keys to make sure files are not filtered out by key
|
|
466
|
+
// ranges.
|
|
467
|
+
ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
|
|
468
|
+
ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
|
|
469
|
+
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
470
|
+
|
|
471
|
+
// Reopen with both of whole key off and prefix extractor enabled.
|
|
472
|
+
// Still no bloom filter should be used.
|
|
473
|
+
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
|
|
474
|
+
bbto.whole_key_filtering = false;
|
|
475
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
476
|
+
Reopen(options);
|
|
462
477
|
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
options.prefix_extractor.reset();
|
|
484
|
-
bbto.whole_key_filtering = true;
|
|
485
|
-
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
486
|
-
Reopen(options);
|
|
478
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
479
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
480
|
+
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
|
481
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
482
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
483
|
+
ASSERT_EQ("NOT_FOUND", Get("bar"));
|
|
484
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
485
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
486
|
+
ASSERT_EQ("foo", Get("foobar"));
|
|
487
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
488
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
489
|
+
|
|
490
|
+
// Try to create a DB with mixed files:
|
|
491
|
+
ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
|
|
492
|
+
// Needs insert some keys to make sure files are not filtered out by key
|
|
493
|
+
// ranges.
|
|
494
|
+
ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
|
|
495
|
+
ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
|
|
496
|
+
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
|
487
497
|
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
|
|
493
|
-
ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
|
|
494
|
-
ASSERT_OK(Flush());
|
|
498
|
+
options.prefix_extractor.reset();
|
|
499
|
+
bbto.whole_key_filtering = true;
|
|
500
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
501
|
+
Reopen(options);
|
|
495
502
|
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
503
|
+
// Try to create a DB with mixed files.
|
|
504
|
+
ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar"));
|
|
505
|
+
// In this case needs insert some keys to make sure files are
|
|
506
|
+
// not filtered out by key ranges.
|
|
507
|
+
ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
|
|
508
|
+
ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
|
|
509
|
+
ASSERT_OK(Flush());
|
|
510
|
+
|
|
511
|
+
// Now we have two files:
|
|
512
|
+
// File 1: An older file with prefix bloom (disabled)
|
|
513
|
+
// File 2: A newer file with whole bloom filter.
|
|
514
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
515
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
516
|
+
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
|
517
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
518
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
|
|
519
|
+
ASSERT_EQ("NOT_FOUND", Get("bar"));
|
|
520
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
521
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
|
|
522
|
+
ASSERT_EQ("foo", Get("foobar"));
|
|
523
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
524
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
|
|
525
|
+
ASSERT_EQ("bar", Get("barfoo"));
|
|
526
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
527
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
528
|
+
|
|
529
|
+
// Reopen with the same setting: only whole key is used
|
|
530
|
+
Reopen(options);
|
|
531
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
532
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
533
|
+
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
|
534
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
535
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
|
|
536
|
+
ASSERT_EQ("NOT_FOUND", Get("bar"));
|
|
537
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
538
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
|
|
539
|
+
ASSERT_EQ("foo", Get("foobar"));
|
|
540
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
541
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
|
|
542
|
+
ASSERT_EQ("bar", Get("barfoo"));
|
|
543
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
544
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
545
|
+
|
|
546
|
+
// Restart with both filters are allowed
|
|
547
|
+
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
|
|
548
|
+
bbto.whole_key_filtering = true;
|
|
549
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
550
|
+
Reopen(options);
|
|
551
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
552
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
553
|
+
// File 1 will has it filtered out.
|
|
554
|
+
// File 2 will not, as prefix `foo` exists in the file.
|
|
555
|
+
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
|
556
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
557
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
|
|
558
|
+
ASSERT_EQ("NOT_FOUND", Get("bar"));
|
|
559
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
560
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
|
|
561
|
+
ASSERT_EQ("foo", Get("foobar"));
|
|
562
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
563
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1);
|
|
564
|
+
ASSERT_EQ("bar", Get("barfoo"));
|
|
565
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
566
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
567
|
+
|
|
568
|
+
// Restart with only prefix bloom is allowed.
|
|
569
|
+
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
|
|
570
|
+
bbto.whole_key_filtering = false;
|
|
571
|
+
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
|
572
|
+
Reopen(options);
|
|
573
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
574
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
575
|
+
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
|
576
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
577
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
578
|
+
ASSERT_EQ("NOT_FOUND", Get("bar"));
|
|
579
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
|
580
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
581
|
+
ASSERT_EQ("foo", Get("foobar"));
|
|
582
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
583
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
584
|
+
ASSERT_EQ("bar", Get("barfoo"));
|
|
585
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
|
586
|
+
EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0);
|
|
587
|
+
uint64_t bloom_filter_useful_all_levels = 0;
|
|
588
|
+
for (auto& kv : (*(get_perf_context()->level_to_perf_context))) {
|
|
589
|
+
if (kv.second.bloom_filter_useful > 0) {
|
|
590
|
+
bloom_filter_useful_all_levels += kv.second.bloom_filter_useful;
|
|
577
591
|
}
|
|
578
|
-
ASSERT_EQ(12, bloom_filter_useful_all_levels);
|
|
579
|
-
get_perf_context()->Reset();
|
|
580
592
|
}
|
|
593
|
+
ASSERT_EQ(12, bloom_filter_useful_all_levels);
|
|
594
|
+
get_perf_context()->Reset();
|
|
581
595
|
}
|
|
582
596
|
|
|
583
|
-
|
|
597
|
+
INSTANTIATE_TEST_CASE_P(
|
|
598
|
+
DBBloomFilterTestWithPartitioningParam,
|
|
599
|
+
DBBloomFilterTestWithPartitioningParam,
|
|
600
|
+
::testing::Values(FilterPartitioning::kUnpartitionedFilter,
|
|
601
|
+
FilterPartitioning::kCoupledPartitionedFilter,
|
|
602
|
+
FilterPartitioning::kDecoupledPartitionedFilter));
|
|
603
|
+
|
|
604
|
+
TEST_P(DBBloomFilterTestWithFormatParams, BloomFilter) {
|
|
584
605
|
do {
|
|
585
606
|
Options options = CurrentOptions();
|
|
586
607
|
env_->count_random_reads_ = true;
|
|
@@ -588,20 +609,23 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
|
|
|
588
609
|
// ChangeCompactOptions() only changes compaction style, which does not
|
|
589
610
|
// trigger reset of table_factory
|
|
590
611
|
BlockBasedTableOptions table_options;
|
|
612
|
+
// When partitioned filters are coupled to index blocks, they tend to get
|
|
613
|
+
// extra fractional bits per key when rounding up to the next cache line
|
|
614
|
+
// size. Here we correct for that to get similar effective bits per key.
|
|
615
|
+
bits_per_key_ = table_options.decouple_partitioned_filters ? 10.5 : 10;
|
|
616
|
+
SetInTableOptions(&table_options);
|
|
591
617
|
table_options.no_block_cache = true;
|
|
592
|
-
table_options.filter_policy = Create(10, bfp_impl_);
|
|
593
618
|
table_options.optimize_filters_for_memory = false;
|
|
594
|
-
table_options.partition_filters = partition_filters_;
|
|
595
|
-
if (partition_filters_) {
|
|
596
|
-
table_options.index_type =
|
|
597
|
-
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
|
598
|
-
}
|
|
599
|
-
table_options.format_version = format_version_;
|
|
600
619
|
if (format_version_ >= 4) {
|
|
601
620
|
// value delta encoding challenged more with index interval > 1
|
|
602
621
|
table_options.index_block_restart_interval = 8;
|
|
603
622
|
}
|
|
604
|
-
|
|
623
|
+
// This test is rather sensitive to the actual filter partition block size,
|
|
624
|
+
// and keeping that consistent between coupled and uncoupled requires a
|
|
625
|
+
// different metadata block size for this example (where it controls index
|
|
626
|
+
// block size).
|
|
627
|
+
table_options.metadata_block_size =
|
|
628
|
+
table_options.decouple_partitioned_filters ? 320 : 32;
|
|
605
629
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
606
630
|
|
|
607
631
|
CreateAndReopenWithCF({"pikachu"}, options);
|
|
@@ -628,7 +652,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
|
|
|
628
652
|
int reads = env_->random_read_counter_.Read();
|
|
629
653
|
fprintf(stderr, "%d present => %d reads\n", N, reads);
|
|
630
654
|
ASSERT_GE(reads, N);
|
|
631
|
-
if (
|
|
655
|
+
if (PartitionFilters()) {
|
|
632
656
|
// Without block cache, we read an extra partition filter per each
|
|
633
657
|
// level*read and a partition index per each read
|
|
634
658
|
ASSERT_LE(reads, 4 * N + 2 * N / 100);
|
|
@@ -643,7 +667,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
|
|
|
643
667
|
}
|
|
644
668
|
reads = env_->random_read_counter_.Read();
|
|
645
669
|
fprintf(stderr, "%d missing => %d reads\n", N, reads);
|
|
646
|
-
if (
|
|
670
|
+
if (PartitionFilters()) {
|
|
647
671
|
// With partitioned filter we read one extra filter per level per each
|
|
648
672
|
// missed read.
|
|
649
673
|
ASSERT_LE(reads, 2 * N + 3 * N / 100);
|
|
@@ -658,7 +682,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
|
|
|
658
682
|
uint64_t nkeys = N + N / 100;
|
|
659
683
|
uint64_t filter_size = ParseUint64(props["filter_size"]);
|
|
660
684
|
EXPECT_LE(filter_size,
|
|
661
|
-
(
|
|
685
|
+
(PartitionFilters() ? 12 : 11) * nkeys / /*bits / byte*/ 8);
|
|
662
686
|
if (bfp_impl_ == kAutoRibbon) {
|
|
663
687
|
// Sometimes using Ribbon filter which is more space-efficient
|
|
664
688
|
EXPECT_GE(filter_size, 7 * nkeys / /*bits / byte*/ 8);
|
|
@@ -679,15 +703,20 @@ namespace {
|
|
|
679
703
|
|
|
680
704
|
class AlwaysTrueBitsBuilder : public FilterBitsBuilder {
|
|
681
705
|
public:
|
|
682
|
-
void AddKey(const Slice&) override {}
|
|
683
|
-
|
|
706
|
+
void AddKey(const Slice&) override { ++count_; }
|
|
707
|
+
void AddKeyAndAlt(const Slice&, const Slice&) override { count_ += 2; }
|
|
708
|
+
size_t EstimateEntriesAdded() override { return count_; }
|
|
684
709
|
Slice Finish(std::unique_ptr<const char[]>* /* buf */) override {
|
|
710
|
+
count_ = 0;
|
|
685
711
|
// Interpreted as "always true" filter (0 probes over 1 byte of
|
|
686
712
|
// payload, 5 bytes metadata)
|
|
687
713
|
return Slice("\0\0\0\0\0\0", 6);
|
|
688
714
|
}
|
|
689
715
|
using FilterBitsBuilder::Finish;
|
|
690
716
|
size_t ApproximateNumEntries(size_t) override { return SIZE_MAX; }
|
|
717
|
+
|
|
718
|
+
private:
|
|
719
|
+
size_t count_ = 0;
|
|
691
720
|
};
|
|
692
721
|
|
|
693
722
|
class AlwaysTrueFilterPolicy : public ReadOnlyBuiltinFilterPolicy {
|
|
@@ -709,7 +738,7 @@ class AlwaysTrueFilterPolicy : public ReadOnlyBuiltinFilterPolicy {
|
|
|
709
738
|
|
|
710
739
|
} // anonymous namespace
|
|
711
740
|
|
|
712
|
-
TEST_P(
|
|
741
|
+
TEST_P(DBBloomFilterTestWithFormatParams, SkipFilterOnEssentiallyZeroBpk) {
|
|
713
742
|
constexpr int maxKey = 10;
|
|
714
743
|
auto PutFn = [&]() {
|
|
715
744
|
int i;
|
|
@@ -740,12 +769,7 @@ TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
|
|
|
740
769
|
Options options = CurrentOptions();
|
|
741
770
|
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
|
742
771
|
BlockBasedTableOptions table_options;
|
|
743
|
-
table_options
|
|
744
|
-
if (partition_filters_) {
|
|
745
|
-
table_options.index_type =
|
|
746
|
-
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
|
747
|
-
}
|
|
748
|
-
table_options.format_version = format_version_;
|
|
772
|
+
SetInTableOptions(&table_options);
|
|
749
773
|
|
|
750
774
|
// Test 1: bits per key < 0.5 means skip filters -> no filter
|
|
751
775
|
// constructed or read.
|
|
@@ -819,27 +843,85 @@ TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
|
|
|
819
843
|
EXPECT_EQ(props["filter_size"], "0");
|
|
820
844
|
}
|
|
821
845
|
|
|
846
|
+
TEST_P(DBBloomFilterTestWithFormatParams, FilterBitsBuilderDedup) {
|
|
847
|
+
BlockBasedTableOptions table_options;
|
|
848
|
+
SetInTableOptions(&table_options);
|
|
849
|
+
FilterBuildingContext context{table_options};
|
|
850
|
+
std::unique_ptr<FilterBitsBuilder> builder{
|
|
851
|
+
table_options.filter_policy->GetBuilderWithContext(context)};
|
|
852
|
+
|
|
853
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 0U);
|
|
854
|
+
// Check for sufficient de-duplication between regular keys and alt keys
|
|
855
|
+
// (prefixes), keeping in mind that the key might equal its prefix.
|
|
856
|
+
|
|
857
|
+
builder->AddKey("abc");
|
|
858
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 1U);
|
|
859
|
+
builder->AddKeyAndAlt("abc1", "abc");
|
|
860
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 2U);
|
|
861
|
+
builder->AddKeyAndAlt("bcd", "bcd");
|
|
862
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 3U);
|
|
863
|
+
builder->AddKeyAndAlt("cde-1", "cde");
|
|
864
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 5U);
|
|
865
|
+
builder->AddKeyAndAlt("cde", "cde");
|
|
866
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 5U);
|
|
867
|
+
builder->AddKeyAndAlt("cde1", "cde");
|
|
868
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 6U);
|
|
869
|
+
builder->AddKeyAndAlt("def-1", "def");
|
|
870
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 8U);
|
|
871
|
+
builder->AddKeyAndAlt("def", "def");
|
|
872
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 8U);
|
|
873
|
+
builder->AddKey("def$$"); // Like not in extractor domain
|
|
874
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 9U);
|
|
875
|
+
builder->AddKey("def$$");
|
|
876
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 9U);
|
|
877
|
+
builder->AddKeyAndAlt("efg42", "efg");
|
|
878
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 11U);
|
|
879
|
+
builder->AddKeyAndAlt("efg", "efg"); // Like extra "alt" on a partition
|
|
880
|
+
ASSERT_EQ(builder->EstimateEntriesAdded(), 11U);
|
|
881
|
+
}
|
|
882
|
+
|
|
822
883
|
#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
|
823
884
|
INSTANTIATE_TEST_CASE_P(
|
|
824
885
|
FormatDef, DBBloomFilterTestDefFormatVersion,
|
|
825
886
|
::testing::Values(
|
|
826
|
-
std::make_tuple(kAutoBloom,
|
|
827
|
-
|
|
828
|
-
|
|
887
|
+
std::make_tuple(kAutoBloom,
|
|
888
|
+
FilterPartitioning::kCoupledPartitionedFilter,
|
|
889
|
+
test::kDefaultFormatVersion),
|
|
890
|
+
std::make_tuple(kAutoBloom,
|
|
891
|
+
FilterPartitioning::kDecoupledPartitionedFilter,
|
|
892
|
+
test::kDefaultFormatVersion),
|
|
893
|
+
std::make_tuple(kAutoBloom, FilterPartitioning::kUnpartitionedFilter,
|
|
894
|
+
test::kDefaultFormatVersion),
|
|
895
|
+
std::make_tuple(kAutoRibbon, FilterPartitioning::kUnpartitionedFilter,
|
|
896
|
+
test::kDefaultFormatVersion)));
|
|
829
897
|
|
|
830
898
|
INSTANTIATE_TEST_CASE_P(
|
|
831
|
-
FormatDef,
|
|
899
|
+
FormatDef, DBBloomFilterTestWithFormatParams,
|
|
832
900
|
::testing::Values(
|
|
833
|
-
std::make_tuple(kAutoBloom,
|
|
834
|
-
|
|
835
|
-
|
|
901
|
+
std::make_tuple(kAutoBloom,
|
|
902
|
+
FilterPartitioning::kCoupledPartitionedFilter,
|
|
903
|
+
test::kDefaultFormatVersion),
|
|
904
|
+
std::make_tuple(kAutoBloom,
|
|
905
|
+
FilterPartitioning::kDecoupledPartitionedFilter,
|
|
906
|
+
test::kDefaultFormatVersion),
|
|
907
|
+
std::make_tuple(kAutoBloom, FilterPartitioning::kUnpartitionedFilter,
|
|
908
|
+
test::kDefaultFormatVersion),
|
|
909
|
+
std::make_tuple(kAutoRibbon, FilterPartitioning::kUnpartitionedFilter,
|
|
910
|
+
test::kDefaultFormatVersion)));
|
|
836
911
|
|
|
837
912
|
INSTANTIATE_TEST_CASE_P(
|
|
838
|
-
FormatLatest,
|
|
839
|
-
::testing::Values(
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
913
|
+
FormatLatest, DBBloomFilterTestWithFormatParams,
|
|
914
|
+
::testing::Values(
|
|
915
|
+
std::make_tuple(kAutoBloom,
|
|
916
|
+
FilterPartitioning::kCoupledPartitionedFilter,
|
|
917
|
+
kLatestFormatVersion),
|
|
918
|
+
std::make_tuple(kAutoBloom,
|
|
919
|
+
FilterPartitioning::kDecoupledPartitionedFilter,
|
|
920
|
+
kLatestFormatVersion),
|
|
921
|
+
std::make_tuple(kAutoBloom, FilterPartitioning::kUnpartitionedFilter,
|
|
922
|
+
kLatestFormatVersion),
|
|
923
|
+
std::make_tuple(kAutoRibbon, FilterPartitioning::kUnpartitionedFilter,
|
|
924
|
+
kLatestFormatVersion)));
|
|
843
925
|
#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
|
844
926
|
|
|
845
927
|
TEST_F(DBBloomFilterTest, BloomFilterRate) {
|
|
@@ -957,18 +1039,17 @@ TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
|
|
|
957
1039
|
using FilterConstructionReserveMemoryHash = uint64_t;
|
|
958
1040
|
|
|
959
1041
|
class ChargeFilterConstructionTestWithParam
|
|
960
|
-
: public
|
|
961
|
-
public testing::WithParamInterface<
|
|
962
|
-
CacheEntryRoleOptions::Decision, std::string,
|
|
1042
|
+
: public DBBloomFilterTest,
|
|
1043
|
+
public testing::WithParamInterface<
|
|
1044
|
+
std::tuple<CacheEntryRoleOptions::Decision, std::string,
|
|
1045
|
+
FilterPartitioning, bool>> {
|
|
963
1046
|
public:
|
|
964
1047
|
ChargeFilterConstructionTestWithParam()
|
|
965
|
-
:
|
|
966
|
-
/*env_do_fsync=*/true),
|
|
967
|
-
num_key_(0),
|
|
1048
|
+
: num_key_(0),
|
|
968
1049
|
charge_filter_construction_(std::get<0>(GetParam())),
|
|
969
1050
|
policy_(std::get<1>(GetParam())),
|
|
970
|
-
partition_filters_(std::get<2>(GetParam())),
|
|
971
1051
|
detect_filter_construct_corruption_(std::get<3>(GetParam())) {
|
|
1052
|
+
filter_partitioning_ = std::get<2>(GetParam());
|
|
972
1053
|
if (charge_filter_construction_ ==
|
|
973
1054
|
CacheEntryRoleOptions::Decision::kDisabled ||
|
|
974
1055
|
policy_ == kLegacyBloom) {
|
|
@@ -976,7 +1057,7 @@ class ChargeFilterConstructionTestWithParam
|
|
|
976
1057
|
// cache charging happens instead of its accuracy. Therefore we don't
|
|
977
1058
|
// need many keys.
|
|
978
1059
|
num_key_ = 5;
|
|
979
|
-
} else if (
|
|
1060
|
+
} else if (PartitionFilters()) {
|
|
980
1061
|
// For PartitionFilter case, since we set
|
|
981
1062
|
// table_options.metadata_block_size big enough such that each partition
|
|
982
1063
|
// trigger at least 1 dummy entry reservation each for hash entries and
|
|
@@ -1013,6 +1094,7 @@ class ChargeFilterConstructionTestWithParam
|
|
|
1013
1094
|
|
|
1014
1095
|
BlockBasedTableOptions GetBlockBasedTableOptions() {
|
|
1015
1096
|
BlockBasedTableOptions table_options;
|
|
1097
|
+
SetInTableOptions(&table_options);
|
|
1016
1098
|
|
|
1017
1099
|
// We set cache capacity big enough to prevent cache full for convenience in
|
|
1018
1100
|
// calculation.
|
|
@@ -1022,10 +1104,7 @@ class ChargeFilterConstructionTestWithParam
|
|
|
1022
1104
|
{CacheEntryRole::kFilterConstruction,
|
|
1023
1105
|
{/*.charged = */ charge_filter_construction_}});
|
|
1024
1106
|
table_options.filter_policy = Create(10, policy_);
|
|
1025
|
-
table_options.partition_filters = partition_filters_;
|
|
1026
1107
|
if (table_options.partition_filters) {
|
|
1027
|
-
table_options.index_type =
|
|
1028
|
-
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
|
1029
1108
|
// We set table_options.metadata_block_size big enough so that each
|
|
1030
1109
|
// partition trigger at least 1 dummy entry insertion each for hash
|
|
1031
1110
|
// entries and final filter.
|
|
@@ -1054,8 +1133,6 @@ class ChargeFilterConstructionTestWithParam
|
|
|
1054
1133
|
|
|
1055
1134
|
std::string GetFilterPolicy() { return policy_; }
|
|
1056
1135
|
|
|
1057
|
-
bool PartitionFilters() { return partition_filters_; }
|
|
1058
|
-
|
|
1059
1136
|
std::shared_ptr<
|
|
1060
1137
|
TargetCacheChargeTrackingCache<CacheEntryRole::kFilterConstruction>>
|
|
1061
1138
|
GetCache() {
|
|
@@ -1066,7 +1143,6 @@ class ChargeFilterConstructionTestWithParam
|
|
|
1066
1143
|
std::size_t num_key_;
|
|
1067
1144
|
CacheEntryRoleOptions::Decision charge_filter_construction_;
|
|
1068
1145
|
std::string policy_;
|
|
1069
|
-
bool partition_filters_;
|
|
1070
1146
|
std::shared_ptr<
|
|
1071
1147
|
TargetCacheChargeTrackingCache<CacheEntryRole::kFilterConstruction>>
|
|
1072
1148
|
cache_;
|
|
@@ -1078,28 +1154,43 @@ INSTANTIATE_TEST_CASE_P(
|
|
|
1078
1154
|
ChargeFilterConstructionTestWithParam,
|
|
1079
1155
|
::testing::Values(
|
|
1080
1156
|
std::make_tuple(CacheEntryRoleOptions::Decision::kDisabled,
|
|
1081
|
-
kFastLocalBloom,
|
|
1157
|
+
kFastLocalBloom,
|
|
1158
|
+
FilterPartitioning::kUnpartitionedFilter, false),
|
|
1082
1159
|
|
|
1083
1160
|
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1084
|
-
kFastLocalBloom,
|
|
1161
|
+
kFastLocalBloom,
|
|
1162
|
+
FilterPartitioning::kUnpartitionedFilter, false),
|
|
1163
|
+
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1164
|
+
kFastLocalBloom,
|
|
1165
|
+
FilterPartitioning::kUnpartitionedFilter, true),
|
|
1085
1166
|
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1086
|
-
kFastLocalBloom,
|
|
1167
|
+
kFastLocalBloom,
|
|
1168
|
+
FilterPartitioning::kCoupledPartitionedFilter, false),
|
|
1087
1169
|
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1088
|
-
kFastLocalBloom,
|
|
1170
|
+
kFastLocalBloom,
|
|
1171
|
+
FilterPartitioning::kCoupledPartitionedFilter, true),
|
|
1089
1172
|
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1090
|
-
kFastLocalBloom,
|
|
1173
|
+
kFastLocalBloom,
|
|
1174
|
+
FilterPartitioning::kDecoupledPartitionedFilter, true),
|
|
1091
1175
|
|
|
1092
1176
|
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1093
|
-
kStandard128Ribbon,
|
|
1177
|
+
kStandard128Ribbon,
|
|
1178
|
+
FilterPartitioning::kUnpartitionedFilter, false),
|
|
1094
1179
|
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1095
|
-
kStandard128Ribbon,
|
|
1180
|
+
kStandard128Ribbon,
|
|
1181
|
+
FilterPartitioning::kUnpartitionedFilter, true),
|
|
1096
1182
|
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1097
|
-
kStandard128Ribbon,
|
|
1183
|
+
kStandard128Ribbon,
|
|
1184
|
+
FilterPartitioning::kCoupledPartitionedFilter, false),
|
|
1098
1185
|
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1099
|
-
kStandard128Ribbon,
|
|
1186
|
+
kStandard128Ribbon,
|
|
1187
|
+
FilterPartitioning::kCoupledPartitionedFilter, true),
|
|
1188
|
+
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled,
|
|
1189
|
+
kStandard128Ribbon,
|
|
1190
|
+
FilterPartitioning::kDecoupledPartitionedFilter, true),
|
|
1100
1191
|
|
|
1101
1192
|
std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, kLegacyBloom,
|
|
1102
|
-
|
|
1193
|
+
FilterPartitioning::kUnpartitionedFilter, false)));
|
|
1103
1194
|
|
|
1104
1195
|
// TODO: Speed up this test, and reduce disk space usage (~700MB)
|
|
1105
1196
|
// The current test inserts many keys (on the scale of dummy entry size)
|
|
@@ -1160,7 +1251,6 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
|
|
|
1160
1251
|
bool charge_filter_construction = (ChargeFilterConstructMemory() ==
|
|
1161
1252
|
CacheEntryRoleOptions::Decision::kEnabled);
|
|
1162
1253
|
std::string policy = GetFilterPolicy();
|
|
1163
|
-
bool partition_filters = PartitionFilters();
|
|
1164
1254
|
bool detect_filter_construct_corruption =
|
|
1165
1255
|
table_options.detect_filter_construct_corruption;
|
|
1166
1256
|
|
|
@@ -1247,7 +1337,7 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
|
|
|
1247
1337
|
* last longer since we release hash entries reservation later.
|
|
1248
1338
|
*
|
|
1249
1339
|
*/
|
|
1250
|
-
if (!
|
|
1340
|
+
if (!PartitionFilters()) {
|
|
1251
1341
|
EXPECT_EQ(filter_construction_cache_res_peaks.size(), 1)
|
|
1252
1342
|
<< "Filter construction cache charging should have only 1 peak in "
|
|
1253
1343
|
"case: kFastLocalBloom + FullFilter";
|
|
@@ -1369,7 +1459,7 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
|
|
|
1369
1459
|
* = hash entries + banding + final filter
|
|
1370
1460
|
*
|
|
1371
1461
|
*/
|
|
1372
|
-
if (!
|
|
1462
|
+
if (!PartitionFilters()) {
|
|
1373
1463
|
ASSERT_GE(
|
|
1374
1464
|
std::floor(
|
|
1375
1465
|
1.0 * predicted_final_filter_cache_res /
|
|
@@ -1444,29 +1534,21 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
|
|
|
1444
1534
|
}
|
|
1445
1535
|
|
|
1446
1536
|
class DBFilterConstructionCorruptionTestWithParam
|
|
1447
|
-
: public
|
|
1537
|
+
: public DBBloomFilterTest,
|
|
1448
1538
|
public testing::WithParamInterface<
|
|
1449
1539
|
std::tuple<bool /* detect_filter_construct_corruption */, std::string,
|
|
1450
|
-
|
|
1540
|
+
FilterPartitioning>> {
|
|
1451
1541
|
public:
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
table_options.detect_filter_construct_corruption = std::get<0>(GetParam());
|
|
1459
|
-
table_options.filter_policy = Create(10, std::get<1>(GetParam()));
|
|
1460
|
-
table_options.partition_filters = std::get<2>(GetParam());
|
|
1461
|
-
if (table_options.partition_filters) {
|
|
1462
|
-
table_options.index_type =
|
|
1463
|
-
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
|
1542
|
+
void SetInTableOptions(BlockBasedTableOptions* table_options) {
|
|
1543
|
+
table_options->detect_filter_construct_corruption = std::get<0>(GetParam());
|
|
1544
|
+
table_options->filter_policy = Create(10, std::get<1>(GetParam()));
|
|
1545
|
+
filter_partitioning_ = std::get<2>(GetParam());
|
|
1546
|
+
DBBloomFilterTest::SetInTableOptions(table_options);
|
|
1547
|
+
if (PartitionFilters()) {
|
|
1464
1548
|
// We set table_options.metadata_block_size small enough so we can
|
|
1465
1549
|
// trigger filter partitioning with GetNumKey() amount of keys
|
|
1466
|
-
table_options
|
|
1550
|
+
table_options->metadata_block_size = 10;
|
|
1467
1551
|
}
|
|
1468
|
-
|
|
1469
|
-
return table_options;
|
|
1470
1552
|
}
|
|
1471
1553
|
|
|
1472
1554
|
// Return an appropriate amount of keys for testing
|
|
@@ -1477,15 +1559,26 @@ class DBFilterConstructionCorruptionTestWithParam
|
|
|
1477
1559
|
INSTANTIATE_TEST_CASE_P(
|
|
1478
1560
|
DBFilterConstructionCorruptionTestWithParam,
|
|
1479
1561
|
DBFilterConstructionCorruptionTestWithParam,
|
|
1480
|
-
::testing::Values(
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1562
|
+
::testing::Values(
|
|
1563
|
+
std::make_tuple(false, kFastLocalBloom,
|
|
1564
|
+
FilterPartitioning::kUnpartitionedFilter),
|
|
1565
|
+
std::make_tuple(true, kFastLocalBloom,
|
|
1566
|
+
FilterPartitioning::kUnpartitionedFilter),
|
|
1567
|
+
std::make_tuple(true, kFastLocalBloom,
|
|
1568
|
+
FilterPartitioning::kCoupledPartitionedFilter),
|
|
1569
|
+
std::make_tuple(true, kFastLocalBloom,
|
|
1570
|
+
FilterPartitioning::kDecoupledPartitionedFilter),
|
|
1571
|
+
std::make_tuple(true, kStandard128Ribbon,
|
|
1572
|
+
FilterPartitioning::kUnpartitionedFilter),
|
|
1573
|
+
std::make_tuple(true, kStandard128Ribbon,
|
|
1574
|
+
FilterPartitioning::kCoupledPartitionedFilter),
|
|
1575
|
+
std::make_tuple(true, kStandard128Ribbon,
|
|
1576
|
+
FilterPartitioning::kDecoupledPartitionedFilter)));
|
|
1485
1577
|
|
|
1486
1578
|
TEST_P(DBFilterConstructionCorruptionTestWithParam, DetectCorruption) {
|
|
1487
1579
|
Options options = CurrentOptions();
|
|
1488
|
-
BlockBasedTableOptions table_options
|
|
1580
|
+
BlockBasedTableOptions table_options;
|
|
1581
|
+
SetInTableOptions(&table_options);
|
|
1489
1582
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
1490
1583
|
options.create_if_missing = true;
|
|
1491
1584
|
options.disable_auto_compactions = true;
|
|
@@ -1572,7 +1665,8 @@ TEST_P(DBFilterConstructionCorruptionTestWithParam, DetectCorruption) {
|
|
|
1572
1665
|
TEST_P(DBFilterConstructionCorruptionTestWithParam,
|
|
1573
1666
|
DynamicallyTurnOnAndOffDetectConstructCorruption) {
|
|
1574
1667
|
Options options = CurrentOptions();
|
|
1575
|
-
BlockBasedTableOptions table_options
|
|
1668
|
+
BlockBasedTableOptions table_options;
|
|
1669
|
+
SetInTableOptions(&table_options);
|
|
1576
1670
|
// We intend to turn on
|
|
1577
1671
|
// table_options.detect_filter_construct_corruption dynamically
|
|
1578
1672
|
// therefore we override this test parmater's value
|
|
@@ -2309,11 +2403,12 @@ static const std::string kPlainTable = "test_PlainTableBloom";
|
|
|
2309
2403
|
|
|
2310
2404
|
class BloomStatsTestWithParam
|
|
2311
2405
|
: public DBBloomFilterTest,
|
|
2312
|
-
public testing::WithParamInterface<
|
|
2406
|
+
public testing::WithParamInterface<
|
|
2407
|
+
std::tuple<std::string, FilterPartitioning>> {
|
|
2313
2408
|
public:
|
|
2314
2409
|
BloomStatsTestWithParam() {
|
|
2315
2410
|
bfp_impl_ = std::get<0>(GetParam());
|
|
2316
|
-
|
|
2411
|
+
filter_partitioning_ = std::get<1>(GetParam());
|
|
2317
2412
|
|
|
2318
2413
|
options_.create_if_missing = true;
|
|
2319
2414
|
options_.prefix_extractor.reset(
|
|
@@ -2321,13 +2416,13 @@ class BloomStatsTestWithParam
|
|
|
2321
2416
|
options_.memtable_prefix_bloom_size_ratio =
|
|
2322
2417
|
8.0 * 1024.0 / static_cast<double>(options_.write_buffer_size);
|
|
2323
2418
|
if (bfp_impl_ == kPlainTable) {
|
|
2324
|
-
assert(!
|
|
2419
|
+
assert(!PartitionFilters()); // not supported in plain table
|
|
2325
2420
|
PlainTableOptions table_options;
|
|
2326
2421
|
options_.table_factory.reset(NewPlainTableFactory(table_options));
|
|
2327
2422
|
} else {
|
|
2328
2423
|
BlockBasedTableOptions table_options;
|
|
2329
|
-
if (
|
|
2330
|
-
table_options.partition_filters =
|
|
2424
|
+
if (PartitionFilters()) {
|
|
2425
|
+
table_options.partition_filters = PartitionFilters();
|
|
2331
2426
|
table_options.index_type =
|
|
2332
2427
|
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
|
2333
2428
|
}
|
|
@@ -2350,7 +2445,6 @@ class BloomStatsTestWithParam
|
|
|
2350
2445
|
static void TearDownTestCase() {}
|
|
2351
2446
|
|
|
2352
2447
|
std::string bfp_impl_;
|
|
2353
|
-
bool partition_filters_;
|
|
2354
2448
|
Options options_;
|
|
2355
2449
|
};
|
|
2356
2450
|
|
|
@@ -2463,11 +2557,20 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
|
|
|
2463
2557
|
|
|
2464
2558
|
INSTANTIATE_TEST_CASE_P(
|
|
2465
2559
|
BloomStatsTestWithParam, BloomStatsTestWithParam,
|
|
2466
|
-
::testing::Values(
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2560
|
+
::testing::Values(
|
|
2561
|
+
std::make_tuple(kLegacyBloom, FilterPartitioning::kUnpartitionedFilter),
|
|
2562
|
+
std::make_tuple(kLegacyBloom,
|
|
2563
|
+
FilterPartitioning::kCoupledPartitionedFilter),
|
|
2564
|
+
std::make_tuple(kLegacyBloom,
|
|
2565
|
+
FilterPartitioning::kDecoupledPartitionedFilter),
|
|
2566
|
+
std::make_tuple(kFastLocalBloom,
|
|
2567
|
+
FilterPartitioning::kUnpartitionedFilter),
|
|
2568
|
+
std::make_tuple(kFastLocalBloom,
|
|
2569
|
+
FilterPartitioning::kCoupledPartitionedFilter),
|
|
2570
|
+
std::make_tuple(kFastLocalBloom,
|
|
2571
|
+
FilterPartitioning::kDecoupledPartitionedFilter),
|
|
2572
|
+
std::make_tuple(kPlainTable,
|
|
2573
|
+
FilterPartitioning::kUnpartitionedFilter)));
|
|
2471
2574
|
|
|
2472
2575
|
namespace {
|
|
2473
2576
|
void PrefixScanInit(DBBloomFilterTest* dbtest) {
|