@nxtedition/rocksdb 5.2.21 → 5.2.28

package/deps/rocksdb/rocksdb/table/block_based/block_based_filter_block.h

@@ -15,6 +15,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
+
 #include <memory>
 #include <string>
 #include <vector>
@@ -23,6 +24,7 @@
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
 #include "table/block_based/filter_block_reader_common.h"
+#include "table/block_based/filter_policy_internal.h"
 #include "table/format.h"
 #include "util/hash.h"
 
@@ -37,7 +39,8 @@ namespace ROCKSDB_NAMESPACE {
 class BlockBasedFilterBlockBuilder : public FilterBlockBuilder {
  public:
   BlockBasedFilterBlockBuilder(const SliceTransform* prefix_extractor,
-                               const BlockBasedTableOptions& table_opt);
+                               const BlockBasedTableOptions& table_opt,
+                               int bits_per_key);
   // No copying allowed
   BlockBasedFilterBlockBuilder(const BlockBasedFilterBlockBuilder&) = delete;
   void operator=(const BlockBasedFilterBlockBuilder&) = delete;
@@ -45,8 +48,13 @@ class BlockBasedFilterBlockBuilder : public FilterBlockBuilder {
   virtual bool IsBlockBased() override { return true; }
   virtual void StartBlock(uint64_t block_offset) override;
   virtual void Add(const Slice& key_without_ts) override;
-  virtual size_t NumAdded() const override { return num_added_; }
-  virtual Slice Finish(const BlockHandle& tmp, Status* status) override;
+  virtual bool IsEmpty() const override {
+    return start_.empty() && filter_offsets_.empty();
+  }
+  virtual size_t EstimateEntriesAdded() override;
+  virtual Slice Finish(
+      const BlockHandle& tmp, Status* status,
+      std::unique_ptr<const char[]>* filter_data = nullptr) override;
   using FilterBlockBuilder::Finish;
 
  private:
@@ -57,9 +65,9 @@ class BlockBasedFilterBlockBuilder : public FilterBlockBuilder {
   // important: all of these might point to invalid addresses
   // at the time of destruction of this filter block. destructor
   // should NOT dereference them.
-  const FilterPolicy* policy_;
   const SliceTransform* prefix_extractor_;
   bool whole_key_filtering_;
+  int bits_per_key_;
 
   size_t prev_prefix_start_;        // the position of the last appended prefix
                                     // to "entries_".
@@ -70,7 +78,7 @@ class BlockBasedFilterBlockBuilder : public FilterBlockBuilder {
   std::string result_;              // Filter data computed so far
   std::vector<Slice> tmp_entries_;  // policy_->CreateFilter() argument
   std::vector<uint32_t> filter_offsets_;
-  size_t num_added_;  // Number of keys added
+  uint64_t total_added_in_built_;  // Total keys added to filters built so far
 };
 
 // A FilterBlockReader is used to parse filter from SST table.

package/deps/rocksdb/rocksdb/table/block_based/block_based_filter_block_test.cc

@@ -19,221 +19,6 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-// For testing: emit an array with one hash value per key
-class TestHashFilter : public FilterPolicy {
- public:
-  const char* Name() const override { return "TestHashFilter"; }
-
-  void CreateFilter(const Slice* keys, int n, std::string* dst) const override {
-    for (int i = 0; i < n; i++) {
-      uint32_t h = Hash(keys[i].data(), keys[i].size(), 1);
-      PutFixed32(dst, h);
-    }
-  }
-
-  bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
-    uint32_t h = Hash(key.data(), key.size(), 1);
-    for (unsigned int i = 0; i + 4 <= filter.size(); i += 4) {
-      if (h == DecodeFixed32(filter.data() + i)) {
-        return true;
-      }
-    }
-    return false;
-  }
-};
-
-class MockBlockBasedTable : public BlockBasedTable {
- public:
-  explicit MockBlockBasedTable(Rep* rep)
-      : BlockBasedTable(rep, nullptr /* block_cache_tracer */) {}
-};
-
-class FilterBlockTest : public mock::MockBlockBasedTableTester,
-                        public testing::Test {
- public:
-  FilterBlockTest() : mock::MockBlockBasedTableTester(new TestHashFilter) {}
-};
-
-TEST_F(FilterBlockTest, EmptyBuilder) {
-  BlockBasedFilterBlockBuilder builder(nullptr, table_options_);
-  Slice slice(builder.Finish());
-  ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(slice));
-
-  CachableEntry<BlockContents> block(
-      new BlockContents(slice), nullptr /* cache */, nullptr /* cache_handle */,
-      true /* own_value */);
-
-  BlockBasedFilterBlockReader reader(table_.get(), std::move(block));
-  ASSERT_TRUE(reader.KeyMayMatch(
-      "foo", /*prefix_extractor=*/nullptr, /*block_offset=*/uint64_t{0},
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-  ASSERT_TRUE(reader.KeyMayMatch(
-      "foo", /*prefix_extractor=*/nullptr, /*block_offset=*/100000,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-}
-
-TEST_F(FilterBlockTest, SingleChunk) {
-  BlockBasedFilterBlockBuilder builder(nullptr, table_options_);
-  ASSERT_EQ(0, builder.NumAdded());
-  builder.StartBlock(100);
-  builder.Add("foo");
-  builder.Add("bar");
-  builder.Add("box");
-  builder.StartBlock(200);
-  builder.Add("box");
-  builder.StartBlock(300);
-  builder.Add("hello");
-  ASSERT_EQ(5, builder.NumAdded());
-  Slice slice(builder.Finish());
-
-  CachableEntry<BlockContents> block(
-      new BlockContents(slice), nullptr /* cache */, nullptr /* cache_handle */,
-      true /* own_value */);
-
-  BlockBasedFilterBlockReader reader(table_.get(), std::move(block));
-  ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/100,
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(reader.KeyMayMatch("bar", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/100,
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(reader.KeyMayMatch("box", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/100,
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(reader.KeyMayMatch("hello", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/100,
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/100,
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "missing", /*prefix_extractor=*/nullptr, /*block_offset=*/100,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "other", /*prefix_extractor=*/nullptr, /*block_offset=*/100,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-}
-
-TEST_F(FilterBlockTest, MultiChunk) {
-  BlockBasedFilterBlockBuilder builder(nullptr, table_options_);
-
-  // First filter
-  builder.StartBlock(0);
-  builder.Add("foo");
-  builder.StartBlock(2000);
-  builder.Add("bar");
-
-  // Second filter
-  builder.StartBlock(3100);
-  builder.Add("box");
-
-  // Third filter is empty
-
-  // Last filter
-  builder.StartBlock(9000);
-  builder.Add("box");
-  builder.Add("hello");
-
-  Slice slice(builder.Finish());
-
-  CachableEntry<BlockContents> block(
-      new BlockContents(slice), nullptr /* cache */, nullptr /* cache_handle */,
-      true /* own_value */);
-
-  BlockBasedFilterBlockReader reader(table_.get(), std::move(block));
-
-  // Check first filter
-  ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/uint64_t{0},
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(reader.KeyMayMatch("bar", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/2000,
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "box", /*prefix_extractor=*/nullptr, /*block_offset=*/uint64_t{0},
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "hello", /*prefix_extractor=*/nullptr, /*block_offset=*/uint64_t{0},
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-
-  // Check second filter
-  ASSERT_TRUE(reader.KeyMayMatch("box", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/3100,
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "foo", /*prefix_extractor=*/nullptr, /*block_offset=*/3100,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "bar", /*prefix_extractor=*/nullptr, /*block_offset=*/3100,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "hello", /*prefix_extractor=*/nullptr, /*block_offset=*/3100,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-
-  // Check third filter (empty)
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "foo", /*prefix_extractor=*/nullptr, /*block_offset=*/4100,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "bar", /*prefix_extractor=*/nullptr, /*block_offset=*/4100,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "box", /*prefix_extractor=*/nullptr, /*block_offset=*/4100,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "hello", /*prefix_extractor=*/nullptr, /*block_offset=*/4100,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-
-  // Check last filter
-  ASSERT_TRUE(reader.KeyMayMatch("box", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/9000,
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(reader.KeyMayMatch("hello", /*prefix_extractor=*/nullptr,
-                                 /*block_offset=*/9000,
-                                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
-                                 /*get_context=*/nullptr,
-                                 /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "foo", /*prefix_extractor=*/nullptr, /*block_offset=*/9000,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-  ASSERT_TRUE(!reader.KeyMayMatch(
-      "bar", /*prefix_extractor=*/nullptr, /*block_offset=*/9000,
-      /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
-      /*lookup_context=*/nullptr));
-}
-
 // Test for block based filter block
 // use new interface in FilterPolicy to create filter builder/reader
 class BlockBasedFilterBlockTest : public mock::MockBlockBasedTableTester,
@@ -245,7 +30,7 @@ class BlockBasedFilterBlockTest : public mock::MockBlockBasedTableTester,
 
 TEST_F(BlockBasedFilterBlockTest, BlockBasedEmptyBuilder) {
   FilterBlockBuilder* builder =
-      new BlockBasedFilterBlockBuilder(nullptr, table_options_);
+      new BlockBasedFilterBlockBuilder(nullptr, table_options_, 10);
   Slice slice(builder->Finish());
   ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(slice));
 
@@ -270,7 +55,7 @@ TEST_F(BlockBasedFilterBlockTest, BlockBasedEmptyBuilder) {
 
 TEST_F(BlockBasedFilterBlockTest, BlockBasedSingleChunk) {
   FilterBlockBuilder* builder =
-      new BlockBasedFilterBlockBuilder(nullptr, table_options_);
+      new BlockBasedFilterBlockBuilder(nullptr, table_options_, 10);
   builder->StartBlock(100);
   builder->Add("foo");
   builder->Add("bar");
@@ -322,7 +107,7 @@ TEST_F(BlockBasedFilterBlockTest, BlockBasedSingleChunk) {
 
 TEST_F(BlockBasedFilterBlockTest, BlockBasedMultiChunk) {
   FilterBlockBuilder* builder =
-      new BlockBasedFilterBlockBuilder(nullptr, table_options_);
+      new BlockBasedFilterBlockBuilder(nullptr, table_options_, 10);
 
   // First filter
   builder->StartBlock(0);