datasketches 0.4.3 → 0.5.0

data/vendor/datasketches-cpp/filters/test/CMakeLists.txt

@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# separate executables for var_opt and ebpps sampling
+
+# BLOOM FILTER
+add_executable(bloom_filter_test)
+
+target_link_libraries(bloom_filter_test filters common_test_lib)
+
+set_target_properties(bloom_filter_test PROPERTIES
+  CXX_STANDARD_REQUIRED YES
+)
+
+file(TO_CMAKE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" FILTERS_TEST_BINARY_PATH)
+string(APPEND FILTERS_TEST_BINARY_PATH "/")
+target_compile_definitions(bloom_filter_test
+  PRIVATE
+    TEST_BINARY_INPUT_PATH="${FILTERS_TEST_BINARY_PATH}"
+)
+
+add_test(
+  NAME bloom_filter_test
+  COMMAND bloom_filter_test
+)
+
+target_sources(bloom_filter_test
+  PRIVATE
+  bit_array_ops_test.cpp
+  bloom_filter_test.cpp
+  bloom_filter_allocation_test.cpp
+)
+
+if (SERDE_COMPAT)
+target_sources(bloom_filter_test
+  PRIVATE
+    bloom_filter_deserialize_from_java_test.cpp
+)
+endif()
+
+if (GENERATE)
+target_sources(bloom_filter_test
+  PRIVATE
+    bloom_filter_serialize_for_java.cpp
+)
+endif()

data/vendor/datasketches-cpp/filters/test/bit_array_ops_test.cpp

@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <algorithm>
+
+#include "bit_array_ops.hpp"
+
+namespace datasketches {
+
+TEST_CASE("bit_array: basic operation", "[bit_array]") {
+  uint8_t* data = new uint8_t[16];
+  std::fill_n(data, 16, 0);
+  REQUIRE(bit_array_ops::get_and_set_bit(data, 1) == false);
+  REQUIRE(bit_array_ops::get_and_set_bit(data, 2) == false);
+  for (int i = 4; i < 64; i <<= 1) {
+    REQUIRE(bit_array_ops::get_and_set_bit(data, 64 + i) == false);
+  }
+
+  REQUIRE(bit_array_ops::count_num_bits_set(data, 16) == 6);
+  REQUIRE(bit_array_ops::get_bit(data, 68));
+
+  REQUIRE(bit_array_ops::get_bit(data, 5) == false);
+  bit_array_ops::set_bit(data, 5);
+  REQUIRE(bit_array_ops::get_and_set_bit(data, 5));
+  REQUIRE(bit_array_ops::count_num_bits_set(data, 16) == 7);
+
+  bit_array_ops::clear_bit(data, 5);
+  REQUIRE(bit_array_ops::get_bit(data, 5) == false);
+  REQUIRE(bit_array_ops::count_num_bits_set(data, 16) == 6);
+
+  std::fill(data, data + 16, 0);
+  REQUIRE(bit_array_ops::count_num_bits_set(data, 16) == 0);
+
+  bit_array_ops::set_bit(data, 35);
+  REQUIRE(bit_array_ops::get_and_set_bit(data, 35));
+  bit_array_ops::assign_bit(data, 35, false);
+  REQUIRE(bit_array_ops::get_bit(data, 35) == false);
+  bit_array_ops::assign_bit(data, 35, true);
+  REQUIRE(bit_array_ops::get_bit(data, 35));
+
+  delete [] data;
+}
+
+TEST_CASE("bit_array: inversion", "[bit_array]") {
+  size_t num_bits = 1024;
+  uint8_t* data = new uint8_t[num_bits / 8];
+  std::fill_n(data, num_bits / 8, 0);
+  for (size_t i = 0; i < num_bits; i += num_bits / 8) {
+    bit_array_ops::get_and_set_bit(data, i);
+  }
+  REQUIRE(bit_array_ops::get_bit(data, 0));
+
+  size_t num_bits_set = bit_array_ops::count_num_bits_set(data, num_bits / 8);
+  bit_array_ops::invert(data, num_bits / 8);
+  REQUIRE(bit_array_ops::count_num_bits_set(data, num_bits / 8) == num_bits - num_bits_set);
+  REQUIRE(bit_array_ops::get_bit(data, 0) == false);
+
+  delete [] data;
+}
+
+TEST_CASE("bit_array: intersection and union", "[bit_array]") {
+  uint8_t* data1 = new uint8_t[8];
+  uint8_t* data2 = new uint8_t[8];
+  uint8_t* data3 = new uint8_t[8];
+  std::fill_n(data1, 8, 0);
+  std::fill_n(data2, 8, 0);
+  std::fill_n(data3, 8, 0);
+
+  size_t n = 10;
+  for (size_t i = 0; i < n; ++i) {
+    bit_array_ops::get_and_set_bit(data1, i);
+    bit_array_ops::get_and_set_bit(data2, i + (n / 2));
+    bit_array_ops::get_and_set_bit(data3, 2 * i);
+  }
+  REQUIRE(bit_array_ops::count_num_bits_set(data1, 8) == n);
+  REQUIRE(bit_array_ops::count_num_bits_set(data2, 8) == n);
+  REQUIRE(bit_array_ops::count_num_bits_set(data3, 8) == n);
+
+  bit_array_ops::intersect(data1, data2, 8);
+  REQUIRE(bit_array_ops::count_num_bits_set(data1, 8) == n / 2);
+
+  bit_array_ops::union_with(data3, data2, 8);
+  REQUIRE(bit_array_ops::count_num_bits_set(data3, 8) == 3 * n / 2);
+
+  delete [] data1;
+  delete [] data2;
+  delete [] data3;
+}
+
+} // namespace datasketches

data/vendor/datasketches-cpp/filters/test/bloom_filter_allocation_test.cpp

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <sstream>
+
+#include <catch2/catch.hpp>
+
+#include "bloom_filter.hpp"
+#include "test_type.hpp"
+#include "test_allocator.hpp"
+
+namespace datasketches {
+
+using bloom_filter_test_alloc = bloom_filter_alloc<test_allocator<test_type>>;
+using alloc = test_allocator<test_type>;
+
+TEST_CASE("bloom filter allocation test", "[bloom_filter][test_type]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    int64_t num_items = 10000;
+    double fpp = 0.01;
+    uint64_t seed = bloom_filter_test_alloc::builder::generate_random_seed();
+    auto bf1 = bloom_filter_test_alloc::builder::create_by_accuracy(num_items,
+                                                                   fpp,
+                                                                   seed,
+                                                                   alloc(0));
+    for (int i = 0; i < num_items; ++i) {
+      if (num_items % 1 == 0) {
+        bf1.update(std::to_string(i));
+      } else {
+        bf1.update(i);
+      }
+    }
+    auto bytes1 = bf1.serialize(0);
+    auto bf2 = bloom_filter_test_alloc::deserialize(bytes1.data(), bytes1.size(), 0);
+
+    std::stringstream ss;
+    bf1.serialize(ss);
+    auto bf3 = bloom_filter_test_alloc::deserialize(ss, alloc(0));
+
+    bf3.reset();
+    for (int i = 0; i < num_items; ++i) {
+      bf1.update(-1.0 * i);
+    }
+
+    bf3.union_with(bf1);
+
+    auto bytes2 = bf3.serialize(0);
+    auto bf4 = bloom_filter_test_alloc::deserialize(bytes2.data(), bytes2.size(), 0);
+
+    auto bf5 = bloom_filter_test_alloc::wrap(bytes2.data(), bytes2.size(), 0);
+    auto bf6 = bloom_filter_test_alloc::writable_wrap(bytes2.data(), bytes2.size(), 0);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+}

data/vendor/datasketches-cpp/filters/test/bloom_filter_deserialize_from_java_test.cpp

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+
+#include "bloom_filter.hpp"
+
+namespace datasketches {
+
+// assume the binary sketches for this test have been generated by datasketches-java code
+// in the subdirectory called "java" in the root directory of this project
+static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../java/";
+
+TEST_CASE("bloom_filter", "[serde_compat]") {
+  const uint64_t n_arr[] = {0, 10000, 2000000, 30000000};
+  const uint16_t h_arr[] = {3, 5};
+  for (const uint64_t n: n_arr) {
+    for (const uint16_t num_hashes: h_arr) {
+      std::ifstream is;
+      is.exceptions(std::ios::failbit | std::ios::badbit);
+      is.open(testBinaryInputPath + "bf_n" + std::to_string(n) + "_h" + std::to_string(num_hashes) + "_java.sk", std::ios::binary);
+      auto bf = bloom_filter::deserialize(is);
+      REQUIRE(bf.is_empty() == (n == 0));
+      REQUIRE((bf.is_empty() || (bf.get_bits_used() > n / 10)));
+
+      for (uint64_t i = 0; i < n / 10; ++i) {
+        REQUIRE(bf.query(i));
+      }
+      if (n > 0) REQUIRE(bf.query(std::nan("1")));
+    }
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/filters/test/bloom_filter_serialize_for_java.cpp

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <algorithm>
+#include <fstream>
+
+#include "bloom_filter.hpp"
+
+namespace datasketches {
+
+TEST_CASE("bloom filter generate", "[serialize_for_java]") {
+  const uint64_t n_arr[] = {0, 10000, 2000000, 30000000};
+  const uint16_t h_arr[] = {3, 5};
+  for (const uint64_t n: n_arr) {
+    for (const uint16_t num_hashes: h_arr) {
+      const uint64_t config_bits = std::max(n, static_cast<uint64_t>(1000)); // so empty still has valid bit size
+      bloom_filter bf = bloom_filter::builder::create_by_size(config_bits, num_hashes);
+      for (uint64_t i = 0; i < n / 10; ++i) bf.update(i); // note: n / 10 items into n bits
+      if (n > 0) bf.update(std::nan("1")); // include a NaN if non-empty
+      REQUIRE(bf.is_empty() == (n == 0));
+      REQUIRE((bf.is_empty() || (bf.get_bits_used() > n / 10)));
+      std::ofstream os("bf_n" + std::to_string(n) + "_h" + std::to_string(num_hashes) + "_cpp.sk", std::ios::binary);
+      bf.serialize(os);
+    }
+  }
+}
+
+} /* namespace datasketches */