datasketches 0.3.1 → 0.4.0

data/vendor/datasketches-cpp/theta/include/theta_union.hpp

@@ -26,6 +26,16 @@
 
 namespace datasketches {
 
+// forward declaration
+template<typename A> class theta_union_alloc;
+
+// alias with default allocator for convenience
+using theta_union = theta_union_alloc<std::allocator<uint64_t>>;
+
+/**
+ * Theta Union.
+ * Computes union of Theta sketches. There is no constructor. Use builder instead.
+ */
 template<typename Allocator = std::allocator<uint64_t>>
 class theta_union_alloc {
 public:
@@ -35,6 +45,7 @@ public:
   using CompactSketch = compact_theta_sketch_alloc<Allocator>;
   using resize_factor = theta_constants::resize_factor;
 
+  // there is no payload in Theta sketch entry
   struct nop_policy {
     void operator()(uint64_t internal_entry, uint64_t incoming_entry) const {
       unused(internal_entry);
@@ -47,22 +58,20 @@ public:
   class builder;
 
   /**
-   * This method is to update the union with a given sketch
+   * Update the union with a given sketch
    * @param sketch to update the union with
    */
   template<typename FwdSketch>
   void update(FwdSketch&& sketch);
 
   /**
-   * This method produces a copy of the current state of the union as a compact sketch.
-   * @param ordered optional flag to specify if ordered sketch should be produced
+   * Produces a copy of the current state of the union as a compact sketch.
+   * @param ordered optional flag to specify if an ordered sketch should be produced
    * @return the result of the union
    */
   CompactSketch get_result(bool ordered = true) const;
 
-  /**
-   * Reset the union to the initial empty state
-   */
+  /// Reset the union to the initial empty state
   void reset();
 
 private:
@@ -72,21 +81,19 @@ private:
   theta_union_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p, uint64_t theta, uint64_t seed, const Allocator& allocator);
 };
 
+/// Theta union builder
 template<typename A>
 class theta_union_alloc<A>::builder: public theta_base_builder<builder, A> {
 public:
   builder(const A& allocator = A());
 
   /**
-   * This is to create an instance of the union with predefined parameters.
+   * Create an instance of the union with predefined parameters.
    * @return an instance of the union
    */
   theta_union_alloc<A> build() const;
 };
 
-// alias with default allocator for convenience
-using theta_union = theta_union_alloc<std::allocator<uint64_t>>;
-
 } /* namespace datasketches */
 
 #include "theta_union_impl.hpp"

data/vendor/datasketches-cpp/theta/include/theta_union_base_impl.hpp

@@ -41,8 +41,8 @@ void theta_union_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
   if (sketch.is_empty()) return;
   if (sketch.get_seed_hash() != compute_seed_hash(table_.seed_)) throw std::invalid_argument("seed hash mismatch");
   table_.is_empty_ = false;
-  if (sketch.get_theta64() < union_theta_) union_theta_ = sketch.get_theta64();
-  for (auto& entry: sketch) {
+  union_theta_ = std::min(union_theta_, sketch.get_theta64());
+  for (auto&& entry: sketch) {
     const uint64_t hash = EK()(entry);
     if (hash < union_theta_ && hash < table_.theta_) {
       auto result = table_.find(hash);
@@ -55,7 +55,7 @@ void theta_union_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
       if (sketch.is_ordered()) break; // early stop
     }
   }
-  if (table_.theta_ < union_theta_) union_theta_ = table_.theta_;
+  union_theta_ = std::min(union_theta_, table_.theta_);
 }
 
 template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
@@ -65,16 +65,16 @@ CS theta_union_base<EN, EK, P, S, CS, A>::get_result(bool ordered) const {
   entries.reserve(table_.num_entries_);
   uint64_t theta = std::min(union_theta_, table_.theta_);
   const uint32_t nominal_num = 1 << table_.lg_nom_size_;
-  if (union_theta_ >= theta && table_.num_entries_ <= nominal_num) {
+  if (union_theta_ >= table_.theta_) {
     std::copy_if(table_.begin(), table_.end(), std::back_inserter(entries), key_not_zero<EN, EK>());
   } else {
     std::copy_if(table_.begin(), table_.end(), std::back_inserter(entries), key_not_zero_less_than<uint64_t, EN, EK>(theta));
-    if (entries.size() > nominal_num) {
-      std::nth_element(entries.begin(), entries.begin() + nominal_num, entries.end(), comparator());
-      theta = EK()(entries[nominal_num]);
-      entries.erase(entries.begin() + nominal_num, entries.end());
-      entries.shrink_to_fit();
-    }
+  }
+  if (entries.size() > nominal_num) {
+    std::nth_element(entries.begin(), entries.begin() + nominal_num, entries.end(), comparator());
+    theta = EK()(entries[nominal_num]);
+    entries.erase(entries.begin() + nominal_num, entries.end());
+    entries.shrink_to_fit();
   }
   if (ordered) std::sort(entries.begin(), entries.end(), comparator());
   return CS(table_.is_empty_, ordered, compute_seed_hash(table_.seed_), theta, std::move(entries));

data/vendor/datasketches-cpp/theta/include/theta_union_impl.hpp

@@ -28,9 +28,9 @@ state_(lg_cur_size, lg_nom_size, rf, p, theta, seed, nop_policy(), allocator)
 {}
 
 template<typename A>
-template<typename SS>
-void theta_union_alloc<A>::update(SS&& sketch) {
-  state_.update(std::forward<SS>(sketch));
+template<typename FwdSketch>
+void theta_union_alloc<A>::update(FwdSketch&& sketch) {
+  state_.update(std::forward<FwdSketch>(sketch));
 }
 
 template<typename A>

data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp

@@ -23,8 +23,8 @@
 #include <vector>
 #include <climits>
 #include <cmath>
+#include <iterator>
 
-#include "common_defs.hpp"
 #include "MurmurHash3.h"
 #include "theta_comparators.hpp"
 #include "theta_constants.hpp"
@@ -91,13 +91,14 @@ struct theta_update_sketch_base {
   static void consolidate_non_empty(Entry* entries, size_t size, size_t num);
 };
 
-// builder
 
+/// Theta base builder
 template<typename Derived, typename Allocator>
 class theta_base_builder {
 public:
   /**
    * Creates and instance of the builder with default parameters.
+   * @param allocator instance of an Allocator to pass to created sketches
    */
   theta_base_builder(const Allocator& allocator);
 
@@ -185,14 +186,21 @@ static inline uint64_t compute_hash(const void* data, size_t length, uint64_t se
 // iterators
 
 template<typename Entry, typename ExtractKey>
-class theta_iterator: public std::iterator<std::input_iterator_tag, Entry> {
+class theta_iterator {
 public:
+  using iterator_category = std::input_iterator_tag;
+  using value_type = Entry;
+  using difference_type = std::ptrdiff_t;
+  using pointer = Entry*;
+  using reference = Entry&;
+
   theta_iterator(Entry* entries, uint32_t size, uint32_t index);
   theta_iterator& operator++();
   theta_iterator operator++(int);
   bool operator==(const theta_iterator& other) const;
   bool operator!=(const theta_iterator& other) const;
-  Entry& operator*() const;
+  reference operator*() const;
+  pointer operator->() const;
 
 private:
   Entry* entries_;
@@ -201,14 +209,21 @@ private:
 };
 
 template<typename Entry, typename ExtractKey>
-class theta_const_iterator: public std::iterator<std::input_iterator_tag, Entry> {
+class theta_const_iterator {
 public:
+  using iterator_category = std::input_iterator_tag;
+  using value_type = const Entry;
+  using difference_type = std::ptrdiff_t;
+  using pointer = const Entry*;
+  using reference = const Entry&;
+
   theta_const_iterator(const Entry* entries, uint32_t size, uint32_t index);
   theta_const_iterator& operator++();
   theta_const_iterator operator++(int);
   bool operator==(const theta_const_iterator& other) const;
   bool operator!=(const theta_const_iterator& other) const;
-  const Entry& operator*() const;
+  reference operator*() const;
+  pointer operator->() const;
 
 private:
   const Entry* entries_;

data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp

@@ -188,7 +188,7 @@ auto theta_update_sketch_base<EN, EK, A>::begin() const -> iterator {
 
 template<typename EN, typename EK, typename A>
 auto theta_update_sketch_base<EN, EK, A>::end() const -> iterator {
-  return &entries_[1ULL << lg_cur_size_];
+  return entries_ + (1ULL << lg_cur_size_);
 }
 
 template<typename EN, typename EK, typename A>
@@ -382,10 +382,15 @@ bool theta_iterator<Entry, ExtractKey>::operator==(const theta_iterator& other)
 }
 
 template<typename Entry, typename ExtractKey>
-auto theta_iterator<Entry, ExtractKey>::operator*() const -> Entry& {
+auto theta_iterator<Entry, ExtractKey>::operator*() const -> reference {
   return entries_[index_];
 }
 
+template<typename Entry, typename ExtractKey>
+auto theta_iterator<Entry, ExtractKey>::operator->() const -> pointer {
+  return entries_ + index_;
+}
+
 // const iterator
 
 template<typename Entry, typename ExtractKey>
@@ -419,10 +424,15 @@ bool theta_const_iterator<Entry, ExtractKey>::operator==(const theta_const_itera
 }
 
 template<typename Entry, typename ExtractKey>
-auto theta_const_iterator<Entry, ExtractKey>::operator*() const -> const Entry& {
+auto theta_const_iterator<Entry, ExtractKey>::operator*() const -> reference {
   return entries_[index_];
 }
 
+template<typename Entry, typename ExtractKey>
+auto theta_const_iterator<Entry, ExtractKey>::operator->() const -> pointer {
+  return entries_ + index_;
+}
+
 } /* namespace datasketches */
 
 #endif

data/vendor/datasketches-cpp/theta/test/CMakeLists.txt

@@ -20,7 +20,6 @@ add_executable(theta_test)
 target_link_libraries(theta_test theta common_test_lib)
 
 set_target_properties(theta_test PROPERTIES
-  CXX_STANDARD 11
   CXX_STANDARD_REQUIRED YES
 )
 
@@ -44,4 +43,19 @@ target_sources(theta_test
     theta_a_not_b_test.cpp
     theta_jaccard_similarity_test.cpp
     theta_setop_test.cpp
+    bit_packing_test.cpp
 )
+
+if (SERDE_COMPAT)
+target_sources(theta_test
+  PRIVATE
+    theta_sketch_deserialize_from_java_test.cpp
+)
+endif()
+
+if (GENERATE)
+target_sources(theta_test
+  PRIVATE
+    theta_sketch_serialize_for_java.cpp
+)
+endif()

data/vendor/datasketches-cpp/theta/test/bit_packing_test.cpp

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <bit_packing.hpp>
+
+namespace datasketches {
+
+// for every number of bits from 1 to 63
+// generate pseudo-random data, pack, unpack and compare
+
+// inverse golden ratio (0.618.. of max uint64_t)
+static const uint64_t IGOLDEN64 = 0x9e3779b97f4a7c13ULL;
+
+TEST_CASE("pack unpack bits") {
+  for (uint8_t bits = 1; bits <= 63; ++bits) {
+    const uint64_t mask = (1ULL << bits) - 1;
+    std::vector<uint64_t> input(8, 0);
+    const uint64_t igolden64 = IGOLDEN64;
+    uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value
+    for (int i = 0; i < 8; ++i) {
+      input[i] = value & mask;
+      value += igolden64;
+    }
+    std::vector<uint8_t> bytes(8 * sizeof(uint64_t), 0);
+    uint8_t offset = 0;
+    uint8_t* ptr = bytes.data();
+    for (int i = 0; i < 8; ++i) {
+      offset = pack_bits(input[i], bits, ptr, offset);
+    }
+
+    std::vector<uint64_t> output(8, 0);
+    offset = 0;
+    const uint8_t* cptr = bytes.data();
+    for (int i = 0; i < 8; ++i) {
+      offset = unpack_bits(output[i], bits, cptr, offset);
+    }
+    for (int i = 0; i < 8; ++i) {
+      REQUIRE((input[i] & mask) == output[i]);
+    }
+  }
+}
+
+TEST_CASE("pack unpack blocks") {
+  for (uint8_t bits = 1; bits <= 63; ++bits) {
+    const uint64_t mask = (1ULL << bits) - 1;
+    std::vector<uint64_t> input(8, 0);
+    const uint64_t igolden64 = IGOLDEN64;
+    uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value
+    for (int i = 0; i < 8; ++i) {
+      input[i] = value & mask;
+      value += igolden64;
+    }
+    std::vector<uint8_t> bytes(8 * sizeof(uint64_t), 0);
+    pack_bits_block8(input.data(), bytes.data(), bits);
+    std::vector<uint64_t> output(8, 0);
+    unpack_bits_block8(output.data(), bytes.data(), bits);
+    for (int i = 0; i < 8; ++i) {
+      REQUIRE((input[i] & mask) == output[i]);
+    }
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/theta/test/theta_sketch_deserialize_from_java_test.cpp

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <theta_sketch.hpp>
+
+namespace datasketches {
+
+// assume the binary sketches for this test have been generated by datasketches-java code
+// in the subdirectory called "java" in the root directory of this project
+static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../java/";
+
+TEST_CASE("theta sketch", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "theta_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = compact_theta_sketch::deserialize(is);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.is_estimation_mode() == (n > 1000));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03));
+    for (const auto hash: sketch) {
+      REQUIRE(hash < sketch.get_theta64());
+    }
+    REQUIRE(sketch.is_ordered());
+    REQUIRE(std::is_sorted(sketch.begin(), sketch.end()));
+  }
+}
+
+TEST_CASE("theta sketch non-empty no entries", "[serde_compat]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(testBinaryInputPath + "theta_non_empty_no_entries_java.sk", std::ios::binary);
+  const auto sketch = compact_theta_sketch::deserialize(is);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_num_retained() == 0);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/theta/test/theta_sketch_serialize_for_java.cpp

@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <theta_sketch.hpp>
+
+namespace datasketches {
+
+TEST_CASE("theta sketch generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    auto sketch = update_theta_sketch::builder().build();
+    for (unsigned i = 0; i < n; ++i) sketch.update(i);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03));
+    std::ofstream os("theta_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.compact().serialize(os);
+  }
+}
+
+TEST_CASE("theta sketch generate compressed", "[serialize_for_java]") {
+  const unsigned n_arr[] = {10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    auto sketch = update_theta_sketch::builder().build();
+    for (unsigned i = 0; i < n; ++i) sketch.update(i);
+    REQUIRE_FALSE(sketch.is_empty());
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03));
+    std::ofstream os("theta_compressed_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.compact().serialize(os);
+  }
+}
+
+TEST_CASE("theta sketch generate non-empty no entries", "[serialize_for_java]") {
+  auto sketch = update_theta_sketch::builder().set_p(0.01).build();
+  // here we rely on the fact that hash of 1 happens to be greater than 0.01 (when normalized)
+  // and therefore gets rejected
+  sketch.update(1);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_num_retained() == 0);
+  std::ofstream os("theta_non_empty_no_entries_cpp.sk", std::ios::binary);
+  sketch.compact().serialize(os);
+}
+
+} /* namespace datasketches */