datasketches 0.4.2 → 0.4.4

data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp

@@ -417,6 +417,20 @@ public:
   virtual uint32_t get_num_retained() const;
   virtual uint16_t get_seed_hash() const;
 
+  /**
+   * Computes maximum serialized size in bytes
+   * @param lg_k nominal number of entries in the sketch
+   */
+  static size_t get_max_serialized_size_bytes(uint8_t lg_k);
+
+  /**
+   * Computes size in bytes required to serialize the current state of the sketch.
+   * Computing compressed size is expensive. It takes iterating over all retained hashes,
+   * and the actual serialization will have to look at them again.
+   * @param compressed if true compressed size is returned (if applicable)
+   */
+  size_t get_serialized_size_bytes(bool compressed = false) const;
+
   /**
    * This method serializes the sketch into a given stream in a binary form
    * @param os output stream
@@ -486,8 +500,11 @@ private:
   uint64_t theta_;
   std::vector<uint64_t, Allocator> entries_;
 
+  uint8_t get_preamble_longs(bool compressed) const;
   bool is_suitable_for_compression() const;
-  uint8_t compute_min_leading_zeros() const;
+  uint8_t compute_entry_bits() const;
+  uint8_t get_num_entries_bytes() const;
+  size_t get_compressed_serialized_size_bytes(uint8_t entry_bits, uint8_t num_entries_bytes) const;
   void serialize_version_4(std::ostream& os) const;
   vector_bytes serialize_version_4(unsigned header_size_bytes = 0) const;
 

data/vendor/datasketches-cpp/theta/include/theta_sketch_impl.hpp

@@ -24,11 +24,11 @@
 #include <vector>
 #include <stdexcept>
 
-#include "serde.hpp"
 #include "binomial_bounds.hpp"
 #include "theta_helpers.hpp"
 #include "count_zeros.hpp"
 #include "bit_packing.hpp"
+#include "memory_operations.hpp"
 
 namespace datasketches {
 
@@ -341,6 +341,39 @@ auto compact_theta_sketch_alloc<A>::end() const -> const_iterator {
 template<typename A>
 void compact_theta_sketch_alloc<A>::print_specifics(std::ostringstream&) const {}
 
+template<typename A>
+uint8_t compact_theta_sketch_alloc<A>::get_preamble_longs(bool compressed) const {
+  if (compressed) {
+    return this->is_estimation_mode() ? 2 : 1;
+  }
+  return this->is_estimation_mode() ? 3 : this->is_empty() || entries_.size() == 1 ? 1 : 2;
+}
+
+template<typename A>
+size_t compact_theta_sketch_alloc<A>::get_max_serialized_size_bytes(uint8_t lg_k) {
+  return sizeof(uint64_t) * (3 + update_theta_sketch_alloc<A>::theta_table::get_capacity(lg_k + 1, lg_k));
+}
+
+template<typename A>
+size_t compact_theta_sketch_alloc<A>::get_serialized_size_bytes(bool compressed) const {
+  if (compressed && is_suitable_for_compression()) {
+    return get_compressed_serialized_size_bytes(compute_entry_bits(), get_num_entries_bytes());
+  }
+  return sizeof(uint64_t) * get_preamble_longs(false) + sizeof(uint64_t) * entries_.size();
+}
+
+// store num_entries as whole bytes since whole-byte blocks will follow (most probably)
+template<typename A>
+uint8_t compact_theta_sketch_alloc<A>::get_num_entries_bytes() const {
+  return whole_bytes_to_hold_bits<uint8_t>(32 - count_leading_zeros_in_u32(static_cast<uint32_t>(entries_.size())));
+}
+
+template<typename A>
+size_t compact_theta_sketch_alloc<A>::get_compressed_serialized_size_bytes(uint8_t entry_bits, uint8_t num_entries_bytes) const {
+  const size_t compressed_bits = entry_bits * entries_.size();
+  return sizeof(uint64_t) * get_preamble_longs(true) + num_entries_bytes + whole_bytes_to_hold_bits(compressed_bits);
+}
+
 template<typename A>
 void compact_theta_sketch_alloc<A>::serialize(std::ostream& os) const {
   const uint8_t preamble_longs = this->is_estimation_mode() ? 3 : this->is_empty() || entries_.size() == 1 ? 1 : 2;
@@ -366,12 +399,10 @@ void compact_theta_sketch_alloc<A>::serialize(std::ostream& os) const {
 
 template<typename A>
 auto compact_theta_sketch_alloc<A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
-  const uint8_t preamble_longs = this->is_estimation_mode() ? 3 : this->is_empty() || entries_.size() == 1 ? 1 : 2;
-  const size_t size = header_size_bytes + sizeof(uint64_t) * preamble_longs
-      + sizeof(uint64_t) * entries_.size();
+  const size_t size = get_serialized_size_bytes() + header_size_bytes;
   vector_bytes bytes(size, 0, entries_.get_allocator());
   uint8_t* ptr = bytes.data() + header_size_bytes;
-
+  const uint8_t preamble_longs = get_preamble_longs(false);
   *ptr++ = preamble_longs;
   *ptr++ = UNCOMPRESSED_SERIAL_VERSION;
   *ptr++ = SKETCH_TYPE;
@@ -413,7 +444,7 @@ auto compact_theta_sketch_alloc<A>::serialize_compressed(unsigned header_size_by
 }
 
 template<typename A>
-uint8_t compact_theta_sketch_alloc<A>::compute_min_leading_zeros() const {
+uint8_t compact_theta_sketch_alloc<A>::compute_entry_bits() const {
   // compression is based on leading zeros in deltas between ordered hash values
   // assumes ordered sketch
   uint64_t previous = 0;
@@ -423,16 +454,14 @@ uint8_t compact_theta_sketch_alloc<A>::compute_min_leading_zeros() const {
     ored |= delta;
     previous = entry;
   }
-  return count_leading_zeros_in_u64(ored);
+  return 64 - count_leading_zeros_in_u64(ored);
 }
 
 template<typename A>
 void compact_theta_sketch_alloc<A>::serialize_version_4(std::ostream& os) const {
   const uint8_t preamble_longs = this->is_estimation_mode() ? 2 : 1;
-  const uint8_t entry_bits = 64 - compute_min_leading_zeros();
-
-  // store num_entries as whole bytes since whole-byte blocks will follow (most probably)
-  const uint8_t num_entries_bytes = whole_bytes_to_hold_bits<uint8_t>(32 - count_leading_zeros_in_u32(static_cast<uint32_t>(entries_.size())));
+  const uint8_t entry_bits = compute_entry_bits();
+  const uint8_t num_entries_bytes = get_num_entries_bytes();
 
   write(os, preamble_longs);
   write(os, COMPRESSED_SERIAL_VERSION);
@@ -477,25 +506,20 @@ void compact_theta_sketch_alloc<A>::serialize_version_4(std::ostream& os) const
       previous = entries_[i];
       offset = pack_bits(delta, entry_bits, ptr, offset);
     }
+    if (offset > 0) ++ptr;
     write(os, buffer.data(), ptr - buffer.data());
   }
 }
 
 template<typename A>
 auto compact_theta_sketch_alloc<A>::serialize_version_4(unsigned header_size_bytes) const -> vector_bytes {
-  const uint8_t preamble_longs = this->is_estimation_mode() ? 2 : 1;
-  const uint8_t entry_bits = 64 - compute_min_leading_zeros();
-  const size_t compressed_bits = entry_bits * entries_.size();
-
-  // store num_entries as whole bytes since whole-byte blocks will follow (most probably)
-  const uint8_t num_entries_bytes = whole_bytes_to_hold_bits<uint8_t>(32 - count_leading_zeros_in_u32(static_cast<uint32_t>(entries_.size())));
-
-  const size_t size = header_size_bytes + sizeof(uint64_t) * preamble_longs + num_entries_bytes
-      + whole_bytes_to_hold_bits(compressed_bits);
+  const uint8_t entry_bits = compute_entry_bits();
+  const uint8_t num_entries_bytes = get_num_entries_bytes();
+  const size_t size = get_compressed_serialized_size_bytes(entry_bits, num_entries_bytes) + header_size_bytes;
   vector_bytes bytes(size, 0, entries_.get_allocator());
   uint8_t* ptr = bytes.data() + header_size_bytes;
 
-  *ptr++ = preamble_longs;
+  *ptr++ = get_preamble_longs(true);
   *ptr++ = COMPRESSED_SERIAL_VERSION;
   *ptr++ = SKETCH_TYPE;
   *ptr++ = entry_bits;

data/vendor/datasketches-cpp/theta/test/bit_packing_test.cpp

@@ -29,50 +29,53 @@ namespace datasketches {
 static const uint64_t IGOLDEN64 = 0x9e3779b97f4a7c13ULL;
 
 TEST_CASE("pack unpack bits") {
-  for (uint8_t bits = 1; bits <= 63; ++bits) {
-    const uint64_t mask = (1ULL << bits) - 1;
-    std::vector<uint64_t> input(8, 0);
-    const uint64_t igolden64 = IGOLDEN64;
-    uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value
-    for (int i = 0; i < 8; ++i) {
-      input[i] = value & mask;
-      value += igolden64;
-    }
-    std::vector<uint8_t> bytes(8 * sizeof(uint64_t), 0);
-    uint8_t offset = 0;
-    uint8_t* ptr = bytes.data();
-    for (int i = 0; i < 8; ++i) {
-      offset = pack_bits(input[i], bits, ptr, offset);
-    }
+  uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value
+  for (int m = 0; m < 10000; ++m) {
+    for (uint8_t bits = 1; bits <= 63; ++bits) {
+      int n = 8;
+      const uint64_t mask = (1ULL << bits) - 1;
+      std::vector<uint64_t> input(n, 0);
+      for (int i = 0; i < n; ++i) {
+        input[i] = value & mask;
+        value += IGOLDEN64;
+      }
+      std::vector<uint8_t> bytes(n * sizeof(uint64_t), 0);
+      uint8_t offset = 0;
+      uint8_t* ptr = bytes.data();
+      for (int i = 0; i < n; ++i) {
+        offset = pack_bits(input[i], bits, ptr, offset);
+      }
 
-    std::vector<uint64_t> output(8, 0);
-    offset = 0;
-    const uint8_t* cptr = bytes.data();
-    for (int i = 0; i < 8; ++i) {
-      offset = unpack_bits(output[i], bits, cptr, offset);
-    }
-    for (int i = 0; i < 8; ++i) {
-      REQUIRE((input[i] & mask) == output[i]);
+      std::vector<uint64_t> output(n, 0);
+      offset = 0;
+      const uint8_t* cptr = bytes.data();
+      for (int i = 0; i < n; ++i) {
+        offset = unpack_bits(output[i], bits, cptr, offset);
+      }
+      for (int i = 0; i < n; ++i) {
+        REQUIRE(input[i] == output[i]);
+      }
     }
   }
 }
 
 TEST_CASE("pack unpack blocks") {
-  for (uint8_t bits = 1; bits <= 63; ++bits) {
-    const uint64_t mask = (1ULL << bits) - 1;
-    std::vector<uint64_t> input(8, 0);
-    const uint64_t igolden64 = IGOLDEN64;
-    uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value
-    for (int i = 0; i < 8; ++i) {
-      input[i] = value & mask;
-      value += igolden64;
-    }
-    std::vector<uint8_t> bytes(8 * sizeof(uint64_t), 0);
-    pack_bits_block8(input.data(), bytes.data(), bits);
-    std::vector<uint64_t> output(8, 0);
-    unpack_bits_block8(output.data(), bytes.data(), bits);
-    for (int i = 0; i < 8; ++i) {
-      REQUIRE((input[i] & mask) == output[i]);
+  uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value
+  for (int n = 0; n < 10000; ++n) {
+    for (uint8_t bits = 1; bits <= 63; ++bits) {
+      const uint64_t mask = (1ULL << bits) - 1;
+      std::vector<uint64_t> input(8, 0);
+      for (int i = 0; i < 8; ++i) {
+        input[i] = value & mask;
+        value += IGOLDEN64;
+      }
+      std::vector<uint8_t> bytes(bits, 0);
+      pack_bits_block8(input.data(), bytes.data(), bits);
+      std::vector<uint64_t> output(8, 0);
+      unpack_bits_block8(output.data(), bytes.data(), bits);
+      for (int i = 0; i < 8; ++i) {
+        REQUIRE(input[i] == output[i]);
+      }
     }
   }
 }

data/vendor/datasketches-cpp/theta/test/theta_sketch_deserialize_from_java_test.cpp

@@ -45,6 +45,23 @@ TEST_CASE("theta sketch", "[serde_compat]") {
   }
 }
 
+TEST_CASE("theta sketch compressed", "[serde_compat]") {
+  const unsigned n_arr[] = {10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "theta_compressed_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = compact_theta_sketch::deserialize(is);
+    REQUIRE(sketch.is_estimation_mode() == (n > 1000));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03));
+    for (const auto hash: sketch) {
+      REQUIRE(hash < sketch.get_theta64());
+    }
+    REQUIRE(sketch.is_ordered());
+    REQUIRE(std::is_sorted(sketch.begin(), sketch.end()));
+  }
+}
+
 TEST_CASE("theta sketch non-empty no entries", "[serde_compat]") {
   std::ifstream is;
   is.exceptions(std::ios::failbit | std::ios::badbit);

data/vendor/datasketches-cpp/theta/test/theta_sketch_serialize_for_java.cpp

@@ -43,7 +43,7 @@ TEST_CASE("theta sketch generate compressed", "[serialize_for_java]") {
     REQUIRE_FALSE(sketch.is_empty());
     REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03));
     std::ofstream os("theta_compressed_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
-    sketch.compact().serialize(os);
+    sketch.compact().serialize_compressed(os);
   }
 }
 

data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp

@@ -273,9 +273,11 @@ TEST_CASE("theta sketch: serialize deserialize stream and bytes equivalence", "[
   for (int i = 0; i < n; i++) update_sketch.update(i);
 
   std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
-  update_sketch.compact().serialize(s);
-  auto bytes = update_sketch.compact().serialize();
+  auto compact_sketch = update_sketch.compact();
+  compact_sketch.serialize(s);
+  auto bytes = compact_sketch.serialize();
   REQUIRE(bytes.size() == static_cast<size_t>(s.tellp()));
+  REQUIRE(bytes.size() == compact_sketch.get_serialized_size_bytes());
   for (size_t i = 0; i < bytes.size(); ++i) {
     REQUIRE(((char*)bytes.data())[i] == (char)s.get());
   }
@@ -515,12 +517,54 @@ TEST_CASE("theta sketch: wrap compact v2 estimation from java", "[theta_sketch]"
   }
 }
 
+TEST_CASE("theta sketch: serialize deserialize small compressed", "[theta_sketch]") {
+  auto update_sketch = update_theta_sketch::builder().build();
+  for (int i = 0; i < 10; i++) update_sketch.update(i);
+  auto compact_sketch = update_sketch.compact();
+
+  auto bytes = compact_sketch.serialize_compressed();
+  REQUIRE(bytes.size() == compact_sketch.get_serialized_size_bytes(true));
+  { // deserialize bytes
+    auto deserialized_sketch = compact_theta_sketch::deserialize(bytes.data(), bytes.size());
+    REQUIRE(deserialized_sketch.get_num_retained() == compact_sketch.get_num_retained());
+    REQUIRE(deserialized_sketch.get_theta() == compact_sketch.get_theta());
+    auto iter = deserialized_sketch.begin();
+    for (const auto key: compact_sketch) {
+      REQUIRE(*iter == key);
+      ++iter;
+    }
+  }
+  { // wrap bytes
+    auto wrapped_sketch = wrapped_compact_theta_sketch::wrap(bytes.data(), bytes.size());
+    REQUIRE(wrapped_sketch.get_num_retained() == compact_sketch.get_num_retained());
+    REQUIRE(wrapped_sketch.get_theta() == compact_sketch.get_theta());
+    auto iter = wrapped_sketch.begin();
+    for (const auto key: compact_sketch) {
+      REQUIRE(*iter == key);
+      ++iter;
+    }
+  }
+
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  compact_sketch.serialize_compressed(s);
+  REQUIRE(static_cast<size_t>(s.tellp()) == compact_sketch.get_serialized_size_bytes(true));
+  auto deserialized_sketch = compact_theta_sketch::deserialize(s);
+  REQUIRE(deserialized_sketch.get_num_retained() == compact_sketch.get_num_retained());
+  REQUIRE(deserialized_sketch.get_theta() == compact_sketch.get_theta());
+  auto iter = deserialized_sketch.begin();
+  for (const auto key: compact_sketch) {
+    REQUIRE(*iter == key);
+    ++iter;
+  }
+}
+
 TEST_CASE("theta sketch: serialize deserialize compressed", "[theta_sketch]") {
   auto update_sketch = update_theta_sketch::builder().build();
   for (int i = 0; i < 10000; i++) update_sketch.update(i);
   auto compact_sketch = update_sketch.compact();
 
   auto bytes = compact_sketch.serialize_compressed();
+  REQUIRE(bytes.size() == compact_sketch.get_serialized_size_bytes(true));
   { // deserialize bytes
     auto deserialized_sketch = compact_theta_sketch::deserialize(bytes.data(), bytes.size());
     REQUIRE(deserialized_sketch.get_num_retained() == compact_sketch.get_num_retained());
@@ -544,6 +588,7 @@ TEST_CASE("theta sketch: serialize deserialize compressed", "[theta_sketch]") {
 
   std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
   compact_sketch.serialize_compressed(s);
+  REQUIRE(static_cast<size_t>(s.tellp()) == compact_sketch.get_serialized_size_bytes(true));
   auto deserialized_sketch = compact_theta_sketch::deserialize(s);
   REQUIRE(deserialized_sketch.get_num_retained() == compact_sketch.get_num_retained());
   REQUIRE(deserialized_sketch.get_theta() == compact_sketch.get_theta());
@@ -554,4 +599,30 @@ TEST_CASE("theta sketch: serialize deserialize compressed", "[theta_sketch]") {
   }
 }
 
+// The sketch reaches capacity for the first time at 2 * K * 15/16,
+// but at that point it is still in exact mode, so the serialized size is not the maximum
+// (theta in not serialized in the exact mode).
+// So we need to catch the second time, but some updates will be ignored in the estimation mode,
+// so we update more than enough times keeping track of the maximum.
+// Potentially the exact number of updates to reach the peak can be figured out given this particular sequence,
+// but not assuming that might be even better (say, in case we change the load factor or hash function
+// or just out of principle not to rely on implementation details too much).
+TEST_CASE("max serialized size", "[theta_sketch]") {
+  const uint8_t lg_k = 10;
+  auto sketch = update_theta_sketch::builder().set_lg_k(lg_k).build();
+  int value = 0;
+
+  // this will go over the first peak, which is not the highest
+  for (int i = 0; i < (1 << lg_k) * 2; ++i) sketch.update(value++);
+
+  // this will to over the second peak keeping track of the max size
+  size_t max_size_bytes = 0;
+  for (int i = 0; i < (1 << lg_k) * 2; ++i) {
+    sketch.update(value++);
+    auto bytes = sketch.compact().serialize();
+    max_size_bytes = std::max(max_size_bytes, bytes.size());
+  }
+  REQUIRE(max_size_bytes == compact_theta_sketch::get_max_serialized_size_bytes(lg_k));
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/tuple/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(tuple
 )
 
 target_link_libraries(tuple INTERFACE common theta)
-target_compile_features(tuple INTERFACE cxx_std_11)
 
 install(TARGETS tuple
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/tuple/include/tuple_sketch.hpp

@@ -381,6 +381,15 @@ public:
    */
   compact_tuple_sketch<Summary, Allocator> compact(bool ordered = true) const;
 
+  /**
+   * Produces a Compact Tuple sketch from this sketch
+   * by applying a given predicate to each entry.
+   * @param predicate should return true for the entries to keep
+   * @return compact sketch with the entries retained according to the predicate
+   */
+  template<typename Predicate>
+  compact_tuple_sketch<Summary, Allocator> filter(const Predicate& predicate) const;
+
   virtual iterator begin();
   virtual iterator end();
   virtual const_iterator begin() const;
@@ -480,6 +489,25 @@ public:
   virtual uint32_t get_num_retained() const;
   virtual uint16_t get_seed_hash() const;
 
+  /**
+   * Produces a Compact Tuple sketch from this sketch
+   * by applying a given predicate to each entry.
+   * @param predicate should return true for the entries to keep
+   * @return compact sketch with the entries retained according to the predicate
+   */
+  template<typename Predicate>
+  compact_tuple_sketch filter(const Predicate& predicate) const;
+
+  /**
+   * Produces a Compact Tuple sketch from a given sketch (Update or Compact)
+   * by applying a given predicate to each entry.
+   * @param sketch input sketch
+   * @param predicate should return true for the entries to keep
+   * @return compact sketch with the entries retained according to the predicate
+   */
+  template<typename Sketch, typename Predicate>
+  static compact_tuple_sketch filter(const Sketch& sketch, const Predicate& predicate);
+
   /**
    * This method serializes the sketch into a given stream in a binary form
    * @param os output stream
@@ -579,7 +607,6 @@ protected:
   template<typename E, typename EK, typename P, typename S, typename CS, typename A> friend class theta_intersection_base;
   template<typename E, typename EK, typename CS, typename A> friend class theta_set_difference_base;
   compact_tuple_sketch(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries);
-
 };
 
 /// Tuple base builder

data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp

@@ -258,6 +258,12 @@ compact_tuple_sketch<S, A> update_tuple_sketch<S, U, P, A>::compact(bool ordered
   return compact_tuple_sketch<S, A>(*this, ordered);
 }
 
+template<typename S, typename U, typename P, typename A>
+template<typename Predicate>
+compact_tuple_sketch<S, A> update_tuple_sketch<S, U, P, A>::filter(const Predicate& predicate) const {
+  return compact_tuple_sketch<S, A>::filter(*this, predicate);
+}
+
 template<typename S, typename U, typename P, typename A>
 void update_tuple_sketch<S, U, P, A>::print_specifics(std::ostringstream& os) const {
   os << "   lg nominal size      : " << (int) map_.lg_nom_size_ << std::endl;
@@ -344,6 +350,33 @@ uint16_t compact_tuple_sketch<S, A>::get_seed_hash() const {
   return seed_hash_;
 }
 
+template<typename S, typename A>
+template<typename Predicate>
+compact_tuple_sketch<S, A> compact_tuple_sketch<S, A>::filter(const Predicate& predicate) const {
+  return filter(*this, predicate);
+}
+
+template<typename S, typename A>
+template<typename Sketch, typename Predicate>
+compact_tuple_sketch<S, A> compact_tuple_sketch<S, A>::filter(const Sketch& sketch, const Predicate& predicate) {
+  std::vector<Entry, AllocEntry> entries(sketch.get_allocator());
+  entries.reserve(sketch.get_num_retained());
+  std::copy_if(
+    sketch.begin(),
+    sketch.end(),
+    std::back_inserter(entries),
+    [&predicate](const Entry& e) {return predicate(e.second);}
+  );
+  entries.shrink_to_fit();
+  return compact_tuple_sketch(
+    !sketch.is_estimation_mode() && entries.empty(),
+    sketch.is_ordered(),
+    sketch.get_seed_hash(),
+    sketch.get_theta64(),
+    std::move(entries)
+  );
+}
+
 // implementation for fixed-size arithmetic types (integral and floating point)
 template<typename S, typename A>
 template<typename SD, typename SS, typename std::enable_if<std::is_arithmetic<SS>::value, int>::type>

data/vendor/datasketches-cpp/tuple/test/tuple_sketch_test.cpp

@@ -310,4 +310,65 @@ TEST_CASE("tuple sketch: float, update with different types of keys", "[tuple_sk
   REQUIRE(sketch.get_num_retained() == 3);
 }
 
+TEST_CASE("filter", "[tuple_sketch]") {
+  auto usk = update_tuple_sketch<int>::builder().build();
+
+  { // empty update sketch
+    auto sk = usk.filter([](int){return true;});
+    REQUIRE(sk.is_empty());
+    REQUIRE(sk.is_ordered());
+    REQUIRE(sk.get_num_retained() == 0);
+  }
+
+  { // empty compact sketch
+    auto sk = usk.compact().filter([](int){return true;});
+    REQUIRE(sk.is_empty());
+    REQUIRE(sk.is_ordered());
+    REQUIRE(sk.get_num_retained() == 0);
+  }
+
+  usk.update(1, 1);
+  usk.update(1, 1);
+  usk.update(2, 1);
+  usk.update(2, 1);
+  usk.update(3, 1);
+
+  { // exact mode update sketch
+    auto sk = usk.filter([](int v){return v > 1;});
+    REQUIRE_FALSE(sk.is_empty());
+    REQUIRE_FALSE(sk.is_ordered());
+    REQUIRE_FALSE(sk.is_estimation_mode());
+    REQUIRE(sk.get_num_retained() == 2);
+  }
+
+  { // exact mode compact sketch
+    auto sk = usk.compact().filter([](int v){return v > 1;});
+    REQUIRE_FALSE(sk.is_empty());
+    REQUIRE(sk.is_ordered());
+    REQUIRE_FALSE(sk.is_estimation_mode());
+    REQUIRE(sk.get_num_retained() == 2);
+  }
+
+  // only keys 1 and 2 had values of 2, which will become 3 after this update
+  // some entries are discarded in estimation mode, but these happen to survive
+  // the process is deterministic, so the test will always work
+  for (int i = 0; i < 10000; ++i) usk.update(i, 1);
+
+  { // estimation mode update sketch
+    auto sk = usk.filter([](int v){return v > 2;});
+    REQUIRE_FALSE(sk.is_empty());
+    REQUIRE_FALSE(sk.is_ordered());
+    REQUIRE(sk.is_estimation_mode());
+    REQUIRE(sk.get_num_retained() == 2);
+  }
+
+  { // estimation mode compact sketch
+    auto sk = usk.compact().filter([](int v){return v > 2;});
+    REQUIRE_FALSE(sk.is_empty());
+    REQUIRE(sk.is_ordered());
+    REQUIRE(sk.is_estimation_mode());
+    REQUIRE(sk.get_num_retained() == 2);
+  }
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/version.cfg.in

@@ -1 +1 @@
-5.0.2
+5.2.0

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: datasketches
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.4
 platform: ruby
 authors:
 - Andrew Kane
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-13 00:00:00.000000000 Z
+date: 2025-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -16,15 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: 4.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.1'
-description:
+        version: 4.3.3
 email: andrew@ankane.org
 executables: []
 extensions:
@@ -72,6 +70,7 @@ files:
 - vendor/datasketches-cpp/common/include/quantiles_sorted_view_impl.hpp
 - vendor/datasketches-cpp/common/include/serde.hpp
 - vendor/datasketches-cpp/common/include/version.hpp.in
+- vendor/datasketches-cpp/common/include/xxhash64.h
 - vendor/datasketches-cpp/common/test/CMakeLists.txt
 - vendor/datasketches-cpp/common/test/catch_runner.cpp
 - vendor/datasketches-cpp/common/test/integration_test.cpp
@@ -124,6 +123,17 @@ files:
 - vendor/datasketches-cpp/fi/test/frequent_items_sketch_serialize_for_java.cpp
 - vendor/datasketches-cpp/fi/test/frequent_items_sketch_test.cpp
 - vendor/datasketches-cpp/fi/test/reverse_purge_hash_map_test.cpp
+- vendor/datasketches-cpp/filters/CMakeLists.txt
+- vendor/datasketches-cpp/filters/include/bit_array_ops.hpp
+- vendor/datasketches-cpp/filters/include/bloom_filter.hpp
+- vendor/datasketches-cpp/filters/include/bloom_filter_builder_impl.hpp
+- vendor/datasketches-cpp/filters/include/bloom_filter_impl.hpp
+- vendor/datasketches-cpp/filters/test/CMakeLists.txt
+- vendor/datasketches-cpp/filters/test/bit_array_ops_test.cpp
+- vendor/datasketches-cpp/filters/test/bloom_filter_allocation_test.cpp
+- vendor/datasketches-cpp/filters/test/bloom_filter_deserialize_from_java_test.cpp
+- vendor/datasketches-cpp/filters/test/bloom_filter_serialize_for_java.cpp
+- vendor/datasketches-cpp/filters/test/bloom_filter_test.cpp
 - vendor/datasketches-cpp/hll/CMakeLists.txt
 - vendor/datasketches-cpp/hll/include/AuxHashMap-internal.hpp
 - vendor/datasketches-cpp/hll/include/AuxHashMap.hpp
@@ -231,6 +241,16 @@ files:
 - vendor/datasketches-cpp/sampling/test/var_opt_union_deserialize_from_java_test.cpp
 - vendor/datasketches-cpp/sampling/test/var_opt_union_serialize_for_java.cpp
 - vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp
+- vendor/datasketches-cpp/tdigest/CMakeLists.txt
+- vendor/datasketches-cpp/tdigest/include/tdigest.hpp
+- vendor/datasketches-cpp/tdigest/include/tdigest_impl.hpp
+- vendor/datasketches-cpp/tdigest/test/CMakeLists.txt
+- vendor/datasketches-cpp/tdigest/test/tdigest_custom_allocator_test.cpp
+- vendor/datasketches-cpp/tdigest/test/tdigest_deserialize_from_java_test.cpp
+- vendor/datasketches-cpp/tdigest/test/tdigest_ref_k100_n10000_double.sk
+- vendor/datasketches-cpp/tdigest/test/tdigest_ref_k100_n10000_float.sk
+- vendor/datasketches-cpp/tdigest/test/tdigest_serialize_for_java.cpp
+- vendor/datasketches-cpp/tdigest/test/tdigest_test.cpp
 - vendor/datasketches-cpp/theta/CMakeLists.txt
 - vendor/datasketches-cpp/theta/include/bit_packing.hpp
 - vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_sampled_sets.hpp
@@ -309,7 +329,6 @@ homepage: https://github.com/ankane/datasketches-ruby
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -324,8 +343,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Sketch data structures for Ruby
 test_files: []