datasketches 0.5.1 → 0.5.3

data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp

@@ -379,4 +379,39 @@ TEST_CASE("cpc sketch: max serialized size", "[cpc_sketch]") {
   REQUIRE(cpc_sketch::get_max_serialized_size_bytes(26) == static_cast<size_t>((0.6 * (1 << 26)) + 40));
 }
 
+TEST_CASE("cpc sketch: deserialize corrupt num_coupons", "[cpc_sketch]") {
+  cpc_sketch sketch(11);
+  for (int i = 0; i < 100; i++) sketch.update(i);
+  auto bytes = sketch.serialize();
+  REQUIRE((bytes[5] & (1 << 3)) != 0); // sparse flavor: table present
+  REQUIRE((bytes[5] & (1 << 4)) == 0); // and no window
+  uint32_t num_coupons;
+  std::memcpy(&num_coupons, bytes.data() + 8, sizeof(num_coupons));
+
+  // more pairs than the compressed table holds: decoder must not read past it
+  auto corrupt = bytes;
+  const uint32_t more_coupons = num_coupons + 50;
+  std::memcpy(corrupt.data() + 8, &more_coupons, sizeof(more_coupons));
+  REQUIRE_THROWS_AS(cpc_sketch::deserialize(corrupt.data(), corrupt.size()), std::out_of_range);
+
+  // more coupons than the k x 64 bit matrix can hold
+  corrupt = bytes;
+  const uint32_t too_many_coupons = 64 * 2048 + 1;
+  std::memcpy(corrupt.data() + 8, &too_many_coupons, sizeof(too_many_coupons));
+  REQUIRE_THROWS_AS(cpc_sketch::deserialize(corrupt.data(), corrupt.size()), std::invalid_argument);
+}
+
+TEST_CASE("cpc sketch: deserialize corrupt lg_k", "[cpc_sketch]") {
+  cpc_sketch sketch(11);
+  for (int i = 0; i < 100; i++) sketch.update(i);
+  auto bytes = sketch.serialize();
+  for (uint8_t lg_k: {0, 3, 27, 255}) {
+    bytes[3] = lg_k;
+    REQUIRE_THROWS_AS(cpc_sketch::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
+    std::stringstream s;
+    s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+    REQUIRE_THROWS_AS(cpc_sketch::deserialize(s), std::invalid_argument);
+  }
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/fi/include/frequent_items_sketch.hpp

@@ -44,6 +44,11 @@ enum frequent_items_error_type {
  * Based on Java implementation here:
  * https://github.com/apache/datasketches-java/blob/master/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java
  * @author Alexander Saydakov
+ *
+ * Sketch that may retain string values.
+ * For sketches containing strings, cross-language portability depends on
+ * using compatible string encodings. This class does not by itself enforce
+ * UTF-8 validity for all string inputs.
  */
 template<
   typename T,
@@ -74,6 +79,8 @@ public:
 
   /**
    * Update this sketch with an item and a positive weight (frequency count).
+   * If cross-language portability is required, callers should ensure that
+   * the input string uses a compatible encoding (valid UTF-8).
    * @param item for which the weight should be increased (lvalue)
    * @param weight the amount by which the weight of the item should be increased
    * A count of zero is a no-op, and a negative count will throw an exception.
@@ -82,6 +89,8 @@ public:
 
   /**
    * Update this sketch with an item and a positive weight (frequency count).
+   * If cross-language portability is required, callers should ensure that
+   * the input string uses a compatible encoding (valid UTF-8).
    * @param item for which the weight should be increased (rvalue)
    * @param weight the amount by which the weight of the item should be increased
    * A count of zero is a no-op, and a negative count will throw an exception.
@@ -91,6 +100,8 @@ public:
   /**
    * This function merges the other sketch into this one.
    * The other sketch may be of a different size.
+   * If sketches contain strings, callers are responsible for ensuring that
+   * both sketches were built using compatible string encodings.
    * @param other sketch to be merged into this (lvalue)
    */
   void merge(const frequent_items_sketch& other);
@@ -98,11 +109,22 @@ public:
   /**
    * This function merges the other sketch into this one.
    * The other sketch may be of a different size.
+   * If sketches contain strings, callers are responsible for ensuring that
+   * both sketches were built using compatible string encodings.
    * @param other sketch to be merged into this (rvalue)
    */
   void merge(frequent_items_sketch&& other);
 
   /**
+   * Resets this sketch to the empty state, as if newly constructed.
+   * The maximum map size, equality operator and allocator are retained.
+   * The internal hash map restarts at its minimum size.
+   */
+  void reset();
+
+  /**
+   * A sketch is empty if it has not been updated with any positive weight.
+   * A non-empty sketch may retain no items if a purge removed all of them.
    * @return true if this sketch is empty
    */
   bool is_empty() const;
@@ -298,13 +320,16 @@ private:
   static const uint8_t PREAMBLE_LONGS_EMPTY = 1;
   static const uint8_t PREAMBLE_LONGS_NONEMPTY = 4;
   static constexpr double EPSILON_FACTOR = 3.5;
-  // due to a mistake different bits were used in C++ and Java to indicate empty sketch
-  // therefore both are set and checked for compatibility with historical binary format
+  // Emptiness of a serialized image is determined by preamble longs (1 for empty, 4 otherwise).
+  // The flags byte is only cross-checked against it. Due to a mistake different bits were used
+  // in C++ and Java to indicate an empty sketch, therefore both are set for compatibility with
+  // the historical binary format, and either one is accepted on read. No other flag bits are defined.
   enum flags { IS_EMPTY_1 = 0, IS_EMPTY_2 = 2 };
   W total_weight;
   W offset;
   reverse_purge_hash_map<T, W, H, E, A> map;
-  static void check_preamble_longs(uint8_t preamble_longs, bool is_empty);
+  static void check_preamble_longs(uint8_t preamble_longs, uint8_t flags_byte);
+  static void check_total_weight(W total_weight);
   static void check_serial_version(uint8_t serial_version);
   static void check_family_id(uint8_t family_id);
   static void check_size(uint8_t lg_cur_size, uint8_t lg_max_size);

data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp

@@ -45,13 +45,13 @@ map(
   allocator
 )
 {
-  if (lg_start_map_size > lg_max_map_size) throw std::invalid_argument("starting size must not be greater than maximum size");
+  if (lg_start_map_size > lg_max_map_size) { throw std::invalid_argument("starting size must not be greater than maximum size"); }
 }
 
 template<typename T, typename W, typename H, typename E, typename A>
 void frequent_items_sketch<T, W, H, E, A>::update(const T& item, W weight) {
   check_weight(weight);
-  if (weight == 0) return;
+  if (weight == 0) { return; }
   total_weight += weight;
   offset += map.adjust_or_insert(item, weight);
 }
@@ -59,14 +59,14 @@ void frequent_items_sketch<T, W, H, E, A>::update(const T& item, W weight) {
 template<typename T, typename W, typename H, typename E, typename A>
 void frequent_items_sketch<T, W, H, E, A>::update(T&& item, W weight) {
   check_weight(weight);
-  if (weight == 0) return;
+  if (weight == 0) { return; }
   total_weight += weight;
   offset += map.adjust_or_insert(std::move(item), weight);
 }
 
 template<typename T, typename W, typename H, typename E, typename A>
 void frequent_items_sketch<T, W, H, E, A>::merge(const frequent_items_sketch& other) {
-  if (other.is_empty()) return;
+  if (other.is_empty()) { return; }
   const W merged_total_weight = total_weight + other.get_total_weight(); // for correction at the end
   for (auto it: other.map) {
     update(it.first, it.second);
@@ -77,7 +77,7 @@ void frequent_items_sketch<T, W, H, E, A>::merge(const frequent_items_sketch& ot
 
 template<typename T, typename W, typename H, typename E, typename A>
 void frequent_items_sketch<T, W, H, E, A>::merge(frequent_items_sketch&& other) {
-  if (other.is_empty()) return;
+  if (other.is_empty()) { return; }
   const W merged_total_weight = total_weight + other.get_total_weight(); // for correction at the end
   for (auto it: other.map) {
     update(std::move(it.first), it.second);
@@ -86,9 +86,18 @@ void frequent_items_sketch<T, W, H, E, A>::merge(frequent_items_sketch&& other)
   total_weight = merged_total_weight;
 }
 
+template<typename T, typename W, typename H, typename E, typename A>
+void frequent_items_sketch<T, W, H, E, A>::reset() {
+  map = reverse_purge_hash_map<T, W, H, E, A>(LG_MIN_MAP_SIZE, map.get_lg_max_size(), map.get_equal(), map.get_allocator());
+  total_weight = 0;
+  offset = 0;
+}
+
 template<typename T, typename W, typename H, typename E, typename A>
 bool frequent_items_sketch<T, W, H, E, A>::is_empty() const {
-  return map.get_num_active() == 0;
+  // a purge may clear all counters while offset and total_weight remain non-zero;
+  // emptiness must mean "no observations", not "no retained items"
+  return total_weight == 0;
 }
 
 template<typename T, typename W, typename H, typename E, typename A>
@@ -105,7 +114,7 @@ template<typename T, typename W, typename H, typename E, typename A>
 W frequent_items_sketch<T, W, H, E, A>::get_estimate(const T& item) const {
   // if item is tracked estimate = weight + offset, otherwise 0
   const W weight = map.get(item);
-  if (weight > 0) return weight + offset;
+  if (weight > 0) { return weight + offset; }
   return 0;
 }
 
@@ -210,7 +219,7 @@ void frequent_items_sketch<T, W, H, E, A>::serialize(std::ostream& os, const Ser
 template<typename T, typename W, typename H, typename E, typename A>
 template<typename SerDe>
 size_t frequent_items_sketch<T, W, H, E, A>::get_serialized_size_bytes(const SerDe& sd) const {
-  if (is_empty()) return PREAMBLE_LONGS_EMPTY * sizeof(uint64_t);
+  if (is_empty()) { return PREAMBLE_LONGS_EMPTY * sizeof(uint64_t); }
   size_t size = PREAMBLE_LONGS_NONEMPTY * sizeof(uint64_t) + map.get_num_active() * sizeof(W);
   for (auto it: map) size += sd.size_of_item(it.first);
   return size;
@@ -300,9 +309,8 @@ frequent_items_sketch<T, W, H, E, A> frequent_items_sketch<T, W, H, E, A>::deser
   const auto flags_byte = read<uint8_t>(is);
   read<uint16_t>(is); // unused
 
-  const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY_1)) | (flags_byte & (1 << flags::IS_EMPTY_2));
-
-  check_preamble_longs(preamble_longs, is_empty);
+  check_preamble_longs(preamble_longs, flags_byte);
+  const bool is_empty = preamble_longs == PREAMBLE_LONGS_EMPTY;
   check_serial_version(serial_version);
   check_family_id(family_id);
   check_size(lg_cur_size, lg_max_size);
@@ -313,6 +321,7 @@ frequent_items_sketch<T, W, H, E, A> frequent_items_sketch<T, W, H, E, A>::deser
     read<uint32_t>(is); // unused
     const auto total_weight = read<W>(is);
     const auto offset = read<W>(is);
+    check_total_weight(total_weight);
 
     // batch deserialization with intermediate array of items and weights
     using AllocW = typename std::allocator_traits<A>::template rebind_alloc<W>;
@@ -328,8 +337,7 @@ frequent_items_sketch<T, W, H, E, A> frequent_items_sketch<T, W, H, E, A>::deser
     sketch.total_weight = total_weight;
     sketch.offset = offset;
   }
-  if (!is.good())
-    throw std::runtime_error("error reading from std::istream"); 
+  if (!is.good()) { throw std::runtime_error("error reading from std::istream"); } 
   return sketch;
 }
 
@@ -354,9 +362,8 @@ frequent_items_sketch<T, W, H, E, A> frequent_items_sketch<T, W, H, E, A>::deser
   ptr += copy_from_mem(ptr, flags_byte);
   ptr += sizeof(uint16_t); // unused
 
-  const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY_1)) | (flags_byte & (1 << flags::IS_EMPTY_2));
-
-  check_preamble_longs(preamble_longs, is_empty);
+  check_preamble_longs(preamble_longs, flags_byte);
+  const bool is_empty = preamble_longs == PREAMBLE_LONGS_EMPTY;
   check_serial_version(serial_version);
   check_family_id(family_id);
   check_size(lg_cur_size, lg_max_size);
@@ -371,6 +378,7 @@ frequent_items_sketch<T, W, H, E, A> frequent_items_sketch<T, W, H, E, A>::deser
     ptr += copy_from_mem(ptr, total_weight);
     W offset;
     ptr += copy_from_mem(ptr, offset);
+    check_total_weight(total_weight);
 
     ensure_minimum_memory(size, ptr - base + (sizeof(W) * num_items));
     // batch deserialization with intermediate array of items and weights
@@ -393,15 +401,23 @@ frequent_items_sketch<T, W, H, E, A> frequent_items_sketch<T, W, H, E, A>::deser
 }
 
 template<typename T, typename W, typename H, typename E, typename A>
-void frequent_items_sketch<T, W, H, E, A>::check_preamble_longs(uint8_t preamble_longs, bool is_empty) {
-  if (is_empty) {
-    if (preamble_longs != PREAMBLE_LONGS_EMPTY) {
-      throw std::invalid_argument("Possible corruption: preamble longs of an empty sketch must be " + std::to_string(PREAMBLE_LONGS_EMPTY) + ": " + std::to_string(preamble_longs));
-    }
-  } else {
-    if (preamble_longs != PREAMBLE_LONGS_NONEMPTY) {
-      throw std::invalid_argument("Possible corruption: preamble longs of an non-empty sketch must be " + std::to_string(PREAMBLE_LONGS_NONEMPTY) + ": " + std::to_string(preamble_longs));
-    }
+void frequent_items_sketch<T, W, H, E, A>::check_preamble_longs(uint8_t preamble_longs, uint8_t flags_byte) {
+  if (preamble_longs != PREAMBLE_LONGS_EMPTY && preamble_longs != PREAMBLE_LONGS_NONEMPTY) {
+    throw std::invalid_argument("Possible corruption: preamble longs must be " + std::to_string(PREAMBLE_LONGS_EMPTY)
+        + " or " + std::to_string(PREAMBLE_LONGS_NONEMPTY) + ": " + std::to_string(preamble_longs));
+  }
+  const bool empty_flag = (flags_byte & ((1 << flags::IS_EMPTY_1) | (1 << flags::IS_EMPTY_2))) != 0;
+  if (empty_flag != (preamble_longs == PREAMBLE_LONGS_EMPTY)) {
+    throw std::invalid_argument("Possible corruption: empty flag does not match preamble longs: flags "
+        + std::to_string(flags_byte) + ", preamble longs " + std::to_string(preamble_longs));
+  }
+}
+
+template<typename T, typename W, typename H, typename E, typename A>
+void frequent_items_sketch<T, W, H, E, A>::check_total_weight(W total_weight) {
+  // written as !(x > 0) to also reject NaN
+  if (!(total_weight > 0)) {
+    throw std::invalid_argument("Possible corruption: total weight of a non-empty sketch must be positive");
   }
 }
 

data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map.hpp

@@ -20,8 +20,9 @@
 #ifndef REVERSE_PURGE_HASH_MAP_HPP_
 #define REVERSE_PURGE_HASH_MAP_HPP_
 
-#include <memory>
+#include <cstdint>
 #include <iterator>
+#include <memory>
 
 namespace datasketches {
 
@@ -61,6 +62,7 @@ public:
   uint32_t get_capacity() const;
   uint32_t get_num_active() const;
   const A& get_allocator() const;
+  const E& get_equal() const;
 
   class iterator;
   iterator begin() const;

data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map_impl.hpp

@@ -74,7 +74,7 @@ states_(nullptr)
       if (other.states_[i] > 0) {
         new (&keys_[i]) K(other.keys_[i]);
         values_[i] = other.values_[i];
-         if (--num == 0) break;
+         if (--num == 0) { break; }
       }
     }
   }
@@ -105,7 +105,7 @@ reverse_purge_hash_map<K, V, H, E, A>::~reverse_purge_hash_map() {
     for (uint32_t i = 0; i < size; i++) {
       if (is_active(i)) {
         keys_[i].~K();
-        if (--num_active_ == 0) break;
+        if (--num_active_ == 0) { break; }
       }
     }
   }
@@ -166,7 +166,7 @@ V reverse_purge_hash_map<K, V, H, E, A>::get(const K& key) const {
   const uint32_t mask = (1 << lg_cur_size_) - 1;
   uint32_t probe = fmix64(H()(key)) & mask;
   while (is_active(probe)) {
-    if (E()(keys_[probe], key)) return values_[probe];
+    if (E()(keys_[probe], key)) { return values_[probe]; }
     probe = (probe + 1) & mask;
   }
   return 0;
@@ -197,6 +197,11 @@ const A& reverse_purge_hash_map<K, V, H, E, A>::get_allocator() const {
   return allocator_;
 }
 
+template<typename K, typename V, typename H, typename E, typename A>
+const E& reverse_purge_hash_map<K, V, H, E, A>::get_equal() const {
+  return equal_;
+}
+
 template<typename K, typename V, typename H, typename E, typename A>
 typename reverse_purge_hash_map<K, V, H, E, A>::iterator reverse_purge_hash_map<K, V, H, E, A>::begin() const {
   const uint32_t size = 1 << lg_cur_size_;
@@ -271,7 +276,7 @@ void reverse_purge_hash_map<K, V, H, E, A>::hash_delete(uint32_t delete_index) {
     probe = (probe + 1) & mask;
     drift++;
     // only used for theoretical analysis
-    if (drift >= DRIFT_LIMIT) throw std::logic_error("drift: " + std::to_string(drift) + " >= DRIFT_LIMIT");
+    if (drift >= DRIFT_LIMIT) { throw std::logic_error("drift: " + std::to_string(drift) + " >= DRIFT_LIMIT"); }
   }
 }
 
@@ -289,7 +294,7 @@ uint32_t reverse_purge_hash_map<K, V, H, E, A>::internal_adjust_or_insert(const
     index = (index + 1) & mask;
     drift++;
     // only used for theoretical analysis
-    if (drift >= DRIFT_LIMIT) throw std::logic_error("drift limit reached");
+    if (drift >= DRIFT_LIMIT) { throw std::logic_error("drift limit reached"); }
   }
   // adding the key and value to the table
   if (num_active_ > get_capacity()) {

data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_serialize_for_java.cpp

@@ -57,6 +57,30 @@ TEST_CASE("frequent strings sketch generate", "[serialize_for_java]") {
   }
 }
 
+// lg_max_map_size=8 -> capacity 192; the 193rd distinct item triggers a purge
+// whose median (1) removes every counter: non-empty with no retained items
+TEST_CASE("frequent longs sketch purged to zero items", "[serialize_for_java]") {
+  frequent_items_sketch<long> sketch(8);
+  for (long i = 1; i <= 193; ++i) sketch.update(i);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_num_active_items() == 0);
+  REQUIRE(sketch.get_total_weight() == 193);
+  REQUIRE(sketch.get_maximum_error() == 1);
+  std::ofstream os("frequent_long_purged_cpp.sk", std::ios::binary);
+  sketch.serialize(os);
+}
+
+TEST_CASE("frequent strings sketch purged to zero items", "[serialize_for_java]") {
+  frequent_items_sketch<std::string> sketch(8);
+  for (unsigned i = 1; i <= 193; ++i) sketch.update(std::to_string(i));
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_num_active_items() == 0);
+  REQUIRE(sketch.get_total_weight() == 193);
+  REQUIRE(sketch.get_maximum_error() == 1);
+  std::ofstream os("frequent_string_purged_cpp.sk", std::ios::binary);
+  sketch.serialize(os);
+}
+
 TEST_CASE("frequent strings sketch ascii", "[serialize_for_java]") {
   frequent_items_sketch<std::string> sketch(6);
   sketch.update("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1);

data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_test.cpp

@@ -131,6 +131,122 @@ TEST_CASE("frequent items: estimation mode", "[frequent_items_sketch]") {
   REQUIRE(12 >= items.size()); // but not more than 12 items
 }
 
+TEST_CASE("frequent items: purge clearing all counters is not empty", "[frequent_items_sketch]") {
+  // lg_k=8 -> capacity 192; the 193rd distinct item triggers a purge whose
+  // median (1) wipes every counter. The sketch still holds observations:
+  // offset=1, total_weight=193. It must not be considered empty, and must
+  // survive serialization and merge without losing weight or error.
+  frequent_items_sketch<uint64_t> sketch(8);
+  for (uint64_t i = 0; i < 193; ++i) sketch.update(i);
+
+  REQUIRE(sketch.get_num_active_items() == 0); // purge wiped all counters
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_total_weight() == 193);
+  REQUIRE(sketch.get_maximum_error() == 1);
+
+  auto bytes = sketch.serialize();
+  auto sketch2 = frequent_items_sketch<uint64_t>::deserialize(bytes.data(), bytes.size());
+  REQUIRE_FALSE(sketch2.is_empty());
+  REQUIRE(sketch2.get_total_weight() == 193);
+  REQUIRE(sketch2.get_maximum_error() == 1);
+
+  frequent_items_sketch<uint64_t> sketch3(8);
+  sketch3.update(999999);
+  sketch3.merge(sketch);
+  REQUIRE(sketch3.get_total_weight() == 194);
+  REQUIRE(sketch3.get_maximum_error() == 1);
+}
+
+TEST_CASE("frequent items: purge clearing all counters serialized form", "[frequent_items_sketch]") {
+  frequent_items_sketch<uint64_t> sketch(8);
+  for (uint64_t i = 0; i < 193; ++i) sketch.update(i);
+  REQUIRE(sketch.get_num_active_items() == 0);
+
+  // full preamble with no items
+  auto bytes = sketch.serialize();
+  REQUIRE(bytes.size() == 32);
+  REQUIRE(bytes[0] == 4); // preamble longs
+  REQUIRE(bytes[5] == 0); // flags
+
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  sketch.serialize(s);
+  auto sketch2 = frequent_items_sketch<uint64_t>::deserialize(s);
+  REQUIRE_FALSE(sketch2.is_empty());
+  REQUIRE(sketch2.get_num_active_items() == 0);
+  REQUIRE(sketch2.get_total_weight() == 193);
+  REQUIRE(sketch2.get_maximum_error() == 1);
+}
+
+TEST_CASE("frequent items: reset", "[frequent_items_sketch]") {
+  frequent_items_sketch<uint64_t> sketch(8);
+  for (uint64_t i = 0; i < 1000; ++i) sketch.update(i % 300, i % 7 + 1);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_maximum_error() > 0);
+  const double epsilon = sketch.get_epsilon();
+
+  sketch.reset();
+  REQUIRE(sketch.is_empty());
+  REQUIRE(sketch.get_num_active_items() == 0);
+  REQUIRE(sketch.get_total_weight() == 0);
+  REQUIRE(sketch.get_maximum_error() == 0);
+  REQUIRE(sketch.get_epsilon() == epsilon); // max map size retained
+  REQUIRE(sketch.get_serialized_size_bytes() == 8);
+
+  // same behavior as a newly constructed sketch
+  frequent_items_sketch<uint64_t> fresh(8);
+  for (uint64_t i = 0; i < 193; ++i) {
+    sketch.update(i);
+    fresh.update(i);
+  }
+  REQUIRE(sketch.serialize() == fresh.serialize());
+}
+
+TEST_CASE("frequent items: empty image with either legacy empty flag", "[frequent_items_sketch]") {
+  frequent_items_sketch<uint64_t> sketch(8);
+  auto bytes = sketch.serialize();
+  REQUIRE(bytes.size() == 8);
+  REQUIRE(bytes[5] == 5); // both empty bits written
+  for (uint8_t flags: {1, 4, 5}) {
+    bytes[5] = flags;
+    auto sketch2 = frequent_items_sketch<uint64_t>::deserialize(bytes.data(), bytes.size());
+    REQUIRE(sketch2.is_empty());
+  }
+}
+
+TEST_CASE("frequent items: corrupt preamble", "[frequent_items_sketch]") {
+  frequent_items_sketch<uint64_t> empty_sketch(8);
+  frequent_items_sketch<uint64_t> sketch(8);
+  sketch.update(1);
+
+  SECTION("invalid preamble longs") {
+    auto bytes = sketch.serialize();
+    bytes[0] = 2;
+    REQUIRE_THROWS_AS(frequent_items_sketch<uint64_t>::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
+  }
+  SECTION("empty preamble longs, not empty flag") {
+    auto bytes = empty_sketch.serialize();
+    bytes[5] = 0;
+    REQUIRE_THROWS_AS(frequent_items_sketch<uint64_t>::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
+  }
+  SECTION("full preamble longs, empty flag") {
+    auto bytes = sketch.serialize();
+    bytes[5] = 5;
+    REQUIRE_THROWS_AS(frequent_items_sketch<uint64_t>::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
+  }
+  SECTION("full preamble longs, zero total weight, bytes") {
+    auto bytes = sketch.serialize();
+    for (size_t i = 16; i < 24; ++i) bytes[i] = 0;
+    REQUIRE_THROWS_AS(frequent_items_sketch<uint64_t>::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
+  }
+  SECTION("full preamble longs, zero total weight, stream") {
+    auto bytes = sketch.serialize();
+    for (size_t i = 16; i < 24; ++i) bytes[i] = 0;
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+    REQUIRE_THROWS_AS(frequent_items_sketch<uint64_t>::deserialize(s), std::invalid_argument);
+  }
+}
+
 TEST_CASE("frequent items: merge exact mode", "[frequent_items_sketch]") {
   frequent_items_sketch<int> sketch1(3);
   sketch1.update(1);

data/vendor/datasketches-cpp/filters/include/bloom_filter.hpp

@@ -624,7 +624,7 @@ private:
   uint64_t capacity_bits_;
   uint64_t num_bits_set_;
   uint8_t* bit_array_;  // data backing bit_array_, regardless of ownership
-  uint8_t* memory_; // if wrapped, pointer to the start of the filter, otheriwse nullptr
+  uint8_t* memory_; // if wrapped, pointer to the start of the filter, otherwise nullptr
 };
 
 /**

data/vendor/datasketches-cpp/filters/include/bloom_filter_impl.hpp

@@ -37,6 +37,19 @@
 
 namespace datasketches {
 
+static uint64_t validate_bloom_filter_num_bits_set(uint64_t num_bits_set,
+                                                   uint8_t* bit_array,
+                                                   uint64_t length_bytes,
+                                                   uint64_t dirty_bits_value)
+{
+  const uint64_t counted_bits_set = bit_array_ops::count_num_bits_set(bit_array, length_bytes);
+  if (num_bits_set != dirty_bits_value && num_bits_set != counted_bits_set) {
+    throw std::invalid_argument("Possible corruption: invalid number of bits set. Expected "
+      + std::to_string(counted_bits_set) + ", found " + std::to_string(num_bits_set));
+  }
+  return counted_bits_set;
+}
+
 template<typename A>
 bloom_filter_alloc<A>::bloom_filter_alloc(uint64_t num_bits, uint16_t num_hashes, uint64_t seed, const A& allocator) :
   allocator_(allocator),
@@ -143,9 +156,6 @@ bloom_filter_alloc<A>::bloom_filter_alloc(uint64_t seed,
 {
   // private constructor
   // no consistency checks since we should have done those prior to calling this
-  if (is_read_only_ && memory_ != nullptr && num_bits_set == DIRTY_BITS_VALUE) {
-    num_bits_set_ = bit_array_ops::count_num_bits_set(bit_array_, capacity_bits_ >> 3);
-  }
 }
 
 template<typename A>
@@ -298,8 +308,7 @@ bloom_filter_alloc<A> bloom_filter_alloc<A>::deserialize(std::istream& is, const
     return bloom_filter_alloc<A>(num_longs << 6, num_hashes, seed, allocator);
   }
 
-  const uint64_t num_bits_set = read<uint64_t>(is);
-  const bool is_dirty = (num_bits_set == DIRTY_BITS_VALUE);
+  const uint64_t raw_num_bits_set = read<uint64_t>(is);
 
   // allocate memory
   const uint64_t num_bytes = num_longs << 3;
@@ -310,8 +319,17 @@ bloom_filter_alloc<A> bloom_filter_alloc<A>::deserialize(std::istream& is, const
   }
   read(is, bit_array, num_bytes);
 
+  uint64_t num_bits_set;
+  try {
+    num_bits_set = validate_bloom_filter_num_bits_set(
+      raw_num_bits_set, bit_array, num_bytes, DIRTY_BITS_VALUE);
+  } catch (...) {
+    alloc.deallocate(bit_array, num_bytes);
+    throw;
+  }
+
   // pass to constructor
-  return bloom_filter_alloc<A>(seed, num_hashes, is_dirty, true, false, num_longs << 6, num_bits_set, bit_array, nullptr, allocator);
+  return bloom_filter_alloc<A>(seed, num_hashes, false, true, false, num_longs << 6, num_bits_set, bit_array, nullptr, allocator);
 }
 
 template<typename A>
@@ -374,9 +392,13 @@ bloom_filter_alloc<A> bloom_filter_alloc<A>::internal_deserialize_or_wrap(void*
     return bloom_filter_alloc<A>(num_longs << 6, num_hashes, seed, allocator);
   }
 
-  uint64_t num_bits_set;
-  ptr += copy_from_mem(ptr, num_bits_set);
-  const bool is_dirty = (num_bits_set == DIRTY_BITS_VALUE);
+  uint64_t raw_num_bits_set;
+  ptr += copy_from_mem(ptr, raw_num_bits_set);
+
+  const uint64_t num_bytes = num_longs << 3;
+  ensure_minimum_memory(end_ptr - ptr, num_bytes);
+  const uint64_t num_bits_set = validate_bloom_filter_num_bits_set(
+    raw_num_bits_set, const_cast<uint8_t*>(ptr), num_bytes, DIRTY_BITS_VALUE);
 
   uint8_t* bit_array;
   uint8_t* memory;
@@ -386,8 +408,6 @@ bloom_filter_alloc<A> bloom_filter_alloc<A>::internal_deserialize_or_wrap(void*
   } else {
     // allocate memory
     memory = nullptr;
-    const uint64_t num_bytes = num_longs << 3;
-    ensure_minimum_memory(end_ptr - ptr, num_bytes);
     AllocUint8 alloc(allocator);
     bit_array = alloc.allocate(num_bytes);
     if (bit_array == nullptr) {
@@ -397,7 +417,7 @@ bloom_filter_alloc<A> bloom_filter_alloc<A>::internal_deserialize_or_wrap(void*
   }
 
   // pass to constructor -- !wrap == is_owned_
-  return bloom_filter_alloc<A>(seed, num_hashes, is_dirty, !wrap, read_only, num_longs << 6, num_bits_set, bit_array, memory, allocator);
+  return bloom_filter_alloc<A>(seed, num_hashes, false, !wrap, read_only, num_longs << 6, num_bits_set, bit_array, memory, allocator);
 }
 
 template<typename A>

data/vendor/datasketches-cpp/filters/test/bloom_filter_test.cpp

@@ -19,6 +19,9 @@
 
 #include <catch2/catch.hpp>
 
+#include <cstring>
+#include <limits>
+
 #include "bloom_filter.hpp"
 
 #ifdef TEST_BINARY_INPUT_PATH
@@ -399,8 +402,32 @@ TEST_CASE("bloom_filter: non-empty serialization", "[bloom_filter]") {
   REQUIRE(bf_writable.query(-1.0));
 
   // not good memory management to do this, but because we wrapped the same bytes as both
-  // read-only adn writable, that update should ahve changed the read-only version, too
+  // read-only and writable, that update should have changed the read-only version, too
   REQUIRE(bf_wrap.query(-1.0));
 }
 
+TEST_CASE("bloom_filter: inconsistent num bits set is rejected", "[bloom_filter]") {
+  const size_t num_bits_set_offset = 24;
+
+  auto bf = bloom_filter::builder::create_by_accuracy(100, 0.01);
+  bf.update("apple");
+  bf.update("banana");
+  const uint64_t actual_bits_set = bf.get_bits_used();
+  REQUIRE(actual_bits_set > 1);
+
+  const uint64_t invalid_counts[] = {0, actual_bits_set - 1, actual_bits_set + 1};
+  for (const uint64_t serialized_count: invalid_counts) {
+    auto bytes = bf.serialize();
+    std::memcpy(bytes.data() + num_bits_set_offset, &serialized_count, sizeof(serialized_count));
+
+    REQUIRE_THROWS_AS(bloom_filter::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
+    REQUIRE_THROWS_AS(bloom_filter::wrap(bytes.data(), bytes.size()), std::invalid_argument);
+    REQUIRE_THROWS_AS(bloom_filter::writable_wrap(bytes.data(), bytes.size()), std::invalid_argument);
+
+    const std::string serialized(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+    std::istringstream is(serialized, std::ios::in | std::ios::binary);
+    REQUIRE_THROWS_AS(bloom_filter::deserialize(is), std::invalid_argument);
+  }
+}
+
 } // namespace datasketches

data/vendor/datasketches-cpp/hll/include/CouponHashSet-internal.hpp

@@ -176,8 +176,7 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(std::istream& is, const A& allocator)
     read(is, sketch->coupons_.data(), sketch->coupons_.size() * sizeof(uint32_t));
   } 
 
-  if (!is.good())
-    throw std::runtime_error("error reading from std::istream"); 
+  if (!is.good()) { throw std::runtime_error("error reading from std::istream"); } 
 
   return ptr.release();
 }