datasketches 0.5.2 → 0.5.3

data/vendor/datasketches-cpp/count/include/count_min.hpp

@@ -21,6 +21,8 @@
 #define COUNT_MIN_HPP_
 
 #include <iterator>
+#include <type_traits>
+#include <vector>
 #include "common_defs.hpp"
 
 namespace datasketches {
@@ -36,6 +38,7 @@ template <typename W,
           typename Allocator = std::allocator<W>>
 class count_min_sketch{
   static_assert(std::is_arithmetic<W>::value, "Arithmetic type expected");
+  static_assert(!std::is_same<W, bool>::value, "Boolean weight type is not supported");
 public:
   using allocator_type = Allocator;
   using const_iterator = typename std::vector<W, Allocator>::const_iterator;
@@ -326,6 +329,15 @@ public:
    */
   void serialize(std::ostream& os) const;
 
+  /**
+   * This method serializes the sketch by passing binary fragments to a callback.
+   * The callback must be callable as write_bytes(const void* data, size_t size).
+   * The data pointer is valid only until the callback returns.
+   * @return size in bytes written to the callback
+   */
+  template<typename Sink>
+  size_t serialize_to(Sink&& sink) const;
+
   // This is a convenience alias for users
   // The type returned by the following serialize method
   using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<Allocator>::template rebind_alloc<uint8_t>>;
@@ -389,12 +401,13 @@ private:
   static void check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id, uint8_t flags_byte);
 
   /*
-   * Obtain the hash values when inserting an item into the sketch.
-   * @param item pointer to the data item to be inserted into the sketch.
+   * Compute the hash locations for an input item
+   * @param item pointer to the data item to be inserted into or queried from the sketch.
    * @param size of the data in bytes
-   * @return vector of uint64_t which each represent the index to which `value' must update in the sketch
+   * @param callback function to invoke for each sketch array location
    */
-  std::vector<uint64_t> get_hashes(const void* item, size_t size) const;
+  template<typename F>
+  void foreach_hash_location(const void* item, size_t size, F callback) const;
 
 };
 

data/vendor/datasketches-cpp/count/include/count_min_impl.hpp

@@ -36,14 +36,16 @@ count_min_sketch<W,A>::count_min_sketch(uint8_t num_hashes, uint32_t num_buckets
 _allocator(allocator),
 _num_hashes(num_hashes),
 _num_buckets(num_buckets),
-_sketch_array((num_hashes*num_buckets < 1<<30) ? num_hashes*num_buckets : 0, 0, _allocator),
+_sketch_array((static_cast<uint64_t>(num_hashes) * num_buckets < 1<<30) ? static_cast<size_t>(num_hashes) * num_buckets : 0, 0, _allocator),
 _seed(seed),
 _total_weight(0) {
-  if (num_buckets < 3) throw std::invalid_argument("Using fewer than 3 buckets incurs relative error greater than 1.");
+  if (num_buckets < 3) {
+    throw std::invalid_argument("Using fewer than 3 buckets incurs relative error greater than 1.");
+  }
 
   // This check is to ensure later compatibility with a Java implementation whose maximum size can only
   // be 2^31-1.  We check only against 2^30 for simplicity.
-  if (num_buckets * num_hashes >= 1 << 30) {
+  if (static_cast<uint64_t>(num_buckets) * num_hashes >= 1 << 30) {
     throw std::invalid_argument("These parameters generate a sketch that exceeds 2^30 elements."
                                 "Try reducing either the number of buckets or the number of hash functions.");
   }
@@ -74,7 +76,7 @@ uint64_t count_min_sketch<W,A>::get_seed() const {
 
 template<typename W, typename A>
 double count_min_sketch<W,A>::get_relative_error() const {
-  return exp(1.0) / double(_num_buckets);
+  return exp(1.0) / static_cast<double>(_num_buckets);
 }
 
 template<typename W, typename A>
@@ -108,9 +110,10 @@ uint8_t count_min_sketch<W,A>::suggest_num_hashes(double confidence) {
 }
 
 template<typename W, typename A>
-std::vector<uint64_t> count_min_sketch<W,A>::get_hashes(const void* item, size_t size) const {
+template<typename F>
+void count_min_sketch<W,A>::foreach_hash_location(const void* item, size_t size, F callback) const {
   /*
-   * Returns the hash locations for the input item using the original hashing
+   * Computes the hash locations for the input item using the original hashing
    * scheme from [1].
    * Generate _num_hashes separate hashes from calls to murmurmhash.
    * This could be optimized by keeping both of the 64bit parts of the hash
@@ -124,8 +127,6 @@ std::vector<uint64_t> count_min_sketch<W,A>::get_hashes(const void* item, size_t
    * https://www.eecs.harvard.edu/~michaelm/postscripts/tr-02-05.pdf
    */
   uint64_t bucket_index;
-  std::vector<uint64_t> sketch_update_locations;
-  sketch_update_locations.reserve(_num_hashes);
 
   uint64_t hash_seed_index = 0;
   for (const auto &it: hash_seeds) {
@@ -133,10 +134,9 @@ std::vector<uint64_t> count_min_sketch<W,A>::get_hashes(const void* item, size_t
     MurmurHash3_x64_128(item, size, it, hashes); // ? BEWARE OVERFLOW.
     uint64_t hash = hashes.h1;
     bucket_index = hash % _num_buckets;
-    sketch_update_locations.push_back((hash_seed_index * _num_buckets) + bucket_index);
+    callback((hash_seed_index * _num_buckets) + bucket_index);
     hash_seed_index += 1;
   }
-  return sketch_update_locations;
 }
 
 template<typename W, typename A>
@@ -147,7 +147,7 @@ W count_min_sketch<W,A>::get_estimate(int64_t item) const {return get_estimate(&
 
 template<typename W, typename A>
 W count_min_sketch<W,A>::get_estimate(const std::string& item) const {
-  if (item.empty()) return 0; // Empty strings are not inserted into the sketch.
+  if (item.empty()) { return 0; } // Empty strings are not inserted into the sketch.
   return get_estimate(item.c_str(), item.length());
 }
 
@@ -156,12 +156,11 @@ W count_min_sketch<W,A>::get_estimate(const void* item, size_t size) const {
   /*
    * Returns the estimated frequency of the item
    */
-  std::vector<uint64_t> hash_locations = get_hashes(item, size);
-  std::vector<W> estimates;
-  for (const auto h: hash_locations) {
-    estimates.push_back(_sketch_array[h]);
-  }
-  return *std::min_element(estimates.begin(), estimates.end());
+  W estimate = std::numeric_limits<W>::max();
+  foreach_hash_location(item, size, [this, &estimate](uint64_t h) {
+    estimate = std::min(estimate, _sketch_array[h]);
+  });
+  return estimate;
 }
 
 template<typename W, typename A>
@@ -176,7 +175,7 @@ void count_min_sketch<W,A>::update(int64_t item, W weight) {
 
 template<typename W, typename A>
 void count_min_sketch<W,A>::update(const std::string& item, W weight) {
-  if (item.empty()) return;
+  if (item.empty()) { return; }
   update(item.c_str(), item.length(), weight);
 }
 
@@ -187,10 +186,9 @@ void count_min_sketch<W,A>::update(const void* item, size_t size, W weight) {
    * locations by the weight.
    */
   _total_weight += weight >= 0 ? weight : -weight;
-  std::vector<uint64_t> hash_locations = get_hashes(item, size);
-  for (const auto h: hash_locations) {
+  foreach_hash_location(item, size, [this, weight](uint64_t h) {
     _sketch_array[h] += weight;
-  }
+  });
 }
 
 template<typename W, typename A>
@@ -201,7 +199,7 @@ W count_min_sketch<W,A>::get_upper_bound(int64_t item) const {return get_upper_b
 
 template<typename W, typename A>
 W count_min_sketch<W,A>::get_upper_bound(const std::string& item) const {
-  if (item.empty()) return 0; // Empty strings are not inserted into the sketch.
+  if (item.empty()) { return 0; } // Empty strings are not inserted into the sketch.
   return get_upper_bound(item.c_str(), item.length());
 }
 
@@ -218,7 +216,7 @@ W count_min_sketch<W,A>::get_lower_bound(int64_t item) const {return get_lower_b
 
 template<typename W, typename A>
 W count_min_sketch<W,A>::get_lower_bound(const std::string& item) const {
-  if (item.empty()) return 0; // Empty strings are not inserted into the sketch.
+  if (item.empty()) { return 0; } // Empty strings are not inserted into the sketch.
   return get_lower_bound(item.c_str(), item.length());
 }
 
@@ -232,17 +230,13 @@ void count_min_sketch<W,A>::merge(const count_min_sketch &other_sketch) {
   /*
   * Merges this sketch into other_sketch sketch by elementwise summing of buckets
   */
-  if (this == &other_sketch) {
-    throw std::invalid_argument( "Cannot merge a sketch with itself." );
-  }
+  if (this == &other_sketch) { throw std::invalid_argument( "Cannot merge a sketch with itself." ); }
 
   bool acceptable_config =
     (get_num_hashes() == other_sketch.get_num_hashes())   &&
     (get_num_buckets() == other_sketch.get_num_buckets()) &&
     (get_seed() == other_sketch.get_seed());
-  if (!acceptable_config) {
-    throw std::invalid_argument( "Incompatible sketch configuration." );
-  }
+  if (!acceptable_config) { throw std::invalid_argument( "Incompatible sketch configuration." ); }
 
   // Merge step - iterate over the other vector and add the weights to this sketch
   auto it = _sketch_array.begin(); // This is a std::vector iterator.
@@ -268,6 +262,22 @@ return _sketch_array.end();
 
 template<typename W, typename A>
 void count_min_sketch<W,A>::serialize(std::ostream& os) const {
+  serialize_to([&os](const void* data, size_t size) {
+    os.write(static_cast<const char*>(data), size);
+  });
+}
+
+template<typename Sink, typename T>
+static inline void write_count_min_value(Sink& sink, size_t& bytes_written, const T& value) {
+  sink(&value, sizeof(value));
+  bytes_written += sizeof(value);
+}
+
+template<typename W, typename A>
+template<typename Sink>
+size_t count_min_sketch<W,A>::serialize_to(Sink&& sink) const {
+  size_t bytes_written = 0;
+
   // Long 0
   //const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_SHORT : PREAMBLE_LONGS_FULL;
   const uint8_t preamble_longs = PREAMBLE_LONGS_SHORT;
@@ -275,32 +285,35 @@ void count_min_sketch<W,A>::serialize(std::ostream& os) const {
   const uint8_t family_id = FAMILY_ID;
   const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
   const uint32_t unused32 = NULL_32;
-  write(os, preamble_longs);
-  write(os, ser_ver);
-  write(os, family_id);
-  write(os, flags_byte);
-  write(os, unused32);
+  write_count_min_value(sink, bytes_written, preamble_longs);
+  write_count_min_value(sink, bytes_written, ser_ver);
+  write_count_min_value(sink, bytes_written, family_id);
+  write_count_min_value(sink, bytes_written, flags_byte);
+  write_count_min_value(sink, bytes_written, unused32);
 
   // Long 1
   const uint32_t nbuckets = _num_buckets;
   const uint8_t nhashes = _num_hashes;
   const uint16_t seed_hash(compute_seed_hash(_seed));
   const uint8_t unused8 =  NULL_8;
-  write(os, nbuckets);
-  write(os, nhashes);
-  write(os, seed_hash);
-  write(os, unused8);
-  if (is_empty()) return; // sketch is empty, no need to write further bytes.
+  write_count_min_value(sink, bytes_written, nbuckets);
+  write_count_min_value(sink, bytes_written, nhashes);
+  write_count_min_value(sink, bytes_written, seed_hash);
+  write_count_min_value(sink, bytes_written, unused8);
+  if (is_empty()) { return bytes_written; } // sketch is empty, no need to write further bytes.
 
   // Long 2
-  write(os, _total_weight);
+  const W t_weight = _total_weight;
+  write_count_min_value(sink, bytes_written, t_weight);
 
   // Long 3 onwards: remaining bytes are consumed by writing the weight and the array values.
-  auto it = _sketch_array.begin();
-  while (it != _sketch_array.end()) {
-    write(os, *it);
-    ++it;
+  const size_t sketch_array_bytes = sizeof(W) * _sketch_array.size();
+  if (sketch_array_bytes > 0) {
+    sink(_sketch_array.data(), sketch_array_bytes);
+    bytes_written += sketch_array_bytes;
   }
+
+  return bytes_written;
 }
 
 template<typename W, typename A>
@@ -327,7 +340,7 @@ auto count_min_sketch<W,A>::deserialize(std::istream& is, uint64_t seed, const A
   }
   count_min_sketch c(nhashes, nbuckets, seed, allocator);
   const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
-  if (is_empty == 1) return c; // sketch is empty, no need to read further.
+  if (is_empty == 1) { return c; } // sketch is empty, no need to read further.
 
   // Set the sketch weight and read in the sketch values
   const auto weight = read<W>(is);
@@ -351,40 +364,9 @@ template<typename W, typename A>
 auto count_min_sketch<W,A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
   vector_bytes bytes(header_size_bytes + get_serialized_size_bytes(), 0, _allocator);
   uint8_t *ptr = bytes.data() + header_size_bytes;
-
-  // Long 0
-  const uint8_t preamble_longs = PREAMBLE_LONGS_SHORT;
-  ptr += copy_to_mem(preamble_longs, ptr);
-  const uint8_t ser_ver = SERIAL_VERSION_1;
-  ptr += copy_to_mem(ser_ver, ptr);
-  const uint8_t family_id = FAMILY_ID;
-  ptr += copy_to_mem(family_id, ptr);
-  const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
-  ptr += copy_to_mem(flags_byte, ptr);
-  const uint32_t unused32 = NULL_32;
-  ptr += copy_to_mem(unused32, ptr);
-
-  // Long 1
-  const uint32_t nbuckets = _num_buckets;
-  const uint8_t nhashes = _num_hashes;
-  const uint16_t seed_hash(compute_seed_hash(_seed));
-  const uint8_t null_characters_8 =  NULL_8;
-  ptr += copy_to_mem(nbuckets, ptr);
-  ptr += copy_to_mem(nhashes, ptr);
-  ptr += copy_to_mem(seed_hash, ptr);
-  ptr += copy_to_mem(null_characters_8, ptr);
-  if (is_empty()) return bytes; // sketch is empty, no need to write further bytes.
-
-  // Long 2
-  const W t_weight = _total_weight;
-  ptr += copy_to_mem(t_weight, ptr);
-
-  // Long  3 onwards: remaining bytes are consumed by writing the weight and the array values.
-  auto it = _sketch_array.begin();
-  while (it != _sketch_array.end()) {
-    ptr += copy_to_mem(*it, ptr);
-    ++it;
-  }
+  serialize_to([&ptr](const void* data, size_t size) {
+    ptr += copy_to_mem(data, ptr, size);
+  });
 
   return bytes;
 }
@@ -423,9 +405,10 @@ auto count_min_sketch<W,A>::deserialize(const void* bytes, size_t size, uint64_t
   }
   count_min_sketch c(nhashes, nbuckets, seed, allocator);
   const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
-  if (is_empty) return c; // sketch is empty, no need to read further.
+  if (is_empty) { return c; } // sketch is empty, no need to read further.
 
-  ensure_minimum_memory(size, sizeof(W) * (1 + nbuckets * nhashes));
+  // preamble (already read) + total weight + table; nbuckets * nhashes < 2^30 was checked by the constructor
+  ensure_minimum_memory(size, PREAMBLE_LONGS_SHORT * sizeof(uint64_t) + sizeof(W) * (1 + c._sketch_array.size()));
 
   // Long 2 is the weight.
   W weight;
@@ -449,8 +432,7 @@ string<A> count_min_sketch<W,A>::to_string() const {
   // count the number of used entries in the sketch
   uint64_t num_nonzero = 0;
   for (const auto entry: _sketch_array) {
-    if (entry != static_cast<W>(0.0))
-      ++num_nonzero;
+    if (entry != static_cast<W>(0.0)) { ++num_nonzero; }
   }
 
   // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.

data/vendor/datasketches-cpp/count/test/count_min_test.cpp

@@ -55,7 +55,7 @@ TEST_CASE("CM init") {
 TEST_CASE("CM parameter suggestions", "[error parameters]") {
 
     // Bucket suggestions
-    REQUIRE_THROWS(count_min_sketch<uint64_t>::suggest_num_buckets(-1.0), "Confidence must be between 0 and 1.0 (inclusive)." );
+    REQUIRE_THROWS_WITH(count_min_sketch<uint64_t>::suggest_num_buckets(-1.0), "Relative error must be at least 0.");
     REQUIRE(count_min_sketch<uint64_t>::suggest_num_buckets(0.2) == 14);
     REQUIRE(count_min_sketch<uint64_t>::suggest_num_buckets(0.1) == 28);
     REQUIRE(count_min_sketch<uint64_t>::suggest_num_buckets(0.05) == 55);
@@ -69,8 +69,8 @@ TEST_CASE("CM parameter suggestions", "[error parameters]") {
     REQUIRE(count_min_sketch<uint64_t>(n_hashes, 272).get_relative_error() <= 0.01);
 
     // Hash suggestions
-    REQUIRE_THROWS(count_min_sketch<uint64_t>::suggest_num_hashes(10.0), "Confidence must be between 0 and 1.0 (inclusive)." );
-    REQUIRE_THROWS(count_min_sketch<uint64_t>::suggest_num_hashes(-1.0), "Confidence must be between 0 and 1.0 (inclusive)." );
+    REQUIRE_THROWS_WITH(count_min_sketch<uint64_t>::suggest_num_hashes(10.0), "Confidence must be between 0 and 1.0 (inclusive)." );
+    REQUIRE_THROWS_WITH(count_min_sketch<uint64_t>::suggest_num_hashes(-1.0), "Confidence must be between 0 and 1.0 (inclusive)." );
     REQUIRE(count_min_sketch<uint64_t>::suggest_num_hashes(0.682689492) == 2); // 1 STDDEV
     REQUIRE(count_min_sketch<uint64_t>::suggest_num_hashes(0.954499736) == 4); // 2 STDDEV
     REQUIRE(count_min_sketch<uint64_t>::suggest_num_hashes(0.997300204) == 6); // 3 STDDEV
@@ -161,9 +161,9 @@ TEST_CASE("CM merge - reject", "[reject cases]") {
     std::vector<count_min_sketch<uint64_t>> sketches = {s1, s2, s3};
 
     // Fail cases
-    REQUIRE_THROWS(s.merge(s), "Cannot merge a sketch with itself." );
+    REQUIRE_THROWS_WITH(s.merge(s), "Cannot merge a sketch with itself." );
     for (count_min_sketch<uint64_t> sk : sketches) {
-      REQUIRE_THROWS(s.merge(sk), "Incompatible sketch config." );
+      REQUIRE_THROWS_WITH(s.merge(sk), "Incompatible sketch configuration." );
     }
 }
 
@@ -215,7 +215,7 @@ TEST_CASE("CountMin sketch: serialize-deserialize empty", "[cm_sketch]") {
     for (auto di: d) {
       REQUIRE(di == 0);
     }
-    std::ofstream os("count_min-empty.bin");
+    std::ofstream os("count_min_empty_cpp.sk", std::ios::binary);
     c.serialize(os);
 }
 
@@ -243,7 +243,7 @@ TEST_CASE("CountMin sketch: serialize-deserialize non-empty", "[cm_sketch]") {
     ++d_it;
   }
 
-  std::ofstream os("count_min-non-empty.bin");
+  std::ofstream os("count_min_non_empty_cpp.sk", std::ios::binary);
   c.serialize(os);
 }
 
@@ -300,4 +300,60 @@ TEST_CASE("CountMin sketch: bytes serialize-deserialize non-empty", "[cm_sketch]
 
 }
 
+TEST_CASE("CountMin sketch: sink serialize-deserialize round trip", "[cm_sketch]") {
+  auto check_sink_serialize = [](const count_min_sketch<uint64_t>& c) {
+    std::vector<uint8_t> sink_bytes;
+    sink_bytes.reserve(c.get_serialized_size_bytes());
+
+    const size_t bytes_written = c.serialize_to([&sink_bytes](const void* data, size_t size) {
+      const auto* begin = static_cast<const uint8_t*>(data);
+      sink_bytes.insert(sink_bytes.end(), begin, begin + size);
+    });
+
+    REQUIRE(bytes_written == c.get_serialized_size_bytes());
+    REQUIRE(sink_bytes.size() == bytes_written);
+
+    auto d = count_min_sketch<uint64_t>::deserialize(sink_bytes.data(), sink_bytes.size(), c.get_seed());
+    REQUIRE(c.get_num_hashes() == d.get_num_hashes());
+    REQUIRE(c.get_num_buckets() == d.get_num_buckets());
+    REQUIRE(c.get_seed() == d.get_seed());
+    REQUIRE(c.get_total_weight() == d.get_total_weight());
+
+    auto c_it = c.begin();
+    auto d_it = d.begin();
+    while (c_it != c.end()) {
+      REQUIRE(*c_it == *d_it);
+      ++c_it;
+      ++d_it;
+    }
+  };
+
+  count_min_sketch<uint64_t> empty(3, 32);
+  check_sink_serialize(empty);
+
+  count_min_sketch<uint64_t> non_empty(5, 64);
+  for (uint64_t i=0; i < 10; ++i) non_empty.update(i, 10 * i * i);
+  check_sink_serialize(non_empty);
+}
+
+TEST_CASE("CountMin sketch: bytes deserialize truncated non-empty", "[cm_sketch]") {
+  count_min_sketch<uint64_t> c(5, 64);
+  for (uint64_t i = 0; i < 10; ++i) c.update(i, 10 * i * i);
+  auto bytes = c.serialize();
+  for (size_t trim = 1; trim <= 24; ++trim) {
+    REQUIRE_THROWS_AS(count_min_sketch<uint64_t>::deserialize(bytes.data(), bytes.size() - trim), std::out_of_range);
+  }
+}
+
+TEST_CASE("CountMin sketch: deserialize rejects overflowing dimensions", "[cm_sketch]") {
+  // num_buckets * num_hashes wraps to 0 in 32-bit arithmetic
+  count_min_sketch<uint64_t> c(2, 64);
+  c.update(uint64_t(1));
+  auto bytes = c.serialize();
+  const uint32_t num_buckets = 1U << 31;
+  std::memcpy(bytes.data() + 8, &num_buckets, sizeof(num_buckets));
+  REQUIRE_THROWS_AS(count_min_sketch<uint64_t>::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
+  REQUIRE_THROWS_AS(count_min_sketch<uint64_t>(2, num_buckets), std::invalid_argument);
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/cpc/include/compression_data.hpp

@@ -22,6 +22,8 @@
 #ifndef CPC_COMPRESSION_DATA_HPP_
 #define CPC_COMPRESSION_DATA_HPP_
 
+#include <cstdint>
+
 namespace datasketches {
 
 /*

data/vendor/datasketches-cpp/cpc/include/cpc_compressor_impl.hpp

@@ -157,7 +157,7 @@ void cpc_compressor<A>::compress(const cpc_sketch_alloc<A>& source, compressed_s
       break;
     case cpc_sketch_alloc<A>::flavor::PINNED:
       compress_pinned_flavor(source, result);
-      if (result.window_data.size() == 0) throw std::logic_error("window is not expected");
+      if (result.window_data.size() == 0) throw std::logic_error("window is expected");
       break;
     case cpc_sketch_alloc<A>::flavor::SLIDING:
       compress_sliding_flavor(source, result);
@@ -354,6 +354,7 @@ void cpc_compressor<A>::uncompress_sliding_flavor(const compressed_state<A>& sou
       const uint32_t row_col = pairs[i];
       const uint32_t row = row_col >> 6;
       uint8_t col = row_col & 63;
+      if (col >= 56) throw std::out_of_range("col out of range");
       // first undo the permutation
       col = permutation[col];
       // then undo the rotation: old = (new + (offset+8)) mod 64
@@ -390,6 +391,9 @@ auto cpc_compressor<A>::uncompress_surprising_values(const uint32_t* data, uint3
   vector_u32 pairs(num_pairs, 0, allocator);
   const uint8_t num_base_bits = golomb_choose_number_of_base_bits(k + num_pairs, num_pairs);
   low_level_uncompress_pairs(pairs.data(), num_pairs, num_base_bits, data, data_words);
+  for (uint32_t i = 0; i < num_pairs; i++) {
+    if ((pairs[i] >> 6) >= k) throw std::out_of_range("row index out of range");
+  }
   return pairs;
 }
 
@@ -472,8 +476,10 @@ static inline void maybe_flush_bitbuf(uint64_t& bitbuf, uint8_t& bufbits, uint32
   }
 }
 
-static inline void maybe_fill_bitbuf(uint64_t& bitbuf, uint8_t& bufbits, const uint32_t* wordarr, uint32_t& wordindex, uint8_t minbits) {
+static inline void maybe_fill_bitbuf(uint64_t& bitbuf, uint8_t& bufbits, const uint32_t* wordarr, uint32_t& wordindex,
+    uint32_t numwords, uint8_t minbits) {
   if (bufbits < minbits) {
+    if (wordindex >= numwords) throw std::out_of_range("compressed data over-run");
     bitbuf |= static_cast<uint64_t>(wordarr[wordindex++]) << bufbits;
     bufbits += 32;
   }
@@ -530,7 +536,7 @@ void cpc_compressor<A>::low_level_uncompress_bytes(
   if (compressed_words == nullptr) throw std::logic_error("compressed_words == NULL");
 
   for (uint32_t byte_index = 0; byte_index < num_bytes_to_decode; byte_index++) {
-    maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, word_index, 12); // ensure 12 bits in bit buffer
+    maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, word_index, num_compressed_words, 12); // ensure 12 bits in bit buffer
 
     const size_t peek12 = bitbuf & 0xfff; // These 12 bits will include an entire Huffman codeword.
     const uint16_t lookup = decoding_table[peek12];
@@ -547,6 +553,7 @@ void cpc_compressor<A>::low_level_uncompress_bytes(
 
 static inline uint64_t read_unary(
     const uint32_t* compressed_words,
+    uint32_t num_compressed_words,
     uint32_t& next_word_index,
     uint64_t& bitbuf,
     uint8_t& bufbits
@@ -646,7 +653,7 @@ void cpc_compressor<A>::low_level_uncompress_pairs(
   // y_delta_lo (basebits)
 
   for (uint32_t pair_index = 0; pair_index < num_pairs_to_decode; pair_index++) {
-    maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, word_index, 12); // ensure 12 bits in bit buffer
+    maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, word_index, num_compressed_words, 12); // ensure 12 bits in bit buffer
     const size_t peek12 = bitbuf & 0xfff;
     const uint16_t lookup = length_limited_unary_decoding_table65[peek12];
     const uint8_t code_word_length = lookup >> 8;
@@ -654,9 +661,9 @@ void cpc_compressor<A>::low_level_uncompress_pairs(
     bitbuf >>= code_word_length;
     bufbits -= code_word_length;
 
-    const uint64_t golomb_hi = read_unary(compressed_words, word_index, bitbuf, bufbits);
+    const uint64_t golomb_hi = read_unary(compressed_words, num_compressed_words, word_index, bitbuf, bufbits);
 
-    maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, word_index, num_base_bits); // ensure num_base_bits in bit buffer
+    maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, word_index, num_compressed_words, num_base_bits); // ensure num_base_bits in bit buffer
     const uint64_t golomb_lo = bitbuf & golomb_lo_mask;
     bitbuf >>= num_base_bits;
     bufbits -= num_base_bits;
@@ -666,6 +673,7 @@ void cpc_compressor<A>::low_level_uncompress_pairs(
     if (y_delta > 0) predicted_col_index = 0;
     const uint32_t row_index = static_cast<uint32_t>(predicted_row_index + y_delta);
     const uint8_t col_index = predicted_col_index + x_delta;
+    if (col_index > 63) throw std::out_of_range("column index out of range");
     const uint32_t row_col = (row_index << 6) | col_index;
     pair_array[pair_index] = row_col;
     predicted_row_index = row_index;
@@ -676,6 +684,7 @@ void cpc_compressor<A>::low_level_uncompress_pairs(
 
 uint64_t read_unary(
     const uint32_t* compressed_words,
+    uint32_t num_compressed_words,
     uint32_t& next_word_index,
     uint64_t& bitbuf,
     uint8_t& bufbits
@@ -683,7 +692,7 @@ uint64_t read_unary(
   if (compressed_words == nullptr) throw std::logic_error("compressed_words == NULL");
   size_t subtotal = 0;
   while (true) {
-    maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, next_word_index, 8); // ensure 8 bits in bit buffer
+    maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, next_word_index, num_compressed_words, 8); // ensure 8 bits in bit buffer
 
     const uint8_t peek8 = bitbuf & 0xff; // These 8 bits include either all or part of the Unary codeword
     const uint8_t trailing_zeros = byte_trailing_zeros_table[peek8];

data/vendor/datasketches-cpp/cpc/include/cpc_sketch.hpp

@@ -315,6 +315,7 @@ private:
   inline size_t copy_hip_to_mem(void* dst) const;
 
   static void check_lg_k(uint8_t lg_k);
+  static void check_num_coupons(uint8_t lg_k, uint32_t num_coupons, uint32_t num_pairs);
 
   friend cpc_compressor<A>;
   friend cpc_union_alloc<A>;