datasketches 0.2.0 → 0.2.4

data/vendor/datasketches-cpp/cpc/include/cpc_util.hpp

@@ -31,9 +31,9 @@ static inline uint64_t divide_longs_rounding_up(uint64_t x, uint64_t y) {
   else return quotient + 1;
 }
 
-static inline uint64_t long_floor_log2_of_long(uint64_t x) {
-  if (x < 1) throw std::invalid_argument("long_floor_log2_of_long: bad argument");
-  uint64_t p = 0;
+static inline uint8_t floor_log2_of_long(uint64_t x) {
+  if (x < 1) throw std::invalid_argument("floor_log2_of_long: bad argument");
+  uint8_t p = 0;
   uint64_t y = 1;
   while (true) {
     if (y == x) return p;
@@ -69,7 +69,7 @@ static inline uint64_t wegner_count_bits_set_in_matrix(const uint64_t* array, si
 // Note: this is an adaptation of the Java code,
 // which is apparently a variation of Figure 5-2 in "Hacker's Delight"
 // by Henry S. Warren.
-static inline uint64_t warren_bit_count(uint64_t i) {
+static inline uint32_t warren_bit_count(uint64_t i) {
   i = i - ((i >> 1) & 0x5555555555555555ULL);
   i = (i & 0x3333333333333333ULL) + ((i >> 2) & 0x3333333333333333ULL);
   i = (i + (i >> 4)) & 0x0f0f0f0f0f0f0f0fULL;
@@ -79,9 +79,9 @@ static inline uint64_t warren_bit_count(uint64_t i) {
   return i & 0x7f;
 }
 
-static inline uint64_t warren_count_bits_set_in_matrix(const uint64_t* array, size_t length) {
-  uint64_t count = 0;
-  for (size_t i = 0; i < length; i++) {
+static inline uint32_t warren_count_bits_set_in_matrix(const uint64_t* array, uint32_t length) {
+  uint32_t count = 0;
+  for (uint32_t i = 0; i < length; i++) {
     count += warren_bit_count(array[i]);
   }
   return count;
@@ -91,13 +91,13 @@ static inline uint64_t warren_count_bits_set_in_matrix(const uint64_t* array, si
 
 #define CSA(h,l,a,b,c) {uint64_t u = a ^ b; uint64_t v = c; h = (a & b) | (u & v); l = u ^ v;}
 
-static inline uint64_t count_bits_set_in_matrix(const uint64_t* a, size_t length) {
+static inline uint32_t count_bits_set_in_matrix(const uint64_t* a, uint32_t length) {
   if ((length & 0x7) != 0) throw std::invalid_argument("the length of the array must be a multiple of 8");
-  uint64_t total = 0;
+  uint32_t total = 0;
   uint64_t ones, twos, twos_a, twos_b, fours, fours_a, fours_b, eights;
   fours = twos = ones = 0;
 
-  for (size_t i = 0; i <= length - 8; i = i + 8) {
+  for (uint32_t i = 0; i <= length - 8; i += 8) {
     CSA(twos_a, ones, ones, a[i+0], a[i+1]);
     CSA(twos_b, ones, ones, a[i+2], a[i+3]);
     CSA(fours_a, twos, twos, twos_a, twos_b);

data/vendor/datasketches-cpp/cpc/include/icon_estimator.hpp

@@ -245,12 +245,12 @@ static inline double icon_exponential_approximation(double k, double c) {
   return (0.7940236163830469 * k * pow(2.0, c / k));
 }
 
-static inline double compute_icon_estimate(uint8_t lg_k, uint64_t c) {
+static inline double compute_icon_estimate(uint8_t lg_k, uint32_t c) {
   if (lg_k < ICON_MIN_LOG_K || lg_k > ICON_MAX_LOG_K) throw std::out_of_range("lg_k out of range");
   if (c < 2) return ((c == 0) ? 0.0 : 1.0);
-  const size_t k = 1 << lg_k;
-  const double double_k = k;
-  const double double_c = c;
+  const uint32_t k = 1 << lg_k;
+  const double double_k = static_cast<double>(k);
+  const double double_c = static_cast<double>(c);
   // Differing thresholds ensure that the approximated estimator is monotonically increasing.
   const double threshold_factor = ((lg_k < 14) ? 5.7 : 5.6);
   if (double_c > (threshold_factor * double_k)) return icon_exponential_approximation(double_k, double_c);

data/vendor/datasketches-cpp/cpc/include/u32_table.hpp

@@ -29,11 +29,11 @@
 
 namespace datasketches {
 
-static const uint64_t U32_TABLE_UPSIZE_NUMER = 3LL;
-static const uint64_t U32_TABLE_UPSIZE_DENOM = 4LL;
+static const uint32_t U32_TABLE_UPSIZE_NUMER = 3LL;
+static const uint32_t U32_TABLE_UPSIZE_DENOM = 4LL;
 
-static const uint64_t U32_TABLE_DOWNSIZE_NUMER = 1LL;
-static const uint64_t U32_TABLE_DOWNSIZE_DENOM = 4LL;
+static const uint32_t U32_TABLE_DOWNSIZE_NUMER = 1LL;
+static const uint32_t U32_TABLE_DOWNSIZE_DENOM = 4LL;
 
 template<typename A>
 class u32_table {
@@ -42,7 +42,7 @@ public:
   u32_table(const A& allocator);
   u32_table(uint8_t lg_size, uint8_t num_valid_bits, const A& allocator);
 
-  inline size_t get_num_items() const;
+  inline uint32_t get_num_items() const;
   inline const uint32_t* get_slots() const;
   inline uint8_t get_lg_size() const;
   inline void clear();
@@ -52,7 +52,7 @@ public:
   // returns true iff the item was present and was therefore removed from the table
   inline bool maybe_delete(uint32_t item);
 
-  static u32_table make_from_pairs(const uint32_t* pairs, size_t num_pairs, uint8_t lg_k, const A& allocator);
+  static u32_table make_from_pairs(const uint32_t* pairs, uint32_t num_pairs, uint8_t lg_k, const A& allocator);
 
   vector_u32<A> unwrapping_get_items() const;
 
@@ -69,10 +69,10 @@ private:
 
   uint8_t lg_size; // log2 of number of slots
   uint8_t num_valid_bits;
-  size_t num_items;
+  uint32_t num_items;
   vector_u32<A> slots;
 
-  inline size_t lookup(uint32_t item) const;
+  inline uint32_t lookup(uint32_t item) const;
   inline void must_insert(uint32_t item);
   inline void rebuild(uint8_t new_lg_size);
 };

data/vendor/datasketches-cpp/cpc/include/u32_table_impl.hpp

@@ -41,14 +41,14 @@ u32_table<A>::u32_table(uint8_t lg_size, uint8_t num_valid_bits, const A& alloca
 lg_size(lg_size),
 num_valid_bits(num_valid_bits),
 num_items(0),
-slots(1 << lg_size, UINT32_MAX, allocator)
+slots(1ULL << lg_size, UINT32_MAX, allocator)
 {
   if (lg_size < 2) throw std::invalid_argument("lg_size must be >= 2");
   if (num_valid_bits < 1 || num_valid_bits > 32) throw std::invalid_argument("num_valid_bits must be between 1 and 32");
 }
 
 template<typename A>
-size_t u32_table<A>::get_num_items() const {
+uint32_t u32_table<A>::get_num_items() const {
   return num_items;
 }
 
@@ -70,7 +70,7 @@ void u32_table<A>::clear() {
 
 template<typename A>
 bool u32_table<A>::maybe_insert(uint32_t item) {
-  const size_t index = lookup(item);
+  const uint32_t index = lookup(item);
   if (slots[index] == item) return false;
   if (slots[index] != UINT32_MAX) throw std::logic_error("could not insert");
   slots[index] = item;
@@ -83,7 +83,7 @@ bool u32_table<A>::maybe_insert(uint32_t item) {
 
 template<typename A>
 bool u32_table<A>::maybe_delete(uint32_t item) {
-  const size_t index = lookup(item);
+  const uint32_t index = lookup(item);
   if (slots[index] == UINT32_MAX) return false;
   if (slots[index] != item) throw std::logic_error("item does not exist");
   if (num_items == 0) throw std::logic_error("delete error");
@@ -110,7 +110,7 @@ bool u32_table<A>::maybe_delete(uint32_t item) {
 
 // this one is specifically tailored to be a part of fm85 decompression scheme
 template<typename A>
-u32_table<A> u32_table<A>::make_from_pairs(const uint32_t* pairs, size_t num_pairs, uint8_t lg_k, const A& allocator) {
+u32_table<A> u32_table<A>::make_from_pairs(const uint32_t* pairs, uint32_t num_pairs, uint8_t lg_k, const A& allocator) {
   uint8_t lg_num_slots = 2;
   while (U32_TABLE_UPSIZE_DENOM * num_pairs > U32_TABLE_UPSIZE_NUMER * (1 << lg_num_slots)) lg_num_slots++;
   u32_table<A> table(lg_num_slots, 6 + lg_k, allocator);
@@ -124,11 +124,11 @@ u32_table<A> u32_table<A>::make_from_pairs(const uint32_t* pairs, size_t num_pai
 }
 
 template<typename A>
-size_t u32_table<A>::lookup(uint32_t item) const {
-  const size_t size = 1 << lg_size;
-  const size_t mask = size - 1;
+uint32_t u32_table<A>::lookup(uint32_t item) const {
+  const uint32_t size = 1 << lg_size;
+  const uint32_t mask = size - 1;
   const uint8_t shift = num_valid_bits - lg_size;
-  size_t probe = item >> shift;
+  uint32_t probe = item >> shift;
   if (probe > mask) throw std::logic_error("probe out of range");
   while (slots[probe] != item && slots[probe] != UINT32_MAX) {
     probe = (probe + 1) & mask;
@@ -139,7 +139,7 @@ size_t u32_table<A>::lookup(uint32_t item) const {
 // counts and resizing must be handled by the caller
 template<typename A>
 void u32_table<A>::must_insert(uint32_t item) {
-  const size_t index = lookup(item);
+  const uint32_t index = lookup(item);
   if (slots[index] == item) throw std::logic_error("item exists");
   if (slots[index] != UINT32_MAX) throw std::logic_error("could not insert");
   slots[index] = item;
@@ -148,13 +148,13 @@ void u32_table<A>::must_insert(uint32_t item) {
 template<typename A>
 void u32_table<A>::rebuild(uint8_t new_lg_size) {
   if (new_lg_size < 2) throw std::logic_error("lg_size must be >= 2");
-  const size_t old_size = 1 << lg_size;
-  const size_t new_size = 1 << new_lg_size;
+  const uint32_t old_size = 1 << lg_size;
+  const uint32_t new_size = 1 << new_lg_size;
   if (new_size <= num_items) throw std::logic_error("new_size <= num_items");
   vector_u32<A> old_slots = std::move(slots);
   slots = vector_u32<A>(new_size, UINT32_MAX, old_slots.get_allocator());
   lg_size = new_lg_size;
-  for (size_t i = 0; i < old_size; i++) {
+  for (uint32_t i = 0; i < old_size; i++) {
     if (old_slots[i] != UINT32_MAX) {
       must_insert(old_slots[i]);
     }
@@ -170,7 +170,7 @@ void u32_table<A>::rebuild(uint8_t new_lg_size) {
 template<typename A>
 vector_u32<A> u32_table<A>::unwrapping_get_items() const {
   if (num_items == 0) return vector_u32<A>(slots.get_allocator());
-  const size_t table_size = 1 << lg_size;
+  const uint32_t table_size = 1 << lg_size;
   vector_u32<A> result(num_items, 0, slots.get_allocator());
   size_t i = 0;
   size_t l = 0;

data/vendor/datasketches-cpp/cpc/test/compression_test.cpp

@@ -27,38 +27,38 @@ namespace datasketches {
 typedef u32_table<std::allocator<void>> table;
 
 TEST_CASE("cpc sketch: compress and decompress pairs", "[cpc_sketch]") {
-  const int N = 200;
-  const int MAXWORDS = 1000;
+  const size_t N = 200;
+  const size_t MAXWORDS = 1000;
 
   HashState twoHashes;
   uint32_t pairArray[N];
   uint32_t pairArray2[N];
   uint64_t value = 35538947; // some arbitrary starting value
   const uint64_t golden64 = 0x9e3779b97f4a7c13ULL; // the golden ratio
-  for (int i = 0; i < N; i++) {
+  for (size_t i = 0; i < N; i++) {
     MurmurHash3_x64_128(&value, sizeof(value), 0, twoHashes);
     uint32_t rand = twoHashes.h1 & 0xffff;
     pairArray[i] = rand;
     value += golden64;
   }
   //table::knuth_shell_sort3(pairArray, 0, N - 1); // unsigned numerical sort
-  std::sort(pairArray, &pairArray[N]);
+  std::sort(pairArray, pairArray + N);
   uint32_t prev = UINT32_MAX;
-  int nxt = 0;
-  for (int i = 0; i < N; i++) { // uniquify
+  uint32_t nxt = 0;
+  for (size_t i = 0; i < N; i++) { // uniquify
     if (pairArray[i] != prev) {
       prev = pairArray[i];
       pairArray[nxt++] = pairArray[i];
     }
   }
-  int numPairs = nxt;
+  uint32_t numPairs = nxt;
 
   uint32_t compressedWords[MAXWORDS];
 
-  for (size_t numBaseBits = 0; numBaseBits <= 11; numBaseBits++) {
-    size_t numWordsWritten = get_compressor<std::allocator<void>>().low_level_compress_pairs(pairArray, numPairs, numBaseBits, compressedWords);
+  for (uint8_t numBaseBits = 0; numBaseBits <= 11; numBaseBits++) {
+    uint32_t numWordsWritten = get_compressor<std::allocator<void>>().low_level_compress_pairs(pairArray, numPairs, numBaseBits, compressedWords);
     get_compressor<std::allocator<void>>().low_level_uncompress_pairs(pairArray2, numPairs, numBaseBits, compressedWords, numWordsWritten);
-    for (int i = 0; i < numPairs; i++) {
+    for (size_t i = 0; i < numPairs; i++) {
       REQUIRE(pairArray[i] == pairArray2[i]);
     }
   }

data/vendor/datasketches-cpp/cpc/test/cpc_sketch_allocation_test.cpp

@@ -25,6 +25,7 @@
 #include <catch.hpp>
 
 #include "cpc_sketch.hpp"
+#include "cpc_union.hpp"
 #include "test_allocator.hpp"
 
 namespace datasketches {
@@ -234,4 +235,20 @@ TEST_CASE("cpc sketch allocation: serialize deserialize sliding, bytes", "[cpc_s
   REQUIRE(test_allocator_net_allocations == 0);
 }
 
+using cpc_union_test_alloc = cpc_union_alloc<test_allocator<uint8_t>>;
+
+TEST_CASE("cpc sketch allocation: union") {
+  cpc_sketch_test_alloc s1(11, DEFAULT_SEED, 0);
+  s1.update(1);
+
+  cpc_sketch_test_alloc s2(11, DEFAULT_SEED, 0);
+  s2.update(2);
+
+  cpc_union_test_alloc u(11, DEFAULT_SEED, 0);
+  u.update(s1);
+  u.update(s2);
+  auto s3 = u.get_result();
+  REQUIRE_FALSE(s3.is_empty());
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp

@@ -283,6 +283,26 @@ TEST_CASE("cpc sketch: serialize deserialize sliding, bytes", "[cpc_sketch]") {
   REQUIRE(deserialized.validate());
 }
 
+TEST_CASE("cpc sketch: serialize deserialize sliding huge", "[cpc_sketch]") {
+  cpc_sketch sketch(26);
+  const int n = 10000000;
+  for (int i = 0; i < n; i++) sketch.update(i);
+  REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.001));
+  auto bytes = sketch.serialize();
+  cpc_sketch deserialized = cpc_sketch::deserialize(bytes.data(), bytes.size());
+  REQUIRE(deserialized.is_empty() == sketch.is_empty());
+  REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+  REQUIRE(deserialized.validate());
+  REQUIRE_THROWS_AS(cpc_sketch::deserialize(bytes.data(), 7), std::out_of_range);
+  REQUIRE_THROWS_AS(cpc_sketch::deserialize(bytes.data(), 15), std::out_of_range);
+  REQUIRE_THROWS_AS(cpc_sketch::deserialize(bytes.data(), bytes.size() - 1), std::out_of_range);
+
+  // updating again with the same values should not change the sketch
+  for (int i = 0; i < n; i++) deserialized.update(i);
+  REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+  REQUIRE(deserialized.validate());
+}
+
 TEST_CASE("cpc sketch: copy", "[cpc_sketch]") {
   cpc_sketch s1(11);
   s1.update(1);
@@ -378,4 +398,9 @@ TEST_CASE("cpc sketch: update string equivalence", "[cpc_sketch]") {
   REQUIRE(sketch.get_estimate() == Approx(1).margin(RELATIVE_ERROR_FOR_LG_K_11));
 }
 
+TEST_CASE("cpc sketch: max serialized size", "[cpc_sketch]") {
+  REQUIRE(cpc_sketch::get_max_serialized_size_bytes(4) == 24 + 40);
+  REQUIRE(cpc_sketch::get_max_serialized_size_bytes(26) == static_cast<size_t>((0.6 * (1 << 26)) + 40));
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp

@@ -81,7 +81,7 @@ TEST_CASE("cpc union: large", "[cpc_union]") {
   cpc_union u(11);
   for (int i = 0; i < 1000; i++) {
     cpc_sketch tmp(11);
-    for (int i = 0; i < 10000; i++) {
+    for (int j = 0; j < 10000; j++) {
       s.update(key);
       tmp.update(key);
       key++;

data/vendor/datasketches-cpp/fi/CMakeLists.txt

@@ -32,23 +32,13 @@ target_include_directories(fi
 target_link_libraries(fi INTERFACE common)
 target_compile_features(fi INTERFACE cxx_std_11)
 
-set(fi_HEADERS "")
-list(APPEND fi_HEADERS "include/frequent_items_sketch.hpp")
-list(APPEND fi_HEADERS "include/frequent_items_sketch_impl.hpp")
-list(APPEND fi_HEADERS "include/reverse_purge_hash_map.hpp")
-list(APPEND fi_HEADERS "include/reverse_purge_hash_map_impl.hpp")
-
 install(TARGETS fi
   EXPORT ${PROJECT_NAME}
 )
 
-install(FILES ${fi_HEADERS}
+install(FILES 
+		include/frequent_items_sketch.hpp
+		include/frequent_items_sketch_impl.hpp
+		include/reverse_purge_hash_map.hpp
+		include/reverse_purge_hash_map_impl.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
-
-target_sources(fi
-  INTERFACE
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/frequent_items_sketch.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/frequent_items_sketch_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/reverse_purge_hash_map.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/reverse_purge_hash_map_impl.hpp
-)