datasketches 0.5.1 → 0.5.3

data/vendor/datasketches-cpp/theta/include/theta_sketch_impl.hpp

@@ -23,6 +23,7 @@
 #include <sstream>
 #include <vector>
 #include <stdexcept>
+#include <algorithm>
 
 #include "binomial_bounds.hpp"
 #include "theta_helpers.hpp"
@@ -239,8 +240,21 @@ auto update_theta_sketch_alloc<A>::end() const -> const_iterator {
 }
 
 template<typename A>
-compact_theta_sketch_alloc<A> update_theta_sketch_alloc<A>::compact(bool ordered) const {
-  return compact_theta_sketch_alloc<A>(*this, ordered);
+compact_theta_sketch_alloc<A> update_theta_sketch_alloc<A>::compact(bool ordered, bool trim) const {
+  if (!trim) return compact_theta_sketch_alloc<A>(*this, ordered);
+
+  std::vector<uint64_t, A> entries(table_.allocator_);
+  if (this->is_empty()) {
+    return compact_theta_sketch_alloc<A>(true, true, this->get_seed_hash(), this->get_theta64(), std::move(entries));
+  }
+  // copy first: this method is const, and quick select would permute the source table
+  entries.reserve(this->get_num_retained());
+  std::copy(this->begin(), this->end(), std::back_inserter(entries));
+  uint64_t theta = this->get_theta64();
+  const uint32_t nominal_size = 1 << table_.lg_nom_size_;
+  theta = trim_to_nominal<trivial_extract_key>(entries, nominal_size, theta);
+  if (ordered) std::sort(entries.begin(), entries.end());
+  return compact_theta_sketch_alloc<A>(false, ordered, this->get_seed_hash(), theta, std::move(entries));
 }
 
 template<typename A>
@@ -315,7 +329,7 @@ uint32_t compact_theta_sketch_alloc<A>::get_num_retained() const {
 
 template<typename A>
 uint16_t compact_theta_sketch_alloc<A>::get_seed_hash() const {
-  return seed_hash_;
+  return is_empty_ ? 0 : seed_hash_;
 }
 
 template<typename A>
@@ -376,7 +390,7 @@ size_t compact_theta_sketch_alloc<A>::get_compressed_serialized_size_bytes(uint8
 
 template<typename A>
 void compact_theta_sketch_alloc<A>::serialize(std::ostream& os) const {
-  const uint8_t preamble_longs = this->is_estimation_mode() ? 3 : this->is_empty() || entries_.size() == 1 ? 1 : 2;
+  const uint8_t preamble_longs = get_preamble_longs(false);
   write(os, preamble_longs);
   write(os, UNCOMPRESSED_SERIAL_VERSION);
   write(os, SKETCH_TYPE);
@@ -385,7 +399,8 @@ void compact_theta_sketch_alloc<A>::serialize(std::ostream& os) const {
     (1 << flags::IS_COMPACT) |
     (1 << flags::IS_READ_ONLY) |
     (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
-    (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+    (this->is_ordered() ? 1 << flags::IS_ORDERED : 0) |
+    (is_single_item() ? 1 << flags::IS_SINGLE_ITEM : 0)
   );
   write(os, flags_byte);
   write(os, get_seed_hash());
@@ -411,7 +426,8 @@ auto compact_theta_sketch_alloc<A>::serialize(unsigned header_size_bytes) const
     (1 << flags::IS_COMPACT) |
     (1 << flags::IS_READ_ONLY) |
     (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
-    (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+    (this->is_ordered() ? 1 << flags::IS_ORDERED : 0) |
+    (is_single_item() ? 1 << flags::IS_SINGLE_ITEM : 0)
   );
   *ptr++ = flags_byte;
   ptr += copy_to_mem(get_seed_hash(), ptr);
@@ -424,6 +440,12 @@ auto compact_theta_sketch_alloc<A>::serialize(unsigned header_size_bytes) const
   return bytes;
 }
 
+template<typename A>
+bool compact_theta_sketch_alloc<A>::is_single_item() const {
+  // one entry in exact mode: the same condition Java uses to set its SingleItem flag
+  return entries_.size() == 1 && !this->is_estimation_mode();
+}
+
 template<typename A>
 bool compact_theta_sketch_alloc<A>::is_suitable_for_compression() const {
   if (!this->is_ordered() || entries_.size() == 0 ||
@@ -459,7 +481,7 @@ uint8_t compact_theta_sketch_alloc<A>::compute_entry_bits() const {
 
 template<typename A>
 void compact_theta_sketch_alloc<A>::serialize_version_4(std::ostream& os) const {
-  const uint8_t preamble_longs = this->is_estimation_mode() ? 2 : 1;
+  const uint8_t preamble_longs = get_preamble_longs(true);
   const uint8_t entry_bits = compute_entry_bits();
   const uint8_t num_entries_bytes = get_num_entries_bytes();
 
@@ -674,7 +696,9 @@ compact_theta_sketch_alloc<A> compact_theta_sketch_alloc<A>::deserialize_v4(
     uint8_t preamble_longs, std::istream& is, uint64_t seed, const A& allocator)
 {
   const auto entry_bits = read<uint8_t>(is);
+  compact_theta_sketch_parser<true>::check_v4_entry_bits(entry_bits);
   const auto num_entries_bytes = read<uint8_t>(is);
+  compact_theta_sketch_parser<true>::check_v4_num_entries_bytes(num_entries_bytes);
   const auto flags_byte = read<uint8_t>(is);
   const auto seed_hash = read<uint16_t>(is);
   const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
@@ -782,7 +806,7 @@ uint32_t wrapped_compact_theta_sketch_alloc<A>::get_num_retained() const {
 
 template<typename A>
 uint16_t wrapped_compact_theta_sketch_alloc<A>::get_seed_hash() const {
-  return data_.seed_hash;
+  return is_empty() ? 0 : data_.seed_hash;
 }
 
 template<typename A>
@@ -817,23 +841,15 @@ num_entries_(num_entries),
 index_(index),
 previous_(0),
 is_block_mode_(num_entries_ >= 8),
-buf_i_(0),
 offset_(0)
 {
   if (entry_bits == 64) { // no compression
     ptr_ = reinterpret_cast<const uint64_t*>(ptr) + index;
   } else if (index < num_entries) {
     if (is_block_mode_) {
-      unpack_bits_block8(buffer_, reinterpret_cast<const uint8_t*>(ptr_), entry_bits_);
-      ptr_ = reinterpret_cast<const uint8_t*>(ptr_) + entry_bits_;
-      for (int i = 0; i < 8; ++i) {
-        buffer_[i] += previous_;
-        previous_ = buffer_[i];
-      }
+      unpack8();
     } else {
-      offset_ = unpack_bits(buffer_[0], entry_bits_, reinterpret_cast<const uint8_t*&>(ptr_), offset_);
-      buffer_[0] += previous_;
-      previous_ = buffer_[0];
+      unpack1();
     }
   }
 }
@@ -844,35 +860,41 @@ auto wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator++()
     ptr_ = reinterpret_cast<const uint64_t*>(ptr_) + 1;
     return *this;
   }
-  ++index_;
-  if (index_ < num_entries_) {
+  if (++index_ < num_entries_) {
     if (is_block_mode_) {
-      ++buf_i_;
-      if (buf_i_ == 8) {
-        buf_i_ = 0;
-        if (index_ + 8 < num_entries_) {
-          unpack_bits_block8(buffer_, reinterpret_cast<const uint8_t*>(ptr_), entry_bits_);
-          ptr_ = reinterpret_cast<const uint8_t*>(ptr_) + entry_bits_;
-          for (int i = 0; i < 8; ++i) {
-            buffer_[i] += previous_;
-            previous_ = buffer_[i];
-          }
+      if ((index_ & 7) == 0) {
+        if (num_entries_ - index_ >= 8) {
+          unpack8();
         } else {
           is_block_mode_ = false;
-          offset_ = unpack_bits(buffer_[0], entry_bits_, reinterpret_cast<const uint8_t*&>(ptr_), offset_);
-          buffer_[0] += previous_;
-          previous_ = buffer_[0];
+          unpack1();
         }
       }
     } else {
-      offset_ = unpack_bits(buffer_[0], entry_bits_, reinterpret_cast<const uint8_t*&>(ptr_), offset_);
-      buffer_[0] += previous_;
-      previous_ = buffer_[0];
+      unpack1();
     }
   }
   return *this;
 }
 
+template<typename Allocator>
+void wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::unpack1() {
+  const uint32_t i = index_ & 7;
+  offset_ = unpack_bits(buffer_[i], entry_bits_, reinterpret_cast<const uint8_t*&>(ptr_), offset_);
+  buffer_[i] += previous_;
+  previous_ = buffer_[i];
+}
+
+template<typename Allocator>
+void wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::unpack8() {
+  unpack_bits_block8(buffer_, reinterpret_cast<const uint8_t*>(ptr_), entry_bits_);
+  ptr_ = reinterpret_cast<const uint8_t*>(ptr_) + entry_bits_;
+  for (int i = 0; i < 8; ++i) {
+    buffer_[i] += previous_;
+    previous_ = buffer_[i];
+  }
+}
+
 template<typename Allocator>
 auto wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator++(int) -> const_iterator {
   const_iterator tmp(*this);
@@ -895,13 +917,13 @@ bool wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator==(c
 template<typename Allocator>
 auto wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator*() const -> reference {
   if (entry_bits_ == 64) return *reinterpret_cast<const uint64_t*>(ptr_);
-  return buffer_[buf_i_];
+  return buffer_[index_ & 7];
 }
 
 template<typename Allocator>
 auto wrapped_compact_theta_sketch_alloc<Allocator>::const_iterator::operator->() const -> pointer {
   if (entry_bits_ == 64) return reinterpret_cast<const uint64_t*>(ptr_);
-  return buffer_ + buf_i_;
+  return buffer_ + (index_ & 7);
 }
 
 } /* namespace datasketches */

data/vendor/datasketches-cpp/theta/include/theta_union_base_impl.hpp

@@ -24,6 +24,7 @@
 #include <stdexcept>
 
 #include "conditional_forward.hpp"
+#include "theta_helpers.hpp"
 
 namespace datasketches {
 
@@ -70,12 +71,7 @@ CS theta_union_base<EN, EK, P, S, CS, A>::get_result(bool ordered) const {
   } else {
     std::copy_if(table_.begin(), table_.end(), std::back_inserter(entries), key_not_zero_less_than<uint64_t, EN, EK>(theta));
   }
-  if (entries.size() > nominal_num) {
-    std::nth_element(entries.begin(), entries.begin() + nominal_num, entries.end(), comparator());
-    theta = EK()(entries[nominal_num]);
-    entries.erase(entries.begin() + nominal_num, entries.end());
-    entries.shrink_to_fit();
-  }
+  theta = trim_to_nominal<EK>(entries, nominal_num, theta);
   if (ordered) std::sort(entries.begin(), entries.end(), comparator());
   return CS(table_.is_empty_, ordered, compute_seed_hash(table_.seed_), theta, std::move(entries));
 }

data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp

@@ -265,7 +265,7 @@ void theta_update_sketch_base<EN, EK, A>::reset() {
     }
   }
   const uint8_t starting_lg_size = theta_build_helper<true>::starting_sub_multiple(
-      lg_nom_size_ + 1, theta_constants::MIN_LG_K, static_cast<uint8_t>(rf_));
+      lg_nom_size_ + 1, theta_constants::MIN_LG_ARR, static_cast<uint8_t>(rf_));
   if (starting_lg_size != lg_cur_size_) {
     allocator_.deallocate(entries_, cur_size);
     lg_cur_size_ = starting_lg_size;
@@ -346,7 +346,7 @@ uint64_t theta_base_builder<Derived, Allocator>::starting_theta() const {
 
 template<typename Derived, typename Allocator>
 uint8_t theta_base_builder<Derived, Allocator>::starting_lg_size() const {
-  return theta_build_helper<true>::starting_sub_multiple(lg_k_ + 1, theta_constants::MIN_LG_K, static_cast<uint8_t>(rf_));
+  return theta_build_helper<true>::starting_sub_multiple(lg_k_ + 1, theta_constants::MIN_LG_ARR, static_cast<uint8_t>(rf_));
 }
 
 // iterator

data/vendor/datasketches-cpp/theta/test/bit_packing_test.cpp

@@ -80,4 +80,54 @@ TEST_CASE("pack unpack blocks") {
   }
 }
 
+TEST_CASE("pack bits unpack blocks") {
+  uint64_t value = 0; // arbitrary starting value
+  for (int m = 0; m < 10000; ++m) {
+    for (uint8_t bits = 1; bits <= 63; ++bits) {
+      const uint64_t mask = (1ULL << bits) - 1;
+      std::vector<uint64_t> input(8, 0);
+      for (int i = 0; i < 8; ++i) {
+        input[i] = value & mask;
+        value += IGOLDEN64;
+      }
+      std::vector<uint8_t> bytes(bits, 0);
+      uint8_t offset = 0;
+      uint8_t* ptr = bytes.data();
+      for (int i = 0; i < 8; ++i) {
+        offset = pack_bits(input[i], bits, ptr, offset);
+      }
+      std::vector<uint64_t> output(8, 0);
+      unpack_bits_block8(output.data(), bytes.data(), bits);
+      for (int i = 0; i < 8; ++i) {
+        REQUIRE(input[i] == output[i]);
+      }
+    }
+  }
+}
+
+TEST_CASE("pack blocks unpack bits") {
+  uint64_t value = 111; // arbitrary starting value
+  for (int m = 0; m < 10000; ++m) {
+    for (uint8_t bits = 1; bits <= 63; ++bits) {
+      const uint64_t mask = (1ULL << bits) - 1;
+      std::vector<uint64_t> input(8, 0);
+      for (int i = 0; i < 8; ++i) {
+        input[i] = value & mask;
+        value += IGOLDEN64;
+      }
+      std::vector<uint8_t> bytes(bits, 0);
+      pack_bits_block8(input.data(), bytes.data(), bits);
+      std::vector<uint64_t> output(8, 0);
+      uint8_t offset = 0;
+      const uint8_t* cptr = bytes.data();
+      for (int i = 0; i < 8; ++i) {
+        offset = unpack_bits(output[i], bits, cptr, offset);
+      }
+      for (int i = 0; i < 8; ++i) {
+        REQUIRE(input[i] == output[i]);
+      }
+    }
+  }
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/theta/test/theta_a_not_b_test.cpp

@@ -241,6 +241,28 @@ TEST_CASE("theta a-not-b: seed mismatch", "[theta_a_not_b]") {
   REQUIRE_THROWS_AS(a_not_b.compute(sketch, sketch), std::invalid_argument);
 }
 
+TEST_CASE("theta a-not-b: empty B with different seed must be ignored", "[theta_a_not_b]") {
+  // An empty sketch carries no hashes, so its seed hash is meaningless. Deserialization,
+  // theta_union::update() and theta_intersection::update() all skip the seed hash check
+  // for empty inputs; a-not-b must do the same. This matters across languages because
+  // datasketches-java serializes every empty compact sketch with a seed hash of 0.
+
+  // A: non-empty, but with zero retained entries (all hashes exceeded theta)
+  update_theta_sketch a = update_theta_sketch::builder().set_p(1e-6f).build();
+  a.update(1);
+  REQUIRE_FALSE(a.is_empty());
+  REQUIRE(a.get_num_retained() == 0);
+
+  // B: empty, built with a different seed
+  compact_theta_sketch b = update_theta_sketch::builder().set_seed(123).build().compact();
+  REQUIRE(b.is_empty());
+
+  theta_a_not_b a_not_b; // default seed
+  compact_theta_sketch result = a_not_b.compute(a.compact(), b);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.get_num_retained() == 0);
+}
+
 TEST_CASE("theta a-not-b: issue #152", "[theta_a_not_b]") {
   update_theta_sketch a = update_theta_sketch::builder().build();
   int value = 0;