datasketches 0.2.3 → 0.2.6

data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser_impl.hpp

@@ -22,39 +22,108 @@
 
 #include <iostream>
 #include <iomanip>
+#include <stdexcept>
 
 namespace datasketches {
 
 template<bool dummy>
 auto compact_theta_sketch_parser<dummy>::parse(const void* ptr, size_t size, uint64_t seed, bool dump_on_error) -> compact_theta_sketch_data {
-  if (size < 8) throw std::invalid_argument("at least 8 bytes expected, actual " + std::to_string(size)
+  if (size < 8) throw std::out_of_range("at least 8 bytes expected, actual " + std::to_string(size)
       + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
-  checker<true>::check_serial_version(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE], COMPACT_SKETCH_SERIAL_VERSION);
-  checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
-  uint64_t theta = theta_constants::MAX_THETA;
-  const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
-  checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
-  if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) {
-    return {true, true, seed_hash, 0, theta, nullptr};
+
+  uint8_t serial_version = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE];
+
+  switch(serial_version) {
+  case COMPACT_SKETCH_SERIAL_VERSION: {
+      checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+      uint64_t theta = theta_constants::MAX_THETA;
+      const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
+      if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) {
+        return {true, true, seed_hash, 0, theta, nullptr};
+      }
+      checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+      const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2;
+      if (has_theta) {
+        if (size < 16) throw std::out_of_range("at least 16 bytes expected, actual " + std::to_string(size));
+        theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+      }
+      if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) {
+        if (size < 16) throw std::out_of_range("at least 16 bytes expected, actual " + std::to_string(size));
+        return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64};
+      }
+      const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+      const size_t entries_start_u64 = has_theta ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 : COMPACT_SKETCH_ENTRIES_EXACT_U64;
+      const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + entries_start_u64;
+      const size_t expected_size_bytes = (entries_start_u64 + num_entries) * sizeof(uint64_t);
+      if (size < expected_size_bytes) {
+        throw std::out_of_range(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
+            + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+      }
+      const bool is_ordered = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_ORDERED_FLAG);
+      return {false, is_ordered, seed_hash, num_entries, theta, entries};
   }
-  const bool has_theta = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2;
-  if (has_theta) {
-    if (size < 16) throw std::invalid_argument("at least 16 bytes expected, actual " + std::to_string(size));
-    theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+  case 1:  {
+      uint16_t seed_hash = compute_seed_hash(seed);
+      checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+      const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+      uint64_t theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+      bool is_empty = (num_entries == 0) && (theta == theta_constants::MAX_THETA);
+      if (is_empty) {
+          return {true, true, seed_hash, 0, theta, nullptr};
+      }
+      const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_ENTRIES_ESTIMATION_U64;
+      const size_t expected_size_bytes = (COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 + num_entries) * sizeof(uint64_t);
+      if (size < expected_size_bytes) {
+        throw std::out_of_range(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
+            + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+      }
+      return {false, true, seed_hash, num_entries, theta, entries};
   }
-  if (reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) {
-    return {false, true, seed_hash, 1, theta, reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_SINGLE_ENTRY_U64};
+  case 2:  {
+      uint8_t preamble_size =  reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE];
+      checker<true>::check_sketch_type(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_TYPE_BYTE], COMPACT_SKETCH_TYPE);
+      const uint16_t seed_hash = reinterpret_cast<const uint16_t*>(ptr)[COMPACT_SKETCH_SEED_HASH_U16];
+      checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+      if (preamble_size == 1) {
+          return {true, true, seed_hash, 0, theta_constants::MAX_THETA, nullptr};
+      } else if (preamble_size == 2) {
+          const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+          if (num_entries == 0) {
+              return {true, true, seed_hash, 0, theta_constants::MAX_THETA, nullptr};
+          } else {
+              const size_t expected_size_bytes = (preamble_size + num_entries) << 3;
+              if (size < expected_size_bytes) {
+                  throw std::out_of_range(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
+                      + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+              }
+              const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_ENTRIES_EXACT_U64;
+              return {false, true, seed_hash, num_entries, theta_constants::MAX_THETA, entries};
+          }
+      } else if (preamble_size == 3) {
+          const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
+          uint64_t theta = reinterpret_cast<const uint64_t*>(ptr)[COMPACT_SKETCH_THETA_U64];
+          bool is_empty = (num_entries == 0) && (theta == theta_constants::MAX_THETA);
+          if (is_empty) {
+              return {true, true, seed_hash, 0, theta, nullptr};
+          }
+          const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + COMPACT_SKETCH_ENTRIES_ESTIMATION_U64;
+          const size_t expected_size_bytes = (COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 + num_entries) * sizeof(uint64_t);
+          if (size < expected_size_bytes) {
+            throw std::out_of_range(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
+                + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+          }
+          return {false, true, seed_hash, num_entries, theta, entries};
+      } else {
+          throw std::invalid_argument(std::to_string(preamble_size) + " longs of premable, but expected 1, 2, or 3");
+      }
   }
-  const uint32_t num_entries = reinterpret_cast<const uint32_t*>(ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32];
-  const size_t entries_start_u64 = has_theta ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 : COMPACT_SKETCH_ENTRIES_EXACT_U64;
-  const uint64_t* entries = reinterpret_cast<const uint64_t*>(ptr) + entries_start_u64;
-  const size_t expected_size_bytes = (entries_start_u64 + num_entries) * sizeof(uint64_t);
-  if (size < expected_size_bytes) {
-    throw std::invalid_argument(std::to_string(expected_size_bytes) + " bytes expected, actual " + std::to_string(size)
-        + (dump_on_error ? (", sketch dump: " + hex_dump(reinterpret_cast<const uint8_t*>(ptr), size)) : ""));
+  default:
+      // this should always fail since the valid cases are handled above
+      checker<true>::check_serial_version(reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE], COMPACT_SKETCH_SERIAL_VERSION);
+      // this throw is never reached, because check_serial_version will throw an informative exception.
+      // This is only here to avoid a compiler warning about a path without a return value.
+      throw std::invalid_argument("unexpected sketch serialization version");
   }
-  const bool is_ordered = reinterpret_cast<const uint8_t*>(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & (1 << COMPACT_SKETCH_IS_ORDERED_FLAG);
-  return {false, is_ordered, seed_hash, num_entries, theta, entries};
 }
 
 template<bool dummy>

data/vendor/datasketches-cpp/theta/include/theta_helpers.hpp

@@ -49,6 +49,21 @@ public:
   }
 };
 
+template<bool dummy>
+class theta_build_helper{
+public:
+  // consistent way of initializing theta from p
+  // avoids multiplication if p == 1 since it might not yield MAX_THETA exactly
+  static uint64_t starting_theta_from_p(float p) {
+    if (p < 1) return static_cast<uint64_t>(theta_constants::MAX_THETA * p);
+    return theta_constants::MAX_THETA;
+  }
+
+  static uint8_t starting_sub_multiple(uint8_t lg_tgt, uint8_t lg_min, uint8_t lg_rf) {
+    return (lg_tgt <= lg_min) ? lg_min : (lg_rf == 0) ? lg_tgt : ((lg_tgt - lg_min) % lg_rf) + lg_min;
+  }
+};
+
 } /* namespace datasketches */
 
 #endif

data/vendor/datasketches-cpp/theta/include/theta_intersection_base_impl.hpp

@@ -20,6 +20,7 @@
 #include <iostream>
 #include <sstream>
 #include <algorithm>
+#include <stdexcept>
 
 #include "conditional_forward.hpp"
 
@@ -29,7 +30,7 @@ template<typename EN, typename EK, typename P, typename S, typename CS, typename
 theta_intersection_base<EN, EK, P, S, CS, A>::theta_intersection_base(uint64_t seed, const P& policy, const A& allocator):
 policy_(policy),
 is_valid_(false),
-table_(0, 0, resize_factor::X1, theta_constants::MAX_THETA, seed, allocator, false)
+table_(0, 0, resize_factor::X1, 1, theta_constants::MAX_THETA, seed, allocator, false)
 {}
 
 template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
@@ -38,17 +39,17 @@ void theta_intersection_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
   if (table_.is_empty_) return;
   if (!sketch.is_empty() && sketch.get_seed_hash() != compute_seed_hash(table_.seed_)) throw std::invalid_argument("seed hash mismatch");
   table_.is_empty_ |= sketch.is_empty();
-  table_.theta_ = std::min(table_.theta_, sketch.get_theta64());
+  table_.theta_ = table_.is_empty_ ? theta_constants::MAX_THETA : std::min(table_.theta_, sketch.get_theta64());
   if (is_valid_ && table_.num_entries_ == 0) return;
   if (sketch.get_num_retained() == 0) {
     is_valid_ = true;
-    table_ = hash_table(0, 0, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+    table_ = hash_table(0, 0, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
     return;
   }
   if (!is_valid_) { // first update, copy or move incoming sketch
     is_valid_ = true;
     const uint8_t lg_size = lg_size_from_count(sketch.get_num_retained(), theta_update_sketch_base<EN, EK, A>::REBUILD_THRESHOLD);
-    table_ = hash_table(lg_size, lg_size, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+    table_ = hash_table(lg_size, lg_size, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
     for (auto& entry: sketch) {
       auto result = table_.find(EK()(entry));
       if (result.second) {
@@ -83,11 +84,11 @@ void theta_intersection_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
       throw std::invalid_argument(" fewer keys than expected, possibly corrupted input sketch");
     }
     if (match_count == 0) {
-      table_ = hash_table(0, 0, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+      table_ = hash_table(0, 0, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
       if (table_.theta_ == theta_constants::MAX_THETA) table_.is_empty_ = true;
     } else {
       const uint8_t lg_size = lg_size_from_count(match_count, theta_update_sketch_base<EN, EK, A>::REBUILD_THRESHOLD);
-      table_ = hash_table(lg_size, lg_size, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+      table_ = hash_table(lg_size, lg_size, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
       for (uint32_t i = 0; i < match_count; i++) {
         auto result = table_.find(EK()(matched_entries[i]));
         table_.insert(result.first, std::move(matched_entries[i]));

data/vendor/datasketches-cpp/theta/include/theta_set_difference_base_impl.hpp

@@ -21,6 +21,7 @@
 #define THETA_A_SET_DIFFERENCE_BASE_IMPL_HPP_
 
 #include <algorithm>
+#include <stdexcept>
 
 #include "conditional_back_inserter.hpp"
 #include "conditional_forward.hpp"
@@ -36,7 +37,7 @@ seed_hash_(compute_seed_hash(seed))
 template<typename EN, typename EK, typename CS, typename A>
 template<typename FwdSketch, typename Sketch>
 CS theta_set_difference_base<EN, EK, CS, A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const {
-  if (a.is_empty() || a.get_num_retained() == 0 || b.is_empty()) return CS(a, ordered);
+  if (a.is_empty() || (a.get_num_retained() > 0 && b.is_empty())) return CS(a, ordered);
   if (a.get_seed_hash() != seed_hash_) throw std::invalid_argument("A seed hash mismatch");
   if (b.get_seed_hash() != seed_hash_) throw std::invalid_argument("B seed hash mismatch");
 
@@ -53,7 +54,7 @@ CS theta_set_difference_base<EN, EK, CS, A>::compute(FwdSketch&& a, const Sketch
           conditional_back_inserter(entries, key_less_than<uint64_t, EN, EK>(theta)), comparator());
     } else { // hash-based
       const uint8_t lg_size = lg_size_from_count(b.get_num_retained(), hash_table::REBUILD_THRESHOLD);
-      hash_table table(lg_size, lg_size, hash_table::resize_factor::X1, 0, 0, allocator_); // theta and seed are not used here
+      hash_table table(lg_size, lg_size, hash_table::resize_factor::X1, 1, 0, 0, allocator_); // theta and seed are not used here
       for (const auto& entry: b) {
         const uint64_t hash = EK()(entry);
         if (hash < theta) {

data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp

@@ -25,14 +25,10 @@
 namespace datasketches {
 
 template<typename Allocator = std::allocator<uint64_t>>
-class theta_sketch_alloc {
+class base_theta_sketch_alloc {
 public:
-  using Entry = uint64_t;
-  using ExtractKey = trivial_extract_key;
-  using iterator = theta_iterator<Entry, ExtractKey>;
-  using const_iterator = theta_const_iterator<Entry, ExtractKey>;
 
-  virtual ~theta_sketch_alloc() = default;
+  virtual ~base_theta_sketch_alloc() = default;
 
   /**
    * @return allocator
@@ -104,6 +100,21 @@ public:
    */
   virtual string<Allocator> to_string(bool print_items = false) const;
 
+protected:
+  virtual void print_specifics(std::ostringstream& os) const = 0;
+  virtual void print_items(std::ostringstream& os) const = 0;
+};
+
+template<typename Allocator = std::allocator<uint64_t>>
+class theta_sketch_alloc: public base_theta_sketch_alloc<Allocator> {
+public:
+  using Entry = uint64_t;
+  using ExtractKey = trivial_extract_key;
+  using iterator = theta_iterator<Entry, ExtractKey>;
+  using const_iterator = theta_const_iterator<Entry, ExtractKey>;
+
+  virtual ~theta_sketch_alloc() = default;
+
   /**
    * Iterator over hash values in this sketch.
    * @return begin iterator
@@ -131,8 +142,7 @@ public:
   virtual const_iterator end() const = 0;
 
 protected:
-  using ostrstream = std::basic_ostringstream<char, std::char_traits<char>, AllocChar<Allocator>>;
-  virtual void print_specifics(ostrstream& os) const = 0;
+  virtual void print_items(std::ostringstream& os) const;
 };
 
 // forward declaration
@@ -269,6 +279,11 @@ public:
    */
   void trim();
 
+  /**
+   * Reset the sketch to the initial empty state
+   */
+  void reset();
+
   /**
    * Converts this sketch to a compact sketch (ordered or unordered).
    * @param ordered optional flag to specify if ordered sketch should be produced
@@ -285,11 +300,10 @@ private:
   theta_table table_;
 
   // for builder
-  update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta,
-      uint64_t seed, const Allocator& allocator);
+  update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p,
+      uint64_t theta, uint64_t seed, const Allocator& allocator);
 
-  using ostrstream = typename Base::ostrstream;
-  virtual void print_specifics(ostrstream& os) const;
+  virtual void print_specifics(std::ostringstream& os) const;
 };
 
 // compact sketch
@@ -377,8 +391,7 @@ private:
   uint64_t theta_;
   std::vector<uint64_t, Allocator> entries_;
 
-  using ostrstream = typename Base::ostrstream;
-  virtual void print_specifics(ostrstream& os) const;
+  virtual void print_specifics(std::ostringstream& os) const;
 };
 
 template<typename Allocator>
@@ -392,7 +405,7 @@ public:
 // It does not take the ownership of the buffer.
 
 template<typename Allocator = std::allocator<uint64_t>>
-class wrapped_compact_theta_sketch_alloc {
+class wrapped_compact_theta_sketch_alloc : public base_theta_sketch_alloc<Allocator> {
 public:
   using const_iterator = const uint64_t*;
 
@@ -415,6 +428,10 @@ public:
    */
   static const wrapped_compact_theta_sketch_alloc wrap(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED, bool dump_on_error = false);
 
+protected:
+  virtual void print_specifics(std::ostringstream& os) const;
+  virtual void print_items(std::ostringstream& os) const;
+
 private:
   bool is_empty_;
   bool is_ordered_;