datasketches 0.2.0 → 0.2.4

data/vendor/datasketches-cpp/theta/include/theta_helpers.hpp

@@ -49,6 +49,21 @@ public:
   }
 };
 
+template<bool dummy>
+class theta_build_helper{
+public:
+  // consistent way of initializing theta from p
+  // avoids multiplication if p == 1 since it might not yield MAX_THETA exactly
+  static uint64_t starting_theta_from_p(float p) {
+    if (p < 1) return static_cast<uint64_t>(theta_constants::MAX_THETA * p);
+    return theta_constants::MAX_THETA;
+  }
+
+  static uint8_t starting_sub_multiple(uint8_t lg_tgt, uint8_t lg_min, uint8_t lg_rf) {
+    return (lg_tgt <= lg_min) ? lg_min : (lg_rf == 0) ? lg_tgt : ((lg_tgt - lg_min) % lg_rf) + lg_min;
+  }
+};
+
 } /* namespace datasketches */
 
 #endif

data/vendor/datasketches-cpp/theta/include/theta_intersection.hpp

@@ -33,14 +33,19 @@ public:
   using Sketch = theta_sketch_alloc<Allocator>;
   using CompactSketch = compact_theta_sketch_alloc<Allocator>;
 
-  struct pass_through_policy {
-    uint64_t operator()(uint64_t internal_entry, uint64_t incoming_entry) const {
+  struct nop_policy {
+    void operator()(uint64_t internal_entry, uint64_t incoming_entry) const {
       unused(incoming_entry);
-      return internal_entry;
+      unused(internal_entry);
     }
   };
-  using State = theta_intersection_base<Entry, ExtractKey, pass_through_policy, Sketch, CompactSketch, Allocator>;
+  using State = theta_intersection_base<Entry, ExtractKey, nop_policy, Sketch, CompactSketch, Allocator>;
 
+  /*
+   * Constructor
+   * @param seed for the hash function that was used to create the sketch
+   * @param allocator to use for allocating and deallocating memory
+   */
   explicit theta_intersection_alloc(uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
 
   /**

data/vendor/datasketches-cpp/theta/include/theta_intersection_base_impl.hpp

@@ -29,7 +29,7 @@ template<typename EN, typename EK, typename P, typename S, typename CS, typename
 theta_intersection_base<EN, EK, P, S, CS, A>::theta_intersection_base(uint64_t seed, const P& policy, const A& allocator):
 policy_(policy),
 is_valid_(false),
-table_(0, 0, resize_factor::X1, theta_constants::MAX_THETA, seed, allocator, false)
+table_(0, 0, resize_factor::X1, 1, theta_constants::MAX_THETA, seed, allocator, false)
 {}
 
 template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
@@ -38,17 +38,17 @@ void theta_intersection_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
   if (table_.is_empty_) return;
   if (!sketch.is_empty() && sketch.get_seed_hash() != compute_seed_hash(table_.seed_)) throw std::invalid_argument("seed hash mismatch");
   table_.is_empty_ |= sketch.is_empty();
-  table_.theta_ = std::min(table_.theta_, sketch.get_theta64());
+  table_.theta_ = table_.is_empty_ ? theta_constants::MAX_THETA : std::min(table_.theta_, sketch.get_theta64());
   if (is_valid_ && table_.num_entries_ == 0) return;
   if (sketch.get_num_retained() == 0) {
     is_valid_ = true;
-    table_ = hash_table(0, 0, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+    table_ = hash_table(0, 0, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
     return;
   }
   if (!is_valid_) { // first update, copy or move incoming sketch
     is_valid_ = true;
     const uint8_t lg_size = lg_size_from_count(sketch.get_num_retained(), theta_update_sketch_base<EN, EK, A>::REBUILD_THRESHOLD);
-    table_ = hash_table(lg_size, lg_size, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+    table_ = hash_table(lg_size, lg_size, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
     for (auto& entry: sketch) {
       auto result = table_.find(EK()(entry));
       if (result.second) {
@@ -83,11 +83,11 @@ void theta_intersection_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
       throw std::invalid_argument(" fewer keys than expected, possibly corrupted input sketch");
     }
     if (match_count == 0) {
-      table_ = hash_table(0, 0, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+      table_ = hash_table(0, 0, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
       if (table_.theta_ == theta_constants::MAX_THETA) table_.is_empty_ = true;
     } else {
       const uint8_t lg_size = lg_size_from_count(match_count, theta_update_sketch_base<EN, EK, A>::REBUILD_THRESHOLD);
-      table_ = hash_table(lg_size, lg_size, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+      table_ = hash_table(lg_size, lg_size, resize_factor::X1, 1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
       for (uint32_t i = 0; i < match_count; i++) {
         auto result = table_.find(EK()(matched_entries[i]));
         table_.insert(result.first, std::move(matched_entries[i]));

data/vendor/datasketches-cpp/theta/include/theta_intersection_impl.hpp

@@ -24,7 +24,7 @@ namespace datasketches {
 
 template<typename A>
 theta_intersection_alloc<A>::theta_intersection_alloc(uint64_t seed, const A& allocator):
-state_(seed, pass_through_policy(), allocator)
+state_(seed, nop_policy(), allocator)
 {}
 
 template<typename A>

data/vendor/datasketches-cpp/theta/include/theta_jaccard_similarity_base.hpp

@@ -46,20 +46,21 @@ public:
    *
    * @param sketch_a given sketch A
    * @param sketch_b given sketch B
+   * @param seed for the hash function that was used to create the sketch
    * @return a double array {LowerBound, Estimate, UpperBound} of the Jaccard index.
    * The Upper and Lower bounds are for a confidence interval of 95.4% or +/- 2 standard deviations.
    */
   template<typename SketchA, typename SketchB>
-  static std::array<double, 3> jaccard(const SketchA& sketch_a, const SketchB& sketch_b) {
+  static std::array<double, 3> jaccard(const SketchA& sketch_a, const SketchB& sketch_b, uint64_t seed = DEFAULT_SEED) {
     if (reinterpret_cast<const void*>(&sketch_a) == reinterpret_cast<const void*>(&sketch_b)) return {1, 1, 1};
     if (sketch_a.is_empty() && sketch_b.is_empty()) return {1, 1, 1};
     if (sketch_a.is_empty() || sketch_b.is_empty()) return {0, 0, 0};
 
-    auto union_ab = compute_union(sketch_a, sketch_b);
+    auto union_ab = compute_union(sketch_a, sketch_b, seed);
     if (identical_sets(sketch_a, sketch_b, union_ab)) return {1, 1, 1};
 
     // intersection
-    Intersection i;
+    Intersection i(seed);
     i.update(sketch_a);
     i.update(sketch_b);
     i.update(union_ab); // ensures that intersection is a subset of the union
@@ -76,15 +77,16 @@ public:
    * Returns true if the two given sketches are equivalent.
    * @param sketch_a the given sketch A
    * @param sketch_b the given sketch B
+   * @param seed for the hash function that was used to create the sketch
    * @return true if the two given sketches are exactly equal
    */
   template<typename SketchA, typename SketchB>
-  static bool exactly_equal(const SketchA& sketch_a, const SketchB& sketch_b) {
+  static bool exactly_equal(const SketchA& sketch_a, const SketchB& sketch_b, uint64_t seed = DEFAULT_SEED) {
     if (reinterpret_cast<const void*>(&sketch_a) == reinterpret_cast<const void*>(&sketch_b)) return true;
     if (sketch_a.is_empty() && sketch_b.is_empty()) return true;
     if (sketch_a.is_empty() || sketch_b.is_empty()) return false;
 
-    auto union_ab = compute_union(sketch_a, sketch_b);
+    auto union_ab = compute_union(sketch_a, sketch_b, seed);
     if (identical_sets(sketch_a, sketch_b, union_ab)) return true;
     return false;
   }
@@ -99,12 +101,13 @@ public:
    * @param actual the sketch to be tested
    * @param expected the reference sketch that is considered to be correct
    * @param threshold a real value between zero and one
+   * @param seed for the hash function that was used to create the sketch
    * @return true if the similarity of the two sketches is greater than the given threshold
    * with at least 97.7% confidence
    */
   template<typename SketchA, typename SketchB>
-  static bool similarity_test(const SketchA& actual, const SketchB& expected, double threshold) {
-    auto jc = jaccard(actual, expected);
+  static bool similarity_test(const SketchA& actual, const SketchB& expected, double threshold, uint64_t seed = DEFAULT_SEED) {
+    auto jc = jaccard(actual, expected, seed);
     return jc[0] >= threshold;
   }
 
@@ -118,23 +121,24 @@ public:
    * @param actual the sketch to be tested
    * @param expected the reference sketch that is considered to be correct
    * @param threshold a real value between zero and one
+   * @param seed for the hash function that was used to create the sketch
    * @return true if the dissimilarity of the two sketches is greater than the given threshold
    * with at least 97.7% confidence
    */
   template<typename SketchA, typename SketchB>
-  static bool dissimilarity_test(const SketchA& actual, const SketchB& expected, double threshold) {
-    auto jc = jaccard(actual, expected);
+  static bool dissimilarity_test(const SketchA& actual, const SketchB& expected, double threshold, uint64_t seed = DEFAULT_SEED) {
+    auto jc = jaccard(actual, expected, seed);
     return jc[2] <= threshold;
   }
 
 private:
 
   template<typename SketchA, typename SketchB>
-  static typename Union::CompactSketch compute_union(const SketchA& sketch_a, const SketchB& sketch_b) {
-    const unsigned count_a = sketch_a.get_num_retained();
-    const unsigned count_b = sketch_b.get_num_retained();
-    const unsigned lg_k = std::min(std::max(log2(ceiling_power_of_2(count_a + count_b)), theta_constants::MIN_LG_K), theta_constants::MAX_LG_K);
-    auto u = typename Union::builder().set_lg_k(lg_k).build();
+  static typename Union::CompactSketch compute_union(const SketchA& sketch_a, const SketchB& sketch_b, uint64_t seed) {
+    const auto count_a = sketch_a.get_num_retained();
+    const auto count_b = sketch_b.get_num_retained();
+    const uint8_t lg_k = std::min(std::max(log2(ceiling_power_of_2(count_a + count_b)), theta_constants::MIN_LG_K), theta_constants::MAX_LG_K);
+    auto u = typename Union::builder().set_lg_k(lg_k).set_seed(seed).build();
     u.update(sketch_a);
     u.update(sketch_b);
     return u.get_result(false);

data/vendor/datasketches-cpp/theta/include/theta_set_difference_base_impl.hpp

@@ -36,7 +36,7 @@ seed_hash_(compute_seed_hash(seed))
 template<typename EN, typename EK, typename CS, typename A>
 template<typename FwdSketch, typename Sketch>
 CS theta_set_difference_base<EN, EK, CS, A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const {
-  if (a.is_empty() || a.get_num_retained() == 0 || b.is_empty()) return CS(a, ordered);
+  if (a.is_empty() || (a.get_num_retained() > 0 && b.is_empty())) return CS(a, ordered);
   if (a.get_seed_hash() != seed_hash_) throw std::invalid_argument("A seed hash mismatch");
   if (b.get_seed_hash() != seed_hash_) throw std::invalid_argument("B seed hash mismatch");
 
@@ -53,7 +53,7 @@ CS theta_set_difference_base<EN, EK, CS, A>::compute(FwdSketch&& a, const Sketch
           conditional_back_inserter(entries, key_less_than<uint64_t, EN, EK>(theta)), comparator());
     } else { // hash-based
       const uint8_t lg_size = lg_size_from_count(b.get_num_retained(), hash_table::REBUILD_THRESHOLD);
-      hash_table table(lg_size, lg_size, hash_table::resize_factor::X1, 0, 0, allocator_); // theta and seed are not used here
+      hash_table table(lg_size, lg_size, hash_table::resize_factor::X1, 1, 0, 0, allocator_); // theta and seed are not used here
       for (const auto& entry: b) {
         const uint64_t hash = EK()(entry);
         if (hash < theta) {

data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp

@@ -25,14 +25,10 @@
 namespace datasketches {
 
 template<typename Allocator = std::allocator<uint64_t>>
-class theta_sketch_alloc {
+class base_theta_sketch_alloc {
 public:
-  using Entry = uint64_t;
-  using ExtractKey = trivial_extract_key;
-  using iterator = theta_iterator<Entry, ExtractKey>;
-  using const_iterator = theta_const_iterator<Entry, ExtractKey>;
 
-  virtual ~theta_sketch_alloc() = default;
+  virtual ~base_theta_sketch_alloc() = default;
 
   /**
    * @return allocator
@@ -104,6 +100,21 @@ public:
    */
   virtual string<Allocator> to_string(bool print_items = false) const;
 
+protected:
+  virtual void print_specifics(std::ostringstream& os) const = 0;
+  virtual void print_items(std::ostringstream& os) const = 0;
+};
+
+template<typename Allocator = std::allocator<uint64_t>>
+class theta_sketch_alloc: public base_theta_sketch_alloc<Allocator> {
+public:
+  using Entry = uint64_t;
+  using ExtractKey = trivial_extract_key;
+  using iterator = theta_iterator<Entry, ExtractKey>;
+  using const_iterator = theta_const_iterator<Entry, ExtractKey>;
+
+  virtual ~theta_sketch_alloc() = default;
+
   /**
    * Iterator over hash values in this sketch.
    * @return begin iterator
@@ -131,8 +142,7 @@ public:
   virtual const_iterator end() const = 0;
 
 protected:
-  using ostrstream = std::basic_ostringstream<char, std::char_traits<char>, AllocChar<Allocator>>;
-  virtual void print_specifics(ostrstream& os) const = 0;
+  virtual void print_items(std::ostringstream& os) const;
 };
 
 // forward declaration
@@ -269,6 +279,11 @@ public:
    */
   void trim();
 
+  /**
+   * Reset the sketch to the initial empty state
+   */
+  void reset();
+
   /**
    * Converts this sketch to a compact sketch (ordered or unordered).
    * @param ordered optional flag to specify if ordered sketch should be produced
@@ -285,11 +300,10 @@ private:
   theta_table table_;
 
   // for builder
-  update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta,
-      uint64_t seed, const Allocator& allocator);
+  update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p,
+      uint64_t theta, uint64_t seed, const Allocator& allocator);
 
-  using ostrstream = typename Base::ostrstream;
-  virtual void print_specifics(ostrstream& os) const;
+  virtual void print_specifics(std::ostringstream& os) const;
 };
 
 // compact sketch
@@ -311,7 +325,8 @@ public:
   // - as a result of a set operation
   // - by deserializing a previously serialized compact sketch
 
-  compact_theta_sketch_alloc(const Base& other, bool ordered);
+  template<typename Other>
+  compact_theta_sketch_alloc(const Other& other, bool ordered);
   compact_theta_sketch_alloc(const compact_theta_sketch_alloc&) = default;
   compact_theta_sketch_alloc(compact_theta_sketch_alloc&&) noexcept = default;
   virtual ~compact_theta_sketch_alloc() = default;
@@ -376,8 +391,7 @@ private:
   uint64_t theta_;
   std::vector<uint64_t, Allocator> entries_;
 
-  using ostrstream = typename Base::ostrstream;
-  virtual void print_specifics(ostrstream& os) const;
+  virtual void print_specifics(std::ostringstream& os) const;
 };
 
 template<typename Allocator>
@@ -387,10 +401,54 @@ public:
     update_theta_sketch_alloc build() const;
 };
 
+// This is to wrap a buffer containing a serialized compact sketch and use it in a set operation avoiding some cost of deserialization.
+// It does not take the ownership of the buffer.
+
+template<typename Allocator = std::allocator<uint64_t>>
+class wrapped_compact_theta_sketch_alloc : public base_theta_sketch_alloc<Allocator> {
+public:
+  using const_iterator = const uint64_t*;
+
+  Allocator get_allocator() const;
+  bool is_empty() const;
+  bool is_ordered() const;
+  uint64_t get_theta64() const;
+  uint32_t get_num_retained() const;
+  uint16_t get_seed_hash() const;
+
+  const_iterator begin() const;
+  const_iterator end() const;
+
+  /**
+   * This method wraps a serialized compact sketch as an array of bytes.
+   * @param bytes pointer to the array of bytes
+   * @param size the size of the array
+   * @param seed the seed for the hash function that was used to create the sketch
+   * @return an instance of the sketch
+   */
+  static const wrapped_compact_theta_sketch_alloc wrap(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED, bool dump_on_error = false);
+
+protected:
+  virtual void print_specifics(std::ostringstream& os) const;
+  virtual void print_items(std::ostringstream& os) const;
+
+private:
+  bool is_empty_;
+  bool is_ordered_;
+  uint16_t seed_hash_;
+  uint32_t num_entries_;
+  uint64_t theta_;
+  const uint64_t* entries_;
+
+  wrapped_compact_theta_sketch_alloc(bool is_empty, bool is_ordered, uint16_t seed_hash, uint32_t num_entries,
+      uint64_t theta, const uint64_t* entries);
+};
+
 // aliases with default allocator for convenience
 using theta_sketch = theta_sketch_alloc<std::allocator<uint64_t>>;
 using update_theta_sketch = update_theta_sketch_alloc<std::allocator<uint64_t>>;
 using compact_theta_sketch = compact_theta_sketch_alloc<std::allocator<uint64_t>>;
+using wrapped_compact_theta_sketch = wrapped_compact_theta_sketch_alloc<std::allocator<uint64_t>>;
 
 } /* namespace datasketches */