datasketches 0.1.2 → 0.2.3

data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp

@@ -20,45 +20,29 @@
 #ifndef THETA_SKETCH_HPP_
 #define THETA_SKETCH_HPP_
 
-#include <memory>
-#include <functional>
-#include <climits>
-#include <vector>
-
-#include "common_defs.hpp"
+#include "theta_update_sketch_base.hpp"
 
 namespace datasketches {
 
-/*
- * author Alexander Saydakov
- * author Lee Rhodes
- * author Kevin Lang
- */
-
-// forward-declarations
-template<typename A> class theta_sketch_alloc;
-template<typename A> class update_theta_sketch_alloc;
-template<typename A> class compact_theta_sketch_alloc;
-template<typename A> class theta_union_alloc;
-template<typename A> class theta_intersection_alloc;
-template<typename A> class theta_a_not_b_alloc;
-
-// for serialization as raw bytes
-template<typename A> using AllocU8 = typename std::allocator_traits<A>::template rebind_alloc<uint8_t>;
-template<typename A> using vector_u8 = std::vector<uint8_t, AllocU8<A>>;
-
-template<typename A>
+template<typename Allocator = std::allocator<uint64_t>>
 class theta_sketch_alloc {
 public:
-  static const uint64_t MAX_THETA = LLONG_MAX; // signed max for compatibility with Java
-  static const uint8_t SERIAL_VERSION = 3;
+  using Entry = uint64_t;
+  using ExtractKey = trivial_extract_key;
+  using iterator = theta_iterator<Entry, ExtractKey>;
+  using const_iterator = theta_const_iterator<Entry, ExtractKey>;
 
   virtual ~theta_sketch_alloc() = default;
 
+  /**
+   * @return allocator
+   */
+  virtual Allocator get_allocator() const = 0;
+
   /**
    * @return true if this sketch represents an empty set (not the same as no retained entries!)
    */
-  bool is_empty() const;
+  virtual bool is_empty() const = 0;
 
   /**
    * @return estimate of the distinct count of the input stream
@@ -96,13 +80,16 @@ public:
   /**
    * @return theta as a positive integer between 0 and LLONG_MAX
    */
-  uint64_t get_theta64() const;
+  virtual uint64_t get_theta64() const = 0;
 
   /**
    * @return the number of retained entries in the sketch
    */
   virtual uint32_t get_num_retained() const = 0;
 
+  /**
+   * @return hash of the seed that was used to hash the input
+   */
   virtual uint16_t get_seed_hash() const = 0;
 
   /**
@@ -111,109 +98,82 @@ public:
   virtual bool is_ordered() const = 0;
 
   /**
-   * Writes a human-readable summary of this sketch to a given stream
+   * Provides a human-readable summary of this sketch as a string
    * @param print_items if true include the list of items retained by the sketch
+   * @return sketch summary as a string
    */
-  virtual string<A> to_string(bool print_items = false) const = 0;
+  virtual string<Allocator> to_string(bool print_items = false) const;
 
   /**
-   * This method serializes the sketch into a given stream in a binary form
-   * @param os output stream
-   */
-  virtual void serialize(std::ostream& os) const = 0;
-
-  // This is a convenience alias for users
-  // The type returned by the following serialize method
-  typedef vector_u8<A> vector_bytes;
-
-  /**
-   * This method serializes the sketch as a vector of bytes.
-   * An optional header can be reserved in front of the sketch.
-   * It is an uninitialized space of a given size.
-   * This header is used in Datasketches PostgreSQL extension.
-   * @param header_size_bytes space to reserve in front of the sketch
-   */
-  virtual vector_bytes serialize(unsigned header_size_bytes = 0) const = 0;
-
-  // This is a convenience alias for users
-  // The type returned by the following deserialize methods
-  // It is not possible to return instances of an abstract type, so this has to be a pointer
-  typedef std::unique_ptr<theta_sketch_alloc<A>, std::function<void(theta_sketch_alloc<A>*)>> unique_ptr;
-
-  /**
-   * This method deserializes a sketch from a given stream.
-   * @param is input stream
-   * @param seed the seed for the hash function that was used to create the sketch
-   * @return an instance of a sketch as a unique_ptr
+   * Iterator over hash values in this sketch.
+   * @return begin iterator
    */
-  static unique_ptr deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED);
+  virtual iterator begin() = 0;
 
   /**
-   * This method deserializes a sketch from a given array of bytes.
-   * @param bytes pointer to the array of bytes
-   * @param size the size of the array
-   * @param seed the seed for the hash function that was used to create the sketch
-   * @return an instance of the sketch
+   * Iterator pointing past the valid range.
+   * Not to be incremented or dereferenced.
+   * @return end iterator
    */
-  static unique_ptr deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED);
-
-  class const_iterator;
+  virtual iterator end() = 0;
 
   /**
-   * Iterator over hash values in this sketch.
+   * Const iterator over hash values in this sketch.
    * @return begin iterator
    */
   virtual const_iterator begin() const = 0;
 
   /**
-   * Iterator pointing past the valid range.
+   * Const iterator pointing past the valid range.
    * Not to be incremented or dereferenced.
    * @return end iterator
    */
   virtual const_iterator end() const = 0;
 
 protected:
-  enum flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED };
-
-  bool is_empty_;
-  uint64_t theta_;
-
-  theta_sketch_alloc(bool is_empty, uint64_t theta);
-
-  static uint16_t get_seed_hash(uint64_t seed);
-
-  static void check_sketch_type(uint8_t actual, uint8_t expected);
-  static void check_serial_version(uint8_t actual, uint8_t expected);
-  static void check_seed_hash(uint16_t actual, uint16_t expected);
-
-  friend theta_intersection_alloc<A>;
-  friend theta_a_not_b_alloc<A>;
+  using ostrstream = std::basic_ostringstream<char, std::char_traits<char>, AllocChar<Allocator>>;
+  virtual void print_specifics(ostrstream& os) const = 0;
 };
 
-// update sketch
-
-template<typename A> using AllocU64 = typename std::allocator_traits<A>::template rebind_alloc<uint64_t>;
-template<typename A> using vector_u64 = std::vector<uint64_t, AllocU64<A>>;
+// forward declaration
+template<typename A> class compact_theta_sketch_alloc;
 
-template<typename A>
-class update_theta_sketch_alloc: public theta_sketch_alloc<A> {
+template<typename Allocator = std::allocator<uint64_t>>
+class update_theta_sketch_alloc: public theta_sketch_alloc<Allocator> {
 public:
-  class builder;
-  enum resize_factor { X1, X2, X4, X8 };
-  static const uint8_t SKETCH_TYPE = 2;
+  using Base = theta_sketch_alloc<Allocator>;
+  using Entry = typename Base::Entry;
+  using ExtractKey = typename Base::ExtractKey;
+  using iterator = typename Base::iterator;
+  using const_iterator = typename Base::const_iterator;
+  using theta_table = theta_update_sketch_base<Entry, ExtractKey, Allocator>;
+  using resize_factor = typename theta_table::resize_factor;
 
   // No constructor here. Use builder instead.
+  class builder;
 
+  update_theta_sketch_alloc(const update_theta_sketch_alloc&) = default;
+  update_theta_sketch_alloc(update_theta_sketch_alloc&&) noexcept = default;
   virtual ~update_theta_sketch_alloc() = default;
+  update_theta_sketch_alloc& operator=(const update_theta_sketch_alloc&) = default;
+  update_theta_sketch_alloc& operator=(update_theta_sketch_alloc&&) = default;
 
-  virtual uint32_t get_num_retained() const;
-  virtual uint16_t get_seed_hash() const;
+  virtual Allocator get_allocator() const;
+  virtual bool is_empty() const;
   virtual bool is_ordered() const;
-  virtual string<A> to_string(bool print_items = false) const;
-  virtual void serialize(std::ostream& os) const;
-  typedef vector_u8<A> vector_bytes; // alias for users
-  // header space is reserved, but not initialized
-  virtual vector_bytes serialize(unsigned header_size_bytes = 0) const;
+  virtual uint16_t get_seed_hash() const;
+  virtual uint64_t get_theta64() const;
+  virtual uint32_t get_num_retained() const;
+
+  /**
+   * @return configured nominal number of entries in the sketch
+   */
+  uint8_t get_lg_k() const;
+
+  /**
+   * @return configured resize factor of the sketch
+   */
+  resize_factor get_rf() const;
 
   /**
    * Update this sketch with a given string.
@@ -302,7 +262,7 @@ public:
    * @param data pointer to the data
    * @param length of the data in bytes
    */
-  void update(const void* data, unsigned length);
+  void update(const void* data, size_t length);
 
   /**
    * Remove retained entries in excess of the nominal size k (if any)
@@ -314,105 +274,86 @@ public:
    * @param ordered optional flag to specify if ordered sketch should be produced
    * @return compact sketch
    */
-  compact_theta_sketch_alloc<A> compact(bool ordered = true) const;
-
-  virtual typename theta_sketch_alloc<A>::const_iterator begin() const;
-  virtual typename theta_sketch_alloc<A>::const_iterator end() const;
-
-  /**
-   * This method deserializes a sketch from a given stream.
-   * @param is input stream
-   * @param seed the seed for the hash function that was used to create the sketch
-   * @return an instance of a sketch
-   */
-  static update_theta_sketch_alloc<A> deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED);
+  compact_theta_sketch_alloc<Allocator> compact(bool ordered = true) const;
 
-  /**
-   * This method deserializes a sketch from a given array of bytes.
-   * @param bytes pointer to the array of bytes
-   * @param size the size of the array
-   * @param seed the seed for the hash function that was used to create the sketch
-   * @return an instance of the sketch
-   */
-  static update_theta_sketch_alloc<A> deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED);
+  virtual iterator begin();
+  virtual iterator end();
+  virtual const_iterator begin() const;
+  virtual const_iterator end() const;
 
 private:
-  // resize threshold = 0.5 tuned for speed
-  static constexpr double RESIZE_THRESHOLD = 0.5;
-  // hash table rebuild threshold = 15/16
-  static constexpr double REBUILD_THRESHOLD = 15.0 / 16.0;
-
-  static constexpr uint8_t STRIDE_HASH_BITS = 7;
-  static constexpr uint32_t STRIDE_MASK = (1 << STRIDE_HASH_BITS) - 1;
-
-  uint8_t lg_cur_size_;
-  uint8_t lg_nom_size_;
-  vector_u64<A> keys_;
-  uint32_t num_keys_;
-  resize_factor rf_;
-  float p_;
-  uint64_t seed_;
-  uint32_t capacity_;
+  theta_table table_;
 
   // for builder
-  update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p, uint64_t seed);
-
-  // for deserialize
-  update_theta_sketch_alloc(bool is_empty, uint64_t theta, uint8_t lg_cur_size, uint8_t lg_nom_size, vector_u64<A>&& keys, uint32_t num_keys, resize_factor rf, float p, uint64_t seed);
-
-  void resize();
-  void rebuild();
+  update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta,
+      uint64_t seed, const Allocator& allocator);
 
-  friend theta_union_alloc<A>;
-  void internal_update(uint64_t hash);
-
-  friend theta_intersection_alloc<A>;
-  friend theta_a_not_b_alloc<A>;
-  static inline uint32_t get_capacity(uint8_t lg_cur_size, uint8_t lg_nom_size);
-  static inline uint32_t get_stride(uint64_t hash, uint8_t lg_size);
-  static bool hash_search_or_insert(uint64_t hash, uint64_t* table, uint8_t lg_size);
-  static bool hash_search(uint64_t hash, const uint64_t* table, uint8_t lg_size);
-
-  friend theta_sketch_alloc<A>;
-  static update_theta_sketch_alloc<A> internal_deserialize(std::istream& is, resize_factor rf, uint8_t lg_cur_size, uint8_t lg_nom_size, uint8_t flags_byte, uint64_t seed);
-  static update_theta_sketch_alloc<A> internal_deserialize(const void* bytes, size_t size, resize_factor rf, uint8_t lg_cur_size, uint8_t lg_nom_size, uint8_t flags_byte, uint64_t seed);
+  using ostrstream = typename Base::ostrstream;
+  virtual void print_specifics(ostrstream& os) const;
 };
 
 // compact sketch
 
-template<typename A>
-class compact_theta_sketch_alloc: public theta_sketch_alloc<A> {
+template<typename Allocator = std::allocator<uint64_t>>
+class compact_theta_sketch_alloc: public theta_sketch_alloc<Allocator> {
 public:
+  using Base = theta_sketch_alloc<Allocator>;
+  using iterator = typename Base::iterator;
+  using const_iterator = typename Base::const_iterator;
+  using AllocBytes = typename std::allocator_traits<Allocator>::template rebind_alloc<uint8_t>;
+  using vector_bytes = std::vector<uint8_t, AllocBytes>;
+
+  static const uint8_t SERIAL_VERSION = 3;
   static const uint8_t SKETCH_TYPE = 3;
 
-  // No constructor here.
   // Instances of this type can be obtained:
-  // - by compacting an update_theta_sketch
+  // - by compacting an update_theta_sketch_alloc
   // - as a result of a set operation
   // - by deserializing a previously serialized compact sketch
 
-  compact_theta_sketch_alloc(const theta_sketch_alloc<A>& other, bool ordered);
+  template<typename Other>
+  compact_theta_sketch_alloc(const Other& other, bool ordered);
+  compact_theta_sketch_alloc(const compact_theta_sketch_alloc&) = default;
+  compact_theta_sketch_alloc(compact_theta_sketch_alloc&&) noexcept = default;
   virtual ~compact_theta_sketch_alloc() = default;
+  compact_theta_sketch_alloc& operator=(const compact_theta_sketch_alloc&) = default;
+  compact_theta_sketch_alloc& operator=(compact_theta_sketch_alloc&&) = default;
 
+  virtual Allocator get_allocator() const;
+  virtual bool is_empty() const;
+  virtual bool is_ordered() const;
+  virtual uint64_t get_theta64() const;
   virtual uint32_t get_num_retained() const;
   virtual uint16_t get_seed_hash() const;
-  virtual bool is_ordered() const;
-  virtual string<A> to_string(bool print_items = false) const;
-  virtual void serialize(std::ostream& os) const;
-  typedef vector_u8<A> vector_bytes; // alias for users
-  // header space is reserved, but not initialized
-  virtual vector_bytes serialize(unsigned header_size_bytes = 0) const;
 
-  virtual typename theta_sketch_alloc<A>::const_iterator begin() const;
-  virtual typename theta_sketch_alloc<A>::const_iterator end() const;
+  /**
+   * This method serializes the sketch into a given stream in a binary form
+   * @param os output stream
+   */
+  void serialize(std::ostream& os) const;
+
+  /**
+   * This method serializes the sketch as a vector of bytes.
+   * An optional header can be reserved in front of the sketch.
+   * It is an uninitialized space of a given size.
+   * This header is used in Datasketches PostgreSQL extension.
+   * @param header_size_bytes space to reserve in front of the sketch
+   */
+  vector_bytes serialize(unsigned header_size_bytes = 0) const;
+
+  virtual iterator begin();
+  virtual iterator end();
+  virtual const_iterator begin() const;
+  virtual const_iterator end() const;
 
   /**
    * This method deserializes a sketch from a given stream.
    * @param is input stream
    * @param seed the seed for the hash function that was used to create the sketch
-   * @return an instance of a sketch
+   * @return an instance of the sketch
    */
-  static compact_theta_sketch_alloc<A> deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED);
+  static compact_theta_sketch_alloc deserialize(std::istream& is,
+      uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
 
   /**
    * This method deserializes a sketch from a given array of bytes.
@@ -421,110 +362,76 @@ public:
    * @param seed the seed for the hash function that was used to create the sketch
    * @return an instance of the sketch
    */
-  static compact_theta_sketch_alloc<A> deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED);
+  static compact_theta_sketch_alloc deserialize(const void* bytes, size_t size,
+      uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+  // for internal use
+  compact_theta_sketch_alloc(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<uint64_t, Allocator>&& entries);
 
 private:
-  typedef typename std::allocator_traits<A>::template rebind_alloc<uint64_t> AllocU64;
+  enum flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED };
 
-  vector_u64<A> keys_;
-  uint16_t seed_hash_;
+  bool is_empty_;
   bool is_ordered_;
+  uint16_t seed_hash_;
+  uint64_t theta_;
+  std::vector<uint64_t, Allocator> entries_;
 
-  friend theta_sketch_alloc<A>;
-  friend update_theta_sketch_alloc<A>;
-  friend theta_union_alloc<A>;
-  friend theta_intersection_alloc<A>;
-  friend theta_a_not_b_alloc<A>;
-  compact_theta_sketch_alloc(bool is_empty, uint64_t theta, vector_u64<A>&& keys, uint16_t seed_hash, bool is_ordered);
-  static compact_theta_sketch_alloc<A> internal_deserialize(std::istream& is, uint8_t preamble_longs, uint8_t flags_byte, uint16_t seed_hash);
-  static compact_theta_sketch_alloc<A> internal_deserialize(const void* bytes, size_t size, uint8_t preamble_longs, uint8_t flags_byte, uint16_t seed_hash);
+  using ostrstream = typename Base::ostrstream;
+  virtual void print_specifics(ostrstream& os) const;
 };
 
-// builder
-
-template<typename A>
-class update_theta_sketch_alloc<A>::builder {
+template<typename Allocator>
+class update_theta_sketch_alloc<Allocator>::builder: public theta_base_builder<builder, Allocator> {
 public:
-  static const uint8_t MIN_LG_K = 5;
-  static const uint8_t DEFAULT_LG_K = 12;
-  static const resize_factor DEFAULT_RESIZE_FACTOR = X8;
-
-  /**
-   * Creates and instance of the builder with default parameters.
-   */
-  builder();
+    builder(const Allocator& allocator = Allocator());
+    update_theta_sketch_alloc build() const;
+};
 
-  /**
-   * Set log2(k), where k is a nominal number of entries in the sketch
-   * @param lg_k base 2 logarithm of nominal number of entries
-   * @return this builder
-   */
-  builder& set_lg_k(uint8_t lg_k);
+// This is to wrap a buffer containing a serialized compact sketch and use it in a set operation avoiding some cost of deserialization.
+// It does not take the ownership of the buffer.
 
-  /**
-   * Set resize factor for the internal hash table (defaults to 8)
-   * @param rf resize factor
-   * @return this builder
-   */
-  builder& set_resize_factor(resize_factor rf);
+template<typename Allocator = std::allocator<uint64_t>>
+class wrapped_compact_theta_sketch_alloc {
+public:
+  using const_iterator = const uint64_t*;
 
-  /**
-   * Set sampling probability (initial theta). The default is 1, so the sketch retains
-   * all entries until it reaches the limit, at which point it goes into the estimation mode
-   * and reduces the effective sampling probability (theta) as necessary.
-   * @param p sampling probability
-   * @return this builder
-   */
-  builder& set_p(float p);
+  Allocator get_allocator() const;
+  bool is_empty() const;
+  bool is_ordered() const;
+  uint64_t get_theta64() const;
+  uint32_t get_num_retained() const;
+  uint16_t get_seed_hash() const;
 
-  /**
-   * Set the seed for the hash function. Should be used carefully if needed.
-   * Sketches produced with different seed are not compatible
-   * and cannot be mixed in set operations.
-   * @param seed hash seed
-   * @return this builder
-   */
-  builder& set_seed(uint64_t seed);
+  const_iterator begin() const;
+  const_iterator end() const;
 
   /**
-   * This is to create an instance of the sketch with predefined parameters.
-   * @return and instance of the sketch
+   * This method wraps a serialized compact sketch as an array of bytes.
+   * @param bytes pointer to the array of bytes
+   * @param size the size of the array
+   * @param seed the seed for the hash function that was used to create the sketch
+   * @return an instance of the sketch
    */
-  update_theta_sketch_alloc<A> build() const;
+  static const wrapped_compact_theta_sketch_alloc wrap(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED, bool dump_on_error = false);
 
 private:
-  uint8_t lg_k_;
-  resize_factor rf_;
-  float p_;
-  uint64_t seed_;
-
-  static uint8_t starting_sub_multiple(uint8_t lg_tgt, uint8_t lg_min, uint8_t lg_rf);
-};
-
-// iterator
-template<typename A>
-class theta_sketch_alloc<A>::const_iterator: public std::iterator<std::input_iterator_tag, uint64_t> {
-public:
-  const_iterator& operator++();
-  const_iterator operator++(int);
-  bool operator==(const const_iterator& other) const;
-  bool operator!=(const const_iterator& other) const;
-  uint64_t operator*() const;
+  bool is_empty_;
+  bool is_ordered_;
+  uint16_t seed_hash_;
+  uint32_t num_entries_;
+  uint64_t theta_;
+  const uint64_t* entries_;
 
-private:
-  const uint64_t* keys_;
-  uint32_t size_;
-  uint32_t index_;
-  const_iterator(const uint64_t* keys, uint32_t size, uint32_t index);
-  friend class update_theta_sketch_alloc<A>;
-  friend class compact_theta_sketch_alloc<A>;
+  wrapped_compact_theta_sketch_alloc(bool is_empty, bool is_ordered, uint16_t seed_hash, uint32_t num_entries,
+      uint64_t theta, const uint64_t* entries);
 };
 
-
 // aliases with default allocator for convenience
-typedef theta_sketch_alloc<std::allocator<void>> theta_sketch;
-typedef update_theta_sketch_alloc<std::allocator<void>> update_theta_sketch;
-typedef compact_theta_sketch_alloc<std::allocator<void>> compact_theta_sketch;
+using theta_sketch = theta_sketch_alloc<std::allocator<uint64_t>>;
+using update_theta_sketch = update_theta_sketch_alloc<std::allocator<uint64_t>>;
+using compact_theta_sketch = compact_theta_sketch_alloc<std::allocator<uint64_t>>;
+using wrapped_compact_theta_sketch = wrapped_compact_theta_sketch_alloc<std::allocator<uint64_t>>;
 
 } /* namespace datasketches */