datasketches 0.1.2 → 0.2.3

data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp

@@ -33,10 +33,14 @@ template<typename T, typename W, typename H, typename E, typename S, typename A>
 const uint8_t frequent_items_sketch<T, W, H, E, S, A>::LG_MIN_MAP_SIZE;
 
 template<typename T, typename W, typename H, typename E, typename S, typename A>
-frequent_items_sketch<T, W, H, E, S, A>::frequent_items_sketch(uint8_t lg_max_map_size, uint8_t lg_start_map_size):
+frequent_items_sketch<T, W, H, E, S, A>::frequent_items_sketch(uint8_t lg_max_map_size, uint8_t lg_start_map_size, const A& allocator):
 total_weight(0),
 offset(0),
-map(std::max(lg_start_map_size, frequent_items_sketch::LG_MIN_MAP_SIZE), std::max(lg_max_map_size, frequent_items_sketch::LG_MIN_MAP_SIZE))
+map(
+  std::max(lg_start_map_size, frequent_items_sketch::LG_MIN_MAP_SIZE),
+  std::max(lg_max_map_size, frequent_items_sketch::LG_MIN_MAP_SIZE),
+  allocator
+)
 {
   if (lg_start_map_size > lg_max_map_size) throw std::invalid_argument("starting size must not be greater than maximum size");
 }
@@ -61,7 +65,7 @@ template<typename T, typename W, typename H, typename E, typename S, typename A>
 void frequent_items_sketch<T, W, H, E, S, A>::merge(const frequent_items_sketch& other) {
   if (other.is_empty()) return;
   const W merged_total_weight = total_weight + other.get_total_weight(); // for correction at the end
-  for (auto &it: other.map) {
+  for (auto it: other.map) {
     update(it.first, it.second);
   }
   offset += other.offset;
@@ -72,7 +76,7 @@ template<typename T, typename W, typename H, typename E, typename S, typename A>
 void frequent_items_sketch<T, W, H, E, S, A>::merge(frequent_items_sketch&& other) {
   if (other.is_empty()) return;
   const W merged_total_weight = total_weight + other.get_total_weight(); // for correction at the end
-  for (auto &it: other.map) {
+  for (auto it: other.map) {
     update(std::move(it.first), it.second);
   }
   offset += other.offset;
@@ -142,8 +146,8 @@ frequent_items_sketch<T, W, H, E, S, A>::get_frequent_items(frequent_items_error
 template<typename T, typename W, typename H, typename E, typename S, typename A>
 typename frequent_items_sketch<T, W, H, E, S, A>::vector_row
 frequent_items_sketch<T, W, H, E, S, A>::get_frequent_items(frequent_items_error_type err_type, W threshold) const {
-  vector_row items;
-  for (auto &it: map) {
+  vector_row items(map.get_allocator());
+  for (auto it: map) {
     const W lb = it.second;
     const W ub = it.second + offset;
     if ((err_type == NO_FALSE_NEGATIVES && ub > threshold) || (err_type == NO_FALSE_POSITIVES && lb > threshold)) {
@@ -158,43 +162,45 @@ frequent_items_sketch<T, W, H, E, S, A>::get_frequent_items(frequent_items_error
 template<typename T, typename W, typename H, typename E, typename S, typename A>
 void frequent_items_sketch<T, W, H, E, S, A>::serialize(std::ostream& os) const {
   const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_EMPTY : PREAMBLE_LONGS_NONEMPTY;
-  os.write((char*)&preamble_longs, sizeof(preamble_longs));
+  write(os, preamble_longs);
   const uint8_t serial_version = SERIAL_VERSION;
-  os.write((char*)&serial_version, sizeof(serial_version));
+  write(os, serial_version);
   const uint8_t family = FAMILY_ID;
-  os.write((char*)&family, sizeof(family));
+  write(os, family);
   const uint8_t lg_max_size = map.get_lg_max_size();
-  os.write((char*)&lg_max_size, sizeof(lg_max_size));
+  write(os, lg_max_size);
   const uint8_t lg_cur_size = map.get_lg_cur_size();
-  os.write((char*)&lg_cur_size, sizeof(lg_cur_size));
+  write(os, lg_cur_size);
   const uint8_t flags_byte(
     (is_empty() ? 1 << flags::IS_EMPTY : 0)
   );
-  os.write((char*)&flags_byte, sizeof(flags_byte));
+  write(os, flags_byte);
   const uint16_t unused16 = 0;
-  os.write((char*)&unused16, sizeof(unused16));
+  write(os, unused16);
   if (!is_empty()) {
     const uint32_t num_items = map.get_num_active();
-    os.write((char*)&num_items, sizeof(num_items));
+    write(os, num_items);
     const uint32_t unused32 = 0;
-    os.write((char*)&unused32, sizeof(unused32));
-    os.write((char*)&total_weight, sizeof(total_weight));
-    os.write((char*)&offset, sizeof(offset));
+    write(os, unused32);
+    write(os, total_weight);
+    write(os, offset);
 
     // copy active items and their weights to use batch serialization
-    typedef typename std::allocator_traits<A>::template rebind_alloc<W> AllocW;
-    W* weights = AllocW().allocate(num_items);
-    T* items = A().allocate(num_items);
+    using AllocW = typename std::allocator_traits<A>::template rebind_alloc<W>;
+    AllocW aw(map.get_allocator());
+    W* weights = aw.allocate(num_items);
+    A alloc(map.get_allocator());
+    T* items = alloc.allocate(num_items);
     uint32_t i = 0;
-    for (auto &it: map) {
+    for (auto it: map) {
       new (&items[i]) T(it.first);
       weights[i++] = it.second;
     }
-    os.write((char*)weights, sizeof(W) * num_items);
-    AllocW().deallocate(weights, num_items);
+    write(os, weights, sizeof(W) * num_items);
+    aw.deallocate(weights, num_items);
     S().serialize(os, items, num_items);
-    for (unsigned i = 0; i < num_items; i++) items[i].~T();
-    A().deallocate(items, num_items);
+    for (i = 0; i < num_items; i++) items[i].~T();
+    alloc.deallocate(items, num_items);
   }
 }
 
@@ -202,56 +208,56 @@ template<typename T, typename W, typename H, typename E, typename S, typename A>
 size_t frequent_items_sketch<T, W, H, E, S, A>::get_serialized_size_bytes() const {
   if (is_empty()) return PREAMBLE_LONGS_EMPTY * sizeof(uint64_t);
   size_t size = PREAMBLE_LONGS_NONEMPTY * sizeof(uint64_t) + map.get_num_active() * sizeof(W);
-  for (auto &it: map) size += S().size_of_item(it.first);
+  for (auto it: map) size += S().size_of_item(it.first);
   return size;
 }
 
 template<typename T, typename W, typename H, typename E, typename S, typename A>
-vector_u8<A> frequent_items_sketch<T, W, H, E, S, A>::serialize(unsigned header_size_bytes) const {
+auto frequent_items_sketch<T, W, H, E, S, A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
   const size_t size = header_size_bytes + get_serialized_size_bytes();
-  vector_u8<A> bytes(size);
+  vector_bytes bytes(size, 0, map.get_allocator());
   uint8_t* ptr = bytes.data() + header_size_bytes;
   uint8_t* end_ptr = ptr + size;
 
   const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_EMPTY : PREAMBLE_LONGS_NONEMPTY;
-  ptr += copy_to_mem(&preamble_longs, ptr, sizeof(uint8_t));
+  ptr += copy_to_mem(preamble_longs, ptr);
   const uint8_t serial_version = SERIAL_VERSION;
-  ptr += copy_to_mem(&serial_version, ptr, sizeof(uint8_t));
+  ptr += copy_to_mem(serial_version, ptr);
   const uint8_t family = FAMILY_ID;
-  ptr += copy_to_mem(&family, ptr, sizeof(uint8_t));
+  ptr += copy_to_mem(family, ptr);
   const uint8_t lg_max_size = map.get_lg_max_size();
-  ptr += copy_to_mem(&lg_max_size, ptr, sizeof(uint8_t));
+  ptr += copy_to_mem(lg_max_size, ptr);
   const uint8_t lg_cur_size = map.get_lg_cur_size();
-  ptr += copy_to_mem(&lg_cur_size, ptr, sizeof(uint8_t));
+  ptr += copy_to_mem(lg_cur_size, ptr);
   const uint8_t flags_byte(
     (is_empty() ? 1 << flags::IS_EMPTY : 0)
   );
-  ptr += copy_to_mem(&flags_byte, ptr, sizeof(uint8_t));
-  const uint16_t unused16 = 0;
-  ptr += copy_to_mem(&unused16, ptr, sizeof(uint16_t));
+  ptr += copy_to_mem(flags_byte, ptr);
+  ptr += sizeof(uint16_t); // unused
   if (!is_empty()) {
     const uint32_t num_items = map.get_num_active();
-    ptr += copy_to_mem(&num_items, ptr, sizeof(uint32_t));
-    const uint32_t unused32 = 0;
-    ptr += copy_to_mem(&unused32, ptr, sizeof(uint32_t));
-    ptr += copy_to_mem(&total_weight, ptr, sizeof(total_weight));
-    ptr += copy_to_mem(&offset, ptr, sizeof(offset));
+    ptr += copy_to_mem(num_items, ptr);
+    ptr += sizeof(uint32_t); // unused
+    ptr += copy_to_mem(total_weight, ptr);
+    ptr += copy_to_mem(offset, ptr);
 
     // copy active items and their weights to use batch serialization
-    typedef typename std::allocator_traits<A>::template rebind_alloc<W> AllocW;
-    W* weights = AllocW().allocate(num_items);
-    T* items = A().allocate(num_items);
+    using AllocW = typename std::allocator_traits<A>::template rebind_alloc<W>;
+    AllocW aw(map.get_allocator());
+    W* weights = aw.allocate(num_items);
+    A alloc(map.get_allocator());
+    T* items = alloc.allocate(num_items);
     uint32_t i = 0;
-    for (auto &it: map) {
+    for (auto it: map) {
       new (&items[i]) T(it.first);
       weights[i++] = it.second;
     }
     ptr += copy_to_mem(weights, ptr, sizeof(W) * num_items);
-    AllocW().deallocate(weights, num_items);
+    aw.deallocate(weights, num_items);
     const size_t bytes_remaining = end_ptr - ptr;
     ptr += S().serialize(ptr, bytes_remaining, items, num_items);
-    for (unsigned i = 0; i < num_items; i++) items[i].~T();
-    A().deallocate(items, num_items);
+    for (i = 0; i < num_items; i++) items[i].~T();
+    alloc.deallocate(items, num_items);
   }
   return bytes;
 }
@@ -259,37 +265,32 @@ vector_u8<A> frequent_items_sketch<T, W, H, E, S, A>::serialize(unsigned header_
 template<typename T, typename W, typename H, typename E, typename S, typename A>
 class frequent_items_sketch<T, W, H, E, S, A>::items_deleter {
 public:
-  items_deleter(uint32_t num, bool destroy): num(num), destroy(destroy) {}
-  void set_destroy(bool destroy) { this->destroy = destroy; }
-  void operator() (T* ptr) const {
+  items_deleter(uint32_t num, bool destroy, const A& allocator):
+    allocator_(allocator), num_(num), destroy_(destroy) {}
+  void set_destroy(bool destroy) { destroy_ = destroy; }
+  void operator() (T* ptr) {
     if (ptr != nullptr) {
-      if (destroy) {
-        for (uint32_t i = 0; i < num; ++i) ptr[i].~T();
+      if (destroy_) {
+        for (uint32_t i = 0; i < num_; ++i) ptr[i].~T();
       }
-      A().deallocate(ptr, num);
+      allocator_.deallocate(ptr, num_);
     }
   }
 private:
-  uint32_t num;
-  bool destroy;
+  A allocator_;
+  uint32_t num_;
+  bool destroy_;
 };
 
 template<typename T, typename W, typename H, typename E, typename S, typename A>
-frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>::deserialize(std::istream& is) {
-  uint8_t preamble_longs;
-  is.read((char*)&preamble_longs, sizeof(preamble_longs));
-  uint8_t serial_version;
-  is.read((char*)&serial_version, sizeof(serial_version));
-  uint8_t family_id;
-  is.read((char*)&family_id, sizeof(family_id));
-  uint8_t lg_max_size;
-  is.read((char*)&lg_max_size, sizeof(lg_max_size));
-  uint8_t lg_cur_size;
-  is.read((char*)&lg_cur_size, sizeof(lg_cur_size));
-  uint8_t flags_byte;
-  is.read((char*)&flags_byte, sizeof(flags_byte));
-  uint16_t unused16;
-  is.read((char*)&unused16, sizeof(unused16));
+frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>::deserialize(std::istream& is, const A& allocator) {
+  const auto preamble_longs = read<uint8_t>(is);
+  const auto serial_version = read<uint8_t>(is);
+  const auto family_id = read<uint8_t>(is);
+  const auto lg_max_size = read<uint8_t>(is);
+  const auto lg_cur_size = read<uint8_t>(is);
+  const auto flags_byte = read<uint8_t>(is);
+  read<uint16_t>(is); // unused
 
   const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
 
@@ -298,22 +299,19 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
   check_family_id(family_id);
   check_size(lg_cur_size, lg_max_size);
 
-  frequent_items_sketch<T, W, H, E, S, A> sketch(lg_max_size, lg_cur_size);
+  frequent_items_sketch<T, W, H, E, S, A> sketch(lg_max_size, lg_cur_size, allocator);
   if (!is_empty) {
-    uint32_t num_items;
-    is.read((char*)&num_items, sizeof(num_items));
-    uint32_t unused32;
-    is.read((char*)&unused32, sizeof(unused32));
-    W total_weight;
-    is.read((char*)&total_weight, sizeof(total_weight));
-    W offset;
-    is.read((char*)&offset, sizeof(offset));
+    const auto num_items = read<uint32_t>(is);
+    read<uint32_t>(is); // unused
+    const auto total_weight = read<W>(is);
+    const auto offset = read<W>(is);
 
     // batch deserialization with intermediate array of items and weights
-    typedef typename std::allocator_traits<A>::template rebind_alloc<W> AllocW;
-    std::vector<W, AllocW> weights(num_items);
-    is.read((char*)weights.data(), sizeof(W) * num_items);
-    std::unique_ptr<T, items_deleter> items(A().allocate(num_items), items_deleter(num_items, false));
+    using AllocW = typename std::allocator_traits<A>::template rebind_alloc<W>;
+    std::vector<W, AllocW> weights(num_items, 0, allocator);
+    read(is, weights.data(), sizeof(W) * num_items);
+    A alloc(allocator);
+    std::unique_ptr<T, items_deleter> items(alloc.allocate(num_items), items_deleter(num_items, false, alloc));
     S().deserialize(is, items.get(), num_items);
     items.get_deleter().set_destroy(true); // serde did not throw, so the items must be constructed
     for (uint32_t i = 0; i < num_items; i++) {
@@ -328,24 +326,23 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
 }
 
 template<typename T, typename W, typename H, typename E, typename S, typename A>
-frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>::deserialize(const void* bytes, size_t size) {
+frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>::deserialize(const void* bytes, size_t size, const A& allocator) {
   ensure_minimum_memory(size, 8);
   const char* ptr = static_cast<const char*>(bytes);
   const char* base = static_cast<const char*>(bytes);
   uint8_t preamble_longs;
-  ptr += copy_from_mem(ptr, &preamble_longs, sizeof(uint8_t));
+  ptr += copy_from_mem(ptr, preamble_longs);
   uint8_t serial_version;
-  ptr += copy_from_mem(ptr, &serial_version, sizeof(uint8_t));
+  ptr += copy_from_mem(ptr, serial_version);
   uint8_t family_id;
-  ptr += copy_from_mem(ptr, &family_id, sizeof(uint8_t));
+  ptr += copy_from_mem(ptr, family_id);
   uint8_t lg_max_size;
-  ptr += copy_from_mem(ptr, &lg_max_size, sizeof(uint8_t));
+  ptr += copy_from_mem(ptr, lg_max_size);
   uint8_t lg_cur_size;
-  ptr += copy_from_mem(ptr, &lg_cur_size, sizeof(uint8_t));
+  ptr += copy_from_mem(ptr, lg_cur_size);
   uint8_t flags_byte;
-  ptr += copy_from_mem(ptr, &flags_byte, sizeof(uint8_t));
-  uint16_t unused16;
-  ptr += copy_from_mem(ptr, &unused16, sizeof(uint16_t));
+  ptr += copy_from_mem(ptr, flags_byte);
+  ptr += sizeof(uint16_t); // unused
 
   const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
 
@@ -353,25 +350,25 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
   check_serial_version(serial_version);
   check_family_id(family_id);
   check_size(lg_cur_size, lg_max_size);
-  ensure_minimum_memory(size, 1 << preamble_longs);
+  ensure_minimum_memory(size, preamble_longs * sizeof(uint64_t));
 
-  frequent_items_sketch<T, W, H, E, S, A> sketch(lg_max_size, lg_cur_size);
+  frequent_items_sketch<T, W, H, E, S, A> sketch(lg_max_size, lg_cur_size, allocator);
   if (!is_empty) {
     uint32_t num_items;
-    ptr += copy_from_mem(ptr, &num_items, sizeof(uint32_t));
-    uint32_t unused32;
-    ptr += copy_from_mem(ptr, &unused32, sizeof(uint32_t));
+    ptr += copy_from_mem(ptr, num_items);
+    ptr += sizeof(uint32_t); // unused
     W total_weight;
-    ptr += copy_from_mem(ptr, &total_weight, sizeof(total_weight));
+    ptr += copy_from_mem(ptr, total_weight);
     W offset;
-    ptr += copy_from_mem(ptr, &offset, sizeof(offset));
+    ptr += copy_from_mem(ptr, offset);
 
     ensure_minimum_memory(size, ptr - base + (sizeof(W) * num_items));
     // batch deserialization with intermediate array of items and weights
-    typedef typename std::allocator_traits<A>::template rebind_alloc<W> AllocW;
-    std::vector<W, AllocW> weights(num_items);
+    using AllocW = typename std::allocator_traits<A>::template rebind_alloc<W>;
+    std::vector<W, AllocW> weights(num_items, 0, allocator);
     ptr += copy_from_mem(ptr, weights.data(), sizeof(W) * num_items);
-    std::unique_ptr<T, items_deleter> items(A().allocate(num_items), items_deleter(num_items, false));
+    A alloc(allocator);
+    std::unique_ptr<T, items_deleter> items(alloc.allocate(num_items), items_deleter(num_items, false, alloc));
     const size_t bytes_remaining = size - (ptr - base);
     ptr += S().deserialize(ptr, bytes_remaining, items.get(), num_items);
     items.get_deleter().set_destroy(true); // serde did not throw, so the items must be constructed
@@ -434,14 +431,14 @@ string<A> frequent_items_sketch<T, W, H, E, S, A>::to_string(bool print_items) c
   os << "### End sketch summary" << std::endl;
   if (print_items) {
     vector_row items;
-    for (auto &it: map) {
+    for (auto it: map) {
       items.push_back(row(&it.first, it.second, offset));
     }
     // sort by estimate in descending order
     std::sort(items.begin(), items.end(), [](row a, row b){ return a.get_estimate() > b.get_estimate(); });
     os << "### Items in descending order by estimate" << std::endl;
     os << "   item, estimate, lower bound, upper bound" << std::endl;
-    for (auto &it: items) {
+    for (auto it: items) {
       os << "   " << it.get_item() << ", " << it.get_estimate() << ", "
          << it.get_lower_bound() << ", " << it.get_upper_bound() << std::endl;
     }

data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map.hpp

@@ -39,33 +39,39 @@ public:
   using AllocV = typename std::allocator_traits<A>::template rebind_alloc<V>;
   using AllocU16 = typename std::allocator_traits<A>::template rebind_alloc<uint16_t>;
 
-  reverse_purge_hash_map(uint8_t lg_size, uint8_t lg_max_size);
+  reverse_purge_hash_map(uint8_t lg_size, uint8_t lg_max_size, const A& allocator);
   reverse_purge_hash_map(const reverse_purge_hash_map& other);
   reverse_purge_hash_map(reverse_purge_hash_map&& other) noexcept;
   ~reverse_purge_hash_map();
   reverse_purge_hash_map& operator=(reverse_purge_hash_map other);
   reverse_purge_hash_map& operator=(reverse_purge_hash_map&& other);
-  V adjust_or_insert(const K& key, V value);
-  V adjust_or_insert(K&& key, V value);
+
+  template<typename FwdK>
+  V adjust_or_insert(FwdK&& key, V value);
+
   V get(const K& key) const;
   uint8_t get_lg_cur_size() const;
   uint8_t get_lg_max_size() const;
   uint32_t get_capacity() const;
   uint32_t get_num_active() const;
+  const A& get_allocator() const;
+
   class iterator;
   iterator begin() const;
   iterator end() const;
+
 private:
   static constexpr double LOAD_FACTOR = 0.75;
   static constexpr uint16_t DRIFT_LIMIT = 1024; // used only for stress testing
   static constexpr uint32_t MAX_SAMPLE_SIZE = 1024; // number of samples to compute approximate median during purge
 
-  uint8_t lg_cur_size;
-  uint8_t lg_max_size;
-  uint32_t num_active;
-  K* keys;
-  V* values;
-  uint16_t* states;
+  A allocator_;
+  uint8_t lg_cur_size_;
+  uint8_t lg_max_size_;
+  uint32_t num_active_;
+  K* keys_;
+  V* values_;
+  uint16_t* states_;
 
   inline bool is_active(uint32_t probe) const;
   void subtract_and_keep_positive_only(V amount);
@@ -83,8 +89,8 @@ public:
   friend class reverse_purge_hash_map<K, V, H, E, A>;
   iterator& operator++() {
     ++count;
-    if (count < map->num_active) {
-      const uint32_t mask = (1 << map->lg_cur_size) - 1;
+    if (count < map->num_active_) {
+      const uint32_t mask = (1 << map->lg_cur_size_) - 1;
       do {
         index = (index + stride) & mask;
       } while (!map->is_active(index));
@@ -95,7 +101,7 @@ public:
   bool operator==(const iterator& rhs) const { return count == rhs.count; }
   bool operator!=(const iterator& rhs) const { return count != rhs.count; }
   const std::pair<K&, V> operator*() const {
-    return std::pair<K&, V>(map->keys[index], map->values[index]);
+    return std::pair<K&, V>(map->keys_[index], map->values_[index]);
   }
 private:
   static constexpr double GOLDEN_RATIO_RECIPROCAL = 0.6180339887498949; // = (sqrt(5) - 1) / 2
@@ -104,7 +110,7 @@ private:
   uint32_t count;
   uint32_t stride;
   iterator(const reverse_purge_hash_map<K, V, H, E, A>* map, uint32_t index, uint32_t count):
-    map(map), index(index), count(count), stride(static_cast<uint32_t>((1 << map->lg_cur_size) * GOLDEN_RATIO_RECIPROCAL) | 1) {}
+    map(map), index(index), count(count), stride(static_cast<uint32_t>((1 << map->lg_cur_size_) * GOLDEN_RATIO_RECIPROCAL) | 1) {}
 };
 
 } /* namespace datasketches */