datasketches 0.1.2 → 0.2.3

data/vendor/datasketches-cpp/cpc/test/cpc_sketch_allocation_test.cpp

@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+#include <cstring>
+#include <sstream>
+#include <fstream>
+
+#include <catch.hpp>
+
+#include "cpc_sketch.hpp"
+#include "test_allocator.hpp"
+
+namespace datasketches {
+
+using cpc_sketch_test_alloc = cpc_sketch_alloc<test_allocator<uint8_t>>;
+using alloc = test_allocator<uint8_t>;
+
+TEST_CASE("cpc sketch allocation: serialize deserialize empty", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    sketch.serialize(s);
+    auto deserialized = cpc_sketch_test_alloc::deserialize(s, DEFAULT_SEED, alloc(0));
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serialize deserialize sparse", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    const int n(100);
+    for (int i = 0; i < n; i++) sketch.update(i);
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    sketch.serialize(s);
+    auto deserialized = cpc_sketch_test_alloc::deserialize(s, DEFAULT_SEED, alloc(0));
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serialize deserialize hybrid", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    const int n(200);
+    for (int i = 0; i < n; i++) sketch.update(i);
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    sketch.serialize(s);
+    auto deserialized = cpc_sketch_test_alloc::deserialize(s, DEFAULT_SEED, alloc(0));
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serialize deserialize pinned", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    const int n(2000);
+    for (int i = 0; i < n; i++) sketch.update(i);
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    sketch.serialize(s);
+    auto deserialized = cpc_sketch_test_alloc::deserialize(s, DEFAULT_SEED, alloc(0));
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serialize deserialize sliding", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    const int n(20000);
+    for (int i = 0; i < n; i++) sketch.update(i);
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    sketch.serialize(s);
+    auto deserialized = cpc_sketch_test_alloc::deserialize(s, DEFAULT_SEED, alloc(0));
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serializing deserialize sliding large", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    const int n(3000000);
+    for (int i = 0; i < n; i++) sketch.update(i);
+    std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+    sketch.serialize(s);
+    auto deserialized = cpc_sketch_test_alloc::deserialize(s, DEFAULT_SEED, alloc(0));
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serialize deserialize empty, bytes", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    auto bytes = sketch.serialize();
+    auto deserialized = cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size(), DEFAULT_SEED, 0);
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size() - 1, DEFAULT_SEED, 0), std::out_of_range);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serialize deserialize sparse, bytes", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    const int n(100);
+    for (int i = 0; i < n; i++) sketch.update(i);
+    auto bytes = sketch.serialize();
+    auto deserialized = cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size(), DEFAULT_SEED, 0);
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), 7, DEFAULT_SEED, 0), std::out_of_range);
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), 15, DEFAULT_SEED, 0), std::out_of_range);
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size() - 1, DEFAULT_SEED, 0), std::out_of_range);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serialize deserialize hybrid, bytes", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    const int n(200);
+    for (int i = 0; i < n; i++) sketch.update(i);
+    auto bytes = sketch.serialize();
+    auto deserialized = cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size(), DEFAULT_SEED, 0);
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), 7, DEFAULT_SEED, 0), std::out_of_range);
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), 15, DEFAULT_SEED, 0), std::out_of_range);
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size() - 1, DEFAULT_SEED, 0), std::out_of_range);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serialize deserialize pinned, bytes", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    const int n(2000);
+    for (int i = 0; i < n; i++) sketch.update(i);
+    auto bytes = sketch.serialize();
+    auto deserialized = cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size(), DEFAULT_SEED, 0);
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), 7, DEFAULT_SEED, 0), std::out_of_range);
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), 15, DEFAULT_SEED, 0), std::out_of_range);
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size() - 1, DEFAULT_SEED, 0), std::out_of_range);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE("cpc sketch allocation: serialize deserialize sliding, bytes", "[cpc_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    cpc_sketch_test_alloc sketch(11, DEFAULT_SEED, 0);
+    const int n(20000);
+    for (int i = 0; i < n; i++) sketch.update(i);
+    auto bytes = sketch.serialize();
+    auto deserialized = cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size(), DEFAULT_SEED, 0);
+    REQUIRE(deserialized.is_empty() == sketch.is_empty());
+    REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+    REQUIRE(deserialized.validate());
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), 7, DEFAULT_SEED, 0), std::out_of_range);
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), 15, DEFAULT_SEED, 0), std::out_of_range);
+    REQUIRE_THROWS_AS(cpc_sketch_test_alloc::deserialize(bytes.data(), bytes.size() - 1, DEFAULT_SEED, 0), std::out_of_range);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp

@@ -283,6 +283,26 @@ TEST_CASE("cpc sketch: serialize deserialize sliding, bytes", "[cpc_sketch]") {
   REQUIRE(deserialized.validate());
 }
 
+TEST_CASE("cpc sketch: serialize deserialize sliding huge", "[cpc_sketch]") {
+  cpc_sketch sketch(26);
+  const int n = 10000000;
+  for (int i = 0; i < n; i++) sketch.update(i);
+  REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.001));
+  auto bytes = sketch.serialize();
+  cpc_sketch deserialized = cpc_sketch::deserialize(bytes.data(), bytes.size());
+  REQUIRE(deserialized.is_empty() == sketch.is_empty());
+  REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+  REQUIRE(deserialized.validate());
+  REQUIRE_THROWS_AS(cpc_sketch::deserialize(bytes.data(), 7), std::out_of_range);
+  REQUIRE_THROWS_AS(cpc_sketch::deserialize(bytes.data(), 15), std::out_of_range);
+  REQUIRE_THROWS_AS(cpc_sketch::deserialize(bytes.data(), bytes.size() - 1), std::out_of_range);
+
+  // updating again with the same values should not change the sketch
+  for (int i = 0; i < n; i++) deserialized.update(i);
+  REQUIRE(deserialized.get_estimate() == sketch.get_estimate());
+  REQUIRE(deserialized.validate());
+}
+
 TEST_CASE("cpc sketch: copy", "[cpc_sketch]") {
   cpc_sketch s1(11);
   s1.update(1);
@@ -378,4 +398,9 @@ TEST_CASE("cpc sketch: update string equivalence", "[cpc_sketch]") {
   REQUIRE(sketch.get_estimate() == Approx(1).margin(RELATIVE_ERROR_FOR_LG_K_11));
 }
 
+TEST_CASE("cpc sketch: max serialized size", "[cpc_sketch]") {
+  REQUIRE(cpc_sketch::get_max_serialized_size_bytes(4) == 24 + 40);
+  REQUIRE(cpc_sketch::get_max_serialized_size_bytes(26) == static_cast<size_t>((0.6 * (1 << 26)) + 40));
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp

@@ -81,7 +81,7 @@ TEST_CASE("cpc union: large", "[cpc_union]") {
   cpc_union u(11);
   for (int i = 0; i < 1000; i++) {
     cpc_sketch tmp(11);
-    for (int i = 0; i < 10000; i++) {
+    for (int j = 0; j < 10000; j++) {
       s.update(key);
       tmp.update(key);
       key++;

data/vendor/datasketches-cpp/fi/include/frequent_items_sketch.hpp

@@ -40,15 +40,20 @@ namespace datasketches {
 
 enum frequent_items_error_type { NO_FALSE_POSITIVES, NO_FALSE_NEGATIVES };
 
-// for serialization as raw bytes
-template<typename A> using AllocU8 = typename std::allocator_traits<A>::template rebind_alloc<uint8_t>;
-template<typename A> using vector_u8 = std::vector<uint8_t, AllocU8<A>>;
-
 // type W for weight must be an arithmetic type (integral or floating point)
-template<typename T, typename W = uint64_t, typename H = std::hash<T>, typename E = std::equal_to<T>, typename S = serde<T>, typename A = std::allocator<T>>
+template<
+  typename T,
+  typename W = uint64_t,
+  typename H = std::hash<T>,
+  typename E = std::equal_to<T>,
+  typename S = serde<T>,
+  typename A = std::allocator<T>
+>
 class frequent_items_sketch {
 public:
 
+  static const uint8_t LG_MIN_MAP_SIZE = 3;
+
   /**
    * Construct this sketch with parameters lg_max_map_size and lg_start_map_size.
    *
@@ -59,7 +64,7 @@ public:
    * @param lg_start_map_size Log2 of the starting physical size of the internal hash
    * map managed by this sketch.
    */
-  explicit frequent_items_sketch(uint8_t lg_max_map_size, uint8_t lg_start_map_size = LG_MIN_MAP_SIZE);
+  explicit frequent_items_sketch(uint8_t lg_max_map_size, uint8_t lg_start_map_size = LG_MIN_MAP_SIZE, const A& allocator = A());
 
   /**
    * Update this sketch with an item and a positive weight (frequency count).
@@ -232,7 +237,8 @@ public:
 
   // This is a convenience alias for users
   // The type returned by the following serialize method
-  typedef vector_u8<A> vector_bytes;
+  using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
+
 
   /**
    * This method serializes the sketch as a vector of bytes.
@@ -249,7 +255,7 @@ public:
    * @param is input stream
    * @return an instance of the sketch
    */
-  static frequent_items_sketch deserialize(std::istream& is);
+  static frequent_items_sketch deserialize(std::istream& is, const A& allocator = A());
 
   /**
    * This method deserializes a sketch from a given array of bytes.
@@ -257,7 +263,7 @@ public:
    * @param size the size of the array
    * @return an instance of the sketch
    */
-  static frequent_items_sketch deserialize(const void* bytes, size_t size);
+  static frequent_items_sketch deserialize(const void* bytes, size_t size, const A& allocator = A());
 
   /**
    * Returns a human readable summary of this sketch
@@ -266,7 +272,6 @@ public:
   string<A> to_string(bool print_items = false) const;
 
 private:
-  static const uint8_t LG_MIN_MAP_SIZE = 3;
   static const uint8_t SERIAL_VERSION = 1;
   static const uint8_t FAMILY_ID = 10;
   static const uint8_t PREAMBLE_LONGS_EMPTY = 1;