datasketches 0.3.2 → 0.4.0

data/vendor/datasketches-cpp/tuple/include/tuple_sketch.hpp

@@ -27,7 +27,7 @@
 
 namespace datasketches {
 
-// forward-declarations
+// forward declarations
 template<typename S, typename A> class tuple_sketch;
 template<typename S, typename U, typename P, typename A> class update_tuple_sketch;
 template<typename S, typename A> class compact_tuple_sketch;
@@ -43,6 +43,10 @@ struct pair_extract_key {
   }
 };
 
+/**
+ * Base class for Tuple sketch.
+ * This is an extension of Theta sketch that allows keeping arbitrary Summary associated with each retained key.
+ */
 template<
   typename Summary,
   typename Allocator = std::allocator<Summary>
@@ -190,7 +194,7 @@ protected:
 
 // for types with defined default constructor and + operation
 template<typename Summary, typename Update>
-struct default_update_policy {
+struct default_tuple_update_policy {
   Summary create() const {
     return Summary();
   }
@@ -199,10 +203,15 @@ struct default_update_policy {
   }
 };
 
+/**
+ * Update Tuple sketch.
+ * The purpose of this class is to build a Tuple sketch from input data via the update() methods.
+ * There is no constructor. Use builder instead.
+ */
 template<
   typename Summary,
   typename Update = Summary,
-  typename Policy = default_update_policy<Summary, Update>,
+  typename Policy = default_tuple_update_policy<Summary, Update>,
   typename Allocator = std::allocator<Summary>
 >
 class update_tuple_sketch: public tuple_sketch<Summary, Allocator> {
@@ -244,21 +253,24 @@ public:
 
   /**
    * Update this sketch with a given string.
-   * @param value string to update the sketch with
+   * @param key string to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(const std::string& key, FwdUpdate&& value);
 
   /**
    * Update this sketch with a given unsigned 64-bit integer.
-   * @param value uint64_t to update the sketch with
+   * @param key uint64_t to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(uint64_t key, FwdUpdate&& value);
 
   /**
    * Update this sketch with a given signed 64-bit integer.
-   * @param value int64_t to update the sketch with
+   * @param key int64_t to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(int64_t key, FwdUpdate&& value);
@@ -266,7 +278,8 @@ public:
   /**
    * Update this sketch with a given unsigned 32-bit integer.
    * For compatibility with Java implementation.
-   * @param value uint32_t to update the sketch with
+   * @param key uint32_t to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(uint32_t key, FwdUpdate&& value);
@@ -274,7 +287,8 @@ public:
   /**
    * Update this sketch with a given signed 32-bit integer.
    * For compatibility with Java implementation.
-   * @param value int32_t to update the sketch with
+   * @param key int32_t to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(int32_t key, FwdUpdate&& value);
@@ -282,7 +296,8 @@ public:
   /**
    * Update this sketch with a given unsigned 16-bit integer.
    * For compatibility with Java implementation.
-   * @param value uint16_t to update the sketch with
+   * @param key uint16_t to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(uint16_t key, FwdUpdate&& value);
@@ -290,7 +305,8 @@ public:
   /**
    * Update this sketch with a given signed 16-bit integer.
    * For compatibility with Java implementation.
-   * @param value int16_t to update the sketch with
+   * @param key int16_t to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(int16_t key, FwdUpdate&& value);
@@ -298,7 +314,8 @@ public:
   /**
    * Update this sketch with a given unsigned 8-bit integer.
    * For compatibility with Java implementation.
-   * @param value uint8_t to update the sketch with
+   * @param key uint8_t to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(uint8_t key, FwdUpdate&& value);
@@ -306,7 +323,8 @@ public:
   /**
    * Update this sketch with a given signed 8-bit integer.
    * For compatibility with Java implementation.
-   * @param value int8_t to update the sketch with
+   * @param key int8_t to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(int8_t key, FwdUpdate&& value);
@@ -314,7 +332,8 @@ public:
   /**
    * Update this sketch with a given double-precision floating point value.
    * For compatibility with Java implementation.
-   * @param value double to update the sketch with
+   * @param key double to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(double key, FwdUpdate&& value);
@@ -322,7 +341,8 @@ public:
   /**
    * Update this sketch with a given floating point value.
    * For compatibility with Java implementation.
-   * @param value float to update the sketch with
+   * @param key float to update the sketch with
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   inline void update(float key, FwdUpdate&& value);
@@ -337,8 +357,9 @@ public:
    * Otherwise two sketches that should represent overlapping sets will be disjoint
    * For instance, for signed 32-bit values call update(int32_t) method above,
    * which does widening conversion to int64_t, if compatibility with Java is expected
-   * @param data pointer to the data
+   * @param key pointer to the data
    * @param length of the data in bytes
+   * @param value to update the sketch with
    */
   template<typename FwdUpdate>
   void update(const void* key, size_t length, FwdUpdate&& value);
@@ -355,7 +376,7 @@ public:
 
   /**
    * Converts this sketch to a compact sketch (ordered or unordered).
-   * @param ordered optional flag to specify if ordered sketch should be produced
+   * @param ordered optional flag to specify if an ordered sketch should be produced
    * @return compact sketch
    */
   compact_tuple_sketch<Summary, Allocator> compact(bool ordered = true) const;
@@ -375,8 +396,10 @@ protected:
   virtual void print_specifics(std::ostringstream& os) const;
 };
 
-// compact sketch
-
+/**
+ * Compact Tuple sketch.
+ * This is an immutable form of the Tuple sketch, the form that can be serialized and deserialized.
+ */
 template<
   typename Summary,
   typename Allocator = std::allocator<Summary>
@@ -406,13 +429,48 @@ public:
   // - as a result of a set operation
   // - by deserializing a previously serialized compact sketch
 
+  /**
+   * Copy constructor.
+   * Constructs a compact sketch from another sketch (either update or compact)
+   * @param other sketch to be copied
+   * @param ordered if true make the resulting sketch ordered
+   */
   compact_tuple_sketch(const Base& other, bool ordered);
-  compact_tuple_sketch(const compact_tuple_sketch&) = default;
+
+  /**
+   * Copy constructor.
+   * @param other sketch to be copied
+   */
+  compact_tuple_sketch(const compact_tuple_sketch& other) = default;
+
+  /**
+   * Move constructor.
+   * @param other sketch to be moved
+   */
   compact_tuple_sketch(compact_tuple_sketch&&) noexcept;
+
   virtual ~compact_tuple_sketch() = default;
-  compact_tuple_sketch& operator=(const compact_tuple_sketch&) = default;
-  compact_tuple_sketch& operator=(compact_tuple_sketch&&) = default;
 
+  /**
+   * Copy assignment
+   * @param other sketch to be copied
+   * @return reference to this sketch
+   */
+  compact_tuple_sketch& operator=(const compact_tuple_sketch& other) = default;
+
+  /**
+   * Move assignment
+   * @param other sketch to be moved
+   * @return reference to this sketch
+   */
+  compact_tuple_sketch& operator=(compact_tuple_sketch&& other) = default;
+
+  /**
+   * Constructor from Theta sketch
+   * @param other Theta sketch to be constructed from
+   * @param summary Summary instance to be associated with each entry
+   * @param ordered if true make the resulting sketch ordered
+   */
   compact_tuple_sketch(const theta_sketch_alloc<AllocU64>& other, const Summary& summary, bool ordered = true);
 
   virtual Allocator get_allocator() const;
@@ -425,7 +483,7 @@ public:
   /**
    * This method serializes the sketch into a given stream in a binary form
    * @param os output stream
-   * @param instance of a SerDe
+   * @param sd instance of a SerDe
    */
   template<typename SerDe = serde<Summary>>
   void serialize(std::ostream& os, const SerDe& sd = SerDe()) const;
@@ -436,7 +494,7 @@ public:
    * It is a blank space of a given size.
    * This header is used in Datasketches PostgreSQL extension.
    * @param header_size_bytes space to reserve in front of the sketch
-   * @param instance of a SerDe
+   * @param sd instance of a SerDe
    * @return serialized sketch as a vector of bytes
    */
   template<typename SerDe = serde<Summary>>
@@ -451,8 +509,8 @@ public:
    * This method deserializes a sketch from a given stream.
    * @param is input stream
    * @param seed the seed for the hash function that was used to create the sketch
-   * @param instance of a SerDe
-   * @param instance of an Allocator
+   * @param sd instance of a SerDe
+   * @param allocator instance of an Allocator
    * @return an instance of a sketch
    */
   template<typename SerDe = serde<Summary>>
@@ -464,17 +522,14 @@ public:
    * @param bytes pointer to the array of bytes
    * @param size the size of the array
    * @param seed the seed for the hash function that was used to create the sketch
-   * @param instance of a SerDe
-   * @param instance of an Allocator
+   * @param sd instance of a SerDe
+   * @param allocator instance of an Allocator
    * @return an instance of the sketch
    */
   template<typename SerDe = serde<Summary>>
   static compact_tuple_sketch deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED,
       const SerDe& sd = SerDe(), const Allocator& allocator = Allocator());
 
-  // for internal use
-  compact_tuple_sketch(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries);
-
 protected:
   bool is_empty_;
   bool is_ordered_;
@@ -520,10 +575,14 @@ protected:
 
   virtual void print_specifics(std::ostringstream& os) const;
 
-};
+  template<typename E, typename EK, typename P, typename S, typename CS, typename A> friend class theta_union_base;
+  template<typename E, typename EK, typename P, typename S, typename CS, typename A> friend class theta_intersection_base;
+  template<typename E, typename EK, typename CS, typename A> friend class theta_set_difference_base;
+  compact_tuple_sketch(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries);
 
-// builder
+};
 
+/// Tuple base builder
 template<typename Derived, typename Policy, typename Allocator>
 class tuple_base_builder: public theta_base_builder<Derived, Allocator> {
 public:
@@ -533,11 +592,15 @@ protected:
   Policy policy_;
 };
 
+/// Update Tuple sketch builder
 template<typename S, typename U, typename P, typename A>
 class update_tuple_sketch<S, U, P, A>::builder: public tuple_base_builder<builder, P, A> {
 public:
   /**
+   * Constructor
    * Creates and instance of the builder with default parameters.
+   * @param policy user-defined way of creating and updating Summary
+   * @param allocator instance of an Allocator to pass to created sketches
    */
   builder(const P& policy = P(), const A& allocator = A());
 

data/vendor/datasketches-cpp/tuple/include/tuple_union.hpp

@@ -27,15 +27,19 @@ namespace datasketches {
 
 // for types with defined + operation
 template<typename Summary>
-struct default_union_policy {
+struct default_tuple_union_policy {
   void operator()(Summary& summary, const Summary& other) const {
     summary += other;
   }
 };
 
+/**
+ * Tuple Union.
+ * Computes union of Tuple sketches. There is no constructor. Use builder instead.
+ */
 template<
   typename Summary,
-  typename Policy = default_union_policy<Summary>,
+  typename Policy = default_tuple_union_policy<Summary>,
   typename Allocator = std::allocator<Summary>
 >
 class tuple_union {
@@ -50,15 +54,15 @@ public:
   // reformulate the external policy that operates on Summary
   // in terms of operations on Entry
   struct internal_policy {
-    internal_policy(const Policy& policy): policy_(policy) {}
+    internal_policy(const Policy& external_policy): external_policy_(external_policy) {}
     void operator()(Entry& internal_entry, const Entry& incoming_entry) const {
-      policy_(internal_entry.second, incoming_entry.second);
+      external_policy_(internal_entry.second, incoming_entry.second);
     }
     void operator()(Entry& internal_entry, Entry&& incoming_entry) const {
-      policy_(internal_entry.second, std::move(incoming_entry.second));
+      external_policy_(internal_entry.second, std::move(incoming_entry.second));
     }
-    const Policy& get_policy() const { return policy_; }
-    Policy policy_;
+    const Policy& get_external_policy() const { return external_policy_; }
+    Policy external_policy_;
   };
 
   using State = theta_union_base<Entry, ExtractKey, internal_policy, Sketch, CompactSketch, AllocEntry>;
@@ -67,15 +71,15 @@ public:
   class builder;
 
   /**
-   * This method is to update the union with a given sketch
+   * Update the union with a given sketch
    * @param sketch to update the union with
    */
   template<typename FwdSketch>
   void update(FwdSketch&& sketch);
 
   /**
-   * This method produces a copy of the current state of the union as a compact sketch.
-   * @param ordered optional flag to specify if ordered sketch should be produced
+   * Produces a copy of the current state of the union as a compact sketch.
+   * @param ordered optional flag to specify if an ordered sketch should be produced
    * @return the result of the union
    */
   CompactSketch get_result(bool ordered = true) const;
@@ -92,16 +96,20 @@ protected:
   tuple_union(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p, uint64_t theta, uint64_t seed, const Policy& policy, const Allocator& allocator);
 };
 
+/// Tuple union builder
 template<typename S, typename P, typename A>
 class tuple_union<S, P, A>::builder: public tuple_base_builder<builder, P, A> {
 public:
   /**
+   * Constructor.
    * Creates and instance of the builder with default parameters.
+   * @param policy
+   * @param allocator
    */
   builder(const P& policy = P(), const A& allocator = A());
 
   /**
-   * This is to create an instance of the union with predefined parameters.
+   * Create an instance of the union with predefined parameters.
    * @return an instance of the union
    */
   tuple_union build() const;

data/vendor/datasketches-cpp/tuple/test/CMakeLists.txt

@@ -20,7 +20,6 @@ add_executable(tuple_test)
 target_link_libraries(tuple_test tuple common_test_lib)
 
 set_target_properties(tuple_test PROPERTIES
-  CXX_STANDARD 11
   CXX_STANDARD_REQUIRED YES
 )
 
@@ -47,3 +46,19 @@ target_sources(tuple_test
     array_of_doubles_sketch_test.cpp
     engagement_test.cpp
 )
+
+if (SERDE_COMPAT)
+target_sources(tuple_test
+  PRIVATE
+    aod_sketch_deserialize_from_java_test.cpp
+    tuple_sketch_deserialize_from_java_test.cpp
+)
+endif()
+
+if (GENERATE)
+target_sources(tuple_test
+  PRIVATE
+    aod_sketch_serialize_for_java.cpp
+    tuple_sketch_serialize_for_java.cpp
+)
+endif()

data/vendor/datasketches-cpp/tuple/test/aod_sketch_deserialize_from_java_test.cpp

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+
+#include "array_of_doubles_sketch.hpp"
+
+namespace datasketches {
+
+// assume the binary sketches for this test have been generated by datasketches-java code
+// in the subdirectory called "java" in the root directory of this project
+static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../java/";
+
+TEST_CASE("aod sketch one value", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "aod_1_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = compact_array_of_doubles_sketch::deserialize(is);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.is_estimation_mode() == (n > 1000));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03));
+    REQUIRE(sketch.get_num_values() == 1);
+    for (const auto& entry: sketch) {
+      REQUIRE(entry.first < sketch.get_theta64());
+    }
+  }
+}
+
+TEST_CASE("aod sketch three values", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "aod_3_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = compact_array_of_doubles_sketch::deserialize(is);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.is_estimation_mode() == (n > 1000));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03));
+    REQUIRE(sketch.get_num_values() == 3);
+    for (const auto& entry: sketch) {
+      REQUIRE(entry.first < sketch.get_theta64());
+      REQUIRE(entry.second[0] == entry.second[1]);
+      REQUIRE(entry.second[0] == entry.second[2]);
+    }
+  }
+}
+
+TEST_CASE("aod sketch non-empty no entries", "[serde_compat]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(testBinaryInputPath + "aod_1_non_empty_no_entries_java.sk", std::ios::binary);
+  const auto sketch = compact_array_of_doubles_sketch::deserialize(is);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_num_retained() == 0);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/tuple/test/aod_sketch_serialize_for_java.cpp

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+
+#include "array_of_doubles_sketch.hpp"
+
+namespace datasketches {
+
+TEST_CASE("aod sketch generate one value", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    auto sketch = update_array_of_doubles_sketch::builder().build();
+    for (unsigned i = 0; i < n; ++i) sketch.update(i, std::vector<double>(1, i));
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03));
+    std::ofstream os("aod_1_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.compact().serialize(os);
+  }
+}
+
+TEST_CASE("aod sketch generate three values", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    auto sketch = update_array_of_doubles_sketch::builder(3).build();
+    for (unsigned i = 0; i < n; ++i) sketch.update(i, std::vector<double>(3, i));
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03));
+    std::ofstream os("aod_3_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.compact().serialize(os);
+  }
+}
+
+TEST_CASE("aod sketch generate non-empty no entries", "[serialize_for_java]") {
+  auto sketch = update_array_of_doubles_sketch::builder().set_p(0.01).build();
+  // here we rely on the fact that hash of 1 happens to be greater than 0.01 (when normalized)
+  // and therefore gets rejected
+  sketch.update(1, std::vector<double>({1}));
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_num_retained() == 0);
+  std::ofstream os("aod_1_non_empty_no_entries_cpp.sk", std::ios::binary);
+  sketch.compact().serialize(os);
+}
+
+} /* namespace datasketches */