datasketches 0.3.2 → 0.4.0

data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch.hpp

@@ -20,160 +20,35 @@
 #ifndef ARRAY_OF_DOUBLES_SKETCH_HPP_
 #define ARRAY_OF_DOUBLES_SKETCH_HPP_
 
-#include <vector>
-#include <memory>
-
-#include "serde.hpp"
-#include "tuple_sketch.hpp"
+#include "array_tuple_sketch.hpp"
+#include "array_tuple_union.hpp"
+#include "array_tuple_intersection.hpp"
+#include "array_tuple_a_not_b.hpp"
 
 namespace datasketches {
 
-// This sketch is equivalent of ArrayOfDoublesSketch in Java
-
-// This simple array of double is faster than std::vector and should be sufficient for this application
-template<typename Allocator = std::allocator<double>>
-class aod {
-public:
-  explicit aod(uint8_t size, const Allocator& allocator = Allocator()):
-  allocator_(allocator), size_(size), array_(allocator_.allocate(size_)) {
-    std::fill(array_, array_ + size_, 0);
-  }
-  aod(const aod& other):
-    allocator_(other.allocator_),
-    size_(other.size_),
-    array_(allocator_.allocate(size_))
-  {
-    std::copy(other.array_, other.array_ + size_, array_);
-  }
-  aod(aod&& other) noexcept:
-    allocator_(std::move(other.allocator_)),
-    size_(other.size_),
-    array_(other.array_)
-  {
-    other.array_ = nullptr;
-  }
-  ~aod() {
-    if (array_ != nullptr) allocator_.deallocate(array_, size_);
-  }
-  aod& operator=(const aod& other) {
-    aod copy(other);
-    std::swap(allocator_, copy.allocator_);
-    std::swap(size_, copy.size_);
-    std::swap(array_, copy.array_);
-    return *this;
-  }
-  aod& operator=(aod&& other) {
-    std::swap(allocator_, other.allocator_);
-    std::swap(size_, other.size_);
-    std::swap(array_, other.array_);
-    return *this;
-  }
-  double& operator[](size_t index) { return array_[index]; }
-  double operator[](size_t index) const { return array_[index]; }
-  uint8_t size() const { return size_; }
-  double* data() { return array_; }
-  const double* data() const { return array_; }
-  bool operator==(const aod& other) const {
-    for (uint8_t i = 0; i < size_; ++i) if (array_[i] != other.array_[i]) return false;
-    return true;
-  }
-private:
-  Allocator allocator_;
-  uint8_t size_;
-  double* array_;
-};
-
-template<typename A = std::allocator<double>>
-class array_of_doubles_update_policy {
-public:
-  array_of_doubles_update_policy(uint8_t num_values = 1, const A& allocator = A()):
-    allocator_(allocator), num_values_(num_values) {}
-  aod<A> create() const {
-    return aod<A>(num_values_, allocator_);
-  }
-  template<typename InputVector> // to allow any type with indexed access (such as double*)
-  void update(aod<A>& summary, const InputVector& update) const {
-    for (uint8_t i = 0; i < num_values_; ++i) summary[i] += update[i];
-  }
-  uint8_t get_num_values() const {
-    return num_values_;
-  }
-
-private:
-  A allocator_;
-  uint8_t num_values_;
-};
-
-// forward declaration
-template<typename A> class compact_array_of_doubles_sketch_alloc;
-
-template<typename A> using AllocAOD = typename std::allocator_traits<A>::template rebind_alloc<aod<A>>;
-
-template<typename A = std::allocator<double>>
-class update_array_of_doubles_sketch_alloc: public update_tuple_sketch<aod<A>, aod<A>, array_of_doubles_update_policy<A>, AllocAOD<A>> {
-public:
-  using Base = update_tuple_sketch<aod<A>, aod<A>, array_of_doubles_update_policy<A>, AllocAOD<A>>;
-  using resize_factor = typename Base::resize_factor;
-
-  class builder;
+/// convenience alias with default allocator, default policy for update_array_of_doubles_sketch
+using default_array_of_doubles_update_policy = default_array_tuple_update_policy<array<double>>;
 
-  compact_array_of_doubles_sketch_alloc<A> compact(bool ordered = true) const;
-  uint8_t get_num_values() const;
+/// convenience alias with default allocator, equivalent to ArrayOfDoublesUpdatableSketch in Java
+using update_array_of_doubles_sketch = update_array_tuple_sketch<array<double>>;
 
-private:
-  // for builder
-  update_array_of_doubles_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p, uint64_t theta,
-      uint64_t seed, const array_of_doubles_update_policy<A>& policy, const A& allocator);
-};
+/// convenience alias with default allocator, equivalent to ArrayOfDoublesCompactSketch in Java
+using compact_array_of_doubles_sketch = compact_array_tuple_sketch<array<double>>;
 
-// alias with the default allocator for convenience
-using update_array_of_doubles_sketch = update_array_of_doubles_sketch_alloc<>;
+/// convenience alias, default policy for array_of_doubles_union
+using default_array_of_doubles_union_policy = default_array_tuple_union_policy<array<double>>;
 
-template<typename A>
-class update_array_of_doubles_sketch_alloc<A>::builder: public tuple_base_builder<builder, array_of_doubles_update_policy<A>, A> {
-public:
-  builder(const array_of_doubles_update_policy<A>& policy = array_of_doubles_update_policy<A>(), const A& allocator = A());
-  update_array_of_doubles_sketch_alloc<A> build() const;
-};
+/// convenience alias with default allocator, equivalent to ArrayOfDoublesUnion in Java
+using array_of_doubles_union = array_tuple_union<array<double>>;
 
-template<typename A = std::allocator<double>>
-class compact_array_of_doubles_sketch_alloc: public compact_tuple_sketch<aod<A>, AllocAOD<A>> {
-public:
-  using Base = compact_tuple_sketch<aod<A>, AllocAOD<A>>;
-  using Entry = typename Base::Entry;
-  using AllocEntry = typename Base::AllocEntry;
-  using AllocU64 = typename Base::AllocU64;
-  using vector_bytes = typename Base::vector_bytes;
+/// convenience alias with default allocator, equivalent to ArrayOfDoublesIntersection in Java
+/// no default policy since it is not clear in general
+template<typename Policy> using array_of_doubles_intersection = array_tuple_intersection<array<double>, Policy>;
 
-  static const uint8_t SERIAL_VERSION = 1;
-  static const uint8_t SKETCH_FAMILY = 9;
-  static const uint8_t SKETCH_TYPE = 3;
-  enum flags { UNUSED1, UNUSED2, IS_EMPTY, HAS_ENTRIES, IS_ORDERED };
-
-  template<typename Sketch>
-  compact_array_of_doubles_sketch_alloc(const Sketch& other, bool ordered = true);
-
-  uint8_t get_num_values() const;
-
-  void serialize(std::ostream& os) const;
-  vector_bytes serialize(unsigned header_size_bytes = 0) const;
-
-  static compact_array_of_doubles_sketch_alloc deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED, const A& allocator = A());
-  static compact_array_of_doubles_sketch_alloc deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED,
-      const A& allocator = A());
-
-  // for internal use
-  compact_array_of_doubles_sketch_alloc(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries, uint8_t num_values);
-  compact_array_of_doubles_sketch_alloc(uint8_t num_values, Base&& base);
-private:
-  uint8_t num_values_;
-};
-
-// alias with the default allocator for convenience
-using compact_array_of_doubles_sketch = compact_array_of_doubles_sketch_alloc<>;
+/// convenience alias with default allocator, equivalent to ArrayOfDoublesAnotB in Java
+using array_of_doubles_a_not_b = array_tuple_a_not_b<array<double>>;
 
 } /* namespace datasketches */
 
-#include "array_of_doubles_sketch_impl.hpp"
-
 #endif

data/vendor/datasketches-cpp/tuple/include/{array_of_doubles_a_not_b.hpp → array_tuple_a_not_b.hpp}

@@ -17,36 +17,44 @@
  * under the License.
  */
 
-#ifndef ARRAY_OF_DOUBLES_A_NOT_B_HPP_
-#define ARRAY_OF_DOUBLES_A_NOT_B_HPP_
+#ifndef ARRAY_TUPLE_A_NOT_B_HPP_
+#define ARRAY_TUPLE_A_NOT_B_HPP_
 
 #include <vector>
 #include <memory>
 
-#include "array_of_doubles_sketch.hpp"
+#include "array_tuple_sketch.hpp"
 #include "tuple_a_not_b.hpp"
 
 namespace datasketches {
 
-template<typename Allocator = std::allocator<double>>
-class array_of_doubles_a_not_b_alloc: tuple_a_not_b<aod<Allocator>, AllocAOD<Allocator>> {
+/// array tuple A-not-B
+template<typename Array, typename Allocator = typename Array::allocator_type>
+class array_tuple_a_not_b: tuple_a_not_b<Array, Allocator> {
 public:
-  using Summary = aod<Allocator>;
-  using AllocSummary = AllocAOD<Allocator>;
-  using Base = tuple_a_not_b<Summary, AllocSummary>;
-  using CompactSketch = compact_array_of_doubles_sketch_alloc<Allocator>;
-
-  explicit array_of_doubles_a_not_b_alloc(uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
-
+  using Base = tuple_a_not_b<Array, Allocator>;
+  using CompactSketch = compact_array_tuple_sketch<Array, Allocator>;
+
+  /**
+   * Constructor
+   * @param seed for the hash function that was used to create the sketch
+   * @param allocator to use for allocating and deallocating memory
+   */
+  explicit array_tuple_a_not_b(uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+  /**
+   * Computes the A-not-B set operation given two sketches.
+   * @param a sketch A
+   * @param b sketch B
+   * @param ordered optional flag to specify if an ordered sketch should be produced
+   * @return the result of A-not-B as a compact sketch
+   */
   template<typename FwdSketch, typename Sketch>
   CompactSketch compute(FwdSketch&& a, const Sketch& b, bool ordered = true) const;
 };
 
-// alias with the default allocator for convenience
-using array_of_doubles_a_not_b = array_of_doubles_a_not_b_alloc<>;
-
 } /* namespace datasketches */
 
-#include "array_of_doubles_a_not_b_impl.hpp"
+#include "array_tuple_a_not_b_impl.hpp"
 
 #endif

data/vendor/datasketches-cpp/tuple/include/{array_of_doubles_a_not_b_impl.hpp → array_tuple_a_not_b_impl.hpp}

@@ -19,13 +19,13 @@
 
 namespace datasketches {
 
-template<typename A>
-array_of_doubles_a_not_b_alloc<A>::array_of_doubles_a_not_b_alloc(uint64_t seed, const A& allocator):
+template<typename Array, typename Allocator>
+array_tuple_a_not_b<Array, Allocator>::array_tuple_a_not_b(uint64_t seed, const Allocator& allocator):
 Base(seed, allocator) {}
 
-template<typename A>
+template<typename Array, typename Allocator>
 template<typename FwdSketch, typename Sketch>
-auto array_of_doubles_a_not_b_alloc<A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const -> CompactSketch {
+auto array_tuple_a_not_b<Array, Allocator>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const -> CompactSketch {
   return CompactSketch(a.get_num_values(), Base::compute(std::forward<FwdSketch>(a), b, ordered));
 }
 

data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection.hpp

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef ARRAY_TUPLE_INTERSECTION_HPP_
+#define ARRAY_TUPLE_INTERSECTION_HPP_
+
+#include <vector>
+#include <memory>
+
+#include "array_tuple_sketch.hpp"
+#include "tuple_intersection.hpp"
+
+namespace datasketches {
+
+/// array tuple intersection
+template<
+  typename Array,
+  typename Policy,
+  typename Allocator = typename Array::allocator_type
+>
+class array_tuple_intersection: public tuple_intersection<Array, Policy, Allocator> {
+public:
+  using Base = tuple_intersection<Array, Policy, Allocator>;
+  using CompactSketch = compact_array_tuple_sketch<Array, Allocator>;
+  using resize_factor = theta_constants::resize_factor;
+
+  /**
+   * Constructor
+   * @param seed for the hash function that was used to create the sketch
+   * @param policy user-defined way of combining Summary during intersection
+   * @param allocator to use for allocating and deallocating memory
+   */
+  explicit array_tuple_intersection(uint64_t seed = DEFAULT_SEED, const Policy& policy = Policy(), const Allocator& allocator = Allocator());
+
+  /**
+   * Produces a copy of the current state of the intersection.
+   * If update() was not called, the state is the infinite "universe",
+   * which is considered an undefined state, and throws an exception.
+   * @param ordered optional flag to specify if an ordered sketch should be produced
+   * @return the result of the intersection as a compact sketch
+   */
+  CompactSketch get_result(bool ordered = true) const;
+};
+
+} /* namespace datasketches */
+
+#include "array_tuple_intersection_impl.hpp"
+
+#endif

data/vendor/datasketches-cpp/{python/include/py_object_ostream.hpp → tuple/include/array_tuple_intersection_impl.hpp}

@@ -17,32 +17,15 @@
  * under the License.
  */
 
-#ifndef _PY_OBJECT_OSTREAM_HPP_
-#define _PY_OBJECT_OSTREAM_HPP_
-
-#include <pybind11/pybind11.h>
-
-#include <string>
-#include <ostream>
-
-/*
-  This header defines an ostream output operator on a generic python
-  object. The implementation calls the object's built-in __str__()
-  method. If that method is not defined, the call may fail.
-
-  NOTE: This header must be included before the inclusion of
-        any sketch classes.
-*/
-
-namespace py = pybind11;
-
 namespace datasketches {
 
-static std::ostream& operator<<(std::ostream& os, const py::object& obj) {
-  os << std::string(pybind11::str(obj));
-  return os;
-}
+template<typename Array, typename Policy, typename Allocator>
+array_tuple_intersection<Array, Policy, Allocator>::array_tuple_intersection(uint64_t seed, const Policy& policy, const Allocator& allocator):
+Base(seed, policy, allocator) {}
 
+template<typename Array, typename Policy, typename Allocator>
+auto array_tuple_intersection<Array, Policy, Allocator>::get_result(bool ordered) const -> CompactSketch {
+  return CompactSketch(this->state_.get_policy().get_external_policy().get_num_values(), Base::get_result(ordered));
 }
 
-#endif // _PY_OBJECT_OSTREAM_HPP_
+} /* namespace datasketches */

data/vendor/datasketches-cpp/tuple/include/array_tuple_sketch.hpp

@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef ARRAY_TUPLE_SKETCH_HPP_
+#define ARRAY_TUPLE_SKETCH_HPP_
+
+#include <vector>
+#include <memory>
+
+#include "serde.hpp"
+#include "tuple_sketch.hpp"
+
+namespace datasketches {
+
+// This simple array is faster than std::vector and should be sufficient for this application
+template<typename T, typename Allocator = std::allocator<T>>
+class array {
+public:
+  using value_type = T;
+  using allocator_type = Allocator;
+
+  explicit array(uint8_t size, T value, const Allocator& allocator = Allocator()):
+  allocator_(allocator), size_(size), array_(allocator_.allocate(size_)) {
+    std::fill(array_, array_ + size_, value);
+  }
+  array(const array& other):
+    allocator_(other.allocator_),
+    size_(other.size_),
+    array_(allocator_.allocate(size_))
+  {
+    std::copy(other.array_, other.array_ + size_, array_);
+  }
+  array(array&& other) noexcept:
+    allocator_(std::move(other.allocator_)),
+    size_(other.size_),
+    array_(other.array_)
+  {
+    other.array_ = nullptr;
+  }
+  ~array() {
+    if (array_ != nullptr) allocator_.deallocate(array_, size_);
+  }
+  array& operator=(const array& other) {
+    array copy(other);
+    std::swap(allocator_, copy.allocator_);
+    std::swap(size_, copy.size_);
+    std::swap(array_, copy.array_);
+    return *this;
+  }
+  array& operator=(array&& other) {
+    std::swap(allocator_, other.allocator_);
+    std::swap(size_, other.size_);
+    std::swap(array_, other.array_);
+    return *this;
+  }
+  T& operator[](size_t index) { return array_[index]; }
+  T operator[](size_t index) const { return array_[index]; }
+  uint8_t size() const { return size_; }
+  T* data() { return array_; }
+  const T* data() const { return array_; }
+  bool operator==(const array& other) const {
+    for (uint8_t i = 0; i < size_; ++i) if (array_[i] != other.array_[i]) return false;
+    return true;
+  }
+private:
+  Allocator allocator_;
+  uint8_t size_;
+  T* array_;
+};
+
+/// default array tuple update policy
+template<typename Array, typename Allocator = typename Array::allocator_type>
+class default_array_tuple_update_policy {
+public:
+  default_array_tuple_update_policy(uint8_t num_values = 1, const Allocator& allocator = Allocator()):
+    allocator_(allocator), num_values_(num_values) {}
+  Array create() const {
+    return Array(num_values_, 0, allocator_);
+  }
+  template<typename InputArray> // to allow any type with indexed access (such as double* or std::vector)
+  void update(Array& array, const InputArray& update) const {
+    for (uint8_t i = 0; i < num_values_; ++i) array[i] += update[i];
+  }
+  uint8_t get_num_values() const {
+    return num_values_;
+  }
+
+private:
+  Allocator allocator_;
+  uint8_t num_values_;
+};
+
+// forward declaration
+template<typename Array, typename Allocator> class compact_array_tuple_sketch;
+
+/**
+ * Update array tuple sketch.
+ * This is a wrapper around tuple sketch to match the functionality and serialization format of ArrayOfDoublesSketch in Java.
+ * For this the sketch must be configured with array<double> or std::vector<double>.
+ * This is a more generic implementation for any arithmetic type (serialization assumes contiguous array size_of(T) * num_values).
+ * A set of type definitions for the ArrayOfDoubles* equivalent is provided in a separate file array_of_doubles_sketch.hpp.
+ * There is no constructor. Use builder instead.
+ */
+template<
+  typename Array,
+  typename Policy = default_array_tuple_update_policy<Array>,
+  typename Allocator = typename Array::allocator_type
+>
+class update_array_tuple_sketch: public update_tuple_sketch<Array, Array, Policy, Allocator> {
+public:
+  using Base = update_tuple_sketch<Array, Array, Policy, Allocator>;
+  using resize_factor = typename Base::resize_factor;
+
+  class builder;
+
+  compact_array_tuple_sketch<Array, Allocator> compact(bool ordered = true) const;
+
+  /// @return number of values in array
+  uint8_t get_num_values() const;
+
+private:
+  // for builder
+  update_array_tuple_sketch(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p, uint64_t theta,
+      uint64_t seed, const Policy& policy, const Allocator& allocator);
+};
+
+/// Update array tuple sketch builder
+template<typename Array, typename Policy, typename Allocator>
+class update_array_tuple_sketch<Array, Policy, Allocator>::builder: public tuple_base_builder<builder, Policy, Allocator> {
+public:
+  /**
+   * Constructor
+   * @param policy
+   * @param allocator
+   */
+  builder(const Policy& policy = Policy(), const Allocator& allocator = Allocator());
+
+  /// @return instance of sketch
+  update_array_tuple_sketch build() const;
+};
+
+/// Compact array tuple sketch
+template<
+  typename Array,
+  typename Allocator = typename Array::allocator_type
+>
+class compact_array_tuple_sketch: public compact_tuple_sketch<Array, Allocator> {
+public:
+  using Base = compact_tuple_sketch<Array, Allocator>;
+  using Entry = typename Base::Entry;
+  using AllocEntry = typename Base::AllocEntry;
+  using AllocU64 = typename Base::AllocU64;
+  using vector_bytes = typename Base::vector_bytes;
+
+  static const uint8_t SERIAL_VERSION = 1;
+  static const uint8_t SKETCH_FAMILY = 9;
+  static const uint8_t SKETCH_TYPE = 3;
+  enum flags { UNUSED1, UNUSED2, IS_EMPTY, HAS_ENTRIES, IS_ORDERED };
+
+  /**
+   * Copy constructor.
+   * Constructs a compact sketch from another sketch (update or compact)
+   * @param other sketch to be constructed from
+   * @param ordered if true make the resulting sketch ordered
+   */
+  template<typename Sketch>
+  compact_array_tuple_sketch(const Sketch& other, bool ordered = true);
+
+  /// @return number of double values in array
+  uint8_t get_num_values() const;
+
+  /**
+   * This method serializes the sketch into a given stream in a binary form
+   * @param os output stream
+   */
+  void serialize(std::ostream& os) const;
+
+  /**
+   * This method serializes the sketch as a vector of bytes.
+   * An optional header can be reserved in front of the sketch.
+   * It is a blank space of a given size.
+   * @param header_size_bytes space to reserve in front of the sketch
+   * @return serialized sketch as a vector of bytes
+   */
+  vector_bytes serialize(unsigned header_size_bytes = 0) const;
+
+  /**
+   * This method deserializes a sketch from a given stream.
+   * @param is input stream
+   * @param seed the seed for the hash function that was used to create the sketch
+   * @param allocator instance of an Allocator
+   * @return an instance of the sketch
+   */
+  static compact_array_tuple_sketch deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+  /**
+   * This method deserializes a sketch from a given array of bytes.
+   * @param bytes pointer to the array of bytes
+   * @param size the size of the array
+   * @param seed the seed for the hash function that was used to create the sketch
+   * @param allocator instance of an Allocator
+   * @return an instance of the sketch
+   */
+  static compact_array_tuple_sketch deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED,
+      const Allocator& allocator = Allocator());
+
+private:
+  uint8_t num_values_;
+
+  template<typename Ar, typename P, typename Al> friend class array_tuple_union;
+  template<typename Ar, typename P, typename Al> friend class array_tuple_intersection;
+  template<typename Ar, typename Al> friend class array_tuple_a_not_b;
+  compact_array_tuple_sketch(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries, uint8_t num_values);
+  compact_array_tuple_sketch(uint8_t num_values, Base&& base);
+};
+
+} /* namespace datasketches */
+
+#include "array_tuple_sketch_impl.hpp"
+
+#endif