datasketches 0.3.1 → 0.4.0

data/vendor/datasketches-cpp/common/include/quantiles_sorted_view.hpp

@@ -27,6 +27,9 @@
 
 namespace datasketches {
 
+/**
+ * Sorted view for quantiles sketches (REQ, KLL and Quantiles)
+ */
 template<
   typename T,
   typename Comparator, // strict weak ordering function (see C++ named requirements: Compare)
@@ -34,30 +37,119 @@ template<
 >
 class quantiles_sorted_view {
 public:
+  /// Entry type
   using Entry = typename std::conditional<std::is_arithmetic<T>::value, std::pair<T, uint64_t>, std::pair<const T*, uint64_t>>::type;
   using AllocEntry = typename std::allocator_traits<Allocator>::template rebind_alloc<Entry>;
   using Container = std::vector<Entry, AllocEntry>;
 
+  /// @private
   quantiles_sorted_view(uint32_t num, const Comparator& comparator, const Allocator& allocator);
 
+  /// @private
   template<typename Iterator>
   void add(Iterator begin, Iterator end, uint64_t weight);
 
+  /// @private
   void convert_to_cummulative();
 
   class const_iterator;
+
+  /**
+   * Iterator pointing to the first entry in the view.
+   * If the view is empty, the returned iterator must not be dereferenced or incremented.
+   * @return iterator pointing to the first entry
+   */
   const_iterator begin() const;
+
+  /**
+   * Iterator pointing to the past-the-end entry in the view.
+   * The past-the-end entry is the hypothetical entry that would follow the last entry.
+   * It does not point to any entry, and must not be dereferenced or incremented.
+   * @return iterator pointing to the past-the-end entry
+   */
   const_iterator end() const;
 
+  /// @return size of the view
   size_t size() const;
 
+  /**
+   * Returns an approximation to the normalized rank of the given item.
+   *
+   * <p>If the view is empty this throws std::runtime_error.
+   *
+   * @param item to be ranked
+   * @param inclusive if true the weight of the given item is included into the rank.
+   * Otherwise the rank equals the sum of the weights of all items that are less than the given item
+   * according to the Comparator.
+   *
+   * @return an approximate normalized rank of the given item (0 to 1 inclusive)
+   */
   double get_rank(const T& item, bool inclusive = true) const;
 
+  /**
+   * Quantile return type.
+   * This is to return quantiles either by value (for arithmetic types) or by const reference (for all other types)
+   */
   using quantile_return_type = typename std::conditional<std::is_arithmetic<T>::value, T, const T&>::type;
+
+  /**
+   * Returns an item from the sketch that is the best approximation to an item
+   * from the original stream with the given normalized rank.
+   *
+   * <p>If the view is empty this throws std::runtime_error.
+   *
+   * @param rank of an item in the hypothetical sorted stream.
+   * @param inclusive if true, the given rank is considered inclusive (includes weight of an item)
+   *
+   * @return approximate quantile associated with the given normalized rank
+   */
   quantile_return_type get_quantile(double rank, bool inclusive = true) const;
 
   using vector_double = std::vector<double, typename std::allocator_traits<Allocator>::template rebind_alloc<double>>;
+
+  /**
+   * Returns an approximation to the Cumulative Distribution Function (CDF), which is the
+   * cumulative analog of the PMF, of the input stream given a set of split points (items).
+   *
+   * <p>If the view is empty this throws std::runtime_error.
+   *
+   * @param split_points an array of <i>m</i> unique, monotonically increasing items
+   * that divide the input domain into <i>m+1</i> consecutive disjoint intervals.
+   *
+   * @param size the number of split points in the array
+   *
+   * @param inclusive if true the rank of an item includes its own weight, and therefore
+   * if the sketch contains items equal to a slit point, then in CDF such items are
+   * included into the interval to the left of split point. Otherwise they are included into
+   * the interval to the right of split point.
+   *
+   * @return an array of m+1 doubles, which are a consecutive approximation to the CDF
+   * of the input stream given the split_points. The value at array position j of the returned
+   * CDF array is the sum of the returned values in positions 0 through j of the returned PMF
+   * array. This can be viewed as array of ranks of the given split points plus one more value
+   * that is always 1.
+   */
   vector_double get_CDF(const T* split_points, uint32_t size, bool inclusive = true) const;
+
+  /**
+   * Returns an approximation to the Probability Mass Function (PMF) of the input stream
+   * given a set of split points (items).
+   *
+   * <p>If the view is empty this throws std::runtime_error.
+   *
+   * @param split_points an array of <i>m</i> unique, monotonically increasing items
+   * that divide the input domain into <i>m+1</i> consecutive disjoint intervals (bins).
+   *
+   * @param size the number of split points in the array
+   *
+   * @param inclusive if true the rank of an item includes its own weight, and therefore
+   * if the sketch contains items equal to a slit point, then in PMF such items are
+   * included into the interval to the left of split point. Otherwise they are included into the interval
+   * to the right of split point.
+   *
+   * @return an array of m+1 doubles each of which is an approximation
+   * to the fraction of the input stream items (the mass) that fall into one of those intervals.
+   */
   vector_double get_PMF(const T* split_points, uint32_t size, bool inclusive = true) const;
 
 private:
@@ -122,8 +214,6 @@ public:
   using Base = typename quantiles_sorted_view<T, C, A>::Container::const_iterator;
   using value_type = typename std::conditional<std::is_arithmetic<T>::value, typename Base::value_type, std::pair<const T&, const uint64_t>>::type;
 
-  const_iterator(const Base& it, const Base& begin): Base(it), begin(begin) {}
-
   template<typename TT = T, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type = 0>
   const value_type operator*() const { return Base::operator*(); }
 
@@ -147,6 +237,9 @@ public:
 
 private:
   Base begin;
+
+  friend class quantiles_sorted_view<T, C, A>;
+  const_iterator(const Base& it, const Base& begin): Base(it), begin(begin) {}
 };
 
 } /* namespace datasketches */

data/vendor/datasketches-cpp/common/include/quantiles_sorted_view_impl.hpp

@@ -75,7 +75,7 @@ double quantiles_sorted_view<T, C, A>::get_rank(const T& item, bool inclusive) c
 template<typename T, typename C, typename A>
 auto quantiles_sorted_view<T, C, A>::get_quantile(double rank, bool inclusive) const -> quantile_return_type {
   if (entries_.empty()) throw std::runtime_error("operation is undefined for an empty sketch");
-  uint64_t weight = inclusive ? std::ceil(rank * total_weight_) : rank * total_weight_;
+  uint64_t weight = static_cast<uint64_t>(inclusive ? std::ceil(rank * total_weight_) : rank * total_weight_);
   auto it = inclusive ?
       std::lower_bound(entries_.begin(), entries_.end(), make_dummy_entry<T>(weight), compare_pairs_by_second())
     : std::upper_bound(entries_.begin(), entries_.end(), make_dummy_entry<T>(weight), compare_pairs_by_second());

data/vendor/datasketches-cpp/common/include/serde.hpp

@@ -30,23 +30,56 @@
 
 namespace datasketches {
 
-// serialize and deserialize
+/// Interface for serializing and deserializing items
 template<typename T, typename Enable = void> struct serde {
-  // stream serialization
+  /**
+   * Stream serialization
+   * @param os output stream
+   * @param items pointer to array of items
+   * @param num number of items
+   */
   void serialize(std::ostream& os, const T* items, unsigned num) const;
-  void deserialize(std::istream& is, T* items, unsigned num) const; // items allocated but not initialized
 
-  // raw bytes serialization
-  size_t size_of_item(const T& item) const;
+  /**
+   * Stream deserialization
+   * @param is input stream
+   * @param items pointer to array of items (items in the array are allocated but not initialized)
+   * @param num number of items
+   */
+  void deserialize(std::istream& is, T* items, unsigned num) const;
+
+  /**
+   * Raw bytes serialization
+   * @param ptr pointer to output buffer
+   * @param capacity size of the buffer in bytes
+   * @param items pointer to array of items
+   * @param num number of items
+   */
   size_t serialize(void* ptr, size_t capacity, const T* items, unsigned num) const;
-  size_t deserialize(const void* ptr, size_t capacity, T* items, unsigned num) const; // items allocated but not initialized
+
+  /**
+   * Raw bytes deserialization
+   * @param ptr pointer to input buffer
+   * @param capacity size of the buffer in bytes
+   * @param items pointer to array of items (items in the array are allocated but not initialized)
+   * @param num number of items
+   */
+  size_t deserialize(const void* ptr, size_t capacity, T* items, unsigned num) const;
+
+  /**
+   * Size of the given item
+   * @param item to be sized
+   * @return size of the given item in bytes
+   */
+  size_t size_of_item(const T& item) const;
 };
 
-// serde for all fixed-size arithmetic types (int and float of different sizes)
-// in particular, kll_sketch<int64_t> should produce sketches binary-compatible
-// with LongsSketch and ItemsSketch<Long> with ArrayOfLongsSerDe in Java
+/// serde for all fixed-size arithmetic types (int and float of different sizes).
+/// in particular, kll_sketch<int64_t> should produce sketches binary-compatible
+/// with LongsSketch and ItemsSketch<Long> with ArrayOfLongsSerDe in Java
 template<typename T>
 struct serde<T, typename std::enable_if<std::is_arithmetic<T>::value>::type> {
+  /// @copydoc serde::serialize
   void serialize(std::ostream& os, const T* items, unsigned num) const {
     bool failure = false;
     try {
@@ -58,6 +91,7 @@ struct serde<T, typename std::enable_if<std::is_arithmetic<T>::value>::type> {
       throw std::runtime_error("error writing to std::ostream with " + std::to_string(num) + " items");
     }
   }
+
   void deserialize(std::istream& is, T* items, unsigned num) const {
     bool failure = false;
     try {
@@ -70,30 +104,37 @@ struct serde<T, typename std::enable_if<std::is_arithmetic<T>::value>::type> {
     }
   }
 
-  size_t size_of_item(const T&) const {
-    return sizeof(T);
-  }
+  /// @copydoc serde::serialize(void*,size_t,const T*,unsigned) const
   size_t serialize(void* ptr, size_t capacity, const T* items, unsigned num) const {
     const size_t bytes_written = sizeof(T) * num;
     check_memory_size(bytes_written, capacity);
     memcpy(ptr, items, bytes_written);
     return bytes_written;
   }
+
+  /// @copydoc serde::deserialize(const void*,size_t,T*,unsigned) const
   size_t deserialize(const void* ptr, size_t capacity, T* items, unsigned num) const {
     const size_t bytes_read = sizeof(T) * num;
     check_memory_size(bytes_read, capacity);
     memcpy(items, ptr, bytes_read);
     return bytes_read;
   }
+
+  /// @copydoc serde::size_of_item
+  size_t size_of_item(const T& item) const {
+    unused(item);
+    return sizeof(T);
+  }
 };
 
-// serde for std::string items
-// This should produce sketches binary-compatible with
-// ItemsSketch<String> with ArrayOfStringsSerDe in Java.
-// The length of each string is stored as a 32-bit integer (historically),
-// which may be too wasteful. Treat this as an example.
+/// serde for std::string items.
+/// This should produce sketches binary-compatible with
+/// ItemsSketch<String> with ArrayOfStringsSerDe in Java.
+/// The length of each string is stored as a 32-bit integer (historically),
+/// which may be too wasteful. Treat this as an example.
 template<>
 struct serde<std::string> {
+  /// @copydoc serde::serialize
   void serialize(std::ostream& os, const std::string* items, unsigned num) const {
     unsigned i = 0;
     bool failure = false;
@@ -110,6 +151,8 @@ struct serde<std::string> {
       throw std::runtime_error("error writing to std::ostream at item " + std::to_string(i));
     }
   }
+
+  /// @copydoc serde::deserialize
   void deserialize(std::istream& is, std::string* items, unsigned num) const {
     unsigned i = 0;
     bool failure = false;
@@ -137,9 +180,8 @@ struct serde<std::string> {
       throw std::runtime_error("error reading from std::istream at item " + std::to_string(i)); 
     }
   }
-  size_t size_of_item(const std::string& item) const {
-    return sizeof(uint32_t) + item.size();
-  }
+
+  /// @copydoc serde::serialize(void*,size_t,const T*,unsigned) const
   size_t serialize(void* ptr, size_t capacity, const std::string* items, unsigned num) const {
     size_t bytes_written = 0;
     for (unsigned i = 0; i < num; ++i) {
@@ -154,6 +196,8 @@ struct serde<std::string> {
     }
     return bytes_written;
   }
+
+  /// @copydoc serde::deserialize(const void*,size_t,T*,unsigned) const
   size_t deserialize(const void* ptr, size_t capacity, std::string* items, unsigned num) const {
     size_t bytes_read = 0;
     unsigned i = 0;
@@ -189,6 +233,11 @@ struct serde<std::string> {
 
     return bytes_read;
   }
+
+  /// @copydoc serde::size_of_item
+  size_t size_of_item(const std::string& item) const {
+    return sizeof(uint32_t) + item.size();
+  }
 };
 
 } /* namespace datasketches */

data/vendor/datasketches-cpp/common/test/CMakeLists.txt

@@ -34,7 +34,6 @@ FetchContent_MakeAvailable(Catch2)
 target_link_libraries(common_test_lib PUBLIC Catch2::Catch2)
 
 set_target_properties(common_test_lib PROPERTIES
-  CXX_STANDARD 11
   CXX_STANDARD_REQUIRED YES
 )
 
@@ -69,12 +68,13 @@ add_test(
 target_sources(common_test
   PRIVATE
     quantiles_sorted_view_test.cpp
+    optional_test.cpp
 )
 
 # now the integration test part
 add_executable(integration_test)
 
-target_link_libraries(integration_test cpc fi hll kll req sampling theta tuple common_test_lib)
+target_link_libraries(integration_test count cpc density fi hll kll req sampling theta tuple common_test_lib)
 
 set_target_properties(integration_test PROPERTIES
   CXX_STANDARD 11

data/vendor/datasketches-cpp/common/test/integration_test.cpp

@@ -19,8 +19,10 @@
 
 #include <catch2/catch.hpp>
 
+#include "count_min.hpp"
 #include "cpc_sketch.hpp"
 #include "cpc_union.hpp"
+#include "density_sketch.hpp"
 #include "frequent_items_sketch.hpp"
 #include "hll.hpp"
 #include "kll_sketch.hpp"
@@ -48,9 +50,13 @@ struct subtracting_intersection_policy {
 using tuple_intersection_float = tuple_intersection<float, subtracting_intersection_policy<float>>;
 
 TEST_CASE("integration: declare all sketches", "[integration]") {
+  count_min_sketch<double> cm(5, 128);
+
   cpc_sketch cpc(12);
   cpc_union cpc_u(12);
 
+  density_sketch<double> ds(32, 3);
+
   frequent_items_sketch<std::string> fi(100);
 
   hll_sketch hll(13);

data/vendor/datasketches-cpp/common/test/optional_test.cpp

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+
+#include <iostream>
+
+#include "optional.hpp"
+
+namespace datasketches {
+
+class tt {
+public:
+  tt() = delete; // make sure it cannot be default constructed
+  tt(int val): val_(val) {}
+  tt(const tt& other): val_(other.val_) { std::cout << "tt copy constructor\n"; }
+  tt(tt&& other): val_(other.val_) { std::cout << "tt move constructor\n"; }
+  tt& operator=(const tt& other) { val_ = other.val_; std::cout << "tt copy assignment\n"; return *this; }
+  tt& operator=(tt&& other) { val_ = other.val_; std::cout << "tt move assignment\n"; return *this; }
+  int get_val() const { return val_; }
+private:
+  int val_;
+};
+
+TEST_CASE("optional", "[common]") {
+  optional<tt> opt;
+  REQUIRE_FALSE(opt);
+  opt.emplace(5);
+  REQUIRE(bool(opt));
+  REQUIRE((*opt).get_val() == 5);
+  REQUIRE(opt->get_val() == 5);
+  opt.reset();
+  REQUIRE_FALSE(opt);
+
+  optional<tt> opt2(opt);
+  REQUIRE_FALSE(opt2);
+
+  opt2.emplace(3);
+  if (opt2) *opt2 = 6; // good if it is initialized
+  REQUIRE(opt2->get_val() == 6);
+
+  opt.reset();
+  REQUIRE_FALSE(opt);
+  optional<tt> opt3(std::move(opt));
+  REQUIRE_FALSE(opt3);
+  *opt3 = 7; // don't do this! may be dangerous for arbitrary T, and it still thinks it is not initialized
+  REQUIRE_FALSE(opt3);
+  opt3.emplace(8);
+  REQUIRE(bool(opt3));
+  REQUIRE(opt3->get_val() == 8);
+
+  std::swap(opt2, opt3);
+  REQUIRE(opt2->get_val() == 8);
+  REQUIRE(opt3->get_val() == 6);
+
+  std::swap(opt2, opt);
+  REQUIRE_FALSE(opt2);
+  REQUIRE(bool(opt));
+  REQUIRE(opt->get_val() == 8);
+}
+
+TEST_CASE("optional conversion", "[common]") {
+  optional<float> opt_f(1);
+  optional<double> opt_d(opt_f);
+  REQUIRE(bool(opt_d));
+  REQUIRE(*opt_d == static_cast<double>(*opt_f));
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/common/test/test_allocator.hpp

@@ -33,16 +33,16 @@ extern long long test_allocator_net_allocations;
 
 template <class T> class test_allocator {
 public:
-  typedef T                 value_type;
-  typedef value_type*       pointer;
-  typedef const value_type* const_pointer;
-  typedef value_type&       reference;
-  typedef const value_type& const_reference;
-  typedef std::size_t       size_type;
-  typedef std::ptrdiff_t    difference_type;
+  using value_type = T;
+  using pointer = value_type*;
+  using const_pointer = const value_type*;
+  using reference = value_type&;
+  using const_reference = const value_type&;
+  using size_type = std::size_t;
+  using difference_type = std::ptrdiff_t;
 
   template <class U>
-  struct rebind { typedef test_allocator<U> other; };
+  struct rebind { using other = test_allocator<U>; };
 
   // this is to test that a given instance of an allocator is used instead of instantiating
   static const bool DISALLOW_DEFAULT_CONSTRUCTOR = true;
@@ -66,7 +66,7 @@ public:
   }
 
   pointer allocate(size_type n, const_pointer = 0) {
-    void* p = new char[n * sizeof(value_type)];
+    void* p = ::operator new(n * sizeof(value_type));
     if (!p) throw std::bad_alloc();
     test_allocator_total_bytes += n * sizeof(value_type);
     ++test_allocator_net_allocations;
@@ -74,7 +74,7 @@ public:
   }
 
   void deallocate(pointer p, size_type n) {
-    if (p) delete[] (char*) p;
+    if (p) ::operator delete(p);
     test_allocator_total_bytes -= n * sizeof(value_type);
     --test_allocator_net_allocations;
   }
@@ -92,12 +92,12 @@ public:
 
 template<> class test_allocator<void> {
 public:
-  typedef void        value_type;
-  typedef void*       pointer;
-  typedef const void* const_pointer;
+  using value_type = void;
+  using pointer = void*;
+  using const_pointer = const void*;
 
   template <class U>
-  struct rebind { typedef test_allocator<U> other; };
+  struct rebind { using other = test_allocator<U>; };
 };
 
 

data/vendor/datasketches-cpp/{python/src/__init__.py → count/CMakeLists.txt}

@@ -15,4 +15,28 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name = "datasketches"
+add_library(count INTERFACE)
+
+add_library(${PROJECT_NAME}::COUNT ALIAS count)
+
+if (BUILD_TESTS)
+    add_subdirectory(test)
+endif()
+
+target_include_directories(count
+        INTERFACE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+        $<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>
+        )
+
+target_link_libraries(count INTERFACE common)
+target_compile_features(count INTERFACE cxx_std_11)
+
+install(TARGETS count
+        EXPORT ${PROJECT_NAME}
+        )
+
+install(FILES
+        include/count_min.hpp
+        include/count_min_impl.hpp
+        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")