datasketches 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 897dbc30f97ce17f0415630b6347a0092dac05196b0ef61e80939410d65cdf17
-  data.tar.gz: 61302f9cadde8a8badc97b455eb5c32d913c3b1fea8ed571e2da93a29e65afa9
+  metadata.gz: 9eaa8a17efdbc591b3e56f94650e887babd30dc79d95db3a7986df0261184191
+  data.tar.gz: 5544326a0edf165d87373a680d8bf5b80acba2894b9048f92cbdb261fcd66d57
 SHA512:
-  metadata.gz: 4d541ba7f96a86f3f8de44f069f6e39d51ba6f28fa5d8c8d1d99a8434a95c5fe1a26470e6b062f348808fe5c0a444134d0dc96385437b4cb946c4a92044a2a5c
-  data.tar.gz: bc1bdacb7cbe69f9bb1382fd2ac7019bec04baf444dc963d63a594e989fd201d9eb9aadd0e463ac4efef8f7ba53915a594d8fb00f74ae295674b9024269a0406
+  metadata.gz: 5a28c093ecda083762367149800770f59fee8e630c0d983d3f29ed32d027fae2e2515dff243ee11bbd41f4875c7cea622f7bc5cc5d7e73176e785503ed19fc0b
+  data.tar.gz: 6b210f2fdca1ae3cbd4e4cbf88e284855014b5a1e1c883085dc96a057da29e370005163ce628e54351c9127b00fae4b7b33a4ca63e6f4b90e0665e93b7742a66

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.5 (2022-05-21)
+
+- Updated DataSketches to 3.4.0
+
 ## 0.2.4 (2021-12-28)
 
 - Updated DataSketches to 3.3.0

data/README.md

@@ -9,7 +9,7 @@
 Add this line to your application’s Gemfile:
 
 ```ruby
-gem 'datasketches'
+gem "datasketches"
 ```
 
 ## Sketch Families

data/ext/datasketches/kll_wrapper.cpp

@@ -33,7 +33,11 @@ void bind_kll_sketch(Rice::Module& m, const char* name) {
     .define_method("estimation_mode?", &kll_sketch<T>::is_estimation_mode)
     .define_method("min_value", &kll_sketch<T>::get_min_value)
     .define_method("max_value", &kll_sketch<T>::get_max_value)
-    .define_method("quantile", &kll_sketch<T>::get_quantile)
+    .define_method(
+      "quantile",
+      [](kll_sketch<T>& self, double fraction) {
+        return self.get_quantile(fraction);
+      })
     .define_method(
       "quantiles",
       [](kll_sketch<T>& self, Rice::Object obj) {

data/lib/datasketches/version.rb

@@ -1,3 +1,3 @@
 module DataSketches
-  VERSION = "0.2.4"
+  VERSION = "0.2.5"
 end

data/vendor/datasketches-cpp/CMakeLists.txt

@@ -15,9 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-cmake_minimum_required(VERSION 3.12.0)
+cmake_minimum_required(VERSION 3.16.0)
 project(DataSketches
-        VERSION 3.2.0
+        VERSION 3.4.0
         LANGUAGES CXX)
 
 include(GNUInstallDirs)
@@ -106,12 +106,13 @@ add_subdirectory(theta)
 add_subdirectory(sampling)
 add_subdirectory(tuple)
 add_subdirectory(req)
+add_subdirectory(quantiles)
 
 if (WITH_PYTHON)
   add_subdirectory(python)
 endif()
 
-target_link_libraries(datasketches INTERFACE hll cpc kll fi theta sampling)
+target_link_libraries(datasketches INTERFACE hll cpc kll fi theta sampling req quantiles)
 
 if (COVERAGE)
   find_program(LCOV_PATH NAMES "lcov")

data/vendor/datasketches-cpp/common/CMakeLists.txt

@@ -43,4 +43,8 @@ install(FILES
 			include/conditional_forward.hpp
 			include/ceiling_power_of_2.hpp
 			include/bounds_binomial_proportions.hpp
+			include/kolmogorov_smirnov.hpp
+			include/kolmogorov_smirnov_impl.hpp
+			include/quantile_sketch_sorted_view.hpp
+			include/quantile_sketch_sorted_view_impl.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")

data/vendor/datasketches-cpp/common/include/binomial_bounds.hpp

@@ -22,6 +22,7 @@
 
 #include <algorithm>
 #include <cmath>
+#include <stdexcept>
 
 /*
  * This class enables the estimation of error bounds given a sample set size, the sampling

data/vendor/datasketches-cpp/common/include/common_defs.hpp

@@ -24,6 +24,8 @@
 #include <string>
 #include <memory>
 #include <iostream>
+#include <random>
+#include <chrono>
 
 namespace datasketches {
 
@@ -34,6 +36,18 @@ enum resize_factor { X1 = 0, X2, X4, X8 };
 template<typename A> using AllocChar = typename std::allocator_traits<A>::template rebind_alloc<char>;
 template<typename A> using string = std::basic_string<char, std::char_traits<char>, AllocChar<A>>;
 
+// random bit
+static std::independent_bits_engine<std::mt19937, 1, uint32_t>
+  random_bit(static_cast<uint32_t>(std::chrono::system_clock::now().time_since_epoch().count()));
+
+// common random declarations
+namespace random_utils {
+  static std::random_device rd; // possibly unsafe in MinGW with GCC < 9.2
+  static std::mt19937_64 rand(rd());
+  static std::uniform_real_distribution<> next_double(0.0, 1.0);
+}
+
+
 // utility function to hide unused compiler warning
 // usually has no additional cost
 template<typename T> void unused(T&&...) {}

data/vendor/datasketches-cpp/{kll → common}/include/kolmogorov_smirnov.hpp

@@ -25,7 +25,8 @@ namespace datasketches {
 class kolmogorov_smirnov {
 public:
   /**
-   * Computes the raw delta area between two KLL quantile sketches for the Kolmogorov-Smirnov Test.
+   * Computes the raw delta area between two quantile sketches for the Kolmogorov-Smirnov Test.
+   * Will work for a type-matched pair of KLL or Quantiles sketches of the same parameterized type T.
    * @param sketch1 KLL sketch 1
    * @param sketch2 KLL sketch 2
    * @return the raw delta between two KLL quantile sketches
@@ -37,6 +38,7 @@ public:
    * Computes the adjusted delta area threshold for the Kolmogorov-Smirnov Test.
    * Adjusts the computed threshold by the error epsilons of the two given sketches.
    * See <a href="https://en.wikipedia.org/wiki/Kolmogorov-Smirnov_test">Kolmogorov–Smirnov Test</a>
+   * Will work for a type-matched pair of KLL or Quantiles sketches of the same parameterized type T.
    * @param sketch1 KLL sketch 1
    * @param sketch2 KLL sketch 2
    * @param p Target p-value. Typically .001 to .1, e.g., .05.
@@ -46,7 +48,8 @@ public:
   static double threshold(const Sketch& sketch1, const Sketch& sketch2, double p);
 
   /**
-   * Performs the Kolmogorov-Smirnov Test between two KLL quantiles sketches.
+   * Performs the Kolmogorov-Smirnov Test between two quantile sketches.
+   * Will work for a type-matched pair of KLL or Quantiles sketches of the same parameterized type T.
    * Note: if the given sketches have insufficient data or if the sketch sizes are too small,
    * this will return false.
    * @param sketch1 KLL sketch 1
@@ -57,7 +60,6 @@ public:
    */
   template<typename Sketch>
   static bool test(const Sketch& sketch1, const Sketch& sketch2, double p);
-
 };
 
 } /* namespace datasketches */

data/vendor/datasketches-cpp/{kll → common}/include/kolmogorov_smirnov_impl.hpp

@@ -20,39 +20,36 @@
 #ifndef KOLMOGOROV_SMIRNOV_IMPL_HPP_
 #define KOLMOGOROV_SMIRNOV_IMPL_HPP_
 
-namespace datasketches {
+#include <cmath>
+#include <algorithm>
 
-// type resolver
-template<typename T, typename C, typename S, typename A>
-kll_quantile_calculator<T, C, A> make_quantile_calculator(const kll_sketch<T, C, S, A>& sketch) {
-  return kll_quantile_calculator<T, C, A>(sketch);
-}
+namespace datasketches {
 
 template<typename Sketch>
 double kolmogorov_smirnov::delta(const Sketch& sketch1, const Sketch& sketch2) {
-  using Comparator = typename Sketch::comparator;
-  auto calc1 = make_quantile_calculator(sketch1);
-  auto calc2 = make_quantile_calculator(sketch2);
-  auto it1 = calc1.begin();
-  auto it2 = calc2.begin();
+  auto comparator = sketch1.get_comparator(); // assuming the same comparator in sketch2
+  auto view1 = sketch1.get_sorted_view(true);
+  auto view2 = sketch2.get_sorted_view(true);
+  auto it1 = view1.begin();
+  auto it2 = view2.begin();
   const auto n1 = sketch1.get_n();
   const auto n2 = sketch2.get_n();
   double delta = 0;
-  while (it1 != calc1.end() && it2 != calc2.end()) {
+  while (it1 != view1.end() && it2 != view2.end()) {
     const double norm_cum_wt1 = static_cast<double>((*it1).second) / n1;
     const double norm_cum_wt2 = static_cast<double>((*it2).second) / n2;
     delta = std::max(delta, std::abs(norm_cum_wt1 - norm_cum_wt2));
-    if (Comparator()((*it1).first, (*it2).first)) {
+    if (comparator((*it1).first, (*it2).first)) {
       ++it1;
-    } else if (Comparator()((*it2).first, (*it1).first)) {
+    } else if (comparator((*it2).first, (*it1).first)) {
       ++it2;
     } else {
       ++it1;
       ++it2;
     }
   }
-  const double norm_cum_wt1 = it1 == calc1.end() ? 1 : static_cast<double>((*it1).second) / n1;
-  const double norm_cum_wt2 = it2 == calc2.end() ? 1 : static_cast<double>((*it2).second) / n2;
+  const double norm_cum_wt1 = it1 == view1.end() ? 1 : static_cast<double>((*it1).second) / n1;
+  const double norm_cum_wt2 = it2 == view2.end() ? 1 : static_cast<double>((*it2).second) / n2;
   delta = std::max(delta, std::abs(norm_cum_wt1 - norm_cum_wt2));
   return delta;
 }

data/vendor/datasketches-cpp/common/include/quantile_sketch_sorted_view.hpp

@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef QUANTILE_SKETCH_SORTED_VIEW_HPP_
+#define QUANTILE_SKETCH_SORTED_VIEW_HPP_
+
+#include <functional>
+
+namespace datasketches {
+
+template<
+  typename T,
+  typename Comparator, // strict weak ordering function (see C++ named requirements: Compare)
+  typename Allocator
+>
+class quantile_sketch_sorted_view {
+public:
+  using Entry = typename std::conditional<std::is_arithmetic<T>::value, std::pair<T, uint64_t>, std::pair<const T*, uint64_t>>::type;
+  using AllocEntry = typename std::allocator_traits<Allocator>::template rebind_alloc<Entry>;
+  using Container = std::vector<Entry, AllocEntry>;
+
+  quantile_sketch_sorted_view(uint32_t num, const Allocator& allocator);
+
+  template<typename Iterator>
+  void add(Iterator begin, Iterator end, uint64_t weight);
+
+  template<bool inclusive>
+  void convert_to_cummulative();
+
+  class const_iterator;
+  const_iterator begin() const;
+  const_iterator end() const;
+
+  size_t size() const;
+
+  // makes sense only with cumulative weight
+  using quantile_return_type = typename std::conditional<std::is_arithmetic<T>::value, T, const T&>::type;
+  quantile_return_type get_quantile(double rank) const;
+
+private:
+  static inline const T& deref_helper(const T* t) { return *t; }
+  static inline T deref_helper(T t) { return t; }
+
+  struct compare_pairs_by_first {
+    bool operator()(const Entry& a, const Entry& b) const {
+      return Comparator()(deref_helper(a.first), deref_helper(b.first));
+    }
+  };
+
+  struct compare_pairs_by_second {
+    bool operator()(const Entry& a, const Entry& b) const {
+      return a.second < b.second;
+    }
+  };
+
+  template<typename TT = T, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type = 0>
+  static inline T ref_helper(const T& t) { return t; }
+
+  template<typename TT = T, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type = 0>
+  static inline const T* ref_helper(const T& t) { return std::addressof(t); }
+
+  template<typename TT = T, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type = 0>
+  static inline Entry make_dummy_entry(uint64_t weight) { return Entry(0, weight); }
+
+  template<typename TT = T, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type = 0>
+  static inline Entry make_dummy_entry(uint64_t weight) { return Entry(nullptr, weight); }
+
+  uint64_t total_weight_;
+  Container entries_;
+};
+
+template<typename T, typename C, typename A>
+class quantile_sketch_sorted_view<T, C, A>::const_iterator: public quantile_sketch_sorted_view<T, C, A>::Container::const_iterator {
+public:
+  using Base = typename quantile_sketch_sorted_view<T, C, A>::Container::const_iterator;
+  using value_type = typename std::conditional<std::is_arithmetic<T>::value, typename Base::value_type, std::pair<const T&, const uint64_t>>::type;
+
+  const_iterator(const Base& it): Base(it) {}
+
+  template<typename TT = T, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type = 0>
+  value_type operator*() const { return Base::operator*(); }
+
+  template<typename TT = T, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type = 0>
+  value_type operator*() const { return value_type(*(Base::operator*().first), Base::operator*().second); }
+
+  class return_value_holder {
+  public:
+    return_value_holder(value_type value): value_(value) {}
+    const value_type* operator->() const { return &value_; }
+  private:
+    value_type value_;
+  };
+
+  template<typename TT = T, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type = 0>
+  const value_type* operator->() const { return Base::operator->(); }
+
+  template<typename TT = T, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type = 0>
+  return_value_holder operator->() const { return **this; }
+};
+
+} /* namespace datasketches */
+
+#include "quantile_sketch_sorted_view_impl.hpp"
+
+#endif

data/vendor/datasketches-cpp/common/include/quantile_sketch_sorted_view_impl.hpp

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef QUANTILE_SKETCH_SORTED_VIEW_IMPL_HPP_
+#define QUANTILE_SKETCH_SORTED_VIEW_IMPL_HPP_
+
+#include <algorithm>
+#include <stdexcept>
+
+namespace datasketches {
+
+template<typename T, typename C, typename A>
+quantile_sketch_sorted_view<T, C, A>::quantile_sketch_sorted_view(uint32_t num, const A& allocator):
+total_weight_(0),
+entries_(allocator)
+{
+  entries_.reserve(num);
+}
+
+template<typename T, typename C, typename A>
+template<typename Iterator>
+void quantile_sketch_sorted_view<T, C, A>::add(Iterator first, Iterator last, uint64_t weight) {
+  const size_t size_before = entries_.size();
+  for (auto it = first; it != last; ++it) entries_.push_back(Entry(ref_helper(*it), weight));
+  if (size_before > 0) {
+    Container tmp(entries_.get_allocator());
+    tmp.reserve(entries_.capacity());
+    std::merge(
+        entries_.begin(), entries_.begin() + size_before,
+        entries_.begin() + size_before, entries_.end(),
+        std::back_inserter(tmp), compare_pairs_by_first()
+    );
+    std::swap(tmp, entries_);
+  }
+}
+
+template<typename T, typename C, typename A>
+template<bool inclusive>
+void quantile_sketch_sorted_view<T, C, A>::convert_to_cummulative() {
+  uint64_t subtotal = 0;
+  for (auto& entry: entries_) {
+    const uint64_t new_subtotal = subtotal + entry.second;
+    entry.second = inclusive ? new_subtotal : subtotal;
+    subtotal = new_subtotal;
+  }
+  total_weight_ = subtotal;
+}
+
+template<typename T, typename C, typename A>
+auto quantile_sketch_sorted_view<T, C, A>::get_quantile(double rank) const -> quantile_return_type {
+  if (total_weight_ == 0) throw std::invalid_argument("supported for cumulative weight only");
+  uint64_t weight = static_cast<uint64_t>(rank * total_weight_);
+  auto it = std::lower_bound(entries_.begin(), entries_.end(), make_dummy_entry<T>(weight), compare_pairs_by_second());
+  if (it == entries_.end()) return deref_helper(entries_[entries_.size() - 1].first);
+  return deref_helper(it->first);
+}
+
+template<typename T, typename C, typename A>
+auto quantile_sketch_sorted_view<T, C, A>::begin() const -> const_iterator {
+  return entries_.begin();
+}
+
+template<typename T, typename C, typename A>
+auto quantile_sketch_sorted_view<T, C, A>::end() const -> const_iterator {
+  return entries_.end();
+}
+
+template<typename T, typename C, typename A>
+size_t quantile_sketch_sorted_view<T, C, A>::size() const {
+  return entries_.size();
+}
+
+} /* namespace datasketches */
+
+#endif

data/vendor/datasketches-cpp/common/test/test_type.hpp

@@ -20,7 +20,9 @@
 #ifndef CLASS_TEST_TYPE_HPP_
 #define CLASS_TEST_TYPE_HPP_
 
+#include <cstring>
 #include <iostream>
+#include "memory_operations.hpp"
 
 namespace datasketches {
 

data/vendor/datasketches-cpp/cpc/include/cpc_compressor_impl.hpp

@@ -23,6 +23,7 @@
 #define CPC_COMPRESSOR_IMPL_HPP_
 
 #include <memory>
+#include <stdexcept>
 
 #include "compression_data.hpp"
 #include "cpc_util.hpp"

data/vendor/datasketches-cpp/cpc/include/cpc_confidence.hpp

@@ -23,6 +23,7 @@
 #define CPC_CONFIDENCE_HPP_
 
 #include <cmath>
+#include <stdexcept>
 
 #include "cpc_sketch.hpp"
 

data/vendor/datasketches-cpp/cpc/include/cpc_union_impl.hpp

@@ -22,6 +22,8 @@
 
 #include "count_zeros.hpp"
 
+#include <stdexcept>
+
 namespace datasketches {
 
 template<typename A>

data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp

@@ -21,6 +21,7 @@
 #include <cstring>
 #include <sstream>
 #include <fstream>
+#include <stdexcept>
 
 #include <catch.hpp>
 

data/vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp

@@ -21,6 +21,8 @@
 
 #include "cpc_union.hpp"
 
+#include <stdexcept>
+
 namespace datasketches {
 
 static const double RELATIVE_ERROR_FOR_LG_K_11 = 0.02;

data/vendor/datasketches-cpp/fi/include/frequent_items_sketch.hpp

@@ -46,7 +46,7 @@ template<
   typename W = uint64_t,
   typename H = std::hash<T>,
   typename E = std::equal_to<T>,
-  typename S = serde<T>,
+  typename S = serde<T>, // deprecated, to be removed in the next major version
   typename A = std::allocator<T>
 >
 class frequent_items_sketch {
@@ -225,46 +225,78 @@ public:
   /**
    * Computes size needed to serialize the current state of the sketch.
    * This can be expensive since every item needs to be looked at.
+   * @param instance of a SerDe
    * @return size in bytes needed to serialize this sketch
    */
-  size_t get_serialized_size_bytes() const;
+  template<typename SerDe = S>
+  size_t get_serialized_size_bytes(const SerDe& sd = SerDe()) const;
 
   /**
    * This method serializes the sketch into a given stream in a binary form
    * @param os output stream
+   * @param instance of a SerDe
    */
-  void serialize(std::ostream& os) const;
+  template<typename SerDe = S>
+  void serialize(std::ostream& os, const SerDe& sd = SerDe()) const;
 
   // This is a convenience alias for users
   // The type returned by the following serialize method
   using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
 
-
   /**
    * This method serializes the sketch as a vector of bytes.
    * An optional header can be reserved in front of the sketch.
    * It is a blank space of a given size.
    * This header is used in Datasketches PostgreSQL extension.
    * @param header_size_bytes space to reserve in front of the sketch
+   * @param instance of a SerDe
    * @return serialized sketch as a vector of bytes
    */
-  vector_bytes serialize(unsigned header_size_bytes = 0) const;
+  template<typename SerDe = S>
+  vector_bytes serialize(unsigned header_size_bytes = 0, const SerDe& sd = SerDe()) const;
 
   /**
    * This method deserializes a sketch from a given stream.
    * @param is input stream
+   * @param instance of an Allocator
    * @return an instance of the sketch
+   *
+   * Deprecated, to be removed in the next major version
    */
   static frequent_items_sketch deserialize(std::istream& is, const A& allocator = A());
 
+  /**
+   * This method deserializes a sketch from a given stream.
+   * @param is input stream
+   * @param instance of a SerDe
+   * @param instance of an Allocator
+   * @return an instance of the sketch
+   */
+  template<typename SerDe = S>
+  static frequent_items_sketch deserialize(std::istream& is, const SerDe& sd = SerDe(), const A& allocator = A());
+
   /**
    * This method deserializes a sketch from a given array of bytes.
    * @param bytes pointer to the array of bytes
    * @param size the size of the array
+   * @param instance of an Allocator
    * @return an instance of the sketch
+   *
+   * Deprecated, to be removed in the next major version
    */
   static frequent_items_sketch deserialize(const void* bytes, size_t size, const A& allocator = A());
 
+  /**
+   * This method deserializes a sketch from a given array of bytes.
+   * @param bytes pointer to the array of bytes
+   * @param size the size of the array
+   * @param instance of a SerDe
+   * @param instance of an Allocator
+   * @return an instance of the sketch
+   */
+  template<typename SerDe = S>
+  static frequent_items_sketch deserialize(const void* bytes, size_t size, const SerDe& sd = SerDe(), const A& allocator = A());
+
   /**
    * Returns a human readable summary of this sketch
    * @param print_items if true include the list of items retained by the sketch