datasketches 0.2.0 → 0.2.4

data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp

@@ -303,7 +303,7 @@ std::vector<T, A> kll_sketch<T, C, S, A>::get_quantiles(const double* fractions,
 }
 
 template<typename T, typename C, typename S, typename A>
-std::vector<T, A> kll_sketch<T, C, S, A>::get_quantiles(size_t num) const {
+std::vector<T, A> kll_sketch<T, C, S, A>::get_quantiles(uint32_t num) const {
   if (is_empty()) return std::vector<T, A>(allocator_);
   if (num == 0) {
     throw std::invalid_argument("num must be > 0");
@@ -380,36 +380,56 @@ size_t kll_sketch<T, C, S, A>::get_serialized_size_bytes() const {
   size_t size = DATA_START + num_levels_ * sizeof(uint32_t);
   size += S().size_of_item(*min_value_);
   size += S().size_of_item(*max_value_);
-  for (auto& it: *this) size += S().size_of_item(it.first);
+  for (auto it: *this) size += S().size_of_item(it.first);
   return size;
 }
 
+// implementation for fixed-size arithmetic types (integral and floating point)
+template<typename T, typename C, typename S, typename A>
+template<typename TT, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type>
+size_t kll_sketch<T, C, S, A>::get_max_serialized_size_bytes(uint16_t k, uint64_t n) {
+  const uint8_t num_levels = kll_helper::ub_on_num_levels(n);
+  const uint32_t max_num_retained = kll_helper::compute_total_capacity(k, DEFAULT_M, num_levels);
+  // the last integer in the levels_ array is not serialized because it can be derived
+  return DATA_START + num_levels * sizeof(uint32_t) + (max_num_retained + 2) * sizeof(TT);
+}
+
+// implementation for all other types
+template<typename T, typename C, typename S, typename A>
+template<typename TT, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type>
+size_t kll_sketch<T, C, S, A>::get_max_serialized_size_bytes(uint16_t k, uint64_t n, size_t max_item_size_bytes) {
+  const uint8_t num_levels = kll_helper::ub_on_num_levels(n);
+  const uint32_t max_num_retained = kll_helper::compute_total_capacity(k, DEFAULT_M, num_levels);
+  // the last integer in the levels_ array is not serialized because it can be derived
+  return DATA_START + num_levels * sizeof(uint32_t) + (max_num_retained + 2) * max_item_size_bytes;
+}
+
 template<typename T, typename C, typename S, typename A>
 void kll_sketch<T, C, S, A>::serialize(std::ostream& os) const {
   const bool is_single_item = n_ == 1;
   const uint8_t preamble_ints(is_empty() || is_single_item ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_FULL);
-  os.write(reinterpret_cast<const char*>(&preamble_ints), sizeof(preamble_ints));
+  write(os, preamble_ints);
   const uint8_t serial_version(is_single_item ? SERIAL_VERSION_2 : SERIAL_VERSION_1);
-  os.write(reinterpret_cast<const char*>(&serial_version), sizeof(serial_version));
+  write(os, serial_version);
   const uint8_t family(FAMILY);
-  os.write(reinterpret_cast<const char*>(&family), sizeof(family));
+  write(os, family);
   const uint8_t flags_byte(
       (is_empty() ? 1 << flags::IS_EMPTY : 0)
     | (is_level_zero_sorted_ ? 1 << flags::IS_LEVEL_ZERO_SORTED : 0)
     | (is_single_item ? 1 << flags::IS_SINGLE_ITEM : 0)
   );
-  os.write(reinterpret_cast<const char*>(&flags_byte), sizeof(flags_byte));
-  os.write((char*)&k_, sizeof(k_));
-  os.write((char*)&m_, sizeof(m_));
+  write(os, flags_byte);
+  write(os, k_);
+  write(os, m_);
   const uint8_t unused = 0;
-  os.write(reinterpret_cast<const char*>(&unused), sizeof(unused));
+  write(os, unused);
   if (is_empty()) return;
   if (!is_single_item) {
-    os.write((char*)&n_, sizeof(n_));
-    os.write((char*)&min_k_, sizeof(min_k_));
-    os.write((char*)&num_levels_, sizeof(num_levels_));
-    os.write((char*)&unused, sizeof(unused));
-    os.write((char*)levels_.data(), sizeof(levels_[0]) * num_levels_);
+    write(os, n_);
+    write(os, min_k_);
+    write(os, num_levels_);
+    write(os, unused);
+    write(os, levels_.data(), sizeof(levels_[0]) * num_levels_);
     S().serialize(os, min_value_, 1);
     S().serialize(os, max_value_, 1);
   }
@@ -424,27 +444,26 @@ vector_u8<A> kll_sketch<T, C, S, A>::serialize(unsigned header_size_bytes) const
   uint8_t* ptr = bytes.data() + header_size_bytes;
   const uint8_t* end_ptr = ptr + size;
   const uint8_t preamble_ints(is_empty() || is_single_item ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_FULL);
-  ptr += copy_to_mem(&preamble_ints, ptr, sizeof(preamble_ints));
+  ptr += copy_to_mem(preamble_ints, ptr);
   const uint8_t serial_version(is_single_item ? SERIAL_VERSION_2 : SERIAL_VERSION_1);
-  ptr += copy_to_mem(&serial_version, ptr, sizeof(serial_version));
+  ptr += copy_to_mem(serial_version, ptr);
   const uint8_t family(FAMILY);
-  ptr += copy_to_mem(&family, ptr, sizeof(family));
+  ptr += copy_to_mem(family, ptr);
   const uint8_t flags_byte(
       (is_empty() ? 1 << flags::IS_EMPTY : 0)
     | (is_level_zero_sorted_ ? 1 << flags::IS_LEVEL_ZERO_SORTED : 0)
     | (is_single_item ? 1 << flags::IS_SINGLE_ITEM : 0)
   );
-  ptr += copy_to_mem(&flags_byte, ptr, sizeof(flags_byte));
-  ptr += copy_to_mem(&k_, ptr, sizeof(k_));
-  ptr += copy_to_mem(&m_, ptr, sizeof(m_));
-  const uint8_t unused = 0;
-  ptr += copy_to_mem(&unused, ptr, sizeof(unused));
+  ptr += copy_to_mem(flags_byte, ptr);
+  ptr += copy_to_mem(k_, ptr);
+  ptr += copy_to_mem(m_, ptr);
+  ptr += sizeof(uint8_t); // unused
   if (!is_empty()) {
     if (!is_single_item) {
-      ptr += copy_to_mem(&n_, ptr, sizeof(n_));
-      ptr += copy_to_mem(&min_k_, ptr, sizeof(min_k_));
-      ptr += copy_to_mem(&num_levels_, ptr, sizeof(num_levels_));
-      ptr += copy_to_mem(&unused, ptr, sizeof(unused));
+      ptr += copy_to_mem(n_, ptr);
+      ptr += copy_to_mem(min_k_, ptr);
+      ptr += copy_to_mem(num_levels_, ptr);
+      ptr += sizeof(uint8_t); // unused
       ptr += copy_to_mem(levels_.data(), ptr, sizeof(levels_[0]) * num_levels_);
       ptr += S().serialize(ptr, end_ptr - ptr, min_value_, 1);
       ptr += S().serialize(ptr, end_ptr - ptr, max_value_, 1);
@@ -459,20 +478,13 @@ vector_u8<A> kll_sketch<T, C, S, A>::serialize(unsigned header_size_bytes) const
 
 template<typename T, typename C, typename S, typename A>
 kll_sketch<T, C, S, A> kll_sketch<T, C, S, A>::deserialize(std::istream& is, const A& allocator) {
-  uint8_t preamble_ints;
-  is.read((char*)&preamble_ints, sizeof(preamble_ints));
-  uint8_t serial_version;
-  is.read((char*)&serial_version, sizeof(serial_version));
-  uint8_t family_id;
-  is.read((char*)&family_id, sizeof(family_id));
-  uint8_t flags_byte;
-  is.read((char*)&flags_byte, sizeof(flags_byte));
-  uint16_t k;
-  is.read((char*)&k, sizeof(k));
-  uint8_t m;
-  is.read((char*)&m, sizeof(m));
-  uint8_t unused;
-  is.read((char*)&unused, sizeof(unused));
+  const auto preamble_ints = read<uint8_t>(is);
+  const auto serial_version = read<uint8_t>(is);
+  const auto family_id = read<uint8_t>(is);
+  const auto flags_byte = read<uint8_t>(is);
+  const auto k = read<uint16_t>(is);
+  const auto m = read<uint8_t>(is);
+  read<uint8_t>(is); // skip unused byte
 
   check_m(m);
   check_preamble_ints(preamble_ints, flags_byte);
@@ -492,10 +504,10 @@ kll_sketch<T, C, S, A> kll_sketch<T, C, S, A>::deserialize(std::istream& is, con
     min_k = k;
     num_levels = 1;
   } else {
-    is.read((char*)&n, sizeof(n_));
-    is.read((char*)&min_k, sizeof(min_k_));
-    is.read((char*)&num_levels, sizeof(num_levels));
-    is.read((char*)&unused, sizeof(unused));
+    n = read<uint64_t>(is);
+    min_k = read<uint16_t>(is);
+    num_levels = read<uint8_t>(is);
+    read<uint8_t>(is); // skip unused byte
   }
   vector_u32<A> levels(num_levels + 1, 0, allocator);
   const uint32_t capacity(kll_helper::compute_total_capacity(k, m, num_levels));
@@ -503,7 +515,7 @@ kll_sketch<T, C, S, A> kll_sketch<T, C, S, A>::deserialize(std::istream& is, con
     levels[0] = capacity - 1;
   } else {
     // the last integer in levels_ is not serialized because it can be derived
-    is.read((char*)levels.data(), sizeof(levels[0]) * num_levels);
+    read(is, levels.data(), sizeof(levels[0]) * num_levels);
   }
   levels[num_levels] = capacity;
   A alloc(allocator);
@@ -546,24 +558,24 @@ kll_sketch<T, C, S, A> kll_sketch<T, C, S, A>::deserialize(const void* bytes, si
   ensure_minimum_memory(size, 8);
   const char* ptr = static_cast<const char*>(bytes);
   uint8_t preamble_ints;
-  ptr += copy_from_mem(ptr, &preamble_ints, sizeof(preamble_ints));
+  ptr += copy_from_mem(ptr, preamble_ints);
   uint8_t serial_version;
-  ptr += copy_from_mem(ptr, &serial_version, sizeof(serial_version));
+  ptr += copy_from_mem(ptr, serial_version);
   uint8_t family_id;
-  ptr += copy_from_mem(ptr, &family_id, sizeof(family_id));
+  ptr += copy_from_mem(ptr, family_id);
   uint8_t flags_byte;
-  ptr += copy_from_mem(ptr, &flags_byte, sizeof(flags_byte));
+  ptr += copy_from_mem(ptr, flags_byte);
   uint16_t k;
-  ptr += copy_from_mem(ptr, &k, sizeof(k));
+  ptr += copy_from_mem(ptr, k);
   uint8_t m;
-  ptr += copy_from_mem(ptr, &m, sizeof(m));
-  ptr++; // skip unused byte
+  ptr += copy_from_mem(ptr, m);
+  ptr += sizeof(uint8_t); // skip unused byte
 
   check_m(m);
   check_preamble_ints(preamble_ints, flags_byte);
   check_serial_version(serial_version);
   check_family_id(family_id);
-  ensure_minimum_memory(size, 1 << preamble_ints);
+  ensure_minimum_memory(size, preamble_ints * sizeof(uint32_t));
 
   const bool is_empty(flags_byte & (1 << flags::IS_EMPTY));
   if (is_empty) return kll_sketch<T, C, S, A>(k, allocator);
@@ -578,10 +590,10 @@ kll_sketch<T, C, S, A> kll_sketch<T, C, S, A>::deserialize(const void* bytes, si
     min_k = k;
     num_levels = 1;
   } else {
-    ptr += copy_from_mem(ptr, &n, sizeof(n));
-    ptr += copy_from_mem(ptr, &min_k, sizeof(min_k));
-    ptr += copy_from_mem(ptr, &num_levels, sizeof(num_levels));
-    ptr++; // skip unused byte
+    ptr += copy_from_mem(ptr, n);
+    ptr += copy_from_mem(ptr, min_k);
+    ptr += copy_from_mem(ptr, num_levels);
+    ptr += sizeof(uint8_t); // skip unused byte
   }
   vector_u32<A> levels(num_levels + 1, 0, allocator);
   const uint32_t capacity(kll_helper::compute_total_capacity(k, m, num_levels));
@@ -779,7 +791,7 @@ std::unique_ptr<kll_quantile_calculator<T, C, A>, std::function<void(kll_quantil
   using AllocCalc = typename std::allocator_traits<A>::template rebind_alloc<kll_quantile_calculator<T, C, A>>;
   AllocCalc alloc(allocator_);
   std::unique_ptr<kll_quantile_calculator<T, C, A>, std::function<void(kll_quantile_calculator<T, C, A>*)>> quantile_calculator(
-    new (alloc.allocate(1)) kll_quantile_calculator<T, C, A>(items_, levels_.data(), num_levels_, n_, allocator_),
+    new (alloc.allocate(1)) kll_quantile_calculator<T, C, A>(*this),
     [&alloc](kll_quantile_calculator<T, C, A>* ptr){ ptr->~kll_quantile_calculator<T, C, A>(); alloc.deallocate(ptr, 1); }
   );
   return quantile_calculator;
@@ -1011,7 +1023,9 @@ void kll_sketch<T, C, S, A>::check_family_id(uint8_t family_id) {
 
 template <typename T, typename C, typename S, typename A>
 string<A> kll_sketch<T, C, S, A>::to_string(bool print_levels, bool print_items) const {
-  std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+  // Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
+  // The stream does not support passing an allocator instance, and alternatives are complicated.
+  std::ostringstream os;
   os << "### KLL sketch summary:" << std::endl;
   os << "   K              : " << k_ << std::endl;
   os << "   min K          : " << min_k_ << std::endl;
@@ -1057,7 +1071,7 @@ string<A> kll_sketch<T, C, S, A>::to_string(bool print_levels, bool print_items)
     }
     os << "### End sketch data" << std::endl;
   }
-  return os.str();
+  return string<A>(os.str().c_str(), allocator_);
 }
 
 template <typename T, typename C, typename S, typename A>
@@ -1067,14 +1081,14 @@ typename kll_sketch<T, C, S, A>::const_iterator kll_sketch<T, C, S, A>::begin()
 
 template <typename T, typename C, typename S, typename A>
 typename kll_sketch<T, C, S, A>::const_iterator kll_sketch<T, C, S, A>::end() const {
-  return kll_sketch<T, C, S, A>::const_iterator(nullptr, nullptr, num_levels_);
+  return kll_sketch<T, C, S, A>::const_iterator(nullptr, levels_.data(), num_levels_);
 }
 
 // kll_sketch::const_iterator implementation
 
 template<typename T, typename C, typename S, typename A>
 kll_sketch<T, C, S, A>::const_iterator::const_iterator(const T* items, const uint32_t* levels, const uint8_t num_levels):
-items(items), levels(levels), num_levels(num_levels), index(levels == nullptr ? 0 : levels[0]), level(levels == nullptr ? num_levels : 0), weight(1)
+items(items), levels(levels), num_levels(num_levels), index(items == nullptr ? levels[num_levels] : levels[0]), level(items == nullptr ? num_levels : 0), weight(1)
 {}
 
 template<typename T, typename C, typename S, typename A>
@@ -1098,8 +1112,6 @@ typename kll_sketch<T, C, S, A>::const_iterator& kll_sketch<T, C, S, A>::const_i
 
 template<typename T, typename C, typename S, typename A>
 bool kll_sketch<T, C, S, A>::const_iterator::operator==(const const_iterator& other) const {
-  if (level != other.level) return false;
-  if (level == num_levels) return true; // end
   return index == other.index;
 }
 

data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov.hpp

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef KOLMOGOROV_SMIRNOV_HPP_
+#define KOLMOGOROV_SMIRNOV_HPP_
+
+namespace datasketches {
+
+class kolmogorov_smirnov {
+public:
+  /**
+   * Computes the raw delta area between two KLL quantile sketches for the Kolmogorov-Smirnov Test.
+   * @param sketch1 KLL sketch 1
+   * @param sketch2 KLL sketch 2
+   * @return the raw delta between two KLL quantile sketches
+   */
+  template<typename Sketch>
+  static double delta(const Sketch& sketch1, const Sketch& sketch2);
+
+  /**
+   * Computes the adjusted delta area threshold for the Kolmogorov-Smirnov Test.
+   * Adjusts the computed threshold by the error epsilons of the two given sketches.
+   * See <a href="https://en.wikipedia.org/wiki/Kolmogorov-Smirnov_test">Kolmogorov–Smirnov Test</a>
+   * @param sketch1 KLL sketch 1
+   * @param sketch2 KLL sketch 2
+   * @param p Target p-value. Typically .001 to .1, e.g., .05.
+   * @return the adjusted threshold to be compared with the raw delta
+   */
+  template<typename Sketch>
+  static double threshold(const Sketch& sketch1, const Sketch& sketch2, double p);
+
+  /**
+   * Performs the Kolmogorov-Smirnov Test between two KLL quantiles sketches.
+   * Note: if the given sketches have insufficient data or if the sketch sizes are too small,
+   * this will return false.
+   * @param sketch1 KLL sketch 1
+   * @param sketch2 KLL sketch 2
+   * @param p Target p-value. Typically .001 to .1, e.g., .05.
+   * @return Boolean indicating whether we can reject the null hypothesis (that the sketches
+   * reflect the same underlying distribution) using the provided p-value.
+   */
+  template<typename Sketch>
+  static bool test(const Sketch& sketch1, const Sketch& sketch2, double p);
+
+};
+
+} /* namespace datasketches */
+
+#include "kolmogorov_smirnov_impl.hpp"
+
+#endif

data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov_impl.hpp

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef KOLMOGOROV_SMIRNOV_IMPL_HPP_
+#define KOLMOGOROV_SMIRNOV_IMPL_HPP_
+
+namespace datasketches {
+
+// type resolver
+template<typename T, typename C, typename S, typename A>
+kll_quantile_calculator<T, C, A> make_quantile_calculator(const kll_sketch<T, C, S, A>& sketch) {
+  return kll_quantile_calculator<T, C, A>(sketch);
+}
+
+template<typename Sketch>
+double kolmogorov_smirnov::delta(const Sketch& sketch1, const Sketch& sketch2) {
+  using Comparator = typename Sketch::comparator;
+  auto calc1 = make_quantile_calculator(sketch1);
+  auto calc2 = make_quantile_calculator(sketch2);
+  auto it1 = calc1.begin();
+  auto it2 = calc2.begin();
+  const auto n1 = sketch1.get_n();
+  const auto n2 = sketch2.get_n();
+  double delta = 0;
+  while (it1 != calc1.end() && it2 != calc2.end()) {
+    const double norm_cum_wt1 = static_cast<double>((*it1).second) / n1;
+    const double norm_cum_wt2 = static_cast<double>((*it2).second) / n2;
+    delta = std::max(delta, std::abs(norm_cum_wt1 - norm_cum_wt2));
+    if (Comparator()((*it1).first, (*it2).first)) {
+      ++it1;
+    } else if (Comparator()((*it2).first, (*it1).first)) {
+      ++it2;
+    } else {
+      ++it1;
+      ++it2;
+    }
+  }
+  const double norm_cum_wt1 = it1 == calc1.end() ? 1 : static_cast<double>((*it1).second) / n1;
+  const double norm_cum_wt2 = it2 == calc2.end() ? 1 : static_cast<double>((*it2).second) / n2;
+  delta = std::max(delta, std::abs(norm_cum_wt1 - norm_cum_wt2));
+  return delta;
+}
+
+template<typename Sketch>
+double kolmogorov_smirnov::threshold(const Sketch& sketch1, const Sketch& sketch2, double p) {
+  const double r1 = sketch1.get_num_retained();
+  const double r2 = sketch2.get_num_retained();
+  const double alpha_factor = sqrt(-0.5 * log(0.5 * p));
+  const double delta_area_threshold = alpha_factor * sqrt((r1 + r2) / (r1 * r2));
+  const double eps1 = sketch1.get_normalized_rank_error(false);
+  const double eps2 = sketch2.get_normalized_rank_error(false);
+  return delta_area_threshold + eps1 + eps2;
+}
+
+template<typename Sketch>
+bool kolmogorov_smirnov::test(const Sketch& sketch1, const Sketch& sketch2, double p) {
+  return delta(sketch1, sketch2) > threshold(sketch1, sketch2, p);
+}
+
+} /* namespace datasketches */
+
+#endif

data/vendor/datasketches-cpp/kll/test/CMakeLists.txt

@@ -41,4 +41,5 @@ target_sources(kll_test
     kll_sketch_test.cpp
     kll_sketch_custom_type_test.cpp
     kll_sketch_validation.cpp
+    kolmogorov_smirnov_test.cpp
 )