datasketches 0.1.2 → 0.2.3

data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov.hpp

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef KOLMOGOROV_SMIRNOV_HPP_
+#define KOLMOGOROV_SMIRNOV_HPP_
+
+namespace datasketches {
+
+class kolmogorov_smirnov {
+public:
+  /**
+   * Computes the raw delta area between two KLL quantile sketches for the Kolmogorov-Smirnov Test.
+   * @param sketch1 KLL sketch 1
+   * @param sketch2 KLL sketch 2
+   * @return the raw delta between two KLL quantile sketches
+   */
+  template<typename Sketch>
+  static double delta(const Sketch& sketch1, const Sketch& sketch2);
+
+  /**
+   * Computes the adjusted delta area threshold for the Kolmogorov-Smirnov Test.
+   * Adjusts the computed threshold by the error epsilons of the two given sketches.
+   * See <a href="https://en.wikipedia.org/wiki/Kolmogorov-Smirnov_test">Kolmogorov–Smirnov Test</a>
+   * @param sketch1 KLL sketch 1
+   * @param sketch2 KLL sketch 2
+   * @param p Target p-value. Typically .001 to .1, e.g., .05.
+   * @return the adjusted threshold to be compared with the raw delta
+   */
+  template<typename Sketch>
+  static double threshold(const Sketch& sketch1, const Sketch& sketch2, double p);
+
+  /**
+   * Performs the Kolmogorov-Smirnov Test between two KLL quantiles sketches.
+   * Note: if the given sketches have insufficient data or if the sketch sizes are too small,
+   * this will return false.
+   * @param sketch1 KLL sketch 1
+   * @param sketch2 KLL sketch 2
+   * @param p Target p-value. Typically .001 to .1, e.g., .05.
+   * @return Boolean indicating whether we can reject the null hypothesis (that the sketches
+   * reflect the same underlying distribution) using the provided p-value.
+   */
+  template<typename Sketch>
+  static bool test(const Sketch& sketch1, const Sketch& sketch2, double p);
+
+};
+
+} /* namespace datasketches */
+
+#include "kolmogorov_smirnov_impl.hpp"
+
+#endif

data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov_impl.hpp

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef KOLMOGOROV_SMIRNOV_IMPL_HPP_
+#define KOLMOGOROV_SMIRNOV_IMPL_HPP_
+
+namespace datasketches {
+
+// type resolver
+template<typename T, typename C, typename S, typename A>
+kll_quantile_calculator<T, C, A> make_quantile_calculator(const kll_sketch<T, C, S, A>& sketch) {
+  return kll_quantile_calculator<T, C, A>(sketch);
+}
+
+template<typename Sketch>
+double kolmogorov_smirnov::delta(const Sketch& sketch1, const Sketch& sketch2) {
+  using Comparator = typename Sketch::comparator;
+  auto calc1 = make_quantile_calculator(sketch1);
+  auto calc2 = make_quantile_calculator(sketch2);
+  auto it1 = calc1.begin();
+  auto it2 = calc2.begin();
+  const auto n1 = sketch1.get_n();
+  const auto n2 = sketch2.get_n();
+  double delta = 0;
+  while (it1 != calc1.end() && it2 != calc2.end()) {
+    const double norm_cum_wt1 = static_cast<double>((*it1).second) / n1;
+    const double norm_cum_wt2 = static_cast<double>((*it2).second) / n2;
+    delta = std::max(delta, std::abs(norm_cum_wt1 - norm_cum_wt2));
+    if (Comparator()((*it1).first, (*it2).first)) {
+      ++it1;
+    } else if (Comparator()((*it2).first, (*it1).first)) {
+      ++it2;
+    } else {
+      ++it1;
+      ++it2;
+    }
+  }
+  const double norm_cum_wt1 = it1 == calc1.end() ? 1 : static_cast<double>((*it1).second) / n1;
+  const double norm_cum_wt2 = it2 == calc2.end() ? 1 : static_cast<double>((*it2).second) / n2;
+  delta = std::max(delta, std::abs(norm_cum_wt1 - norm_cum_wt2));
+  return delta;
+}
+
+template<typename Sketch>
+double kolmogorov_smirnov::threshold(const Sketch& sketch1, const Sketch& sketch2, double p) {
+  const double r1 = sketch1.get_num_retained();
+  const double r2 = sketch2.get_num_retained();
+  const double alpha_factor = sqrt(-0.5 * log(0.5 * p));
+  const double delta_area_threshold = alpha_factor * sqrt((r1 + r2) / (r1 * r2));
+  const double eps1 = sketch1.get_normalized_rank_error(false);
+  const double eps2 = sketch2.get_normalized_rank_error(false);
+  return delta_area_threshold + eps1 + eps2;
+}
+
+template<typename Sketch>
+bool kolmogorov_smirnov::test(const Sketch& sketch1, const Sketch& sketch2, double p) {
+  return delta(sketch1, sketch2) > threshold(sketch1, sketch2, p);
+}
+
+} /* namespace datasketches */
+
+#endif

data/vendor/datasketches-cpp/kll/test/CMakeLists.txt

@@ -41,4 +41,5 @@ target_sources(kll_test
     kll_sketch_test.cpp
     kll_sketch_custom_type_test.cpp
     kll_sketch_validation.cpp
+    kolmogorov_smirnov_test.cpp
 )

data/vendor/datasketches-cpp/kll/test/kll_sketch_custom_type_test.cpp

@@ -26,7 +26,8 @@
 
 namespace datasketches {
 
-typedef kll_sketch<test_type, test_type_less, test_type_serde, test_allocator<test_type>> kll_test_type_sketch;
+using kll_test_type_sketch = kll_sketch<test_type, test_type_less, test_type_serde, test_allocator<test_type>>;
+using alloc = test_allocator<test_type>;
 
 TEST_CASE("kll sketch custom type", "[kll_sketch]") {
 
@@ -34,7 +35,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   test_allocator_total_bytes = 0;
 
   SECTION("compact level zero") {
-    kll_test_type_sketch sketch(8);
+    kll_test_type_sketch sketch(8, 0);
     REQUIRE_THROWS_AS(sketch.get_quantile(0), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_min_value(), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_max_value(), std::runtime_error);
@@ -59,10 +60,10 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   }
 
   SECTION("merge small") {
-    kll_test_type_sketch sketch1(8);
+    kll_test_type_sketch sketch1(8, 0);
     sketch1.update(1);
 
-    kll_test_type_sketch sketch2(8);
+    kll_test_type_sketch sketch2(8, 0);
     sketch2.update(2);
 
     sketch2.merge(sketch1);
@@ -76,7 +77,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   }
 
   SECTION("merge higher levels") {
-    kll_test_type_sketch sketch1(8);
+    kll_test_type_sketch sketch1(8, 0);
     sketch1.update(1);
     sketch1.update(2);
     sketch1.update(3);
@@ -87,7 +88,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
     sketch1.update(8);
     sketch1.update(9);
 
-    kll_test_type_sketch sketch2(8);
+    kll_test_type_sketch sketch2(8, 0);
     sketch2.update(10);
     sketch2.update(11);
     sketch2.update(12);
@@ -109,7 +110,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   }
 
   SECTION("serialize deserialize") {
-    kll_test_type_sketch sketch1;
+    kll_test_type_sketch sketch1(200, 0);
 
     const int n = 1000;
     for (int i = 0; i < n; i++) sketch1.update(i);
@@ -117,7 +118,7 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
     sketch1.serialize(s);
     REQUIRE((size_t) s.tellp() == sketch1.get_serialized_size_bytes());
-    auto sketch2 = kll_test_type_sketch::deserialize(s);
+    auto sketch2 = kll_test_type_sketch::deserialize(s, alloc(0));
     REQUIRE((size_t) s.tellg() == sketch2.get_serialized_size_bytes());
     REQUIRE(s.tellg() == s.tellp());
     REQUIRE(sketch2.is_empty() == sketch1.is_empty());
@@ -135,9 +136,9 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") {
   }
 
   SECTION("moving merge") {
-    kll_test_type_sketch sketch1(8);
+    kll_test_type_sketch sketch1(8, 0);
     for (int i = 0; i < 10; i++) sketch1.update(i);
-    kll_test_type_sketch sketch2(8);
+    kll_test_type_sketch sketch2(8, 0);
     sketch2.update(10);
     sketch2.merge(std::move(sketch1));
     REQUIRE(sketch2.get_min_value().get_value() == 0);