datasketches 0.4.2 → 0.4.4

data/vendor/datasketches-cpp/tdigest/test/tdigest_custom_allocator_test.cpp

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+
+#include "tdigest.hpp"
+#include "test_allocator.hpp"
+
+namespace datasketches {
+
+using alloc_d = test_allocator<double>;
+using tdigest_d = tdigest<double, alloc_d>;
+
+TEST_CASE("tdigest custom allocator", "[tdigest]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    tdigest_d td(100, alloc_d(0));
+    for (int i = 0; i < 10000; ++i) td.update(static_cast<double>(i));
+    REQUIRE(test_allocator_total_bytes != 0);
+    REQUIRE(test_allocator_net_allocations != 0);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/tdigest/test/tdigest_deserialize_from_java_test.cpp

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+
+#include "tdigest.hpp"
+
+namespace datasketches {
+
+// assume the binary sketches for this test have been generated by datasketches-java code
+// in the subdirectory called "java" in the root directory of this project
+static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../java/";
+
+TEST_CASE("tdigest double", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "tdigest_double_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto td = tdigest<double>::deserialize(is);
+    REQUIRE(td.is_empty() == (n == 0));
+    REQUIRE(td.get_total_weight() == n);
+    if (n > 0) {
+      REQUIRE(td.get_min_value() == 1.0);
+      REQUIRE(td.get_max_value() == static_cast<double>(n));
+      REQUIRE(td.get_rank(0) == 0);
+      REQUIRE(td.get_rank(n + 1) == 1);
+      if (n == 1) {
+        REQUIRE(td.get_rank(n) == 0.5);
+      } else {
+        REQUIRE(td.get_rank(n / 2) == Approx(0.5).margin(0.05));
+      }
+    }
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/tdigest/test/tdigest_serialize_for_java.cpp

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+
+#include "tdigest.hpp"
+
+namespace datasketches {
+
+TEST_CASE("tdigest double generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    tdigest_double td(100);
+    for (unsigned i = 1; i <= n; ++i) td.update(i);
+    std::ofstream os("tdigest_double_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    td.serialize(os);
+  }
+}
+
+TEST_CASE("tdigest double generate with buffer", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    tdigest_double td(100);
+    for (unsigned i = 1; i <= n; ++i) td.update(i);
+    std::ofstream os("tdigest_double_buf_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    td.serialize(os, true);
+  }
+}
+
+TEST_CASE("tdigest float generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    tdigest_float td(100);
+    for (unsigned i = 1; i <= n; ++i) td.update(i);
+    std::ofstream os("tdigest_float_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    td.serialize(os);
+  }
+}
+
+TEST_CASE("tdigest float generate with buffer", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    tdigest_float td(100);
+    for (unsigned i = 1; i <= n; ++i) td.update(i);
+    std::ofstream os("tdigest_float_buf_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    td.serialize(os, true);
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/tdigest/test/tdigest_test.cpp

@@ -0,0 +1,456 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <iostream>
+#include <fstream>
+
+#include "tdigest.hpp"
+
+namespace datasketches {
+
+TEST_CASE("empty", "[tdigest]") {
+  tdigest_double td(10);
+//  std::cout << td.to_string();
+  REQUIRE(td.is_empty());
+  REQUIRE(td.get_k() == 10);
+  REQUIRE(td.get_total_weight() == 0);
+  REQUIRE_THROWS_AS(td.get_min_value(), std::runtime_error);
+  REQUIRE_THROWS_AS(td.get_max_value(), std::runtime_error);
+  REQUIRE_THROWS_AS(td.get_rank(0), std::runtime_error);
+  REQUIRE_THROWS_AS(td.get_quantile(0.5), std::runtime_error);
+  const double split_points[1] {0};
+  REQUIRE_THROWS_AS(td.get_PMF(split_points, 1), std::runtime_error);
+  REQUIRE_THROWS_AS(td.get_CDF(split_points, 1), std::runtime_error);
+}
+
+TEST_CASE("one value", "[tdigest]") {
+  tdigest_double td(100);
+  td.update(1);
+  REQUIRE(td.get_k() == 100);
+  REQUIRE(td.get_total_weight() == 1);
+  REQUIRE(td.get_min_value() == 1);
+  REQUIRE(td.get_max_value() == 1);
+  REQUIRE(td.get_rank(0.99) == 0);
+  REQUIRE(td.get_rank(1) == 0.5);
+  REQUIRE(td.get_rank(1.01) == 1);
+  REQUIRE(td.get_quantile(0) == 1);
+  REQUIRE(td.get_quantile(0.5) == 1);
+  REQUIRE(td.get_quantile(1) == 1);
+}
+
+TEST_CASE("many values", "[tdigest]") {
+  const size_t n = 10000;
+  tdigest_double td;
+  for (size_t i = 0; i < n; ++i) td.update(i);
+  REQUIRE_FALSE(td.is_empty());
+  REQUIRE(td.get_total_weight() == n);
+  REQUIRE(td.get_min_value() == 0);
+  REQUIRE(td.get_max_value() == n - 1);
+  REQUIRE(td.get_rank(0) == Approx(0).margin(0.0001));
+  REQUIRE(td.get_rank(n / 4) == Approx(0.25).margin(0.0001));
+  REQUIRE(td.get_rank(n / 2) == Approx(0.5).margin(0.0001));
+  REQUIRE(td.get_rank(n * 3 / 4) == Approx(0.75).margin(0.0001));
+  REQUIRE(td.get_rank(n) == 1);
+  REQUIRE(td.get_quantile(0) == 0);
+  REQUIRE(td.get_quantile(0.5) == Approx(n / 2).epsilon(0.03));
+  REQUIRE(td.get_quantile(0.9) == Approx(n * 0.9).epsilon(0.01));
+  REQUIRE(td.get_quantile(0.95) == Approx(n * 0.95).epsilon(0.01));
+  REQUIRE(td.get_quantile(1) == n - 1);
+  const double split_points[1] {n / 2};
+  const auto pmf = td.get_PMF(split_points, 1);
+  REQUIRE(pmf.size() == 2);
+  REQUIRE(pmf[0] == Approx(0.5).margin(0.0001));
+  REQUIRE(pmf[1] == Approx(0.5).margin(0.0001));
+  const auto cdf = td.get_CDF(split_points, 1);
+  REQUIRE(cdf.size() == 2);
+  REQUIRE(cdf[0] == Approx(0.5).margin(0.0001));
+  REQUIRE(cdf[1] == 1);
+}
+
+TEST_CASE("rank - two values", "[tdigest]") {
+  tdigest_double td(100);
+  td.update(1);
+  td.update(2);
+//  td.compress();
+//  std::cout << td.to_string(true);
+  REQUIRE(td.get_rank(0.99) == 0);
+  REQUIRE(td.get_rank(1) == 0.25);
+  REQUIRE(td.get_rank(1.25) == 0.375);
+  REQUIRE(td.get_rank(1.5) == 0.5);
+  REQUIRE(td.get_rank(1.75) == 0.625);
+  REQUIRE(td.get_rank(2) == 0.75);
+  REQUIRE(td.get_rank(2.01) == 1);
+}
+
+TEST_CASE("rank - repeated value", "[tdigest]") {
+  tdigest_double td(100);
+  td.update(1);
+  td.update(1);
+  td.update(1);
+  td.update(1);
+//  td.compress();
+//  std::cout << td.to_string(true);
+  REQUIRE(td.get_rank(0.99) == 0);
+  REQUIRE(td.get_rank(1) == 0.5);
+  REQUIRE(td.get_rank(1.01) == 1);
+}
+
+TEST_CASE("rank - repeated block", "[tdigest]") {
+  tdigest_double td(100);
+  td.update(1);
+  td.update(2);
+  td.update(2);
+  td.update(3);
+//  td.compress();
+//  std::cout << td.to_string(true);
+  REQUIRE(td.get_rank(0.99) == 0);
+  REQUIRE(td.get_rank(1) == 0.125);
+  REQUIRE(td.get_rank(2) == 0.5);
+  REQUIRE(td.get_rank(3) == 0.875);
+  REQUIRE(td.get_rank(3.01) == 1);
+}
+
+TEST_CASE("merge small", "[tdigest]") {
+  tdigest_double td1(10);
+  td1.update(1);
+  td1.update(2);
+  tdigest_double td2(10);
+  td2.update(2);
+  td2.update(3);
+  td1.merge(td2);
+  REQUIRE(td1.get_min_value() == 1);
+  REQUIRE(td1.get_max_value() == 3);
+  REQUIRE(td1.get_total_weight() == 4);
+  REQUIRE(td1.get_rank(0.99) == 0);
+  REQUIRE(td1.get_rank(1) == 0.125);
+  REQUIRE(td1.get_rank(2) == 0.5);
+  REQUIRE(td1.get_rank(3) == 0.875);
+  REQUIRE(td1.get_rank(3.01) == 1);
+}
+
+TEST_CASE("merge large", "[tdigest]") {
+  const size_t n = 10000;
+  tdigest_double td1;
+  tdigest_double td2;
+  for (size_t i = 0; i < n / 2; ++i) {
+    td1.update(i);
+    td2.update(n / 2 + i);
+  }
+//  std::cout << td1.to_string();
+//  std::cout << td2.to_string();
+  td1.merge(td2);
+//  td1.compress();
+//  std::cout << td1.to_string(true);
+  REQUIRE(td1.get_total_weight() == n);
+  REQUIRE(td1.get_min_value() == 0);
+  REQUIRE(td1.get_max_value() == n - 1);
+  REQUIRE(td1.get_rank(0) == Approx(0).margin(0.0001));
+  REQUIRE(td1.get_rank(n / 4) == Approx(0.25).margin(0.0001));
+  REQUIRE(td1.get_rank(n / 2) == Approx(0.5).margin(0.0001));
+  REQUIRE(td1.get_rank(n * 3 / 4) == Approx(0.75).margin(0.0001));
+  REQUIRE(td1.get_rank(n) == 1);
+}
+
+TEST_CASE("serialize deserialize stream empty", "[tdigest]") {
+  tdigest<double> td(100);
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  td.serialize(s);
+  auto deserialized_td = tdigest<double>::deserialize(s);
+  REQUIRE(td.get_k() == deserialized_td.get_k());
+  REQUIRE(td.get_total_weight() == deserialized_td.get_total_weight());
+  REQUIRE(td.is_empty() == deserialized_td.is_empty());
+}
+
+TEST_CASE("serialize deserialize stream single value", "[tdigest]") {
+  tdigest<double> td;
+  td.update(123);
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  td.serialize(s);
+  auto deserialized_td = tdigest<double>::deserialize(s);
+  REQUIRE(deserialized_td.get_k() == 200);
+  REQUIRE(deserialized_td.get_total_weight() == 1);
+  REQUIRE_FALSE(deserialized_td.is_empty());
+  REQUIRE(deserialized_td.get_min_value() == 123);
+  REQUIRE(deserialized_td.get_max_value() == 123);
+}
+
+TEST_CASE("serialize deserialize stream single value buffered", "[tdigest]") {
+  tdigest<double> td;
+  td.update(123);
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  td.serialize(s, true);
+  auto deserialized_td = tdigest<double>::deserialize(s);
+  REQUIRE(deserialized_td.get_k() == 200);
+  REQUIRE(deserialized_td.get_total_weight() == 1);
+  REQUIRE_FALSE(deserialized_td.is_empty());
+  REQUIRE(deserialized_td.get_min_value() == 123);
+  REQUIRE(deserialized_td.get_max_value() == 123);
+}
+
+TEST_CASE("serialize deserialize stream many values", "[tdigest]") {
+  tdigest<double> td(100);
+  for (int i = 0; i < 1000; ++i) td.update(i);
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  td.serialize(s);
+  auto deserialized_td = tdigest<double>::deserialize(s);
+  REQUIRE(td.get_k() == deserialized_td.get_k());
+  REQUIRE(td.get_total_weight() == deserialized_td.get_total_weight());
+  REQUIRE(td.is_empty() == deserialized_td.is_empty());
+  REQUIRE(td.get_min_value() == deserialized_td.get_min_value());
+  REQUIRE(td.get_max_value() == deserialized_td.get_max_value());
+  REQUIRE(td.get_rank(500) == deserialized_td.get_rank(500));
+  REQUIRE(td.get_quantile(0.5) == deserialized_td.get_quantile(0.5));
+}
+
+TEST_CASE("serialize deserialize stream many values with buffer", "[tdigest]") {
+  tdigest<double> td(100);
+  for (int i = 0; i < 10000; ++i) td.update(i);
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  td.serialize(s, true);
+  auto deserialized_td = tdigest<double>::deserialize(s);
+  REQUIRE(td.get_k() == deserialized_td.get_k());
+  REQUIRE(td.get_total_weight() == deserialized_td.get_total_weight());
+  REQUIRE(td.is_empty() == deserialized_td.is_empty());
+  REQUIRE(td.get_min_value() == deserialized_td.get_min_value());
+  REQUIRE(td.get_max_value() == deserialized_td.get_max_value());
+  REQUIRE(td.get_rank(500) == deserialized_td.get_rank(500));
+  REQUIRE(td.get_quantile(0.5) == deserialized_td.get_quantile(0.5));
+}
+
+TEST_CASE("serialize deserialize bytes empty", "[tdigest]") {
+  tdigest<double> td(100);
+  auto bytes = td.serialize();
+  auto deserialized_td = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(td.get_k() == deserialized_td.get_k());
+  REQUIRE(td.get_total_weight() == deserialized_td.get_total_weight());
+  REQUIRE(td.is_empty() == deserialized_td.is_empty());
+}
+
+TEST_CASE("serialize deserialize bytes single value", "[tdigest]") {
+  tdigest<double> td(200);
+  td.update(123);
+  auto bytes = td.serialize();
+  auto deserialized_td = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(deserialized_td.get_k() == 200);
+  REQUIRE(deserialized_td.get_total_weight() == 1);
+  REQUIRE_FALSE(deserialized_td.is_empty());
+  REQUIRE(deserialized_td.get_min_value() == 123);
+  REQUIRE(deserialized_td.get_max_value() == 123);
+}
+
+TEST_CASE("serialize deserialize bytes single value buffered", "[tdigest]") {
+  tdigest<double> td(200);
+  td.update(123);
+  auto bytes = td.serialize(0, true);
+  auto deserialized_td = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(deserialized_td.get_k() == 200);
+  REQUIRE(deserialized_td.get_total_weight() == 1);
+  REQUIRE_FALSE(deserialized_td.is_empty());
+  REQUIRE(deserialized_td.get_min_value() == 123);
+  REQUIRE(deserialized_td.get_max_value() == 123);
+}
+
+TEST_CASE("serialize deserialize bytes many values", "[tdigest]") {
+  tdigest<double> td(100);
+  for (int i = 0; i < 1000; ++i) td.update(i);
+  auto bytes = td.serialize();
+  auto deserialized_td = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(td.get_k() == deserialized_td.get_k());
+  REQUIRE(td.get_total_weight() == deserialized_td.get_total_weight());
+  REQUIRE(td.is_empty() == deserialized_td.is_empty());
+  REQUIRE(td.get_min_value() == deserialized_td.get_min_value());
+  REQUIRE(td.get_max_value() == deserialized_td.get_max_value());
+  REQUIRE(td.get_rank(500) == deserialized_td.get_rank(500));
+  REQUIRE(td.get_quantile(0.5) == deserialized_td.get_quantile(0.5));
+}
+
+TEST_CASE("serialize deserialize bytes many values with buffer", "[tdigest]") {
+  tdigest<double> td(100);
+  for (int i = 0; i < 10000; ++i) td.update(i);
+  auto bytes = td.serialize();
+  auto deserialized_td = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(td.get_k() == deserialized_td.get_k());
+  REQUIRE(td.get_total_weight() == deserialized_td.get_total_weight());
+  REQUIRE(td.is_empty() == deserialized_td.is_empty());
+  REQUIRE(td.get_min_value() == deserialized_td.get_min_value());
+  REQUIRE(td.get_max_value() == deserialized_td.get_max_value());
+  REQUIRE(td.get_rank(500) == deserialized_td.get_rank(500));
+  REQUIRE(td.get_quantile(0.5) == deserialized_td.get_quantile(0.5));
+}
+
+TEST_CASE("serialize deserialize steam and bytes equivalence empty", "[tdigest]") {
+  tdigest<double> td(100);
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  td.serialize(s);
+  auto bytes = td.serialize();
+
+  REQUIRE(bytes.size() == static_cast<size_t>(s.tellp()));
+  for (size_t i = 0; i < bytes.size(); ++i) {
+    REQUIRE(((char*)bytes.data())[i] == (char)s.get());
+  }
+
+  s.seekg(0); // rewind
+  auto deserialized_td1 = tdigest<double>::deserialize(s);
+  auto deserialized_td2 = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(bytes.size() == static_cast<size_t>(s.tellg()));
+
+  REQUIRE(deserialized_td1.is_empty());
+  REQUIRE(deserialized_td2.is_empty());
+  REQUIRE(deserialized_td1.get_k() == 100);
+  REQUIRE(deserialized_td2.get_k() == 100);
+  REQUIRE(deserialized_td1.get_total_weight() == 0);
+  REQUIRE(deserialized_td2.get_total_weight() == 0);
+}
+
+TEST_CASE("serialize deserialize steam and bytes equivalence", "[tdigest]") {
+  tdigest<double> td(100);
+  const int n = 1000;
+  for (int i = 0; i < n; ++i) td.update(i);
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  td.serialize(s);
+  auto bytes = td.serialize();
+
+  REQUIRE(bytes.size() == static_cast<size_t>(s.tellp()));
+  for (size_t i = 0; i < bytes.size(); ++i) {
+    REQUIRE(((char*)bytes.data())[i] == (char)s.get());
+  }
+
+  s.seekg(0); // rewind
+  auto deserialized_td1 = tdigest<double>::deserialize(s);
+  auto deserialized_td2 = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(bytes.size() == static_cast<size_t>(s.tellg()));
+
+  REQUIRE_FALSE(deserialized_td1.is_empty());
+  REQUIRE(deserialized_td1.get_k() == 100);
+  REQUIRE(deserialized_td1.get_total_weight() == n);
+  REQUIRE(deserialized_td1.get_min_value() == 0);
+  REQUIRE(deserialized_td1.get_max_value() == n - 1);
+
+  REQUIRE_FALSE(deserialized_td2.is_empty());
+  REQUIRE(deserialized_td2.get_k() == 100);
+  REQUIRE(deserialized_td2.get_total_weight() == n);
+  REQUIRE(deserialized_td2.get_min_value() == 0);
+  REQUIRE(deserialized_td2.get_max_value() == n - 1);
+
+  REQUIRE(deserialized_td1.get_rank(n / 2) == deserialized_td2.get_rank(n / 2));
+  REQUIRE(deserialized_td1.get_quantile(0.5) == deserialized_td2.get_quantile(0.5));
+}
+
+TEST_CASE("serialize deserialize steam and bytes equivalence with buffer", "[tdigest]") {
+  tdigest<double> td(100);
+  const int n = 10000;
+  for (int i = 0; i < n; ++i) td.update(i);
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  td.serialize(s, true);
+  auto bytes = td.serialize(0, true);
+
+  REQUIRE(bytes.size() == static_cast<size_t>(s.tellp()));
+  for (size_t i = 0; i < bytes.size(); ++i) {
+    REQUIRE(((char*)bytes.data())[i] == (char)s.get());
+  }
+
+  s.seekg(0); // rewind
+  auto deserialized_td1 = tdigest<double>::deserialize(s);
+  auto deserialized_td2 = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  REQUIRE(bytes.size() == static_cast<size_t>(s.tellg()));
+
+  REQUIRE_FALSE(deserialized_td1.is_empty());
+  REQUIRE(deserialized_td1.get_k() == 100);
+  REQUIRE(deserialized_td1.get_total_weight() == n);
+  REQUIRE(deserialized_td1.get_min_value() == 0);
+  REQUIRE(deserialized_td1.get_max_value() == n - 1);
+
+  REQUIRE_FALSE(deserialized_td2.is_empty());
+  REQUIRE(deserialized_td2.get_k() == 100);
+  REQUIRE(deserialized_td2.get_total_weight() == n);
+  REQUIRE(deserialized_td2.get_min_value() == 0);
+  REQUIRE(deserialized_td2.get_max_value() == n - 1);
+
+  REQUIRE(deserialized_td1.get_rank(n / 2) == deserialized_td2.get_rank(n / 2));
+  REQUIRE(deserialized_td1.get_quantile(0.5) == deserialized_td2.get_quantile(0.5));
+}
+
+TEST_CASE("deserialize from reference implementation stream double", "[tdigest]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(std::string(TEST_BINARY_INPUT_PATH) + "tdigest_ref_k100_n10000_double.sk", std::ios::binary);
+  const auto td = tdigest<double>::deserialize(is);
+  const size_t n = 10000;
+  REQUIRE(td.get_total_weight() == n);
+  REQUIRE(td.get_min_value() == 0);
+  REQUIRE(td.get_max_value() == n - 1);
+  REQUIRE(td.get_rank(0) == Approx(0).margin(0.0001));
+  REQUIRE(td.get_rank(n / 4) == Approx(0.25).margin(0.0001));
+  REQUIRE(td.get_rank(n / 2) == Approx(0.5).margin(0.0001));
+  REQUIRE(td.get_rank(n * 3 / 4) == Approx(0.75).margin(0.0001));
+  REQUIRE(td.get_rank(n) == 1);
+}
+
+TEST_CASE("deserialize from reference implementation stream float", "[tdigest]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(std::string(TEST_BINARY_INPUT_PATH) + "tdigest_ref_k100_n10000_float.sk", std::ios::binary);
+  const auto td = tdigest<float>::deserialize(is);
+  const size_t n = 10000;
+  REQUIRE(td.get_total_weight() == n);
+  REQUIRE(td.get_min_value() == 0);
+  REQUIRE(td.get_max_value() == n - 1);
+  REQUIRE(td.get_rank(0) == Approx(0).margin(0.0001));
+  REQUIRE(td.get_rank(n / 4) == Approx(0.25).margin(0.0001));
+  REQUIRE(td.get_rank(n / 2) == Approx(0.5).margin(0.0001));
+  REQUIRE(td.get_rank(n * 3 / 4) == Approx(0.75).margin(0.0001));
+  REQUIRE(td.get_rank(n) == 1);
+}
+
+TEST_CASE("deserialize from reference implementation bytes double", "[tdigest]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(std::string(TEST_BINARY_INPUT_PATH) + "tdigest_ref_k100_n10000_double.sk", std::ios::binary);
+  std::vector<char> bytes((std::istreambuf_iterator<char>(is)), (std::istreambuf_iterator<char>()));
+  const auto td = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  const size_t n = 10000;
+  REQUIRE(td.get_total_weight() == n);
+  REQUIRE(td.get_min_value() == 0);
+  REQUIRE(td.get_max_value() == n - 1);
+  REQUIRE(td.get_rank(0) == Approx(0).margin(0.0001));
+  REQUIRE(td.get_rank(n / 4) == Approx(0.25).margin(0.0001));
+  REQUIRE(td.get_rank(n / 2) == Approx(0.5).margin(0.0001));
+  REQUIRE(td.get_rank(n * 3 / 4) == Approx(0.75).margin(0.0001));
+  REQUIRE(td.get_rank(n) == 1);
+}
+
+TEST_CASE("deserialize from reference implementation bytes float", "[tdigest]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(std::string(TEST_BINARY_INPUT_PATH) + "tdigest_ref_k100_n10000_float.sk", std::ios::binary);
+  std::vector<char> bytes((std::istreambuf_iterator<char>(is)), (std::istreambuf_iterator<char>()));
+  const auto td = tdigest<double>::deserialize(bytes.data(), bytes.size());
+  const size_t n = 10000;
+  REQUIRE(td.get_total_weight() == n);
+  REQUIRE(td.get_min_value() == 0);
+  REQUIRE(td.get_max_value() == n - 1);
+  REQUIRE(td.get_rank(0) == Approx(0).margin(0.0001));
+  REQUIRE(td.get_rank(n / 4) == Approx(0.25).margin(0.0001));
+  REQUIRE(td.get_rank(n / 2) == Approx(0.5).margin(0.0001));
+  REQUIRE(td.get_rank(n * 3 / 4) == Approx(0.75).margin(0.0001));
+  REQUIRE(td.get_rank(n) == 1);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/theta/CMakeLists.txt

@@ -30,7 +30,6 @@ target_include_directories(theta
 )
 
 target_link_libraries(theta INTERFACE common)
-target_compile_features(theta INTERFACE cxx_std_11)
 
 install(TARGETS theta
   EXPORT ${PROJECT_NAME}

data/vendor/datasketches-cpp/theta/include/bit_packing.hpp

@@ -329,7 +329,7 @@ static inline void pack_bits_13(const uint64_t* values, uint8_t* ptr) {
 
   *ptr++ = static_cast<uint8_t>(values[3] >> 4);
 
-  *ptr = static_cast<uint8_t>(values[3] >> 4);
+  *ptr = static_cast<uint8_t>(values[3] << 4);
   *ptr++ |= static_cast<uint8_t>(values[4] >> 9);
 
   *ptr++ = static_cast<uint8_t>(values[4] >> 1);
@@ -4227,7 +4227,7 @@ static inline void unpack_bits_33(uint64_t* values, const uint8_t* ptr) {
   values[6] |= *ptr >> 1;
 
   values[7] = static_cast<uint64_t>(*ptr++ & 1) << 32;
-  values[7] |= *ptr++ << 24;
+  values[7] |= static_cast<uint64_t>(*ptr++) << 24;
   values[7] |= *ptr++ << 16;
   values[7] |= *ptr++ << 8;
   values[7] |= *ptr;
@@ -4296,7 +4296,7 @@ static inline void unpack_bits_35(uint64_t* values, const uint8_t* ptr) {
   values[1] |= *ptr++ << 6;
   values[1] |= *ptr >> 2;
 
-  values[2] = static_cast<uint64_t>(*ptr++ & 2) << 33;
+  values[2] = static_cast<uint64_t>(*ptr++ & 3) << 33;
   values[2] |= static_cast<uint64_t>(*ptr++) << 25;
   values[2] |= *ptr++ << 17;
   values[2] |= *ptr++ << 9;
@@ -6201,7 +6201,7 @@ static inline void pack_bits_block8(const uint64_t* values, uint8_t* ptr, uint8_
     case 61: pack_bits_61(values, ptr); break;
     case 62: pack_bits_62(values, ptr); break;
     case 63: pack_bits_63(values, ptr); break;
-    default: throw std::logic_error("wrong number of bits " + std::to_string(bits));
+    default: throw std::logic_error("wrong number of bits in pack_bits_block8: " + std::to_string(bits));
   }
 }
 
@@ -6270,7 +6270,7 @@ static inline void unpack_bits_block8(uint64_t* values, const uint8_t* ptr, uint
     case 61: unpack_bits_61(values, ptr); break;
     case 62: unpack_bits_62(values, ptr); break;
     case 63: unpack_bits_63(values, ptr); break;
-    default: throw std::logic_error("wrong number of bits " + std::to_string(bits));
+    default: throw std::logic_error("wrong number of bits in unpack_bits_block8: " + std::to_string(bits));
   }
 }
 

data/vendor/datasketches-cpp/theta/include/theta_helpers.hpp

@@ -57,7 +57,7 @@ public:
   // consistent way of initializing theta from p
   // avoids multiplication if p == 1 since it might not yield MAX_THETA exactly
   static uint64_t starting_theta_from_p(float p) {
-    if (p < 1) return static_cast<uint64_t>(theta_constants::MAX_THETA * p);
+    if (p < 1) return static_cast<uint64_t>(static_cast<double>(theta_constants::MAX_THETA) * p);
     return theta_constants::MAX_THETA;
   }