datasketches 0.2.0 → 0.2.4

data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp

@@ -17,8 +17,10 @@
  * under the License.
  */
 
+#include <istream>
 #include <fstream>
 #include <sstream>
+#include <vector>
 
 #include <catch.hpp>
 #include <theta_sketch.hpp>
@@ -39,6 +41,7 @@ TEST_CASE("theta sketch: empty", "[theta_sketch]") {
   REQUIRE(update_sketch.get_estimate() == 0.0);
   REQUIRE(update_sketch.get_lower_bound(1) == 0.0);
   REQUIRE(update_sketch.get_upper_bound(1) == 0.0);
+  REQUIRE(update_sketch.is_ordered());
 
   compact_theta_sketch compact_sketch = update_sketch.compact();
   REQUIRE(compact_sketch.is_empty());
@@ -47,10 +50,14 @@ TEST_CASE("theta sketch: empty", "[theta_sketch]") {
   REQUIRE(compact_sketch.get_estimate() == 0.0);
   REQUIRE(compact_sketch.get_lower_bound(1) == 0.0);
   REQUIRE(compact_sketch.get_upper_bound(1) == 0.0);
+  REQUIRE(compact_sketch.is_ordered());
+
+  // empty is forced to be ordered
+  REQUIRE(update_sketch.compact(false).is_ordered());
 }
 
 TEST_CASE("theta sketch: non empty no retained keys", "[theta_sketch]") {
-  update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001).build();
+  update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001f).build();
   update_sketch.update(1);
   //std::cerr << update_sketch.to_string();
   REQUIRE(update_sketch.get_num_retained() == 0);
@@ -67,6 +74,14 @@ TEST_CASE("theta sketch: non empty no retained keys", "[theta_sketch]") {
   REQUIRE(compact_sketch.get_estimate() == 0.0);
   REQUIRE(compact_sketch.get_lower_bound(1) == 0.0);
   REQUIRE(compact_sketch.get_upper_bound(1) > 0);
+
+  update_sketch.reset();
+  REQUIRE(update_sketch.is_empty());
+  REQUIRE_FALSE(update_sketch.is_estimation_mode());
+  REQUIRE(update_sketch.get_theta() == 1.0);
+  REQUIRE(update_sketch.get_estimate() == 0.0);
+  REQUIRE(update_sketch.get_lower_bound(1) == 0.0);
+  REQUIRE(update_sketch.get_upper_bound(1) == 0.0);
 }
 
 TEST_CASE("theta sketch: single item", "[theta_sketch]") {
@@ -78,6 +93,7 @@ TEST_CASE("theta sketch: single item", "[theta_sketch]") {
   REQUIRE(update_sketch.get_estimate() == 1.0);
   REQUIRE(update_sketch.get_lower_bound(1) == 1.0);
   REQUIRE(update_sketch.get_upper_bound(1) == 1.0);
+  REQUIRE(update_sketch.is_ordered()); // one item is ordered
 
   compact_theta_sketch compact_sketch = update_sketch.compact();
   REQUIRE_FALSE(compact_sketch.is_empty());
@@ -86,6 +102,10 @@ TEST_CASE("theta sketch: single item", "[theta_sketch]") {
   REQUIRE(compact_sketch.get_estimate() == 1.0);
   REQUIRE(compact_sketch.get_lower_bound(1) == 1.0);
   REQUIRE(compact_sketch.get_upper_bound(1) == 1.0);
+  REQUIRE(compact_sketch.is_ordered());
+
+  // single item is forced to be ordered
+  REQUIRE(update_sketch.compact(false).is_ordered());
 }
 
 TEST_CASE("theta sketch: resize exact", "[theta_sketch]") {
@@ -97,6 +117,7 @@ TEST_CASE("theta sketch: resize exact", "[theta_sketch]") {
   REQUIRE(update_sketch.get_estimate() == 2000.0);
   REQUIRE(update_sketch.get_lower_bound(1) == 2000.0);
   REQUIRE(update_sketch.get_upper_bound(1) == 2000.0);
+  REQUIRE_FALSE(update_sketch.is_ordered());
 
   compact_theta_sketch compact_sketch = update_sketch.compact();
   REQUIRE_FALSE(compact_sketch.is_empty());
@@ -105,6 +126,17 @@ TEST_CASE("theta sketch: resize exact", "[theta_sketch]") {
   REQUIRE(compact_sketch.get_estimate() == 2000.0);
   REQUIRE(compact_sketch.get_lower_bound(1) == 2000.0);
   REQUIRE(compact_sketch.get_upper_bound(1) == 2000.0);
+  REQUIRE(compact_sketch.is_ordered());
+
+  update_sketch.reset();
+  REQUIRE(update_sketch.is_empty());
+  REQUIRE_FALSE(update_sketch.is_estimation_mode());
+  REQUIRE(update_sketch.get_theta() == 1.0);
+  REQUIRE(update_sketch.get_estimate() == 0.0);
+  REQUIRE(update_sketch.get_lower_bound(1) == 0.0);
+  REQUIRE(update_sketch.get_upper_bound(1) == 0.0);
+  REQUIRE(update_sketch.is_ordered());
+
 }
 
 TEST_CASE("theta sketch: estimation", "[theta_sketch]") {
@@ -148,6 +180,34 @@ TEST_CASE("theta sketch: deserialize compact empty from java", "[theta_sketch]")
   REQUIRE(sketch.get_upper_bound(1) == 0.0);
 }
 
+TEST_CASE("theta sketch: deserialize compact v1 empty from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_empty_from_java_v1.sk", std::ios::binary);
+  auto sketch = compact_theta_sketch::deserialize(is);
+  REQUIRE(sketch.is_empty());
+  REQUIRE_FALSE(sketch.is_estimation_mode());
+  REQUIRE(sketch.get_num_retained() == 0);
+  REQUIRE(sketch.get_theta() == 1.0);
+  REQUIRE(sketch.get_estimate() == 0.0);
+  REQUIRE(sketch.get_lower_bound(1) == 0.0);
+  REQUIRE(sketch.get_upper_bound(1) == 0.0);
+}
+
+TEST_CASE("theta sketch: deserialize compact v2 empty from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_empty_from_java_v2.sk", std::ios::binary);
+  auto sketch = compact_theta_sketch::deserialize(is);
+  REQUIRE(sketch.is_empty());
+  REQUIRE_FALSE(sketch.is_estimation_mode());
+  REQUIRE(sketch.get_num_retained() == 0);
+  REQUIRE(sketch.get_theta() == 1.0);
+  REQUIRE(sketch.get_estimate() == 0.0);
+  REQUIRE(sketch.get_lower_bound(1) == 0.0);
+  REQUIRE(sketch.get_upper_bound(1) == 0.0);
+}
+
 TEST_CASE("theta sketch: deserialize single item from java", "[theta_sketch]") {
   std::ifstream is;
   is.exceptions(std::ios::failbit | std::ios::badbit);
@@ -162,6 +222,38 @@ TEST_CASE("theta sketch: deserialize single item from java", "[theta_sketch]") {
   REQUIRE(sketch.get_upper_bound(1) == 1.0);
 }
 
+TEST_CASE("theta sketch: deserialize compact exact from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_exact_from_java.sk", std::ios::binary);
+  auto sketch = compact_theta_sketch::deserialize(is);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE_FALSE(sketch.is_estimation_mode());
+  REQUIRE(sketch.is_ordered());
+  REQUIRE(sketch.get_num_retained() == 100);
+
+  // the same construction process in Java must have produced exactly the same sketch
+  auto update_sketch = update_theta_sketch::builder().build();
+  const int n = 100;
+  for (int i = 0; i < n; i++) update_sketch.update(i);
+  REQUIRE(sketch.get_num_retained() == update_sketch.get_num_retained());
+  REQUIRE(sketch.get_theta() == Approx(update_sketch.get_theta()).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(update_sketch.get_estimate()).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(1) == Approx(update_sketch.get_lower_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(1) == Approx(update_sketch.get_upper_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(update_sketch.get_lower_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(update_sketch.get_upper_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(3) == Approx(update_sketch.get_lower_bound(3)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(3) == Approx(update_sketch.get_upper_bound(3)).margin(1e-10));
+  compact_theta_sketch compact_sketch = update_sketch.compact();
+  // the sketches are ordered, so the iteration sequence must match exactly
+  auto iter = sketch.begin();
+  for (const auto& key: compact_sketch) {
+    REQUIRE(*iter == key);
+    ++iter;
+  }
+}
+
 TEST_CASE("theta sketch: deserialize compact estimation from java", "[theta_sketch]") {
   std::ifstream is;
   is.exceptions(std::ios::failbit | std::ios::badbit);
@@ -198,6 +290,78 @@ TEST_CASE("theta sketch: deserialize compact estimation from java", "[theta_sket
   }
 }
 
+TEST_CASE("theta sketch: deserialize compact v1 estimation from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_estimation_from_java_v1.sk", std::ios::binary);
+  auto sketch = compact_theta_sketch::deserialize(is);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.is_estimation_mode());
+  REQUIRE(sketch.is_ordered());
+  REQUIRE(sketch.get_num_retained() == 4342);
+  REQUIRE(sketch.get_theta() == Approx(0.531700444213199).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(8166.25234614053).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(7996.956955317471).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(8339.090301078124).margin(1e-10));
+
+  // the same construction process in Java must have produced exactly the same sketch
+  update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+  const int n = 8192;
+  for (int i = 0; i < n; i++) update_sketch.update(i);
+  REQUIRE(sketch.get_num_retained() == update_sketch.get_num_retained());
+  REQUIRE(sketch.get_theta() == Approx(update_sketch.get_theta()).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(update_sketch.get_estimate()).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(1) == Approx(update_sketch.get_lower_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(1) == Approx(update_sketch.get_upper_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(update_sketch.get_lower_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(update_sketch.get_upper_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(3) == Approx(update_sketch.get_lower_bound(3)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(3) == Approx(update_sketch.get_upper_bound(3)).margin(1e-10));
+  compact_theta_sketch compact_sketch = update_sketch.compact();
+  // the sketches are ordered, so the iteration sequence must match exactly
+  auto iter = sketch.begin();
+  for (const auto& key: compact_sketch) {
+    REQUIRE(*iter == key);
+    ++iter;
+  }
+}
+
+TEST_CASE("theta sketch: deserialize compact v2 estimation from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_estimation_from_java_v2.sk", std::ios::binary);
+  auto sketch = compact_theta_sketch::deserialize(is);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.is_estimation_mode());
+  REQUIRE(sketch.is_ordered());
+  REQUIRE(sketch.get_num_retained() == 4342);
+  REQUIRE(sketch.get_theta() == Approx(0.531700444213199).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(8166.25234614053).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(7996.956955317471).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(8339.090301078124).margin(1e-10));
+
+  // the same construction process in Java must have produced exactly the same sketch
+  update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+  const int n = 8192;
+  for (int i = 0; i < n; i++) update_sketch.update(i);
+  REQUIRE(sketch.get_num_retained() == update_sketch.get_num_retained());
+  REQUIRE(sketch.get_theta() == Approx(update_sketch.get_theta()).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(update_sketch.get_estimate()).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(1) == Approx(update_sketch.get_lower_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(1) == Approx(update_sketch.get_upper_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(update_sketch.get_lower_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(update_sketch.get_upper_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(3) == Approx(update_sketch.get_lower_bound(3)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(3) == Approx(update_sketch.get_upper_bound(3)).margin(1e-10));
+  compact_theta_sketch compact_sketch = update_sketch.compact();
+  // the sketches are ordered, so the iteration sequence must match exactly
+  auto iter = sketch.begin();
+  for (const auto& key: compact_sketch) {
+    REQUIRE(*iter == key);
+    ++iter;
+  }
+}
+
 TEST_CASE("theta sketch: serialize deserialize stream and bytes equivalence", "[theta_sketch]") {
   update_theta_sketch update_sketch = update_theta_sketch::builder().build();
   const int n = 8192;
@@ -238,4 +402,276 @@ TEST_CASE("theta sketch: deserialize compact single item buffer overrun", "[thet
   REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(bytes.data(), bytes.size() - 1), std::out_of_range);
 }
 
+TEST_CASE("theta sketch: conversion constructor and wrapped compact", "[theta_sketch]") {
+  update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+  const int n = 8192;
+  for (int i = 0; i < n; i++) update_sketch.update(i);
+
+  // unordered
+  auto unordered_compact1 = update_sketch.compact(false);
+  compact_theta_sketch unordered_compact2(update_sketch, false);
+  auto it = unordered_compact1.begin();
+  for (auto entry: unordered_compact2) {
+    REQUIRE(*it == entry);
+    ++it;
+  }
+
+  // ordered
+  auto ordered_compact1 = update_sketch.compact();
+  compact_theta_sketch ordered_compact2(update_sketch, true);
+  it = ordered_compact1.begin();
+  for (auto entry: ordered_compact2) {
+    REQUIRE(*it == entry);
+    ++it;
+  }
+
+  // wrapped compact
+  auto bytes = ordered_compact1.serialize();
+  auto ordered_compact3 = wrapped_compact_theta_sketch::wrap(bytes.data(), bytes.size());
+  it = ordered_compact1.begin();
+  for (auto entry: ordered_compact3) {
+    REQUIRE(*it == entry);
+    ++it;
+  }
+  REQUIRE(ordered_compact3.get_estimate() == ordered_compact1.get_estimate());
+  REQUIRE(ordered_compact3.get_lower_bound(1) == ordered_compact1.get_lower_bound(1));
+  REQUIRE(ordered_compact3.get_upper_bound(1) == ordered_compact1.get_upper_bound(1));
+  REQUIRE(ordered_compact3.is_estimation_mode() == ordered_compact1.is_estimation_mode());
+  REQUIRE(ordered_compact3.get_theta() == ordered_compact1.get_theta());
+
+
+  // seed mismatch
+  REQUIRE_THROWS_AS(wrapped_compact_theta_sketch::wrap(bytes.data(), bytes.size(), 0), std::invalid_argument);
+}
+
+TEST_CASE("theta sketch: wrap compact empty from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_empty_from_java.sk", std::ios::binary | std::ios::ate);
+
+  std::vector<uint8_t> buf;
+  if(is) {
+      auto size = is.tellg();
+      buf.reserve(size);
+      buf.assign(size, 0);
+      is.seekg(0, std::ios_base::beg);
+      is.read((char*)(buf.data()), buf.size());
+  }
+
+  auto sketch = wrapped_compact_theta_sketch::wrap(buf.data(), buf.size());
+  REQUIRE(sketch.is_empty());
+  REQUIRE_FALSE(sketch.is_estimation_mode());
+  REQUIRE(sketch.get_num_retained() == 0);
+  REQUIRE(sketch.get_theta() == 1.0);
+  REQUIRE(sketch.get_estimate() == 0.0);
+  REQUIRE(sketch.get_lower_bound(1) == 0.0);
+  REQUIRE(sketch.get_upper_bound(1) == 0.0);
+}
+
+TEST_CASE("theta sketch: wrap compact v1 empty from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_empty_from_java_v1.sk", std::ios::binary | std::ios::ate);
+
+  std::vector<uint8_t> buf;
+  if(is) {
+      auto size = is.tellg();
+      buf.reserve(size);
+      buf.assign(size, 0);
+      is.seekg(0, std::ios_base::beg);
+      is.read((char*)(buf.data()), buf.size());
+  }
+
+  auto sketch = wrapped_compact_theta_sketch::wrap(buf.data(), buf.size());
+  REQUIRE(sketch.is_empty());
+  REQUIRE_FALSE(sketch.is_estimation_mode());
+  REQUIRE(sketch.get_num_retained() == 0);
+  REQUIRE(sketch.get_theta() == 1.0);
+  REQUIRE(sketch.get_estimate() == 0.0);
+  REQUIRE(sketch.get_lower_bound(1) == 0.0);
+  REQUIRE(sketch.get_upper_bound(1) == 0.0);
+}
+
+TEST_CASE("theta sketch: wrap compact v2 empty from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_empty_from_java_v2.sk", std::ios::binary | std::ios::ate);
+
+  std::vector<uint8_t> buf;
+  if(is) {
+      auto size = is.tellg();
+      buf.reserve(size);
+      buf.assign(size, 0);
+      is.seekg(0, std::ios_base::beg);
+      is.read((char*)(buf.data()), buf.size());
+  }
+
+  auto sketch = wrapped_compact_theta_sketch::wrap(buf.data(), buf.size());
+  REQUIRE(sketch.is_empty());
+  REQUIRE_FALSE(sketch.is_estimation_mode());
+  REQUIRE(sketch.get_num_retained() == 0);
+  REQUIRE(sketch.get_theta() == 1.0);
+  REQUIRE(sketch.get_estimate() == 0.0);
+  REQUIRE(sketch.get_lower_bound(1) == 0.0);
+  REQUIRE(sketch.get_upper_bound(1) == 0.0);
+}
+
+TEST_CASE("theta sketch: wrap single item from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_single_item_from_java.sk", std::ios::binary | std::ios::ate);
+  std::vector<uint8_t> buf;
+  if(is) {
+      auto size = is.tellg();
+      buf.reserve(size);
+      buf.assign(size, 0);
+      is.seekg(0, std::ios_base::beg);
+      is.read((char*)(buf.data()), buf.size());
+  }
+
+  auto sketch = wrapped_compact_theta_sketch::wrap(buf.data(), buf.size());
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE_FALSE(sketch.is_estimation_mode());
+  REQUIRE(sketch.get_num_retained() == 1);
+  REQUIRE(sketch.get_theta() == 1.0);
+  REQUIRE(sketch.get_estimate() == 1.0);
+  REQUIRE(sketch.get_lower_bound(1) == 1.0);
+  REQUIRE(sketch.get_upper_bound(1) == 1.0);
+}
+
+TEST_CASE("theta sketch: wrap compact estimation from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_estimation_from_java.sk", std::ios::binary | std::ios::ate);
+  std::vector<uint8_t> buf;
+  if(is) {
+      auto size = is.tellg();
+      buf.reserve(size);
+      buf.assign(size, 0);
+      is.seekg(0, std::ios_base::beg);
+      is.read((char*)(buf.data()), buf.size());
+  }
+
+  auto sketch = wrapped_compact_theta_sketch::wrap(buf.data(), buf.size());
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.is_estimation_mode());
+  REQUIRE(sketch.is_ordered());
+  REQUIRE(sketch.get_num_retained() == 4342);
+  REQUIRE(sketch.get_theta() == Approx(0.531700444213199).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(8166.25234614053).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(7996.956955317471).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(8339.090301078124).margin(1e-10));
+
+  // the same construction process in Java must have produced exactly the same sketch
+  update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+  const int n = 8192;
+  for (int i = 0; i < n; i++) update_sketch.update(i);
+  REQUIRE(sketch.get_num_retained() == update_sketch.get_num_retained());
+  REQUIRE(sketch.get_theta() == Approx(update_sketch.get_theta()).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(update_sketch.get_estimate()).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(1) == Approx(update_sketch.get_lower_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(1) == Approx(update_sketch.get_upper_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(update_sketch.get_lower_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(update_sketch.get_upper_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(3) == Approx(update_sketch.get_lower_bound(3)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(3) == Approx(update_sketch.get_upper_bound(3)).margin(1e-10));
+  compact_theta_sketch compact_sketch = update_sketch.compact();
+  // the sketches are ordered, so the iteration sequence must match exactly
+  auto iter = sketch.begin();
+  for (const auto& key: compact_sketch) {
+    REQUIRE(*iter == key);
+    ++iter;
+  }
+}
+
+TEST_CASE("theta sketch: wrap compact v1 estimation from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_estimation_from_java_v1.sk", std::ios::binary | std::ios::ate);
+  std::vector<uint8_t> buf;
+  if(is) {
+      auto size = is.tellg();
+      buf.reserve(size);
+      buf.assign(size, 0);
+      is.seekg(0, std::ios_base::beg);
+      is.read((char*)(buf.data()), buf.size());
+  }
+
+  auto sketch = wrapped_compact_theta_sketch::wrap(buf.data(), buf.size());
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.is_estimation_mode());
+//  REQUIRE(sketch.is_ordered());       // v1 may not be ordered
+  REQUIRE(sketch.get_num_retained() == 4342);
+  REQUIRE(sketch.get_theta() == Approx(0.531700444213199).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(8166.25234614053).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(7996.956955317471).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(8339.090301078124).margin(1e-10));
+
+  // the same construction process in Java must have produced exactly the same sketch
+  update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+  const int n = 8192;
+  for (int i = 0; i < n; i++) update_sketch.update(i);
+  REQUIRE(sketch.get_num_retained() == update_sketch.get_num_retained());
+  REQUIRE(sketch.get_theta() == Approx(update_sketch.get_theta()).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(update_sketch.get_estimate()).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(1) == Approx(update_sketch.get_lower_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(1) == Approx(update_sketch.get_upper_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(update_sketch.get_lower_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(update_sketch.get_upper_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(3) == Approx(update_sketch.get_lower_bound(3)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(3) == Approx(update_sketch.get_upper_bound(3)).margin(1e-10));
+  compact_theta_sketch compact_sketch = update_sketch.compact();
+  // the sketches are ordered, so the iteration sequence must match exactly
+  auto iter = sketch.begin();
+  for (const auto& key: compact_sketch) {
+    REQUIRE(*iter == key);
+    ++iter;
+  }
+}
+
+TEST_CASE("theta sketch: wrap compact v2 estimation from java", "[theta_sketch]") {
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(inputPath + "theta_compact_estimation_from_java_v2.sk", std::ios::binary | std::ios::ate);
+  std::vector<uint8_t> buf;
+  if(is) {
+      auto size = is.tellg();
+      buf.reserve(size);
+      buf.assign(size, 0);
+      is.seekg(0, std::ios_base::beg);
+      is.read((char*)(buf.data()), buf.size());
+  }
+
+  auto sketch = wrapped_compact_theta_sketch::wrap(buf.data(), buf.size());
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.is_estimation_mode());
+//  REQUIRE(sketch.is_ordered());       // v1 may not be ordered
+  REQUIRE(sketch.get_num_retained() == 4342);
+  REQUIRE(sketch.get_theta() == Approx(0.531700444213199).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(8166.25234614053).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(7996.956955317471).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(8339.090301078124).margin(1e-10));
+
+  // the same construction process in Java must have produced exactly the same sketch
+  update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+  const int n = 8192;
+  for (int i = 0; i < n; i++) update_sketch.update(i);
+  REQUIRE(sketch.get_num_retained() == update_sketch.get_num_retained());
+  REQUIRE(sketch.get_theta() == Approx(update_sketch.get_theta()).margin(1e-10));
+  REQUIRE(sketch.get_estimate() == Approx(update_sketch.get_estimate()).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(1) == Approx(update_sketch.get_lower_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(1) == Approx(update_sketch.get_upper_bound(1)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(2) == Approx(update_sketch.get_lower_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(2) == Approx(update_sketch.get_upper_bound(2)).margin(1e-10));
+  REQUIRE(sketch.get_lower_bound(3) == Approx(update_sketch.get_lower_bound(3)).margin(1e-10));
+  REQUIRE(sketch.get_upper_bound(3) == Approx(update_sketch.get_upper_bound(3)).margin(1e-10));
+  compact_theta_sketch compact_sketch = update_sketch.compact();
+  // the sketches are ordered, so the iteration sequence must match exactly
+  auto iter = sketch.begin();
+  for (const auto& key: compact_sketch) {
+    REQUIRE(*iter == key);
+    ++iter;
+  }
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/theta/test/theta_union_test.cpp

@@ -39,7 +39,7 @@ TEST_CASE("theta union: empty", "[theta_union]") {
 }
 
 TEST_CASE("theta union: non empty no retained keys", "[theta_union]") {
-  update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001).build();
+  update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001f).build();
   update_sketch.update(1);
   theta_union u = theta_union::builder().build();
   u.update(update_sketch);
@@ -51,40 +51,72 @@ TEST_CASE("theta union: non empty no retained keys", "[theta_union]") {
 }
 
 TEST_CASE("theta union: exact mode half overlap", "[theta_union]") {
-  update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+  auto sketch1 = update_theta_sketch::builder().build();
   int value = 0;
   for (int i = 0; i < 1000; i++) sketch1.update(value++);
 
-  update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+  auto sketch2 = update_theta_sketch::builder().build();
   value = 500;
   for (int i = 0; i < 1000; i++) sketch2.update(value++);
 
-  theta_union u = theta_union::builder().build();
+  auto u = theta_union::builder().build();
   u.update(sketch1);
   u.update(sketch2);
+  auto sketch3 = u.get_result();
+  REQUIRE_FALSE(sketch3.is_empty());
+  REQUIRE_FALSE(sketch3.is_estimation_mode());
+  REQUIRE(sketch3.get_estimate() == 1500.0);
+
+  u.reset();
+  sketch3 = u.get_result();
+  REQUIRE(sketch3.get_num_retained() == 0);
+  REQUIRE(sketch3.is_empty());
+  REQUIRE_FALSE(sketch3.is_estimation_mode());
+}
+
+TEST_CASE("theta union: exact mode half overlap wrapped compact", "[theta_union]") {
+  auto sketch1 = update_theta_sketch::builder().build();
+  int value = 0;
+  for (int i = 0; i < 1000; i++) sketch1.update(value++);
+  auto bytes1 = sketch1.compact().serialize();
+
+  auto sketch2 = update_theta_sketch::builder().build();
+  value = 500;
+  for (int i = 0; i < 1000; i++) sketch2.update(value++);
+  auto bytes2 = sketch2.compact().serialize();
+
+  auto u = theta_union::builder().build();
+  u.update(wrapped_compact_theta_sketch::wrap(bytes1.data(), bytes1.size()));
+  u.update(wrapped_compact_theta_sketch::wrap(bytes2.data(), bytes2.size()));
   compact_theta_sketch sketch3 = u.get_result();
   REQUIRE_FALSE(sketch3.is_empty());
   REQUIRE_FALSE(sketch3.is_estimation_mode());
-  REQUIRE(sketch3.get_estimate() == Approx(1500).margin(1500 * 0.01));
+  REQUIRE(sketch3.get_estimate() == 1500.0);
 }
 
 TEST_CASE("theta union: estimation mode half overlap", "[theta_union]") {
-  update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+  auto sketch1 = update_theta_sketch::builder().build();
   int value = 0;
   for (int i = 0; i < 10000; i++) sketch1.update(value++);
 
-  update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+  auto sketch2 = update_theta_sketch::builder().build();
   value = 5000;
   for (int i = 0; i < 10000; i++) sketch2.update(value++);
 
-  theta_union u = theta_union::builder().build();
+  auto u = theta_union::builder().build();
   u.update(sketch1);
   u.update(sketch2);
-  compact_theta_sketch sketch3 = u.get_result();
+  auto sketch3 = u.get_result();
   REQUIRE_FALSE(sketch3.is_empty());
   REQUIRE(sketch3.is_estimation_mode());
   REQUIRE(sketch3.get_estimate() == Approx(15000).margin(15000 * 0.01));
   //std::cerr << sketch3.to_string(true);
+
+  u.reset();
+  sketch3 = u.get_result();
+  REQUIRE(sketch3.get_num_retained() == 0);
+  REQUIRE(sketch3.is_empty());
+  REQUIRE_FALSE(sketch3.is_estimation_mode());
 }
 
 TEST_CASE("theta union: seed mismatch", "[theta_union]") {

data/vendor/datasketches-cpp/tuple/CMakeLists.txt

@@ -32,41 +32,26 @@ target_include_directories(tuple
 target_link_libraries(tuple INTERFACE common theta)
 target_compile_features(tuple INTERFACE cxx_std_11)
 
-set(tuple_HEADERS "")
-list(APPEND tuple_HEADERS "include/tuple_sketch.hpp;include/tuple_sketch_impl.hpp")
-list(APPEND tuple_HEADERS "include/tuple_union.hpp;include/tuple_union_impl.hpp")
-list(APPEND tuple_HEADERS "include/tuple_intersection.hpp;include/tuple_intersection_impl.hpp")
-list(APPEND tuple_HEADERS "include/tuple_a_not_b.hpp;include/tuple_a_not_b_impl.hpp")
-list(APPEND tuple_HEADERS "include/tuple_jaccard_similarity.hpp")
-list(APPEND tuple_HEADERS "include/array_of_doubles_sketch.hpp;include/array_of_doubles_sketch_impl.hpp")
-list(APPEND tuple_HEADERS "include/array_of_doubles_union.hpp;include/array_of_doubles_union_impl.hpp")
-list(APPEND tuple_HEADERS "include/array_of_doubles_intersection.hpp;include/array_of_doubles_intersection_impl.hpp")
-list(APPEND tuple_HEADERS "include/array_of_doubles_a_not_b.hpp;include/array_of_doubles_a_not_b_impl.hpp")
-
 install(TARGETS tuple
   EXPORT ${PROJECT_NAME}
 )
 
-install(FILES ${tuple_HEADERS}
+install(FILES 
+		include/tuple_sketch.hpp
+		include/tuple_sketch_impl.hpp
+		include/tuple_union.hpp
+		include/tuple_union_impl.hpp
+		include/tuple_intersection.hpp
+		include/tuple_intersection_impl.hpp
+		include/tuple_a_not_b.hpp
+		include/tuple_a_not_b_impl.hpp
+		include/tuple_jaccard_similarity.hpp
+		include/array_of_doubles_sketch.hpp
+		include/array_of_doubles_sketch_impl.hpp
+		include/array_of_doubles_union.hpp
+		include/array_of_doubles_union_impl.hpp
+		include/array_of_doubles_intersection.hpp
+		include/array_of_doubles_intersection_impl.hpp
+		include/array_of_doubles_a_not_b.hpp
+		include/array_of_doubles_a_not_b_impl.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
-
-target_sources(tuple
-  INTERFACE
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_sketch.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_sketch_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_union.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_union_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_intersection.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_intersection_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_a_not_b.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_a_not_b_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_jaccard_similarity.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_sketch.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_sketch_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_union.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_union_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_intersection.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_intersection_impl.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_a_not_b.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_a_not_b_impl.hpp
-)

data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch.hpp

@@ -122,7 +122,7 @@ public:
 
 private:
   // for builder
-  update_array_of_doubles_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta,
+  update_array_of_doubles_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p, uint64_t theta,
       uint64_t seed, const array_of_doubles_update_policy<A>& policy, const A& allocator);
 };