datasketches 0.3.1 → 0.4.0

data/vendor/datasketches-cpp/sampling/test/CMakeLists.txt

@@ -15,30 +15,77 @@
 # specific language governing permissions and limitations
 # under the License.
 
-add_executable(sampling_test)
+# separate executables for var_opt and ebpps sampling
 
-target_link_libraries(sampling_test sampling common_test_lib)
+# VAR OPT SAMPLING
+add_executable(var_opt_sampling_test)
 
-set_target_properties(sampling_test PROPERTIES
-  CXX_STANDARD 11
+target_link_libraries(var_opt_sampling_test sampling common_test_lib)
+
+set_target_properties(var_opt_sampling_test PROPERTIES
   CXX_STANDARD_REQUIRED YES
 )
 
 file(TO_CMAKE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" SAMPLING_TEST_BINARY_PATH)
 string(APPEND SAMPLING_TEST_BINARY_PATH "/")
-target_compile_definitions(sampling_test
+target_compile_definitions(var_opt_sampling_test
   PRIVATE
     TEST_BINARY_INPUT_PATH="${SAMPLING_TEST_BINARY_PATH}"
 )
 
 add_test(
-  NAME sampling_test
-  COMMAND sampling_test
+  NAME var_opt_sampling_test
+  COMMAND var_opt_sampling_test
 )
 
-target_sources(sampling_test
+target_sources(var_opt_sampling_test
   PRIVATE
   var_opt_sketch_test.cpp
   var_opt_union_test.cpp
   var_opt_allocation_test.cpp
 )
+
+
+# EBPPS SAMPLING
+add_executable(ebpps_sampling_test)
+
+target_link_libraries(ebpps_sampling_test sampling common_test_lib)
+
+set_target_properties(ebpps_sampling_test PROPERTIES
+  CXX_STANDARD_REQUIRED YES
+)
+
+target_compile_definitions(ebpps_sampling_test
+  PRIVATE
+    TEST_BINARY_INPUT_PATH="${SAMPLING_TEST_BINARY_PATH}"
+)
+
+add_test(
+  NAME ebpps_sampling_test
+  COMMAND ebpps_sampling_test
+)
+
+target_sources(ebpps_sampling_test
+  PRIVATE
+  ebpps_sample_test.cpp
+  ebpps_sketch_test.cpp
+  ebpps_allocation_test.cpp
+)
+
+
+# Compatibility
+if (SERDE_COMPAT)
+target_sources(var_opt_sampling_test
+  PRIVATE
+   var_opt_sketch_deserialize_from_java_test.cpp
+   var_opt_union_deserialize_from_java_test.cpp
+)
+endif()
+
+if (GENERATE)
+target_sources(var_opt_sampling_test
+  PRIVATE
+   var_opt_sketch_serialize_for_java.cpp
+   var_opt_union_serialize_for_java.cpp
+)
+endif()

data/vendor/datasketches-cpp/sampling/test/ebpps_allocation_test.cpp

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <ebpps_sketch.hpp>
+#include <test_type.hpp>
+#include <test_allocator.hpp>
+
+#include <catch2/catch.hpp>
+
+#include <sstream>
+
+namespace datasketches {
+
+using ebpps_test_sketch = ebpps_sketch<test_type, test_allocator<test_type>>;
+using alloc = test_allocator<test_type>;
+
+TEST_CASE("ebpps allocation test", "[ebpps_sketch][test_type]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    ebpps_test_sketch sk1(10, 0);
+    for (int i = 0; i < 100; ++i)
+      sk1.update(i);
+    auto bytes1 = sk1.serialize(0, test_type_serde());
+    auto sk2 = ebpps_test_sketch::deserialize(bytes1.data(), bytes1.size(), test_type_serde(), 0);
+
+    std::stringstream ss;
+    sk1.serialize(ss, test_type_serde());
+    auto sk3 = ebpps_test_sketch::deserialize(ss, test_type_serde(), alloc(0));
+
+    sk1.merge(sk2); // same size into sk1
+    sk3.merge(sk1); // larger into sk3
+
+    auto bytes2 = sk1.serialize(0, test_type_serde());
+    auto sk4 = ebpps_test_sketch::deserialize(bytes2.data(), bytes2.size(), test_type_serde(), 0);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE( "ebpps merge", "[ebpps_sketch][test_type]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    uint32_t n = 20;
+    uint32_t k = 5;
+    ebpps_test_sketch sk1(k, 0);
+    ebpps_test_sketch sk2(k, 0);
+
+    // move udpates
+    for (int i = 0; i < (int) n; ++i) {
+      sk1.update(i);
+      sk2.update(-i);
+      sk1.update(n + i); // sk1 heavier than sk2
+    }
+    REQUIRE(sk1.get_n() == 2 * n);
+    REQUIRE(sk2.get_n() == n);
+
+    // move merge -- lighter into heavier
+    sk1.merge(std::move(sk2));
+    REQUIRE(sk1.get_n() == 3 * n);
+
+    // move constructor
+    ebpps_test_sketch sk3(std::move(sk1));
+    REQUIRE(sk3.get_n() == 3 * n);
+
+    // move assignment
+    ebpps_test_sketch sk4(k, 0);
+    sk4 = std::move(sk2);
+    REQUIRE(sk4.get_n() == n);
+
+    // move merge -- heavier into lighter
+    sk4.merge(sk3);
+    REQUIRE(sk4.get_n() == 4 * n);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+}

data/vendor/datasketches-cpp/sampling/test/ebpps_sample_test.cpp

@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <ebpps_sample.hpp>
+
+#include <catch2/catch.hpp>
+
+#include <vector>
+#include <string>
+#include <sstream>
+#include <fstream>
+#include <cmath>
+#include <random>
+#include <stdexcept>
+
+namespace datasketches {
+
+static constexpr double EPS = 1e-15;
+
+TEST_CASE("ebpps sample: basic initialization", "[ebpps_sketch]") {
+  ebpps_sample<int> sample = ebpps_sample<int>(0);
+  REQUIRE(sample.get_c() == 0.0);
+  REQUIRE(sample.get_num_retained_items() == 0);
+  REQUIRE(sample.get_sample().size() == 0);
+}
+
+TEST_CASE("ebpps sample: pre-initialized", "[ebpps_sketch]") {
+  double theta = 1.0;
+  ebpps_sample<int> sample = ebpps_sample<int>(-1, theta);
+  REQUIRE(sample.get_c() == theta);
+  REQUIRE(sample.get_num_retained_items() == 1);
+  REQUIRE(sample.get_sample().size() == 1);
+  REQUIRE(sample.has_partial_item() == false);
+  
+  theta = 1e-300;
+  sample = ebpps_sample<int>(-1, theta);
+  REQUIRE(sample.get_c() == theta);
+  REQUIRE(sample.get_num_retained_items() == 1);
+  REQUIRE(sample.get_sample().size() == 0); // assuming the random number is > 1e-300
+  REQUIRE(sample.has_partial_item());
+}
+
+TEST_CASE("ebpps sample: downsampling", "[ebpps_sketch]") {
+  ebpps_sample<char> sample = ebpps_sample<char>('a', 1.0);
+
+  sample.downsample(2.0); // no-op
+  REQUIRE(sample.get_c() == 1.0);
+  REQUIRE(sample.get_num_retained_items() == 1);
+  REQUIRE(sample.has_partial_item() == false);
+
+  // downsample and result in an empty sample
+  random_utils::override_seed(12);
+  std::vector<char> items = {'a', 'b'};
+  optional<char> opt; // empty
+  sample = ebpps_sample<char>(std::move(items), std::move(opt), 1.8);
+  sample.downsample(0.5);
+  REQUIRE(sample.get_c() == 0.9);
+  REQUIRE(sample.get_num_retained_items() == 0);
+  REQUIRE(sample.has_partial_item() == false);
+
+  // downsample and result in a sample with a partial item
+  items = {'a', 'b'};
+  opt.reset();
+  sample = ebpps_sample<char>(std::move(items), std::move(opt), 1.5);
+  sample.downsample(0.5);
+  REQUIRE(sample.get_c() == 0.75);
+  REQUIRE(sample.get_num_retained_items() == 1);
+  REQUIRE(sample.has_partial_item() == true);
+  for (char c : sample) {
+    REQUIRE((c == 'a' || c == 'b'));
+  }
+
+  // downsample to an exact integer c (7.5 * 0.8 = 6.0)
+  items = {'a', 'b', 'c', 'd', 'e', 'f', 'g'};
+  opt.emplace('h');
+  auto ref_items = items; // copy to check contents
+  ref_items.emplace_back('h'); // include partial item
+  sample = ebpps_sample<char>(std::move(items), std::move(opt), 7.5);
+  sample.downsample(0.8);
+  REQUIRE(sample.get_c() == 6.0);
+  REQUIRE(sample.get_num_retained_items() == 6);
+  REQUIRE(sample.has_partial_item() == false);
+  for (char c : sample) {
+    REQUIRE(std::find(ref_items.begin(), ref_items.end(), c) != ref_items.end());
+  }
+
+  // downsample to c > 1 with partial item
+  items = ref_items; // includes previous optional item
+  opt.emplace('i');
+  sample = ebpps_sample<char>(std::move(items), std::move(opt), 8.5);
+  REQUIRE(sample.get_partial_item() == 'i');
+  sample.downsample(0.8);
+  REQUIRE(sample.get_c() == Approx(6.8).margin(EPS));
+  REQUIRE(sample.get_num_retained_items() == 7);
+  REQUIRE(sample.has_partial_item() == true);
+  ref_items.emplace_back('i');
+  for (char c : sample) {
+    REQUIRE(std::find(ref_items.begin(), ref_items.end(), c) != ref_items.end());
+  }
+
+  random_utils::override_seed(random_utils::rd());
+}
+
+TEST_CASE("ebpps sample: merge unit samples", "[ebpps_sketch]") {
+  uint32_t k = 8;
+  ebpps_sample<int> sample = ebpps_sample<int>(k);
+  
+  for (uint32_t i = 1; i <= k; ++i) {
+    ebpps_sample<int> s = ebpps_sample<int>(i, 1.0);
+    sample.merge(s);
+    REQUIRE(sample.get_c() == static_cast<double>(i));
+    REQUIRE(sample.get_num_retained_items() == i);
+  }
+
+  sample.reset();
+  REQUIRE(sample.get_c() == 0);
+  REQUIRE(sample.get_num_retained_items() == 0);
+  REQUIRE(sample.has_partial_item() == false);
+}
+
+} // namespace datasketches

data/vendor/datasketches-cpp/sampling/test/ebpps_sketch_test.cpp

@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <ebpps_sketch.hpp>
+
+#include <catch2/catch.hpp>
+
+#include <vector>
+#include <string>
+#include <sstream>
+#include <fstream>
+#include <cmath>
+#include <random>
+#include <stdexcept>
+
+#ifdef TEST_BINARY_INPUT_PATH
+static std::string testBinaryInputPath = TEST_BINARY_INPUT_PATH;
+#else
+static std::string testBinaryInputPath = "test/";
+#endif
+
+namespace datasketches {
+
+static constexpr double EPS = 1e-13;
+
+static ebpps_sketch<int> create_unweighted_sketch(uint32_t k, uint64_t n) {
+  ebpps_sketch<int> sk(k);
+  for (uint64_t i = 0; i < n; ++i) {
+    sk.update(static_cast<int>(i), 1.0);
+  }
+  return sk;
+}
+
+template<typename T, typename A>
+static void check_if_equal(ebpps_sketch<T, A>& sk1, ebpps_sketch<T, A>& sk2) {
+  REQUIRE(sk1.get_k() == sk2.get_k());
+  REQUIRE(sk1.get_n() == sk2.get_n());
+  REQUIRE(sk1.get_c() == sk2.get_c());
+  REQUIRE(sk1.get_cumulative_weight() == sk2.get_cumulative_weight());
+
+  auto it1 = sk1.begin();
+  auto it2 = sk2.begin();
+  size_t count = 0;
+
+  while ((it1 != sk1.end()) && (it2 != sk2.end())) {
+    REQUIRE(*it1 == *it2);
+    ++it1;
+    ++it2;
+    ++count;
+  }
+
+  REQUIRE(((count == std::floor(sk1.get_c())) || (count == std::ceil(sk1.get_c()))));
+
+  // if c != floor(c) one sketch may not have reached the end,
+  // but that's not testable from the external API
+}
+
+TEST_CASE("ebpps sketch: invalid k", "[ebpps_sketch]") {
+  REQUIRE_THROWS_AS(ebpps_sketch<int>(0), std::invalid_argument);
+  REQUIRE_THROWS_AS(ebpps_sketch<int>(ebpps_constants::MAX_K + 1), std::invalid_argument);
+}
+
+TEST_CASE("ebpps sketch: invalid weights", "[ebpps_sketch]") {
+  uint32_t k = 100;
+  ebpps_sketch<int> sk = create_unweighted_sketch(k, 3);
+  REQUIRE(sk.get_n() == 3);
+  REQUIRE(sk.get_cumulative_weight() == 3.0);
+  sk.update(-1, 0.0); // no-op
+  REQUIRE(sk.get_n() == 3);
+  REQUIRE(sk.get_cumulative_weight() == 3.0);
+
+  REQUIRE_THROWS_AS(sk.update(-2, -1.0), std::invalid_argument);
+
+  ebpps_sketch<float> sk2(k);
+  REQUIRE_THROWS_AS(sk2.update(-2, std::numeric_limits<float>::infinity()), std::invalid_argument);
+  REQUIRE_THROWS_AS(sk2.update(-2, nanf("")), std::invalid_argument);
+}
+
+TEST_CASE("ebpps sketch: insert items", "[ebpps_sketch]") {
+  size_t n = 0;
+  uint32_t k = 5;
+  ebpps_sketch<int> sk = create_unweighted_sketch(k, n);
+  REQUIRE(sk.get_allocator() == std::allocator<int>());
+  REQUIRE(sk.get_k() == k);
+  REQUIRE(sk.get_n() == 0);
+  REQUIRE(sk.get_c() == 0.0);
+  REQUIRE(sk.get_cumulative_weight() == 0.0);
+  REQUIRE(sk.is_empty());
+
+  n = k;
+  sk = create_unweighted_sketch(k, n);
+  REQUIRE_FALSE(sk.is_empty());
+  REQUIRE(sk.get_n() == n);
+  REQUIRE(sk.get_cumulative_weight() == static_cast<double>(n));
+  for (int val : sk.get_result())
+    REQUIRE(val < static_cast<int>(n));
+
+  n = k * 10;
+  sk = create_unweighted_sketch(k, n);
+  REQUIRE_FALSE(sk.is_empty());
+  REQUIRE(sk.get_n() == n);
+  REQUIRE(sk.get_cumulative_weight() == static_cast<double>(n));
+  
+  auto result = sk.get_result();
+  REQUIRE(result.size() == sk.get_k()); // uniform weights so should be exactly k
+  for (int val : sk.get_result())
+    REQUIRE(val < static_cast<int>(n));
+}
+
+TEST_CASE("ebpps sketch: serialize/deserialize string", "[ebpps_sketch]") {
+  // since C <= k we don't have the usual sketch notion of exact vs estimation
+  // mode at any time. The only real serializaiton cases are empty and non-empty
+  // with and without a partial item
+  uint32_t k = 10;
+  ebpps_sketch<std::string> sk(k);
+
+  // empty  
+  auto bytes = sk.serialize();
+  REQUIRE(bytes.size() == sk.get_serialized_size_bytes());
+  REQUIRE_THROWS_AS(ebpps_sketch<std::string>::deserialize(bytes.data(), bytes.size()-1), std::out_of_range);
+  auto sk_bytes = ebpps_sketch<std::string>::deserialize(bytes.data(), bytes.size());
+  check_if_equal(sk, sk_bytes);
+
+  std::stringstream ss(std::ios::in | std::ios::out | std::ios::binary);
+  sk.serialize(ss);
+  auto sk_stream = ebpps_sketch<std::string>::deserialize(ss);
+  check_if_equal(sk, sk_stream);
+  check_if_equal(sk_bytes, sk_stream); // should be redundant
+
+  for (uint32_t i = 0; i < k; ++i)
+    sk.update(std::to_string(i));
+
+  // non-empty, no partial item
+  bytes = sk.serialize();
+  REQUIRE(bytes.size() == sk.get_serialized_size_bytes());
+  REQUIRE_THROWS_AS(ebpps_sketch<std::string>::deserialize(bytes.data(), bytes.size()-1), std::out_of_range);
+  sk_bytes = ebpps_sketch<std::string>::deserialize(bytes.data(), bytes.size());
+  check_if_equal(sk, sk_bytes);
+
+  ss.str("");
+  sk.serialize(ss);
+  sk_stream = ebpps_sketch<std::string>::deserialize(ss);
+  check_if_equal(sk, sk_stream);
+  check_if_equal(sk_bytes, sk_stream); // should be redundant
+
+  // non-empty with partial item
+  sk.update(std::to_string(2 * k), 2.5);
+  REQUIRE(sk.get_cumulative_weight() == Approx(k + 2.5).margin(EPS));
+  bytes = sk.serialize();
+  REQUIRE(bytes.size() == sk.get_serialized_size_bytes());
+  REQUIRE_THROWS_AS(ebpps_sketch<std::string>::deserialize(bytes.data(), bytes.size()-1), std::out_of_range);
+  sk_bytes = ebpps_sketch<std::string>::deserialize(bytes.data(), bytes.size());
+  check_if_equal(sk, sk_bytes);
+
+  ss.str("");
+  sk.serialize(ss);
+  sk_stream = ebpps_sketch<std::string>::deserialize(ss);
+  check_if_equal(sk, sk_stream);
+  check_if_equal(sk_bytes, sk_stream); // should be redundant
+}
+
+TEST_CASE("ebpps sketch: serialize/deserialize ints", "[ebpps_sketch]") {
+  uint32_t k = 10;
+  ebpps_sketch<uint32_t> sk(k);
+
+  for (uint32_t i = 0; i < k; ++i)
+    sk.update(i);
+  sk.update(2 * k, 3.5);
+  REQUIRE(sk.get_cumulative_weight() == Approx(k + 3.5).margin(EPS));
+
+  auto bytes = sk.serialize();
+  REQUIRE(bytes.size() == sk.get_serialized_size_bytes());
+  REQUIRE_THROWS_AS(ebpps_sketch<uint32_t>::deserialize(bytes.data(), bytes.size()-1), std::out_of_range);
+  auto sk_bytes = ebpps_sketch<uint32_t>::deserialize(bytes.data(), bytes.size());
+  check_if_equal(sk, sk_bytes);
+
+  std::stringstream ss(std::ios::in | std::ios::out | std::ios::binary);
+  sk.serialize(ss);
+  auto sk_stream = ebpps_sketch<uint32_t>::deserialize(ss);
+  check_if_equal(sk, sk_stream);
+  check_if_equal(sk_bytes, sk_stream); // should be redundant
+
+  sk.reset();
+  REQUIRE(sk.get_k() == k);
+  REQUIRE(sk.get_n() == 0);
+  REQUIRE(sk.get_c() == 0.0);
+  REQUIRE(sk.get_cumulative_weight() == 0.0);
+  REQUIRE(sk.is_empty());
+}
+
+TEST_CASE("ebpps sketch: merge large into small", "[ebpps_sketch]") {
+  uint32_t k = 100;
+
+  // lvalue merge  
+  ebpps_sketch<int> sk1(k / 2);
+  sk1.update(-1, k / 10.0); // one heavy item, but less than sk2 weight  
+  ebpps_sketch<int> sk2 = create_unweighted_sketch(k, k);
+
+  sk1.merge(sk2);
+  REQUIRE(sk1.get_k() == k / 2);
+  REQUIRE(sk1.get_n() == k + 1);
+  REQUIRE(sk1.get_c() < k);
+  REQUIRE(sk1.get_cumulative_weight() == Approx(1.1 * k).margin(EPS));
+  
+  // rvalue merge
+  sk1 = create_unweighted_sketch(k / 2, 0);
+  sk1.update(-1, k / 4.0);
+  sk1.update(-2, k / 8.0);
+  // sk2 should have been unchaged
+  REQUIRE(sk2.get_n() == k);
+  REQUIRE(sk2.get_c() == Approx(k).margin(EPS));
+
+  sk1.merge(std::move(sk2));
+  REQUIRE(sk1.get_k() == k / 2);
+  REQUIRE(sk1.get_n() == k + 2);
+  REQUIRE(sk1.get_c() < k);
+  // cumulative weight is now (1.5 + 0.2) k
+  REQUIRE(sk1.get_cumulative_weight() == Approx(1.375 * k).margin(EPS));
+}
+
+TEST_CASE("ebpps sketch: merge small into large", "[ebpps_sketch]") {
+  uint32_t k = 100;
+
+  // lvalue merge  
+  ebpps_sketch<int> sk1 = create_unweighted_sketch(k, k);
+  ebpps_sketch<int> sk2(k / 2);
+  sk2.update(-1, k / 10.0); // one heavy item, but less than sk1 weight
+
+  sk1.merge(sk2);
+  REQUIRE(sk1.get_k() == k / 2);
+  REQUIRE(sk1.get_n() == k + 1);
+  REQUIRE(sk1.get_c() < k);
+  REQUIRE(sk1.get_cumulative_weight() == Approx(1.1 * k).margin(EPS));
+  
+  // rvalue merge
+  sk1 = create_unweighted_sketch(k, 3 * k / 2);
+  // sk2 should have been unchaged
+  REQUIRE(sk2.get_n() == 1);
+  REQUIRE(sk2.get_c() == 1.0);
+  sk2.update(-2, k / 10.0);
+
+  sk1.merge(std::move(sk2));
+  REQUIRE(sk1.get_k() == k / 2);
+  REQUIRE(sk1.get_n() == (3 * k / 2) + 2);
+  REQUIRE(sk1.get_c() < k);
+  // cumulative weight is now (1.5 + 0.2) k
+  REQUIRE(sk1.get_cumulative_weight() == Approx(1.7 * k).margin(EPS));
+}
+
+}

data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_deserialize_from_java_test.cpp

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <var_opt_sketch.hpp>
+
+namespace datasketches {
+
+// assume the binary sketches for this test have been generated by datasketches-java code
+// in the subdirectory called "java" in the root directory of this project
+static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../java/";
+
+TEST_CASE("var opt sketch long", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "varopt_sketch_long_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = var_opt_sketch<long>::deserialize(is);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.get_num_samples() == (n > 10 ? 32 : n));
+  }
+}
+
+TEST_CASE("var opt sketch: deserialize exact from java", "[serde_compat]") {
+  const double EPS = 1e-13;
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(testBinaryInputPath + "varopt_sketch_string_exact_java.sk", std::ios::binary);
+  const auto sketch = var_opt_sketch<std::string>::deserialize(is);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_k() == 1024);
+  REQUIRE(sketch.get_n() == 200);
+  REQUIRE(sketch.get_num_samples() == 200);
+  const subset_summary ss = sketch.estimate_subset_sum([](std::string){ return true; });
+
+  double tgt_wt = 0.0;
+  for (int i = 1; i <= 200; ++i) { tgt_wt += 1000.0 / i; }
+  REQUIRE(ss.total_sketch_weight == Approx(tgt_wt).margin(EPS));
+}
+
+
+TEST_CASE("var opt sketch: deserialize sampling from java", "[serde_compat]") {
+  const double EPS = 1e-13;
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(testBinaryInputPath + "varopt_sketch_long_sampling_java.sk", std::ios::binary);
+  const auto sketch = var_opt_sketch<int64_t>::deserialize(is);
+  REQUIRE_FALSE(sketch.is_empty());
+  REQUIRE(sketch.get_k() == 1024);
+  REQUIRE(sketch.get_n() == 2003);
+  REQUIRE(sketch.get_num_samples() == sketch.get_k());
+  subset_summary ss = sketch.estimate_subset_sum([](int64_t){ return true; });
+  REQUIRE(ss.estimate == Approx(332000.0).margin(EPS));
+  REQUIRE(ss.total_sketch_weight == Approx(332000.0).margin(EPS));
+
+  ss = sketch.estimate_subset_sum([](int64_t x){ return x < 0; });
+  REQUIRE(ss.estimate == 330000.0); // heavy item, weight is exact
+
+  ss = sketch.estimate_subset_sum([](int64_t x){ return x >= 0; });
+  REQUIRE(ss.estimate == Approx(2000.0).margin(EPS));
+}
+
+} /* namespace datasketches */