datasketches 0.3.1 → 0.4.0

data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_serialize_for_java.cpp

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <var_opt_sketch.hpp>
+
+namespace datasketches {
+
+TEST_CASE("varopt sketch long generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    var_opt_sketch<long> sketch(32);
+    for (unsigned i = 1; i <= n; ++i) sketch.update(i);
+    std::ofstream os("varopt_sketch_long_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.serialize(os);
+  }
+}
+
+TEST_CASE("varopt sketch string exact", "[serialize_for_java]") {
+  var_opt_sketch<std::string> sketch(1024);
+  for (unsigned i = 1; i <= 200; ++i) sketch.update(std::to_string(i), 1000.0 / i);
+  std::ofstream os("varopt_sketch_string_exact_cpp.sk", std::ios::binary);
+  sketch.serialize(os);
+}
+
+TEST_CASE("varopt sketch long sampling", "[serialize_for_java]") {
+  var_opt_sketch<long> sketch(1024);
+  for (unsigned i = 0; i < 2000; ++i) sketch.update(i);
+  // negative heavy items to allow a simple predicate to filter
+  sketch.update(-1L, 100000.0);
+  sketch.update(-2L, 110000.0);
+  sketch.update(-3L, 120000.0);
+  std::ofstream os("varopt_sketch_long_sampling_cpp.sk", std::ios::binary);
+  sketch.serialize(os);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp

@@ -52,17 +52,15 @@ static void check_if_equal(var_opt_sketch<T, A>& sk1, var_opt_sketch<T, A>& sk2)
   REQUIRE(sk1.get_k() == sk2.get_k());
   REQUIRE(sk1.get_n() == sk2.get_n());
   REQUIRE(sk1.get_num_samples() == sk2.get_num_samples());
-      
+
   auto it1 = sk1.begin();
   auto it2 = sk2.begin();
-  size_t i = 0;
 
   while ((it1 != sk1.end()) && (it2 != sk2.end())) {
-    const std::pair<const T&, const double> p1 = *it1;
-    const std::pair<const T&, const double> p2 = *it2;
+    auto p1 = *it1;
+    auto p2 = *it2;
     REQUIRE(p1.first == p2.first);   // data values
     REQUIRE(p1.second == p2.second); // weights
-    ++i;
     ++it1;
     ++it2;
   }
@@ -182,7 +180,7 @@ TEST_CASE("varopt sketch: invalid weight", "[var_opt_sketch]") {
   var_opt_sketch<std::string> sk(100, resize_factor::X2);
   REQUIRE_THROWS_AS(sk.update("invalid_weight", -1.0), std::invalid_argument);
 
-  // should not throw but sketch shoulds till be empty
+  // should not throw but sketch should still be empty
   sk.update("zero weight", 0.0);
   REQUIRE(sk.is_empty());
 }
@@ -213,7 +211,7 @@ TEST_CASE("varopt sketch: cumulative weight", "[var_opt_sketch]") {
 
   double input_sum = 0.0;
   for (size_t i = 0; i < n; ++i) {
-    // generate weights aboev and below 1.0 using w ~ exp(5*N(0,1))
+    // generate weights above and below 1.0 using w ~ exp(5*N(0,1))
     // which covers about 10 orders of magnitude
     double w = std::exp(5 * N(rand));
     input_sum += w;
@@ -221,12 +219,12 @@ TEST_CASE("varopt sketch: cumulative weight", "[var_opt_sketch]") {
   }
 
   double output_sum = 0.0;
-  for (auto it : sk) { // std::pair<int, weight>
-    output_sum += it.second;
+  for (auto pair : sk) { // std::pair<int, weight>
+    output_sum += pair.second;
   }
     
   double weight_ratio = output_sum / input_sum;
-  REQUIRE(std::abs(weight_ratio - 1.0) == Approx(0).margin(EPS));
+  REQUIRE(weight_ratio == Approx(1.0).margin(EPS));
 }
 
 TEST_CASE("varopt sketch: under-full sketch serialization", "[var_opt_sketch]") {
@@ -275,26 +273,38 @@ TEST_CASE("varopt sketch: full sketch serialization", "[var_opt_sketch]") {
   sk.update(100, 100.0);
   sk.update(101, 101.0);
 
+  subset_summary summary = sk.estimate_subset_sum([](int){ return true; });
+  double total_weight = summary.total_sketch_weight;
+  double cum_weight = 0.0;
+  for (auto pair : sk) {
+    cum_weight += pair.second;
+  }
+  double weight_ratio = cum_weight / total_weight;
+  REQUIRE(weight_ratio == Approx(1.0).margin(EPS));
+
   // first 2 entries should be heavy and in heap order (smallest at root)
   auto it = sk.begin();
-  const std::pair<const int, const double> p1 = *it;
+  auto p1 = *it;
   ++it;
-  const std::pair<const int, const double> p2 = *it;
+  auto p2 = *it;
   REQUIRE(p1.second == Approx(100.0).margin(EPS));
   REQUIRE(p2.second == Approx(101.0).margin(EPS));
   REQUIRE(p1.first == 100);
   REQUIRE(p2.first == 101);
+  // using operator ->
+  REQUIRE(it->first == p2.first);
+  REQUIRE(it->second == p2.second);
 
   // check for 4 preamble longs
   auto bytes = sk.serialize();
   REQUIRE((bytes.data()[0] & 0x3f) == 4);; // PREAMBLE_LONGS_WARMUP
 
-  var_opt_sketch<int> sk_from_bytes = var_opt_sketch<int>::deserialize(bytes.data(), bytes.size());
+  auto sk_from_bytes = var_opt_sketch<int>::deserialize(bytes.data(), bytes.size());
   check_if_equal(sk, sk_from_bytes);
 
   std::stringstream ss(std::ios::in | std::ios::out | std::ios::binary);
   sk.serialize(ss);
-  var_opt_sketch<int> sk_from_stream = var_opt_sketch<int>::deserialize(ss);
+  auto sk_from_stream = var_opt_sketch<int>::deserialize(ss);
   check_if_equal(sk, sk_from_stream);
 
   // ensure we unroll properly
@@ -340,6 +350,15 @@ TEST_CASE("varopt sketch: pseudo-light update", "[var_opt_sketch]") {
   auto it = sk.begin();
   double wt = (*it).second;
   REQUIRE(wt == Approx((k + 2.0) / k).margin(EPS));
+
+  subset_summary summary = sk.estimate_subset_sum([](int){ return true; });
+  double total_weight = summary.total_sketch_weight;
+  double cum_weight = 0.0;
+  for (auto pair : sk) {
+    cum_weight += pair.second;
+  }
+  double weight_ratio = cum_weight / total_weight;
+  REQUIRE(weight_ratio == Approx(1.0).margin(EPS));
 }
 
 TEST_CASE("varopt sketch: pseudo-heavy update", "[var_opt_sketch]") {
@@ -470,41 +489,4 @@ TEST_CASE("varopt sketch: estimate subset sum", "[var_opt_sketch]") {
   REQUIRE(summary.estimate < total_weight); // exact mode, so know it must be strictly less
 }
 
-TEST_CASE("varopt sketch: deserialize exact from java", "[var_opt_sketch]") {
-  std::ifstream is;
-  is.exceptions(std::ios::failbit | std::ios::badbit);
-  is.open(testBinaryInputPath + "varopt_sketch_string_exact.sk", std::ios::binary);
-  var_opt_sketch<std::string> sketch = var_opt_sketch<std::string>::deserialize(is);
-  REQUIRE_FALSE(sketch.is_empty());
-  REQUIRE(sketch.get_k() == 1024);
-  REQUIRE(sketch.get_n() == 200);
-  REQUIRE(sketch.get_num_samples() == 200);
-  subset_summary ss = sketch.estimate_subset_sum([](std::string){ return true; });
-
-  double tgt_wt = 0.0;
-  for (int i = 1; i <= 200; ++i) { tgt_wt += 1000.0 / i; }
-  REQUIRE(ss.total_sketch_weight == Approx(tgt_wt).margin(EPS));
-}
-
-
-TEST_CASE("varopt sketch: deserialize sampling from java", "[var_opt_sketch]") {
-  std::ifstream is;
-  is.exceptions(std::ios::failbit | std::ios::badbit);
-  is.open(testBinaryInputPath + "varopt_sketch_long_sampling.sk", std::ios::binary);
-  var_opt_sketch<int64_t> sketch = var_opt_sketch<int64_t>::deserialize(is);
-  REQUIRE_FALSE(sketch.is_empty());
-  REQUIRE(sketch.get_k() == 1024);
-  REQUIRE(sketch.get_n() == 2003);
-  REQUIRE(sketch.get_num_samples() == sketch.get_k());
-  subset_summary ss = sketch.estimate_subset_sum([](int64_t){ return true; });
-  REQUIRE(ss.estimate == Approx(332000.0).margin(EPS));
-  REQUIRE(ss.total_sketch_weight == Approx(332000.0).margin(EPS));
-
-  ss = sketch.estimate_subset_sum([](int64_t x){ return x < 0; });
-  REQUIRE(ss.estimate == 330000.0); // heavy item, weight is exact
-
-  ss = sketch.estimate_subset_sum([](int64_t x){ return x >= 0; });
-  REQUIRE(ss.estimate == Approx(2000.0).margin(EPS));
-}
-
 }

data/vendor/datasketches-cpp/sampling/test/var_opt_union_deserialize_from_java_test.cpp

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <var_opt_union.hpp>
+
+namespace datasketches {
+
+// assume the binary sketches for this test have been generated by datasketches-java code
+// in the subdirectory called "java" in the root directory of this project
+static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../java/";
+
+TEST_CASE("var opt union double", "[serde_compat]") {
+  const double EPS = 1e-13;
+
+  std::ifstream is;
+  is.exceptions(std::ios::failbit | std::ios::badbit);
+  is.open(testBinaryInputPath + "varopt_union_double_sampling_java.sk", std::ios::binary);
+  auto u = var_opt_union<double>::deserialize(is);
+
+  // must reduce k in the process
+  const auto result = u.get_result();
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.get_n() == 97);
+
+  const double expected_wt = 96.0; // light items -- ignoring the heavy one
+  const subset_summary ss = result.estimate_subset_sum([](double x){return x >= 0;});
+  REQUIRE(ss.estimate == Approx(expected_wt).margin(EPS));
+  REQUIRE(ss.total_sketch_weight == Approx(expected_wt + 1024.0).margin(EPS));
+  REQUIRE(result.get_k() < 128);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/sampling/test/var_opt_union_serialize_for_java.cpp

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <var_opt_union.hpp>
+
+namespace datasketches {
+
+TEST_CASE("var opt union double sampling", "[serialize_for_java]") {
+  const unsigned k_small = 16;
+  const unsigned k_max = 128;
+  const unsigned n1 = 32;
+  const unsigned n2 = 64;
+
+  // small k sketch, but sampling
+  var_opt_sketch<double> sketch1(k_small);
+  for (unsigned i = 0; i < n1; ++i) sketch1.update(i);
+  // negative heavy item to allow a simple predicate to filter
+  sketch1.update(-1, n1 * n1);
+
+  // another one, but different n to get a different per-item weight
+  var_opt_sketch<double> sketch2(k_small);
+  for (unsigned i = 0; i < n2; ++i) sketch2.update(i);
+
+  var_opt_union<double> u(k_max);
+  u.update(sketch1);
+  u.update(sketch2);
+
+  // must reduce k in the process
+  auto result = u.get_result();
+  REQUIRE(result.get_k() < k_max);
+  REQUIRE(result.get_k() >= k_small);
+  REQUIRE(result.get_n() == 97);
+
+  std::ofstream os("varopt_union_double_sampling_cpp.sk", std::ios::binary);
+  u.serialize(os);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp

@@ -57,7 +57,6 @@ static void check_if_equal(var_opt_sketch<T, A>& sk1, var_opt_sketch<T, A>& sk2,
       
   auto it1 = sk1.begin();
   auto it2 = sk2.begin();
-  size_t i = 0;
 
   while ((it1 != sk1.end()) && (it2 != sk2.end())) {
     const std::pair<const T&, const double> p1 = *it1;
@@ -66,7 +65,6 @@ static void check_if_equal(var_opt_sketch<T, A>& sk1, var_opt_sketch<T, A>& sk2,
       REQUIRE(p1.first == p2.first); // data values
     }
     REQUIRE(p1.second == p2.second); // weight values
-    ++i;
     ++it1;
     ++it2;
   }
@@ -307,22 +305,4 @@ TEST_CASE("varopt union: serialize sampling", "[var_opt_union]") {
   compare_serialization_deserialization(u);
 }
 
-TEST_CASE("varopt union: deserialize from java", "[var_opt_union]") {
-  std::ifstream is;
-  is.exceptions(std::ios::failbit | std::ios::badbit);
-  is.open(testBinaryInputPath + "varopt_union_double_sampling.sk", std::ios::binary);
-  var_opt_union<double> u = var_opt_union<double>::deserialize(is);
-    
-  // must reduce k in the process, like in small_sampling_sketch()
-  var_opt_sketch<double> result = u.get_result();
-  REQUIRE_FALSE(result.is_empty());
-  REQUIRE(result.get_n() == 97);
-  
-  double expected_wt = 96.0;// light items -- ignoring the heavy one
-  subset_summary ss = result.estimate_subset_sum([](double x){return x >= 0;});
-  REQUIRE(ss.estimate == Approx(expected_wt).margin(EPS));
-  REQUIRE(ss.total_sketch_weight == Approx(expected_wt + 1024.0).margin(EPS));
-  REQUIRE(result.get_k() < 128);
-}
-
 }

data/vendor/datasketches-cpp/theta/CMakeLists.txt

@@ -62,4 +62,5 @@ install(FILES
 			include/bounds_on_ratios_in_theta_sketched_sets.hpp
 			include/compact_theta_sketch_parser.hpp
 			include/compact_theta_sketch_parser_impl.hpp
+			include/bit_packing.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")