datasketches 0.3.1 → 0.3.2

data/vendor/datasketches-cpp/python/src/tuple_wrapper.cpp

@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <memory>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+#include "theta_sketch.hpp"
+#include "tuple_sketch.hpp"
+#include "tuple_union.hpp"
+#include "tuple_intersection.hpp"
+#include "tuple_a_not_b.hpp"
+#include "theta_jaccard_similarity_base.hpp"
+#include "common_defs.hpp"
+
+#include "py_serde.hpp"
+#include "tuple_policy.hpp"
+
+namespace py = pybind11;
+
+void init_tuple(py::module &m) {
+  using namespace datasketches;
+
+  // generic tuple_policy:
+  // * update sketch policy uses create_summary and update_summary
+  // * set operation policies all use __call__
+  py::class_<tuple_policy, TuplePolicy, std::shared_ptr<tuple_policy>>(m, "TuplePolicy")
+    .def(py::init())
+    .def("create_summary", &tuple_policy::create_summary)
+    .def("update_summary", &tuple_policy::update_summary, py::arg("summary"), py::arg("update"))
+    .def("__call__", &tuple_policy::operator(), py::arg("summary"), py::arg("update"))
+  ;
+
+  // potentially useful for debugging but not needed as a permanent
+  // object type in the library
+  /*
+  py::class_<tuple_policy_holder>(m, "TuplePolicyHolder")
+    .def(py::init<std::shared_ptr<tuple_policy>>(), py::arg("policy"))
+    .def("create", &tuple_policy_holder::create, "Creates a new Summary object")
+    .def("update", &tuple_policy_holder::update, py::arg("summary"), py::arg("update"),
+         "Updates the provided summary using the data in update")
+  ;
+  */
+
+  using py_tuple_sketch = tuple_sketch<py::object>;
+  using py_update_tuple = update_tuple_sketch<py::object, py::object, tuple_policy_holder>;
+  using py_compact_tuple = compact_tuple_sketch<py::object>;
+  using py_tuple_union = tuple_union<py::object, tuple_policy_holder>;
+  using py_tuple_intersection = tuple_intersection<py::object, tuple_policy_holder>;
+  using py_tuple_a_not_b = tuple_a_not_b<py::object>;
+  using py_tuple_jaccard_similarity = jaccard_similarity_base<tuple_union<py::object, dummy_jaccard_policy>, tuple_intersection<py::object, dummy_jaccard_policy>, pair_extract_key<uint64_t, py::object>>;
+
+  py::class_<py_tuple_sketch>(m, "_tuple_sketch")
+    .def("__str__", &py_tuple_sketch::to_string, py::arg("print_items")=false,
+         "Produces a string summary of the sketch")
+    .def("to_string", &py_tuple_sketch::to_string, py::arg("print_items")=false,
+         "Produces a string summary of the sketch")
+    .def("is_empty", &py_tuple_sketch::is_empty,
+         "Returns True if the sketch is empty, otherwise False")
+    .def("get_estimate", &py_tuple_sketch::get_estimate,
+         "Estimate of the distinct count of the input stream")
+    .def("get_upper_bound", static_cast<double (py_tuple_sketch::*)(uint8_t) const>(&py_tuple_sketch::get_upper_bound), py::arg("num_std_devs"),
+         "Returns an approximate upper bound on the estimate at standard deviations in {1, 2, 3}")
+    .def("get_lower_bound", static_cast<double (py_tuple_sketch::*)(uint8_t) const>(&py_tuple_sketch::get_lower_bound), py::arg("num_std_devs"),
+         "Returns an approximate lower bound on the estimate at standard deviations in {1, 2, 3}")
+    .def("is_estimation_mode", &py_tuple_sketch::is_estimation_mode,
+         "Returns True if sketch is in estimation mode, otherwise False")
+    .def("get_theta", &py_tuple_sketch::get_theta,
+         "Returns theta (effective sampling rate) as a fraction from 0 to 1")
+    .def("get_theta64", &py_tuple_sketch::get_theta64,
+         "Returns theta as 64-bit value")
+    .def("get_num_retained", &py_tuple_sketch::get_num_retained,
+         "Returns the number of items currently in the sketch")
+    .def("get_seed_hash", [](const py_tuple_sketch& sk) { return sk.get_seed_hash(); }, // why does regular call not work??
+         "Returns a hash of the seed used in the sketch")
+    .def("is_ordered", &py_tuple_sketch::is_ordered,
+         "Returns True if the sketch entries are sorted, otherwise False")
+    .def("__iter__", [](const py_tuple_sketch& s) { return py::make_iterator(s.begin(), s.end()); })
+    .def_property_readonly_static("DEFAULT_SEED", [](py::object /* self */) { return DEFAULT_SEED; });
+  ;
+
+  py::class_<py_compact_tuple, py_tuple_sketch>(m, "_compact_tuple_sketch")
+    .def(py::init<const py_compact_tuple&>(), py::arg("other"))
+    .def(py::init<const py_tuple_sketch&, bool>(), py::arg("other"), py::arg("ordered")=true)
+    .def(py::init<const theta_sketch&, py::object&>(), py::arg("other"), py::arg("summary"),
+         "Creates a compact tuple sketch from a theta sketch using a fixed summary value.")
+    .def(
+        "serialize",
+        [](const py_compact_tuple& sk, py_object_serde& serde) {
+          auto bytes = sk.serialize(0, serde);
+          return py::bytes(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+        }, py::arg("serde"),
+        "Serializes the sketch into a bytes object"
+    )
+    .def_static(
+        "deserialize",
+        [](const std::string& bytes, py_object_serde& serde, uint64_t seed) {
+          return py_compact_tuple::deserialize(bytes.data(), bytes.size(), seed, serde);
+        },
+        py::arg("bytes"), py::arg("serde"), py::arg("seed")=DEFAULT_SEED,
+        "Reads a bytes object and returns the corresponding compact_tuple_sketch"
+    );
+
+  py::class_<py_update_tuple, py_tuple_sketch>(m, "_update_tuple_sketch")
+    .def(
+        py::init([](std::shared_ptr<tuple_policy> policy, uint8_t lg_k, double p, uint64_t seed) {
+          tuple_policy_holder holder(policy);
+          return py_update_tuple::builder(holder).set_lg_k(lg_k).set_p(p).set_seed(seed).build();
+        }),
+        py::arg("policy"), py::arg("lg_k")=theta_constants::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED
+    )
+    .def(py::init<const py_update_tuple&>())
+    .def("update", static_cast<void (py_update_tuple::*)(int64_t, py::object&)>(&py_update_tuple::update),
+         py::arg("datum"), py::arg("value"),
+         "Updates the sketch with the given integral item and summary value")
+    .def("update", static_cast<void (py_update_tuple::*)(double, py::object&)>(&py_update_tuple::update),
+         py::arg("datum"), py::arg("value"),
+         "Updates the sketch with the given floating point item and summary value")
+    .def("update", static_cast<void (py_update_tuple::*)(const std::string&, py::object&)>(&py_update_tuple::update),
+         py::arg("datum"), py::arg("value"),
+         "Updates the sketch with the given string item and summary value")
+    .def("compact", &py_update_tuple::compact, py::arg("ordered")=true,
+         "Returns a compacted form of the sketch, optionally sorting it")
+    .def("reset", &py_update_tuple::reset, "Resets the sketch to the initial empty state")
+  ;
+
+  py::class_<py_tuple_union>(m, "_tuple_union")
+    .def(
+        py::init([](std::shared_ptr<tuple_policy> policy, uint8_t lg_k, double p, uint64_t seed) {
+          tuple_policy_holder holder(policy);
+          return py_tuple_union::builder(holder).set_lg_k(lg_k).set_p(p).set_seed(seed).build();
+        }),
+        py::arg("policy"), py::arg("lg_k")=theta_constants::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED
+    )
+    .def("update", &py_tuple_union::update<const py_tuple_sketch&>, py::arg("sketch"),
+         "Updates the union with the given sketch")
+    .def("get_result", &py_tuple_union::get_result, py::arg("ordered")=true,
+         "Returns the sketch corresponding to the union result")
+    .def("reset", &py_tuple_union::reset,
+         "Resets the sketch to the initial empty")
+  ;
+
+  py::class_<py_tuple_intersection>(m, "_tuple_intersection")
+    .def(
+        py::init([](std::shared_ptr<tuple_policy> policy, uint64_t seed) {
+          tuple_policy_holder holder(policy);
+          return py_tuple_intersection(seed, holder);
+        }),
+        py::arg("policy"), py::arg("seed")=DEFAULT_SEED)
+    .def("update", &py_tuple_intersection::update<const py_tuple_sketch&>, py::arg("sketch"),
+         "Intersects the provided sketch with the current intersection state")
+    .def("get_result", &py_tuple_intersection::get_result, py::arg("ordered")=true,
+         "Returns the sketch corresponding to the intersection result")
+    .def("has_result", &py_tuple_intersection::has_result,
+         "Returns True if the intersection has a valid result, otherwise False")
+  ;
+
+  py::class_<py_tuple_a_not_b>(m, "_tuple_a_not_b")
+    .def(py::init<uint64_t>(), py::arg("seed")=DEFAULT_SEED)
+    .def(
+        "compute",
+        &py_tuple_a_not_b::compute<const py_tuple_sketch&, const py_tuple_sketch&>,
+        py::arg("a"), py::arg("b"), py::arg("ordered")=true,
+        "Returns a sketch with the result of applying the A-not-B operation on the given inputs"
+    )
+  ;
+
+  py::class_<py_tuple_jaccard_similarity>(m, "_tuple_jaccard_similarity")
+    .def_static(
+        "jaccard",
+        [](const py_tuple_sketch& sketch_a, const py_tuple_sketch& sketch_b, uint64_t seed) {
+          return py_tuple_jaccard_similarity::jaccard(sketch_a, sketch_b, seed);
+        },
+        py::arg("sketch_a"), py::arg("sketch_b"), py::arg("seed")=DEFAULT_SEED,
+        "Returns a list with {lower_bound, estimate, upper_bound} of the Jaccard similarity between sketches"
+    )
+    .def_static(
+        "exactly_equal",
+        &py_tuple_jaccard_similarity::exactly_equal<const py_tuple_sketch&, const py_tuple_sketch&>,
+        py::arg("sketch_a"), py::arg("sketch_b"), py::arg("seed")=DEFAULT_SEED,
+        "Returns True if sketch_a and sketch_b are equivalent, otherwise False"
+    )
+    .def_static(
+        "similarity_test",
+        &py_tuple_jaccard_similarity::similarity_test<const py_tuple_sketch&, const py_tuple_sketch&>,
+        py::arg("actual"), py::arg("expected"), py::arg("threshold"), py::arg("seed")=DEFAULT_SEED,
+        "Tests similarity of an actual sketch against an expected sketch. Computes the lower bound of the Jaccard "
+        "index J_{LB} of the actual and expected sketches. If J_{LB} >= threshold, then the sketches are considered "
+        "to be similar with a confidence of 97.7% and returns True, otherwise False.")
+    .def_static(
+        "dissimilarity_test",
+        &py_tuple_jaccard_similarity::dissimilarity_test<const py_tuple_sketch&, const py_tuple_sketch&>,
+        py::arg("actual"), py::arg("expected"), py::arg("threshold"), py::arg("seed")=DEFAULT_SEED,
+        "Tests dissimilarity of an actual sketch against an expected sketch. Computes the upper bound of the Jaccard "
+        "index J_{UB} of the actual and expected sketches. If J_{UB} <= threshold, then the sketches are considered "
+        "to be dissimilar with a confidence of 97.7% and returns True, otherwise False."
+    )
+  ;
+}

data/vendor/datasketches-cpp/python/src/vo_wrapper.cpp

@@ -140,7 +140,7 @@ void bind_vo_sketch(py::module &m, const char* name) {
     .def("serialize", &dspy::vo_sketch_serialize<T>, py::arg("serde"), "Serialize the var opt sketch using the provided serde")
     .def_static("deserialize", &dspy::vo_sketch_deserialize<T>, py::arg("bytes"), py::arg("serde"),
         "Constructs a var opt sketch from the given bytes using the provided serde")
-    ;
+    .def("__iter__", [](const var_opt_sketch<T>& sk) { return py::make_iterator(sk.begin(), sk.end()); });
 }
 
 template<typename T>

data/vendor/datasketches-cpp/python/tests/count_min_test.py

@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+  
+import unittest
+from datasketches import count_min_sketch
+
+class CountMinTest(unittest.TestCase):
+  def test_count_min_example(self):
+    # we'll define target confidence and relative error and use the built-in
+    # methods to determine how many hashes and buckets to use
+    confidence = 0.95
+    num_hashes = count_min_sketch.suggest_num_hashes(confidence)
+    relative_error = 0.01
+    num_buckets = count_min_sketch.suggest_num_buckets(relative_error)
+
+    # now we can create a few empty sketches
+    cm = count_min_sketch(num_hashes, num_buckets)
+    cm2 = count_min_sketch(num_hashes, num_buckets)
+    self.assertTrue(cm.is_empty())
+
+    # we'll use a moderate number of distinct items with
+    # increasing weights, with each item's weight being
+    # equal to its value
+    n = 1000
+    total_wt = 0
+    for i in range(1, n+1):
+      cm.update(i, i)
+      total_wt += i
+    self.assertFalse(cm.is_empty())
+    self.assertEqual(cm.get_total_weight(), total_wt)
+
+    # querying the items, each of them should
+    # have a non-zero count.  the estimate should
+    # be at least i with appropriately behaved bounds.
+    for i in range(1, n+1):
+      val = cm.get_estimate(i)
+      self.assertGreaterEqual(val, i)
+      self.assertGreaterEqual(val, cm.get_lower_bound(i))
+      self.assertGreater(cm.get_upper_bound(i), val)
+
+    # values not in the sketch should have lower estimates, but
+    # are not guaranteed to be zero and will succeed
+    self.assertIsNotNone(cm.get_estimate("not in set"))
+
+    # we can create another sketch with partial overlap
+    # and merge them
+    for i in range(int(n / 2), int(3 * n / 2)):
+      cm2.update(i, i)
+    cm.merge(cm2)
+
+    # and the estimated weight for the overlapped meerged values
+    # (n/2 to n) should now be at least 2x the value
+    self.assertGreaterEqual(cm.get_estimate(n), 2 * n)
+
+    # finally, serialize and reconstruct
+    cm_bytes = cm.serialize()
+    self.assertEqual(cm.get_serialized_size_bytes(), len(cm_bytes))
+    new_cm = count_min_sketch.deserialize(cm_bytes)
+
+    # and now interrogate the sketch
+    self.assertFalse(new_cm.is_empty())
+    self.assertEqual(new_cm.get_num_hashes(), cm.get_num_hashes())
+    self.assertEqual(new_cm.get_num_buckets(), cm.get_num_buckets())
+    self.assertEqual(new_cm.get_total_weight(), cm.get_total_weight())
+    
+    # we can also iterate through values in and out of the sketch to ensure
+    # the estimates match
+    for i in range(0, 2 * n):
+      self.assertEqual(cm.get_estimate(i), new_cm.get_estimate(i))
+
+if __name__ == '__main__':
+    unittest.main()

data/vendor/datasketches-cpp/python/tests/cpc_test.py

@@ -14,26 +14,26 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-  
+
 import unittest
 from datasketches import cpc_sketch, cpc_union
 
 class CpcTest(unittest.TestCase):
   def test_cpc_example(self):
-    k = 12      # 2^k = 4096 rows in the table
-    n = 1 << 18 # ~256k unique values
+    lgk = 12    # 2^k = 4096 rows in the table
+    n = 1 << 18 # ~256k distinct values
 
     # create a couple sketches and inject some values
     # we'll have 1/4 of the values overlap
-    cpc  = cpc_sketch(k)
-    cpc2 = cpc_sketch(k)
+    cpc  = cpc_sketch(lgk)
+    cpc2 = cpc_sketch(lgk)
     offset = int(3 * n / 4) # it's a float w/o cast
     # because we hash on the bits, not an abstract numeric value,
     # cpc.update(1) and cpc.update(1.0) give different results.
     for i in range(0, n):
         cpc.update(i)
         cpc2.update(i + offset)
-        
+
     # although we provide get_composite_estimate() and get_estimate(),
     # the latter will always give the best available estimate.  we
     # recommend using get_estimate().
@@ -42,9 +42,9 @@ class CpcTest(unittest.TestCase):
     self.assertLessEqual(cpc.get_lower_bound(1), cpc.get_estimate())
     self.assertGreaterEqual(cpc.get_upper_bound(1), cpc.get_estimate())
 
-    # unioning uses a separate class, but we need to get_result()
-    # tp query the unioned sketches
-    union = cpc_union(k)
+    # union is a separate class, so we need to get_result()
+    # to query the unioned sketches
+    union = cpc_union(lgk)
     union.update(cpc)
     union.update(cpc2)
     result = union.get_result()
@@ -54,7 +54,7 @@ class CpcTest(unittest.TestCase):
     # answer is within one standard deviation of the estimate
     self.assertLessEqual(result.get_lower_bound(1), 7 * n / 4)
     self.assertGreaterEqual(result.get_upper_bound(1), 7 * n / 4)
-     
+
     # serialize for storage and reconstruct
     sk_bytes = result.serialize()
     new_cpc = cpc_sketch.deserialize(sk_bytes)

data/vendor/datasketches-cpp/python/tests/density_test.py

@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from datasketches import density_sketch, KernelFunction
+import numpy as np
+
+class UnitSphereKernel(KernelFunction):
+  def __call__(self, a: np.array, b: np.array) -> float:
+    if np.linalg.norm(a - b) < 1.0:
+      return 1.0
+    else:
+      return 0.0
+
+class densityTest(unittest.TestCase):
+  def test_density_sketch(self):
+    k = 10
+    dim = 3
+    n = 1000
+
+    sketch = density_sketch(k, dim)
+
+    self.assertEqual(sketch.get_k(), k)
+    self.assertEqual(sketch.get_dim(), dim)
+    self.assertTrue(sketch.is_empty())
+    self.assertFalse(sketch.is_estimation_mode())
+    self.assertEqual(sketch.get_n(), 0)
+    self.assertEqual(sketch.get_num_retained(), 0)
+
+    for i in range(n):
+      sketch.update([i, i, i])
+
+    self.assertFalse(sketch.is_empty())
+    self.assertTrue(sketch.is_estimation_mode())
+    self.assertEqual(sketch.get_n(), n)
+    self.assertGreater(sketch.get_num_retained(), k)
+    self.assertLess(sketch.get_num_retained(), n)
+    self.assertGreater(sketch.get_estimate([n - 1, n - 1, n - 1]), 0)
+
+    for tuple in sketch:
+      vector = tuple[0]
+      weight = tuple[1]
+      self.assertEqual(len(vector), dim)
+      self.assertGreaterEqual(weight, 1)
+
+    sk_bytes = sketch.serialize()
+    sketch2 = density_sketch.deserialize(sk_bytes)
+    self.assertEqual(sketch.get_estimate([1.5, 2.5, 3.5]), sketch2.get_estimate([1.5, 2.5, 3.5]))
+
+  def test_density_merge(self):
+    sketch1 = density_sketch(10, 2)
+    sketch1.update([0, 0])
+    sketch2 = density_sketch(10, 2)
+    sketch2.update([0, 1])
+    sketch1.merge(sketch2)
+    self.assertEqual(sketch1.get_n(), 2)
+    self.assertEqual(sketch1.get_num_retained(), 2)
+
+  def test_custom_kernel(self):
+    gaussianSketch = density_sketch(10, 2) # default kernel
+    sphericalSketch = density_sketch(10, 2, UnitSphereKernel())
+
+    p = [1, 1]
+    gaussianSketch.update(p)
+    sphericalSketch.update(p)
+
+    # Spherical kernel should return 1.0 for a nearby point, 0 farther
+    # Gaussian kernel should return something nonzero when farther away
+    self.assertEqual(sphericalSketch.get_estimate([1.001, 1]), 1.0)
+    self.assertEqual(sphericalSketch.get_estimate([2, 2]), 0.0)
+    self.assertGreater(gaussianSketch.get_estimate([2, 2]), 0.0)
+
+    # We can also use a custom kernel when deserializing
+    sk_bytes = sphericalSketch.serialize()
+    sphericalRebuilt = density_sketch.deserialize(sk_bytes, UnitSphereKernel())
+    self.assertEqual(sphericalSketch.get_estimate([1.001, 1]), sphericalRebuilt.get_estimate([1.001, 1]))
+
+if __name__ == '__main__':
+    unittest.main()

data/vendor/datasketches-cpp/python/tests/fi_test.py

@@ -16,10 +16,11 @@
 # under the License.
  
 import unittest
-from datasketches import frequent_strings_sketch, frequent_items_error_type
+from datasketches import frequent_strings_sketch, frequent_items_sketch
+from datasketches import frequent_items_error_type, PyIntsSerDe
 
 class FiTest(unittest.TestCase):
-  def test_fi_example(self):
+  def test_fi_strings_example(self):
     k = 3  # a small value so we can easily fill the sketch
     fi = frequent_strings_sketch(k)
 
@@ -93,6 +94,44 @@ class FiTest(unittest.TestCase):
     self.assertGreater(new_fi.get_num_active_items(), 0)
     self.assertEqual(5 * wt, new_fi.get_total_weight())
 
+  # This example uses generic objects but is otherwise identical
+  def test_fi_items_example(self):
+    k = 3  # a small value so we can easily fill the sketch
+    fi = frequent_items_sketch(k)
+
+    # as above, but in this case inserting ints
+    n = 8
+    for i in range(0, n):
+      fi.update(i, 2 ** (n - i))
+
+    # everything else works identically, so let's jump straight
+    # to merging and serialization
+
+    # now create a second sketch with a lot of unique
+    # values but all with equal weight (of 1) such that
+    # the total weight is much larger than the first sketch
+    fi2 = frequent_items_sketch(k)
+    wt = fi.get_total_weight()
+    for i in range(0, 4*wt):
+      fi2.update(i)
+
+    # merge the second sketch into the first
+    fi.merge(fi2)
+
+    # we can see that the weight is much larger
+    self.assertEqual(5 * wt, fi.get_total_weight())
+
+    # finally, serialize and reconstruct -- now we need a serde to tell
+    # (de)serialization how to interpret the objects
+    fi_bytes = fi.serialize(PyIntsSerDe())
+    self.assertEqual(len(fi_bytes), fi.get_serialized_size_bytes(PyIntsSerDe()))
+    new_fi = frequent_items_sketch.deserialize(fi_bytes, PyIntsSerDe())
+
+    # and again interrogate the sketch to check that it's what we serialized
+    self.assertFalse(new_fi.is_empty())
+    self.assertGreater(new_fi.get_num_active_items(), 0)
+    self.assertEqual(5 * wt, new_fi.get_total_weight())
+
 
   def test_fi_sketch(self):
     # only testing a few things not used in the above example

data/vendor/datasketches-cpp/python/tests/hll_test.py

@@ -14,34 +14,34 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
- 
+
 import unittest
 from datasketches import hll_sketch, hll_union, tgt_hll_type
 
 class HllTest(unittest.TestCase):
     def test_hll_example(self):
-        k = 12      # 2^k = 4096 rows in the table
+        lgk = 12    # 2^k = 4096 rows in the table
         n = 1 << 18 # ~256k unique values
 
         # create a couple sketches and inject some values
         # we'll have 1/4 of the values overlap
-        hll  = hll_sketch(k, tgt_hll_type.HLL_8)
-        hll2 = hll_sketch(k, tgt_hll_type.HLL_6)
+        hll  = hll_sketch(lgk, tgt_hll_type.HLL_8)
+        hll2 = hll_sketch(lgk, tgt_hll_type.HLL_6)
         offset = int(3 * n / 4) # it's a float w/o cast
         # because we hash on the bits, not an abstract numeric value,
         # hll.update(1) and hll.update(1.0) give different results.
         for i in range(0, n):
             hll.update(i)
             hll2.update(i + offset)
-        
+
         # we can check that the upper and lower bounds bracket the
         # estimate, without needing to know the exact value.
         self.assertLessEqual(hll.get_lower_bound(1), hll.get_estimate())
         self.assertGreaterEqual(hll.get_upper_bound(1), hll.get_estimate())
 
-        # unioning uses a separate class, and we can either get a result
+        # union is a separate class, and we can either get a result
         # sketch or query the union object directly
-        union = hll_union(k)
+        union = hll_union(lgk)
         union.update(hll)
         union.update(hll2)
         result = union.get_result()
@@ -59,7 +59,7 @@ class HllTest(unittest.TestCase):
         new_hll = hll_sketch.deserialize(sk_bytes)
 
         # the sketch can self-report its configuration and status
-        self.assertEqual(new_hll.lg_config_k, k)
+        self.assertEqual(new_hll.lg_config_k, lgk)
         self.assertEqual(new_hll.tgt_type, tgt_hll_type.HLL_4)
         self.assertFalse(new_hll.is_empty())
 
@@ -68,16 +68,16 @@ class HllTest(unittest.TestCase):
         self.assertTrue(new_hll.is_empty())
 
     def test_hll_sketch(self):
-        k = 8
+        lgk = 8
         n = 117
-        hll = self.generate_sketch(n, k, tgt_hll_type.HLL_6)
+        hll = self.generate_sketch(n, lgk, tgt_hll_type.HLL_6)
         hll.update('string data')
         hll.update(3.14159) # double data
 
         self.assertLessEqual(hll.get_lower_bound(1), hll.get_estimate())
         self.assertGreaterEqual(hll.get_upper_bound(1), hll.get_estimate())
 
-        self.assertEqual(hll.lg_config_k, k)
+        self.assertEqual(hll.lg_config_k, lgk)
         self.assertEqual(hll.tgt_type, tgt_hll_type.HLL_6)
 
         bytes_compact = hll.serialize_compact()
@@ -98,13 +98,13 @@ class HllTest(unittest.TestCase):
         self.assertTrue(hll.is_empty())
 
     def test_hll_union(self):
-        k = 7
+        lgk = 7
         n = 53
-        union = hll_union(k)
+        union = hll_union(lgk)
 
-        sk = self.generate_sketch(n, k, tgt_hll_type.HLL_4, 0)
+        sk = self.generate_sketch(n, lgk, tgt_hll_type.HLL_4, 0)
         union.update(sk)
-        sk = self.generate_sketch(3 * n, k, tgt_hll_type.HLL_4, n)
+        sk = self.generate_sketch(3 * n, lgk, tgt_hll_type.HLL_4, n)
         union.update(sk)
         union.update('string data')
         union.update(1.4142136)
@@ -112,19 +112,18 @@ class HllTest(unittest.TestCase):
         self.assertLessEqual(union.get_lower_bound(1), union.get_estimate())
         self.assertGreaterEqual(union.get_upper_bound(1), union.get_estimate())
 
-        self.assertEqual(union.lg_config_k, k)
+        self.assertEqual(union.lg_config_k, lgk)
         self.assertFalse(union.is_empty())
 
         sk = union.get_result()
         self.assertTrue(isinstance(sk, hll_sketch))
         self.assertEqual(sk.tgt_type, tgt_hll_type.HLL_4)
         
-    def generate_sketch(self, n, k, sk_type=tgt_hll_type.HLL_4, st_idx=0):
-        sk = hll_sketch(k, sk_type)
+    def generate_sketch(self, n, lgk, sk_type=tgt_hll_type.HLL_4, st_idx=0):
+        sk = hll_sketch(lgk, sk_type)
         for i in range(st_idx, st_idx + n):
             sk.update(i)
         return sk
-        
-        
+
 if __name__ == '__main__':
     unittest.main()