datasketches 0.1.0

data/vendor/datasketches-cpp/python/src/vo_wrapper.cpp

@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "var_opt_sketch.hpp"
+#include "var_opt_union.hpp"
+
+#include <pybind11/pybind11.h>
+#include <pybind11/functional.h>
+#include <sstream>
+
+namespace py = pybind11;
+
+namespace datasketches {
+namespace python {
+
+template<typename T>
+py::list vo_sketch_get_samples(const var_opt_sketch<T>& sk) {
+  py::list list;
+  for (auto& item : sk) {
+    py::tuple t = py::make_tuple(item.first, item.second);
+    list.append(t);
+  }
+  return list;
+}
+
+template<typename T>
+py::dict vo_sketch_estimate_subset_sum(const var_opt_sketch<T>& sk, const std::function<bool(T)> func) {
+  subset_summary summary = sk.estimate_subset_sum(func);
+  py::dict d;
+  d["estimate"] = summary.estimate;
+  d["lower_bound"] = summary.lower_bound;
+  d["upper_bound"] = summary.upper_bound;
+  d["total_sketch_weight"] = summary.total_sketch_weight;
+  return d;
+}
+
+template<typename T>
+std::string vo_sketch_to_string(const var_opt_sketch<T>& sk, bool print_items) {
+  if (print_items) {
+    std::ostringstream ss;
+    ss << sk.to_string();
+    ss << "### VarOpt Sketch Items" << std::endl;
+    int i = 0;
+    for (auto& item : sk) {
+      // item.second is always a double
+      // item.first is an arbitrary py::object, so get the value by
+      // using internal str() method then casting to C++ std::string
+      py::str item_pystr(item.first);
+      std::string item_str = py::cast<std::string>(item_pystr);
+      // item.second is guaranteed to be a double
+      ss << i++ << ": " << item_str << "\twt = " << item.second << std::endl;
+    }
+    return ss.str();
+  } else {
+    return sk.to_string();
+  }
+}
+
+}
+}
+
+namespace dspy = datasketches::python;
+
+template<typename T>
+void bind_vo_sketch(py::module &m, const char* name) {
+  using namespace datasketches;
+
+  py::class_<var_opt_sketch<T>>(m, name)
+    .def(py::init<uint32_t>(), py::arg("k"))
+    .def("__str__", &dspy::vo_sketch_to_string<T>, py::arg("print_items")=false,
+         "Produces a string summary of the sketch")
+    .def("to_string", &dspy::vo_sketch_to_string<T>, py::arg("print_items")=false,
+         "Produces a string summary of the sketch")
+    .def("update", (void (var_opt_sketch<T>::*)(const T&, double)) &var_opt_sketch<T>::update, py::arg("item"), py::arg("weight")=1.0,
+         "Updates the sketch with the given value and weight")
+    .def_property_readonly("k", &var_opt_sketch<T>::get_k,
+         "Returns the sketch's maximum configured sample size")
+    .def_property_readonly("n", &var_opt_sketch<T>::get_n,
+         "Returns the total stream length")
+    .def_property_readonly("num_samples", &var_opt_sketch<T>::get_num_samples,
+         "Returns the number of samples currently in the sketch")
+    .def("get_samples", &dspy::vo_sketch_get_samples<T>,
+         "Retyrns the set of samples in the sketch")
+    .def("is_empty", &var_opt_sketch<T>::is_empty,
+         "Returns True if the sketch is empty, otherwise False")
+    .def("estimate_subset_sum", &dspy::vo_sketch_estimate_subset_sum<T>,
+         "Applies a provided predicate to the sketch and returns the estimated total weight matching the predicate, as well "
+         "as upper and lower bounds on the estimate and the total weight processed by the sketch")
+    // As of writing, not yet clear how to serialize arbitrary python objects,
+    // especially in any sort of language-portable way
+    //.def("get_serialized_size_bytes", &var_opt_sketch<T>::get_serialized_size_bytes)
+    //.def("serialize", &dspy::vo_sketch_serialize<T>)
+    //.def_static("deserialize", &dspy::vo_sketch_deserialize<T>)
+    ;
+}
+
+template<typename T>
+void bind_vo_union(py::module &m, const char* name) {
+  using namespace datasketches;
+
+  py::class_<var_opt_union<T>>(m, name)
+    .def(py::init<uint32_t>(), py::arg("max_k"))
+    .def("__str__", &var_opt_union<T>::to_string,
+         "Produces a string summary of the sketch")
+    .def("to_string", &var_opt_union<T>::to_string,
+         "Produces a string summary of the sketch")
+    .def("update", (void (var_opt_union<T>::*)(const var_opt_sketch<T>& sk)) &var_opt_union<T>::update, py::arg("sketch"),
+         "Updates the union with the given sketch")
+    .def("get_result", &var_opt_union<T>::get_result,
+         "Returns a sketch corresponding to the union result")
+    .def("reset", &var_opt_union<T>::reset,
+         "Resets the union to the empty state")
+    // As of writing, not yet clear how to serialize arbitrary python objects,
+    // especially in any sort of language-portable way
+    //.def("get_serialized_size_bytes", &var_opt_sketch<T>::get_serialized_size_bytes)
+    //.def("serialize", &dspy::vo_union_serialize<T>)
+    //.def_static("deserialize", &dspy::vo_union_deserialize<T>)
+    ;
+}
+
+void init_vo(py::module &m) {
+  bind_vo_sketch<py::object>(m, "var_opt_sketch");
+  bind_vo_union<py::object>(m, "var_opt_union");
+}

data/vendor/datasketches-cpp/python/tests/cpc_test.py

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+  
+import unittest
+from datasketches import cpc_sketch, cpc_union
+
+class CpcTest(unittest.TestCase):
+  def test_cpc_example(self):
+    k = 12      # 2^k = 4096 rows in the table
+    n = 1 << 18 # ~256k unique values
+
+    # create a couple sketches and inject some values
+    # we'll have 1/4 of the values overlap
+    cpc  = cpc_sketch(k)
+    cpc2 = cpc_sketch(k)
+    offset = int(3 * n / 4) # it's a float w/o cast
+    # because we hash on the bits, not an abstract numeric value,
+    # cpc.update(1) and cpc.update(1.0) give different results.
+    for i in range(0, n):
+        cpc.update(i)
+        cpc2.update(i + offset)
+        
+    # although we provide get_composite_estimate() and get_estimate(),
+    # the latter will always give the best available estimate.  we
+    # recommend using get_estimate().
+    # we can check that the upper and lower bounds bracket the
+    # estimate, without needing to know the exact value.
+    self.assertLessEqual(cpc.get_lower_bound(1), cpc.get_estimate())
+    self.assertGreaterEqual(cpc.get_upper_bound(1), cpc.get_estimate())
+
+    # unioning uses a separate class, but we need to get_result()
+    # tp query the unioned sketches
+    union = cpc_union(k)
+    union.update(cpc)
+    union.update(cpc2)
+    result = union.get_result()
+
+    # since our process here (including post-union CPC) is
+    # deterministic, we have checked and know the exact
+    # answer is within one standard deviation of the estimate
+    self.assertLessEqual(result.get_lower_bound(1), 7 * n / 4)
+    self.assertGreaterEqual(result.get_upper_bound(1), 7 * n / 4)
+     
+    # serialize for storage and reconstruct
+    sk_bytes = result.serialize()
+    new_cpc = cpc_sketch.deserialize(sk_bytes)
+    self.assertFalse(new_cpc.is_empty())
+
+if __name__ == '__main__':
+    unittest.main()

data/vendor/datasketches-cpp/python/tests/fi_test.py

@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ 
+import unittest
+from datasketches import frequent_strings_sketch, frequent_items_error_type
+
+class FiTest(unittest.TestCase):
+  def test_fi_example(self):
+    k = 3  # a small value so we can easily fill the sketch
+    fi = frequent_strings_sketch(k)
+
+    # we'll use a small number of distinct items so we
+    # can use exponentially increasing weights and have
+    # some frequent items, decreasing so we have some
+    # small items inserted after a purge
+    n = 8
+    for i in range(0, n):
+      fi.update(str(i), 2 ** (n - i))
+
+    # there are two ways to extract items :
+    # * NO_FALSE_POSITIVES includes all items with a lower bound
+    #   above the a posteriori error
+    # * NO_FALSE_NEGATIVES includes all items with an uper bound
+    #   above the a posteriori error
+    # a more complete discussion may be found at
+    # https://datasketches.github.io/docs/Frequency/FrequentItemsOverview.html
+    items_no_fp = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_POSITIVES)
+    items_no_fn = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_NEGATIVES)
+    self.assertLessEqual(len(items_no_fp), len(items_no_fn))
+
+    # the items list returns a decreasing weight-sorted list, and
+    # for each item we have (item, estimate, lower_bound, upper_bound)
+    item = items_no_fp[1]
+    self.assertLessEqual(item[2], item[1]) # lower bound vs estimate 
+    self.assertLessEqual(item[1], item[3]) # estimate vs upper bound
+
+    # we can also query directly for a specific item
+    id = items_no_fn[0][0]
+    est = fi.get_estimate(id)
+    lb = fi.get_lower_bound(id)
+    ub = fi.get_upper_bound(id)
+    self.assertLessEqual(lb, est)
+    self.assertLessEqual(est, ub)
+
+    # the values are zero if the item isn't in our list
+    self.assertEqual(fi.get_estimate("NaN"), 0)
+
+    # now create a second sketch with a lot of unique
+    # values but all with equal weight (of 1) such that
+    # the total weight is much larger than the first sketch
+    fi2 = frequent_strings_sketch(k)
+    wt = fi.get_total_weight()
+    for i in range(0, 4*wt):
+      fi2.update(str(i))
+
+    # merge the second sketch into the first
+    fi.merge(fi2)
+
+    # we can see that the weight is much larger
+    self.assertEqual(5 * wt, fi.get_total_weight())
+
+    # querying with NO_FALSE_POSITIVES means we don't find anything
+    # heavy enough to return
+    items_no_fp = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_POSITIVES)
+    self.assertEqual(len(items_no_fp), 0)
+
+    # we do, however, find a few potential heavy items
+    # if querying with NO_FALSE_NEGATIVES
+    items_no_fn = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_NEGATIVES)
+    self.assertGreater(len(items_no_fn), 0)
+
+    # finally, serialize and reconstruct
+    fi_bytes = fi.serialize()
+    self.assertEqual(len(fi_bytes), fi.get_serialized_size_bytes())
+    new_fi = frequent_strings_sketch.deserialize(fi_bytes)
+
+    # and now interrogate the sketch
+    self.assertFalse(new_fi.is_empty())
+    self.assertGreater(new_fi.get_num_active_items(), 0)
+    self.assertEqual(5 * wt, new_fi.get_total_weight())
+
+
+  def test_fi_sketch(self):
+    # only testing a few things not used in the above example
+    k = 12
+    wt = 10000
+    fi = frequent_strings_sketch(k)
+
+    self.assertAlmostEqual(fi.get_sketch_epsilon(), 0.0008545, delta=1e-6)
+
+    sk_apriori_error = fi.get_sketch_epsilon() * wt
+    reference_apriori_error = frequent_strings_sketch.get_apriori_error(k, wt)
+    self.assertAlmostEqual(sk_apriori_error, reference_apriori_error, delta=1e-6)
+
+if __name__ == '__main__':
+  unittest.main()

data/vendor/datasketches-cpp/python/tests/hll_test.py

@@ -0,0 +1,131 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ 
+import unittest
+from datasketches import hll_sketch, hll_union, tgt_hll_type
+
+class HllTest(unittest.TestCase):
+    def test_hll_example(self):
+        k = 12      # 2^k = 4096 rows in the table
+        n = 1 << 18 # ~256k unique values
+
+        # create a couple sketches and inject some values
+        # we'll have 1/4 of the values overlap
+        hll  = hll_sketch(k, tgt_hll_type.HLL_8)
+        hll2 = hll_sketch(k, tgt_hll_type.HLL_6)
+        offset = int(3 * n / 4) # it's a float w/o cast
+        # because we hash on the bits, not an abstract numeric value,
+        # hll.update(1) and hll.update(1.0) give different results.
+        for i in range(0, n):
+            hll.update(i)
+            hll2.update(i + offset)
+        
+        # we can check that the upper and lower bounds bracket the
+        # estimate, without needing to know the exact value.
+        self.assertLessEqual(hll.get_lower_bound(1), hll.get_estimate())
+        self.assertGreaterEqual(hll.get_upper_bound(1), hll.get_estimate())
+
+        # unioning uses a separate class, and we can either get a result
+        # sketch or query the union object directly
+        union = hll_union(k)
+        union.update(hll)
+        union.update(hll2)
+        result = union.get_result()
+        self.assertEqual(result.get_estimate(), union.get_estimate())
+
+        # since our process here (including post-union HLL) is
+        # deterministic, we have checked and know the exact
+        # answer is within one standard deviation of the estimate
+        self.assertLessEqual(union.get_lower_bound(1), 7 * n / 4)
+        self.assertGreaterEqual(union.get_upper_bound(1), 7 * n / 4)
+
+        # serialize for storage and reconstruct
+        sk_bytes = result.serialize_compact()
+        self.assertEqual(len(sk_bytes), result.get_compact_serialization_bytes())
+        new_hll = hll_sketch.deserialize(sk_bytes)
+
+        # the sketch can self-report its configuation and status
+        self.assertEqual(new_hll.lg_config_k, k)
+        self.assertEqual(new_hll.tgt_type, tgt_hll_type.HLL_4)
+        self.assertFalse(new_hll.is_empty())
+
+        # if we want to reduce some object overhead, we can also reset
+        new_hll.reset()
+        self.assertTrue(new_hll.is_empty())
+
+    def test_hll_sketch(self):
+        k = 8
+        n = 117
+        hll = self.generate_sketch(n, k, tgt_hll_type.HLL_6)
+        hll.update('string data')
+        hll.update(3.14159) # double data
+
+        self.assertLessEqual(hll.get_lower_bound(1), hll.get_estimate())
+        self.assertGreaterEqual(hll.get_upper_bound(1), hll.get_estimate())
+
+        self.assertEqual(hll.lg_config_k, k)
+        self.assertEqual(hll.tgt_type, tgt_hll_type.HLL_6)
+
+        bytes_compact = hll.serialize_compact()
+        bytes_update = hll.serialize_updatable()
+        self.assertEqual(len(bytes_compact), hll.get_compact_serialization_bytes())
+        self.assertEqual(len(bytes_update), hll.get_updatable_serialization_bytes())
+
+        self.assertFalse(hll.is_compact())
+        self.assertFalse(hll.is_empty())
+
+        self.assertTrue(isinstance(hll_sketch.deserialize(bytes_compact), hll_sketch))
+        self.assertTrue(isinstance(hll_sketch.deserialize(bytes_update), hll_sketch))
+
+        self.assertIsNotNone(hll_sketch.get_rel_err(True, False, 12, 1))
+        self.assertIsNotNone(hll_sketch.get_max_updatable_serialization_bytes(20, tgt_hll_type.HLL_6))
+
+        hll.reset()
+        self.assertTrue(hll.is_empty())
+
+    def test_hll_union(self):
+        k = 7
+        n = 53
+        union = hll_union(k)
+
+        sk = self.generate_sketch(n, k, tgt_hll_type.HLL_4, 0)
+        union.update(sk)
+        sk = self.generate_sketch(3 * n, k, tgt_hll_type.HLL_4, n)
+        union.update(sk)
+        union.update('string data')
+        union.update(1.4142136)
+
+        self.assertLessEqual(union.get_lower_bound(1), union.get_estimate())
+        self.assertGreaterEqual(union.get_upper_bound(1), union.get_estimate())
+
+        self.assertEqual(union.lg_config_k, k)
+        self.assertFalse(union.is_compact())
+        self.assertFalse(union.is_empty())
+
+        sk = union.get_result()
+        self.assertTrue(isinstance(sk, hll_sketch))
+        self.assertEqual(sk.tgt_type, tgt_hll_type.HLL_4)
+        
+    def generate_sketch(self, n, k, sk_type=tgt_hll_type.HLL_4, st_idx=0):
+        sk = hll_sketch(k, sk_type)
+        for i in range(st_idx, st_idx + n):
+            sk.update(i)
+        return sk
+        
+        
+if __name__ == '__main__':
+    unittest.main()