datasketches 0.3.1 → 0.3.2

data/vendor/datasketches-cpp/python/src/fi_wrapper.cpp

@@ -17,105 +17,158 @@
  * under the License.
  */
 
+
+#include "py_serde.hpp"
+#include "py_object_ostream.hpp"
 #include "frequent_items_sketch.hpp"
 
 #include <pybind11/pybind11.h>
-#include <sstream>
-
-namespace py = pybind11;
-
-namespace datasketches {
-namespace python {
-
-template<typename T>
-frequent_items_sketch<T> fi_sketch_deserialize(py::bytes skBytes) {
-  std::string skStr = skBytes; // implicit cast  
-  return frequent_items_sketch<T>::deserialize(skStr.c_str(), skStr.length());
-}
 
-template<typename T>
-py::object fi_sketch_serialize(const frequent_items_sketch<T>& sk) {
-  auto serResult = sk.serialize();
-  return py::bytes((char*)serResult.data(), serResult.size());
-}
-
-// maybe possible to disambiguate the static vs method get_epsilon calls, but
-// this is easier for now
-template<typename T>
-double fi_sketch_get_generic_epsilon(uint8_t lg_max_map_size) {
-  return frequent_items_sketch<T>::get_epsilon(lg_max_map_size);
-}
-
-template<typename T>
-py::list fi_sketch_get_frequent_items(const frequent_items_sketch<T>& sk,
-                                   frequent_items_error_type err_type,
-                                   uint64_t threshold = 0) {
-  if (threshold == 0) { threshold = sk.get_maximum_error(); }
-
-  py::list list;
-  auto items = sk.get_frequent_items(err_type, threshold);
-  for (auto iter = items.begin(); iter != items.end(); ++iter) {
-    py::tuple t = py::make_tuple(iter->get_item(),
-                                 iter->get_estimate(),
-                                 iter->get_lower_bound(),
-                                 iter->get_upper_bound());
-    list.append(t);
-  }
-  return list;
-}
+#include <ostream>
 
-template<typename T>
-size_t fi_sketch_get_serialized_size_bytes(const frequent_items_sketch<T>& sk) {
-  return sk.get_serialized_size_bytes();
-}
+namespace py = pybind11;
 
-}
-}
+// forward declarations
+// std::string and arithmetic types, where we don't need a separate serde
+template<typename T, typename W, typename H, typename E, typename std::enable_if<std::is_arithmetic<T>::value || std::is_same<std::string, T>::value, bool>::type = 0>
+void add_serialization(py::class_<datasketches::frequent_items_sketch<T, W, H, E>>& clazz);
 
-namespace dspy = datasketches::python;
+// py::object and other types where the caller must provide a serde
+template<typename T, typename W, typename H, typename E, typename std::enable_if<!std::is_arithmetic<T>::value && !std::is_same<std::string, T>::value, bool>::type = 0>
+void add_serialization(py::class_<datasketches::frequent_items_sketch<T, W, H, E>>& clazz);
 
-template<typename T>
+template<typename T, typename W, typename H, typename E>
 void bind_fi_sketch(py::module &m, const char* name) {
   using namespace datasketches;
 
-  py::class_<frequent_items_sketch<T>>(m, name)
+  auto fi_class = py::class_<frequent_items_sketch<T, W, H, E>>(m, name)
     .def(py::init<uint8_t>(), py::arg("lg_max_k"))
-    .def("__str__", &frequent_items_sketch<T>::to_string, py::arg("print_items")=false,
+    .def("__str__", &frequent_items_sketch<T, W, H, E>::to_string, py::arg("print_items")=false,
          "Produces a string summary of the sketch")
-    .def("to_string", &frequent_items_sketch<T>::to_string, py::arg("print_items")=false,
+    .def("to_string", &frequent_items_sketch<T, W, H, E>::to_string, py::arg("print_items")=false,
          "Produces a string summary of the sketch")
-    .def("update", (void (frequent_items_sketch<T>::*)(const T&, uint64_t)) &frequent_items_sketch<T>::update, py::arg("item"), py::arg("weight")=1,
+    .def("update", (void (frequent_items_sketch<T, W, H, E>::*)(const T&, uint64_t)) &frequent_items_sketch<T, W, H, E>::update, py::arg("item"), py::arg("weight")=1,
          "Updates the sketch with the given string and, optionally, a weight")
-    .def("get_frequent_items", &dspy::fi_sketch_get_frequent_items<T>, py::arg("err_type"), py::arg("threshold")=0)
-    .def("merge", (void (frequent_items_sketch<T>::*)(const frequent_items_sketch<T>&)) &frequent_items_sketch<T>::merge,
+    .def("merge", (void (frequent_items_sketch<T, W, H, E>::*)(const frequent_items_sketch<T, W, H, E>&)) &frequent_items_sketch<T, W, H, E>::merge,
          "Merges the given sketch into this one")
-    .def("is_empty", &frequent_items_sketch<T>::is_empty,
+    .def("is_empty", &frequent_items_sketch<T, W, H, E>::is_empty,
          "Returns True if the sketch is empty, otherwise False")
-    .def("get_num_active_items", &frequent_items_sketch<T>::get_num_active_items,
+    .def("get_num_active_items", &frequent_items_sketch<T, W, H, E>::get_num_active_items,
          "Returns the number of active items in the sketch")
-    .def("get_total_weight", &frequent_items_sketch<T>::get_total_weight,
+    .def("get_total_weight", &frequent_items_sketch<T, W, H, E>::get_total_weight,
          "Returns the sum of the weights (frequencies) in the stream seen so far by the sketch")
-    .def("get_estimate", &frequent_items_sketch<T>::get_estimate, py::arg("item"),
+    .def("get_estimate", &frequent_items_sketch<T, W, H, E>::get_estimate, py::arg("item"),
          "Returns the estimate of the weight (frequency) of the given item.\n"
          "Note: The true frequency of a item would be the sum of the counts as a result of the "
          "two update functions.")
-    .def("get_lower_bound", &frequent_items_sketch<T>::get_lower_bound, py::arg("item"),
+    .def("get_lower_bound", &frequent_items_sketch<T, W, H, E>::get_lower_bound, py::arg("item"),
          "Returns the guaranteed lower bound weight (frequency) of the given item.")
-    .def("get_upper_bound", &frequent_items_sketch<T>::get_upper_bound, py::arg("item"),
+    .def("get_upper_bound", &frequent_items_sketch<T, W, H, E>::get_upper_bound, py::arg("item"),
          "Returns the guaranteed upper bound weight (frequency) of the given item.")
-    .def("get_sketch_epsilon", (double (frequent_items_sketch<T>::*)(void) const) &frequent_items_sketch<T>::get_epsilon,
+    .def("get_sketch_epsilon", (double (frequent_items_sketch<T, W, H, E>::*)(void) const) &frequent_items_sketch<T, W, H, E>::get_epsilon,
          "Returns the epsilon value used by the sketch to compute error")
-    .def_static("get_epsilon_for_lg_size", &dspy::fi_sketch_get_generic_epsilon<T>, py::arg("lg_max_map_size"),
-         "Returns the epsilon value used to compute a priori error for a given log2(max_map_size)")
-    .def_static("get_apriori_error", &frequent_items_sketch<T>::get_apriori_error, py::arg("lg_max_map_size"), py::arg("estimated_total_weight"),
-         "Returns the estimated a priori error given the max_map_size for the sketch and the estimated_total_stream_weight.")
-    .def("get_serialized_size_bytes", &dspy::fi_sketch_get_serialized_size_bytes<T>,
-         "Computes the size needed to serialize the current state of the sketch. This can be expensive since every item needs to be looked at.")
-    .def("serialize", &dspy::fi_sketch_serialize<T>, "Serializes the sketch into a bytes object")
-    .def_static("deserialize", &dspy::fi_sketch_deserialize<T>, "Reads a bytes object and returns the corresponding frequent_strings_sketch")
-    ;
+    .def(
+        "get_frequent_items",
+        [](const frequent_items_sketch<T, W, H, E>& sk, frequent_items_error_type err_type, uint64_t threshold) {
+          if (threshold == 0) threshold = sk.get_maximum_error();
+          py::list list;
+          auto rows = sk.get_frequent_items(err_type, threshold);
+          for (auto row: rows) {
+            list.append(py::make_tuple(
+                row.get_item(),
+                row.get_estimate(),
+                row.get_lower_bound(),
+                row.get_upper_bound())
+            );
+          }
+          return list;
+        },
+        py::arg("err_type"), py::arg("threshold")=0
+    )
+    .def_static(
+        "get_epsilon_for_lg_size",
+        [](uint8_t lg_max_map_size) { return frequent_items_sketch<T, W, H, E>::get_epsilon(lg_max_map_size); },
+        py::arg("lg_max_map_size"),
+        "Returns the epsilon value used to compute a priori error for a given log2(max_map_size)"
+    )
+    .def_static(
+        "get_apriori_error",
+        &frequent_items_sketch<T, W, H, E>::get_apriori_error,
+        py::arg("lg_max_map_size"), py::arg("estimated_total_weight"),
+        "Returns the estimated a priori error given the max_map_size for the sketch and the estimated_total_stream_weight."
+    );
+
+    // serialization may need a caller-provided serde depending on the sketch type, so
+    // we use a separate method to handle that appropriately based on type T.
+    add_serialization(fi_class);
+}
+
+// std::string or arithmetic types, for which we have a built-in serde
+template<typename T, typename W, typename H, typename E, typename std::enable_if<std::is_arithmetic<T>::value || std::is_same<std::string, T>::value, bool>::type>
+void add_serialization(py::class_<datasketches::frequent_items_sketch<T, W, H, E>>& clazz) {
+    using namespace datasketches;
+    clazz.def(
+        "get_serialized_size_bytes",
+        [](const frequent_items_sketch<T, W, H, E>& sk) { return sk.get_serialized_size_bytes(); },
+        "Computes the size needed to serialize the current state of the sketch. This can be expensive since every item needs to be looked at."
+    )
+    .def(
+        "serialize",
+        [](const frequent_items_sketch<T, W, H, E>& sk) {
+          auto bytes = sk.serialize();
+          return py::bytes(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+        },
+        "Serializes the sketch into a bytes object."
+    )
+    .def_static(
+        "deserialize",
+        [](const std::string& bytes) { return frequent_items_sketch<T, W, H, E>::deserialize(bytes.data(), bytes.size()); },
+        py::arg("bytes"),
+        "Reads a bytes object and returns the corresponding frequent_strings_sketch."
+    );
 }
 
+// py::object or any other type that requires a provided serde
+template<typename T, typename W, typename H, typename E, typename std::enable_if<!std::is_arithmetic<T>::value && !std::is_same<std::string, T>::value, bool>::type>
+void add_serialization(py::class_<datasketches::frequent_items_sketch<T, W, H, E>>& clazz) {
+    using namespace datasketches;
+    clazz.def(
+        "get_serialized_size_bytes",
+        [](const frequent_items_sketch<T, W, H, E>& sk, py_object_serde& serde) { return sk.get_serialized_size_bytes(serde); },
+        py::arg("serde"),
+        "Computes the size needed to serialize the current state of the sketch using the provided serde. This can be expensive since every item needs to be looked at."
+    )
+    .def(
+        "serialize",
+        [](const frequent_items_sketch<T, W, H, E>& sk, py_object_serde& serde) {
+          auto bytes = sk.serialize(0, serde);
+          return py::bytes(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+        }, py::arg("serde"),
+        "Serializes the sketch into a bytes object using the provided serde."
+    )
+    .def_static(
+        "deserialize",
+        [](const std::string& bytes, py_object_serde& serde) {
+          return frequent_items_sketch<T, W, H, E>::deserialize(bytes.data(), bytes.size(), serde);
+        }, py::arg("bytes"), py::arg("serde"),
+        "Reads a bytes object using the provided serde and returns the corresponding frequent_strings_sketch."
+    );
+}
+
+// calls class __hash__ method
+struct py_hash_caller {
+  size_t operator()(const py::object& a) const {
+    return py::hash(a);
+  }
+};
+
+// calls class __eq__ method
+struct py_equal_caller {
+  bool operator()(const py::object& a, const py::object& b) const {
+    return a.equal(b);
+  }
+};
+
 void init_fi(py::module &m) {
   using namespace datasketches;
 
@@ -124,5 +177,6 @@ void init_fi(py::module &m) {
     .value("NO_FALSE_NEGATIVES", NO_FALSE_NEGATIVES)
     .export_values();
 
-  bind_fi_sketch<std::string>(m, "frequent_strings_sketch");
+  bind_fi_sketch<std::string, uint64_t, std::hash<std::string>, std::equal_to<std::string>>(m, "frequent_strings_sketch");
+  bind_fi_sketch<py::object, uint64_t, py_hash_caller, py_equal_caller>(m, "frequent_items_sketch"); 
 }

data/vendor/datasketches-cpp/python/src/hll_wrapper.cpp

@@ -17,34 +17,11 @@
  * under the License.
  */
 
-#include "hll.hpp"
-
 #include <pybind11/pybind11.h>
 
-namespace py = pybind11;
-
-namespace datasketches {
-namespace python {
-
-hll_sketch hll_sketch_deserialize(py::bytes skBytes) {
-  std::string skStr = skBytes; // implicit cast  
-  return hll_sketch::deserialize(skStr.c_str(), skStr.length());
-}
-
-py::object hll_sketch_serialize_compact(const hll_sketch& sk) {
-  auto serResult = sk.serialize_compact();
-  return py::bytes((char*)serResult.data(), serResult.size());
-}
-
-py::object hll_sketch_serialize_updatable(const hll_sketch& sk) {
-  auto serResult = sk.serialize_updatable();
-  return py::bytes((char*)serResult.data(), serResult.size());
-}
-
-}
-}
+#include "hll.hpp"
 
-namespace dspy = datasketches::python;
+namespace py = pybind11;
 
 void init_hll(py::module &m) {
   using namespace datasketches;
@@ -59,12 +36,6 @@ void init_hll(py::module &m) {
     .def(py::init<uint8_t>(), py::arg("lg_k"))
     .def(py::init<uint8_t, target_hll_type>(), py::arg("lg_k"), py::arg("tgt_type"))
     .def(py::init<uint8_t, target_hll_type, bool>(), py::arg("lg_k"), py::arg("tgt_type"), py::arg("start_max_size")=false)
-    .def_static("deserialize", &dspy::hll_sketch_deserialize,
-         "Reads a bytes object and returns the corresponding hll_sketch")
-    .def("serialize_compact", &dspy::hll_sketch_serialize_compact,
-         "Serializes the sketch into a bytes object, compressiong the exception table if HLL_4")
-    .def("serialize_updatable", &dspy::hll_sketch_serialize_updatable,
-         "Serializes the sketch into a bytes object")
     .def("__str__", (std::string (hll_sketch::*)(bool,bool,bool,bool) const) &hll_sketch::to_string,
          py::arg("summary")=true, py::arg("detail")=false, py::arg("aux_detail")=false, py::arg("all")=false,
          "Produces a string summary of the sketch")
@@ -88,7 +59,7 @@ void init_hll(py::module &m) {
     .def("get_compact_serialization_bytes", &hll_sketch::get_compact_serialization_bytes,
          "Returns the size of the serialized sketch when compressing the exception table if HLL_4")
     .def("reset", &hll_sketch::reset,
-         "Resets the sketch to the empty state in coupon colleciton mode")
+         "Resets the sketch to the empty state in coupon collection mode")
     .def("update", (void (hll_sketch::*)(int64_t)) &hll_sketch::update, py::arg("datum"),
          "Updates the sketch with the given integral value")
     .def("update", (void (hll_sketch::*)(double)) &hll_sketch::update, py::arg("datum"),
@@ -97,11 +68,32 @@ void init_hll(py::module &m) {
          "Updates the sketch with the given string value")
     .def_static("get_max_updatable_serialization_bytes", &hll_sketch::get_max_updatable_serialization_bytes,
          py::arg("lg_k"), py::arg("tgt_type"),
-         "Provides a likely upper bound on serialization size for the given paramters")
+         "Provides a likely upper bound on serialization size for the given parameters")
     .def_static("get_rel_err", &hll_sketch::get_rel_err,
          py::arg("upper_bound"), py::arg("unioned"), py::arg("lg_k"), py::arg("num_std_devs"),
-         "Retuns the a priori relative error bound for the given parameters")
-    ;
+         "Returns the a priori relative error bound for the given parameters")
+    .def(
+        "serialize_compact",
+        [](const hll_sketch& sk) {
+          auto bytes = sk.serialize_compact();
+          return py::bytes(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+        },
+        "Serializes the sketch into a bytes object, compressing the exception table if HLL_4"
+    )
+    .def(
+        "serialize_updatable",
+        [](const hll_sketch& sk) {
+          auto bytes = sk.serialize_updatable();
+          return py::bytes(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+        },
+        "Serializes the sketch into a bytes object"
+    )
+    .def_static(
+        "deserialize",
+        [](const std::string& bytes) { return hll_sketch::deserialize(bytes.data(), bytes.size()); },
+        py::arg("bytes"),
+        "Reads a bytes object and returns the corresponding hll_sketch"
+    );
 
   py::class_<hll_union>(m, "hll_union")
     .def(py::init<uint8_t>(), py::arg("lg_max_k"))
@@ -129,6 +121,6 @@ void init_hll(py::module &m) {
          "Updates the union with the given string value")
     .def_static("get_rel_err", &hll_union::get_rel_err,
          py::arg("upper_bound"), py::arg("unioned"), py::arg("lg_k"), py::arg("num_std_devs"),
-         "Retuns the a priori relative error bound for the given parameters")
+         "Returns the a priori relative error bound for the given parameters")
     ;
 }