datasketches 0.3.1 → 0.4.0

data/vendor/datasketches-cpp/python/jupyter/ThetaSketchNotebook.ipynb

@@ -1,403 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Theta Sketch Examples"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Basic Sketch Usage"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datasketches import theta_sketch, update_theta_sketch, compact_theta_sketch\n",
-    "from datasketches import theta_union, theta_intersection, theta_a_not_b"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "To start, we'll create a sketch with 1 million points in order to demonstrate basic sketch operations."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "### Theta sketch summary:\n",
-      "   num retained entries : 6560\n",
-      "   seed hash            : 37836\n",
-      "   empty?               : false\n",
-      "   ordered?             : false\n",
-      "   estimation mode?     : true\n",
-      "   theta (fraction)     : 0.00654224\n",
-      "   theta (raw 64-bit)   : 60341508738660257\n",
-      "   estimate             : 1.00271e+06\n",
-      "   lower bound 95% conf : 978261\n",
-      "   upper bound 95% conf : 1.02778e+06\n",
-      "   lg nominal size      : 12\n",
-      "   lg current size      : 13\n",
-      "   resize factor        : 8\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "n = 1000000\n",
-    "k = 12\n",
-    "sk1 = update_theta_sketch(k)\n",
-    "for i in range(0, n):\n",
-    "    sk1.update(i)\n",
-    "print(sk1)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The summary contains most data fo interest, but we can also query for specific information. And in this case, since we know the exact number of distinct items presented ot the sketch, we can look at the estimate, upper, and lower bounds as a percentage of the exact value."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Upper bound (1 std. dev) as % of true value:\t 101.5208\n",
-      "Sketch estimate as % of true value:\t\t 100.2715\n",
-      "Lower bound (1 std. dev) as % of true value:\t 99.0374\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Upper bound (1 std. dev) as % of true value:\\t\", round(100*sk1.get_upper_bound(1) / n, 4))\n",
-    "print(\"Sketch estimate as % of true value:\\t\\t\", round(100*sk1.get_estimate() / n, 4))\n",
-    "print(\"Lower bound (1 std. dev) as % of true value:\\t\", round(100*sk1.get_lower_bound(1) / n, 4))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We can serialize and reconstruct the sketch. Serialization necessarily produces a compact sketch, meaning the sketch can be deserialized and queried or used for further unions or set operations but can not be updated directly."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "52504"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "sk1_bytes = sk1.compact().serialize()\n",
-    "len(sk1_bytes)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Estimate: \t\t 1002714.745231455\n",
-      "Estimation mode: \t True\n"
-     ]
-    }
-   ],
-   "source": [
-    "new_sk1 = compact_theta_sketch.deserialize(sk1_bytes)\n",
-    "print(\"Estimate: \\t\\t\", new_sk1.get_estimate())\n",
-    "print(\"Estimation mode: \\t\", new_sk1.is_estimation_mode())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Sketch Unions"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Theta Sketch unions make use of a separate union object. The union will accept input sketches with different values of $k$.\n",
-    "\n",
-    "For this example, we will create a sketch with distinct values that partially overlap those in `sk1`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "### Theta sketch summary:\n",
-      "   num retained entries : 12488\n",
-      "   seed hash            : 37836\n",
-      "   empty?               : false\n",
-      "   ordered?             : false\n",
-      "   estimation mode?     : true\n",
-      "   theta (fraction)     : 0.0123336\n",
-      "   theta (raw 64-bit)   : 113757656857900725\n",
-      "   estimate             : 1.01252e+06\n",
-      "   lower bound 95% conf : 994626\n",
-      "   upper bound 95% conf : 1.03073e+06\n",
-      "   lg nominal size      : 13\n",
-      "   lg current size      : 14\n",
-      "   resize factor        : 8\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "offset = int(3 * n / 4)\n",
-    "sk2 = update_theta_sketch(k+1)\n",
-    "for i in range(0, n):\n",
-    "    sk2.update(i + offset)\n",
-    "print(sk2)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We can now feed the sketches into the union. As constructed, the exact number of unique values presented to the two sketches is $\\frac{7}{4}n$."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Union estimate as % of true value:  99.6787\n"
-     ]
-    }
-   ],
-   "source": [
-    "union = theta_union(k)\n",
-    "union.update(sk1)\n",
-    "union.update(sk2)\n",
-    "result = union.get_result()\n",
-    "print(\"Union estimate as % of true value: \", round(100*result.get_estimate()/(1.75*n), 4))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Sketch Intersections"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Beyond unions, theta sketches also support intersctions through the use of an intersection object. These set intersections can have vastly superior error bounds than the classic inclusion-exclusion rule used with sketches like HLL."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Has result:  True\n",
-      "### Theta sketch summary:\n",
-      "   num retained entries : 1668\n",
-      "   seed hash            : 37836\n",
-      "   empty?               : false\n",
-      "   ordered?             : true\n",
-      "   estimation mode?     : true\n",
-      "   theta (fraction)     : 0.00654224\n",
-      "   theta (raw 64-bit)   : 60341508738660257\n",
-      "   estimate             : 254959\n",
-      "   lower bound 95% conf : 242739\n",
-      "   upper bound 95% conf : 267789\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "intersection = theta_intersection()\n",
-    "intersection.update(sk1)\n",
-    "intersection.update(sk2)\n",
-    "print(\"Has result: \", intersection.has_result())\n",
-    "result = intersection.get_result()\n",
-    "print(result)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "In this case, we expect the sets to have an overlap of $\\frac{1}{4}n$."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Intersection estimate as % of true value:  101.9834\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Intersection estimate as % of true value: \", round(100*result.get_estimate()/(0.25*n), 4))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Set Subtraction (A-not-B)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Finally, we have the set subtraction operation. Unlike `theta_union` and `theta_intersection`, `theta_a_not_b` always takes as input 2 sketches at a time, namely $a$ and $b$, and directly returns the result as a sketch."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "### Theta sketch summary:\n",
-      "   num retained entries : 4892\n",
-      "   seed hash            : 37836\n",
-      "   empty?               : false\n",
-      "   ordered?             : true\n",
-      "   estimation mode?     : true\n",
-      "   theta (fraction)     : 0.00654224\n",
-      "   theta (raw 64-bit)   : 60341508738660257\n",
-      "   estimate             : 747756\n",
-      "   lower bound 95% conf : 726670\n",
-      "   upper bound 95% conf : 769452\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "anb = theta_a_not_b()\n",
-    "result = anb.compute(sk1, sk2)\n",
-    "print(result)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "By using the same two sketches as before, the expected result here is $\\frac{3}{4}n$."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "A-not-B estimate as % of true value:  99.7008\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"A-not-B estimate as % of true value: \", round(100*result.get_estimate()/(0.75*n), 4))"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3.10.6 64-bit",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.6"
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

data/vendor/datasketches-cpp/python/pybind11Path.cmd

@@ -1,21 +0,0 @@
-:: Licensed to the Apache Software Foundation (ASF) under one
-:: or more contributor license agreements.  See the NOTICE file
-:: distributed with this work for additional information
-:: regarding copyright ownership.  The ASF licenses this file
-:: to you under the Apache License, Version 2.0 (the
-:: "License"); you may not use this file except in compliance
-:: with the License.  You may obtain a copy of the License at
-::
-::   http://www.apache.org/licenses/LICENSE-2.0
-::
-:: Unless required by applicable law or agreed to in writing,
-:: software distributed under the License is distributed on an
-:: "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-:: KIND, either express or implied.  See the License for the
-:: specific language governing permissions and limitations
-:: under the License.
-
-
-@echo off
-:: Takes path to the Python interpreter and returns the path to pybind11
-%1 -c "import pybind11,sys;sys.stdout.write(pybind11.get_cmake_dir())"

data/vendor/datasketches-cpp/python/src/cpc_wrapper.cpp

@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include <sstream>
-#include <pybind11/pybind11.h>
-
-#include "cpc_sketch.hpp"
-#include "cpc_union.hpp"
-#include "cpc_common.hpp"
-#include "common_defs.hpp"
-
-namespace py = pybind11;
-
-namespace datasketches {
-namespace python {
-
-cpc_sketch* cpc_sketch_deserialize(py::bytes skBytes) {
-  std::string skStr = skBytes; // implicit cast
-  return new cpc_sketch(cpc_sketch::deserialize(skStr.c_str(), skStr.length()));
-}
-
-py::object cpc_sketch_serialize(const cpc_sketch& sk) {
-  auto serResult = sk.serialize();
-  return py::bytes((char*)serResult.data(), serResult.size());
-}
-
-cpc_sketch* cpc_union_get_result(const cpc_union& u) {
-  return new cpc_sketch(u.get_result());
-}
-
-}
-}
-
-namespace dspy = datasketches::python;
-
-void init_cpc(py::module &m) {
-  using namespace datasketches;
-
-  py::class_<cpc_sketch>(m, "cpc_sketch")
-    .def(py::init<uint8_t, uint64_t>(), py::arg("lg_k")=cpc_constants::DEFAULT_LG_K, py::arg("seed")=DEFAULT_SEED)
-    .def(py::init<const cpc_sketch&>())
-    .def("__str__", &cpc_sketch::to_string,
-         "Produces a string summary of the sketch")
-    .def("to_string", &cpc_sketch::to_string,
-         "Produces a string summary of the sketch")
-    .def("serialize", &dspy::cpc_sketch_serialize,
-         "Serializes the sketch into a bytes object")
-    .def_static("deserialize", &dspy::cpc_sketch_deserialize,
-         "Reads a bytes object and returns the corresponding cpc_sketch")
-    .def<void (cpc_sketch::*)(uint64_t)>("update", &cpc_sketch::update, py::arg("datum"),
-         "Updates the sketch with the given 64-bit integer value")
-    .def<void (cpc_sketch::*)(double)>("update", &cpc_sketch::update, py::arg("datum"),
-         "Updates the sketch with the given 64-bit floating point")
-    .def<void (cpc_sketch::*)(const std::string&)>("update", &cpc_sketch::update, py::arg("datum"),
-         "Updates the sketch with the given string")
-    .def("is_empty", &cpc_sketch::is_empty,
-         "Returns True if the sketch is empty, otherwise Dalse")
-    .def("get_estimate", &cpc_sketch::get_estimate,
-         "Estimate of the distinct count of the input stream")
-    .def("get_lower_bound", &cpc_sketch::get_lower_bound, py::arg("kappa"),
-         "Returns an approximate lower bound on the estimate for kappa values in {1, 2, 3}, roughly corresponding to standard deviations")
-    .def("get_upper_bound", &cpc_sketch::get_upper_bound, py::arg("kappa"),
-         "Returns an approximate upper bound on the estimate for kappa values in {1, 2, 3}, roughly corresponding to standard deviations")
-    ;
-
-  py::class_<cpc_union>(m, "cpc_union")
-    .def(py::init<uint8_t, uint64_t>(), py::arg("lg_k"), py::arg("seed")=DEFAULT_SEED)
-    .def(py::init<const cpc_union&>())
-    .def("update", (void (cpc_union::*)(const cpc_sketch&)) &cpc_union::update, py::arg("sketch"),
-         "Updates the union with the provided CPC sketch")
-    .def("get_result", &dspy::cpc_union_get_result,
-         "Returns a CPC sketch with the result of the union")
-    ;
-}

data/vendor/datasketches-cpp/python/src/fi_wrapper.cpp

@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "frequent_items_sketch.hpp"
-
-#include <pybind11/pybind11.h>
-#include <sstream>
-
-namespace py = pybind11;
-
-namespace datasketches {
-namespace python {
-
-template<typename T>
-frequent_items_sketch<T> fi_sketch_deserialize(py::bytes skBytes) {
-  std::string skStr = skBytes; // implicit cast  
-  return frequent_items_sketch<T>::deserialize(skStr.c_str(), skStr.length());
-}
-
-template<typename T>
-py::object fi_sketch_serialize(const frequent_items_sketch<T>& sk) {
-  auto serResult = sk.serialize();
-  return py::bytes((char*)serResult.data(), serResult.size());
-}
-
-// maybe possible to disambiguate the static vs method get_epsilon calls, but
-// this is easier for now
-template<typename T>
-double fi_sketch_get_generic_epsilon(uint8_t lg_max_map_size) {
-  return frequent_items_sketch<T>::get_epsilon(lg_max_map_size);
-}
-
-template<typename T>
-py::list fi_sketch_get_frequent_items(const frequent_items_sketch<T>& sk,
-                                   frequent_items_error_type err_type,
-                                   uint64_t threshold = 0) {
-  if (threshold == 0) { threshold = sk.get_maximum_error(); }
-
-  py::list list;
-  auto items = sk.get_frequent_items(err_type, threshold);
-  for (auto iter = items.begin(); iter != items.end(); ++iter) {
-    py::tuple t = py::make_tuple(iter->get_item(),
-                                 iter->get_estimate(),
-                                 iter->get_lower_bound(),
-                                 iter->get_upper_bound());
-    list.append(t);
-  }
-  return list;
-}
-
-template<typename T>
-size_t fi_sketch_get_serialized_size_bytes(const frequent_items_sketch<T>& sk) {
-  return sk.get_serialized_size_bytes();
-}
-
-}
-}
-
-namespace dspy = datasketches::python;
-
-template<typename T>
-void bind_fi_sketch(py::module &m, const char* name) {
-  using namespace datasketches;
-
-  py::class_<frequent_items_sketch<T>>(m, name)
-    .def(py::init<uint8_t>(), py::arg("lg_max_k"))
-    .def("__str__", &frequent_items_sketch<T>::to_string, py::arg("print_items")=false,
-         "Produces a string summary of the sketch")
-    .def("to_string", &frequent_items_sketch<T>::to_string, py::arg("print_items")=false,
-         "Produces a string summary of the sketch")
-    .def("update", (void (frequent_items_sketch<T>::*)(const T&, uint64_t)) &frequent_items_sketch<T>::update, py::arg("item"), py::arg("weight")=1,
-         "Updates the sketch with the given string and, optionally, a weight")
-    .def("get_frequent_items", &dspy::fi_sketch_get_frequent_items<T>, py::arg("err_type"), py::arg("threshold")=0)
-    .def("merge", (void (frequent_items_sketch<T>::*)(const frequent_items_sketch<T>&)) &frequent_items_sketch<T>::merge,
-         "Merges the given sketch into this one")
-    .def("is_empty", &frequent_items_sketch<T>::is_empty,
-         "Returns True if the sketch is empty, otherwise False")
-    .def("get_num_active_items", &frequent_items_sketch<T>::get_num_active_items,
-         "Returns the number of active items in the sketch")
-    .def("get_total_weight", &frequent_items_sketch<T>::get_total_weight,
-         "Returns the sum of the weights (frequencies) in the stream seen so far by the sketch")
-    .def("get_estimate", &frequent_items_sketch<T>::get_estimate, py::arg("item"),
-         "Returns the estimate of the weight (frequency) of the given item.\n"
-         "Note: The true frequency of a item would be the sum of the counts as a result of the "
-         "two update functions.")
-    .def("get_lower_bound", &frequent_items_sketch<T>::get_lower_bound, py::arg("item"),
-         "Returns the guaranteed lower bound weight (frequency) of the given item.")
-    .def("get_upper_bound", &frequent_items_sketch<T>::get_upper_bound, py::arg("item"),
-         "Returns the guaranteed upper bound weight (frequency) of the given item.")
-    .def("get_sketch_epsilon", (double (frequent_items_sketch<T>::*)(void) const) &frequent_items_sketch<T>::get_epsilon,
-         "Returns the epsilon value used by the sketch to compute error")
-    .def_static("get_epsilon_for_lg_size", &dspy::fi_sketch_get_generic_epsilon<T>, py::arg("lg_max_map_size"),
-         "Returns the epsilon value used to compute a priori error for a given log2(max_map_size)")
-    .def_static("get_apriori_error", &frequent_items_sketch<T>::get_apriori_error, py::arg("lg_max_map_size"), py::arg("estimated_total_weight"),
-         "Returns the estimated a priori error given the max_map_size for the sketch and the estimated_total_stream_weight.")
-    .def("get_serialized_size_bytes", &dspy::fi_sketch_get_serialized_size_bytes<T>,
-         "Computes the size needed to serialize the current state of the sketch. This can be expensive since every item needs to be looked at.")
-    .def("serialize", &dspy::fi_sketch_serialize<T>, "Serializes the sketch into a bytes object")
-    .def_static("deserialize", &dspy::fi_sketch_deserialize<T>, "Reads a bytes object and returns the corresponding frequent_strings_sketch")
-    ;
-}
-
-void init_fi(py::module &m) {
-  using namespace datasketches;
-
-  py::enum_<frequent_items_error_type>(m, "frequent_items_error_type")
-    .value("NO_FALSE_POSITIVES", NO_FALSE_POSITIVES)
-    .value("NO_FALSE_NEGATIVES", NO_FALSE_NEGATIVES)
-    .export_values();
-
-  bind_fi_sketch<std::string>(m, "frequent_strings_sketch");
-}