datasketches 0.3.2 → 0.4.0

data/vendor/datasketches-cpp/python/include/quantile_conditional.hpp

@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef _QUANTILE_CONDITIONAL_HPP_
-#define _QUANTILE_CONDITIONAL_HPP_
-
-/*
-  This header defines conditionally compiled functions shared
-  across the set of quantile family sketches.
-*/
-
-#include "common_defs.hpp"
-#include "py_serde.hpp"
-
-#include <pybind11/pybind11.h>
-#include <pybind11/numpy.h>
-
-namespace py = pybind11;
-
-// Serialization
-// std::string and arithmetic types, where we don't need a separate serde
-template<typename T, typename SK, typename std::enable_if<std::is_arithmetic<T>::value || std::is_same<std::string, T>::value, bool>::type = 0>
-void add_serialization(py::class_<SK>& clazz) {
-  clazz.def(
-        "serialize",
-        [](const SK& sk) {
-          auto bytes = sk.serialize();
-          return py::bytes(reinterpret_cast<const char*>(bytes.data()), bytes.size());
-        },
-        "Serializes the sketch into a bytes object."
-    )
-    .def_static(
-        "deserialize",
-        [](const std::string& bytes) { return SK::deserialize(bytes.data(), bytes.size()); },
-        py::arg("bytes"),
-        "Deserializes the sketch from a bytes object."
-    );
-}
-
-// py::object and other types where the caller must provide a serde
-template<typename T, typename SK, typename std::enable_if<!std::is_arithmetic<T>::value && !std::is_same<std::string, T>::value, bool>::type = 0>
-void add_serialization(py::class_<SK>& clazz) {
-  clazz.def(
-        "serialize",
-        [](const SK& sk, datasketches::py_object_serde& serde) {
-          auto bytes = sk.serialize(0, serde);
-          return py::bytes(reinterpret_cast<const char*>(bytes.data()), bytes.size());
-        }, py::arg("serde"),
-        "Serializes the sketch into a bytes object using the provided serde."
-    )
-    .def_static(
-        "deserialize",
-        [](const std::string& bytes, datasketches::py_object_serde& serde) {
-            return SK::deserialize(bytes.data(), bytes.size(), serde);
-        }, py::arg("bytes"), py::arg("serde"),
-        "Deserializes the sketch from a bytes object using the provided serde."
-    );
-}
-
-// Vector Updates
-// * Only allowed for POD types based on numpy restriction, which
-//   is equivalent to both std::is_trivial and std::is_standard_layout.
-// * Nothing is added to types that are not PODs.
-// POD type
-template<typename T, typename SK, typename std::enable_if<std::is_trivial<T>::value && std::is_standard_layout<T>::value, bool>::type = 0>
-void add_vector_update(py::class_<SK>& clazz) {
-  clazz.def(
-    "update",
-    [](SK& sk, py::array_t<T, py::array::c_style | py::array::forcecast> items) {
-      if (items.ndim() != 1) {
-        throw std::invalid_argument("input data must have only one dimension. Found: "
-          + std::to_string(items.ndim()));
-      }
-      auto array = items.template unchecked<1>();
-      for (uint32_t i = 0; i < array.size(); ++i) sk.update(array(i));
-    },
-    py::arg("array"),
-    "Updates the sketch with the values in the given array"
-  );
-}
-
-// non-POD type
-template<typename T, typename SK, typename std::enable_if<!std::is_trivial<T>::value || !std::is_standard_layout<T>::value, bool>::type = 0>
-void add_vector_update(py::class_<SK>& clazz) {
-  unused(clazz);
-}
-
-#endif // _QUANTILE_CONDITIONAL_HPP_

data/vendor/datasketches-cpp/python/include/tuple_policy.hpp

@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include <memory>
-#include <pybind11/pybind11.h>
-
-#ifndef _TUPLE_POLICY_HPP_
-#define _TUPLE_POLICY_HPP_
-
-namespace py = pybind11;
-
-namespace datasketches {
-
-/**
- * @brief tuple_policy provides the underlying base class from
- *        which native Python policies ultimately inherit. The actual
- *        policies implement TuplePolicy, as shown in TuplePolicy.py
- */
-struct tuple_policy {
-  virtual py::object create_summary() const = 0;
-  virtual py::object update_summary(py::object& summary, const py::object& update) const = 0;
-  virtual py::object operator()(py::object& summary, const py::object& update) const = 0;
-  virtual ~tuple_policy() = default;
-};
-
-/**
- * @brief TuplePolicy provides the "trampoline" class for pybind11
- *        that allows for a native Python implementation of tuple
- *        sketch policies.
- */
-struct TuplePolicy : public tuple_policy {
-  using tuple_policy::tuple_policy;
-
-  /**
-   * @brief Create a summary object
-   * 
-   * @return py::object representing a new summary
-   */
-  py::object create_summary() const override {
-    PYBIND11_OVERRIDE_PURE(
-      py::object,          // Return type
-      tuple_policy,        // Parent class
-      create_summary,      // Name of function in C++ (must match Python name)
-                           // Argument(s) -- if any
-    );
-  }
-
-  /**
-   * @brief Update a summary object using this policy
-   * 
-   * @param summary The current summary to update
-   * @param update The new value with which to update the summary
-   * @return py::object The updated summary
-   */
-  py::object update_summary(py::object& summary, const py::object& update) const override {
-    PYBIND11_OVERRIDE_PURE(
-      py::object,          // Return type
-      tuple_policy,        // Parent class
-      update_summary,      // Name of function in C++ (must match Python name)
-      summary, update      // Arguments
-    );
-  }
-
-  /**
-   * @brief Applies this policy to summary with the provided update
-   * 
-   * @param summary The current summary on which to apply the policy
-   * @param update An update to apply to the current summary
-   * @return py::object The potentially modified summary
-   */
-  py::object operator()(py::object& summary, const py::object& update) const override {
-    PYBIND11_OVERRIDE_PURE_NAME(
-      py::object,          // Return type
-      tuple_policy,        // Parent class
-      "__call__",          // Name of function in python
-      operator(),          // Name of function in C++
-      summary, update      // Arguemnts
-    );
-  }
-};
-
-/* The tuple_policy_holder provides a concrete class that dispatches calls
- * from the sketch to the tuple_policy. This class is needed to provide a
- * concrete object to produce a compiled library, but library users should
- * never need to use this directly.
- */
-struct tuple_policy_holder {
-  explicit tuple_policy_holder(std::shared_ptr<tuple_policy> policy) : _policy(policy) {}
-  tuple_policy_holder(const tuple_policy_holder& other) : _policy(other._policy) {}
-  tuple_policy_holder(tuple_policy_holder&& other) : _policy(std::move(other._policy)) {}
-  tuple_policy_holder& operator=(const tuple_policy_holder& other) { _policy = other._policy; return *this; }
-  tuple_policy_holder& operator=(tuple_policy_holder&& other) { std::swap(_policy, other._policy); return *this; }
-
-  py::object create() const { return _policy->create_summary(); }
-  
-  void update(py::object& summary, const py::object& update) const {
-    summary = _policy->update_summary(summary, update);
-  }
-
-  void operator()(py::object& summary, const py::object& update) const {
-    summary = _policy->operator()(summary, update);
-  }
-
-  private:
-    std::shared_ptr<tuple_policy> _policy;
-};
-
-/* A degenerate policy used to enable Jaccard Similarity on tuple sketches,
- * where the computation requires a union and intersection over the keys but
- * does not need to observe the summaries.
- */
-struct dummy_jaccard_policy {
-  void operator()(py::object&, const py::object&) const {
-    return;
-  }
-};
-
-}
-
-#endif // _TUPLE_POLICY_HPP_

data/vendor/datasketches-cpp/python/jupyter/CPCSketch.ipynb

@@ -1,345 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## CPC Sketch Examples"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Basic Sketch Usage"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datasketches import cpc_sketch, cpc_union"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We'll create a sketch with log2(k) = 12"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sk = cpc_sketch(12)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Insert ~2 million points. Values are hashed, so using sequential integers is fine for demonstration purposes."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "### CPC sketch summary:\n",
-      "   lgK            : 12\n",
-      "   seed hash      : 93cc\n",
-      "   C              : 38212\n",
-      "   flavor         : 4\n",
-      "   merged         : false\n",
-      "   compressed     : false\n",
-      "   intresting col : 5\n",
-      "   HIP estimate   : 2.09721e+06\n",
-      "   kxp            : 11.4725\n",
-      "   offset         : 6\n",
-      "   table          : allocated\n",
-      "   num SV         : 135\n",
-      "   window         : allocated\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "n = 1 << 21\n",
-    "for i in range(0, n):\n",
-    "    sk.update(i)\n",
-    "print(sk)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Since we know the exact value of n we can look at the estimate and upper/lower bounds as a % of the true value. We'll look at the bounds at 1 standard deviation. In this case, the true value does lie within the bounds, but since these are probabilistic bounds the true value will sometimes be outside them (especially at 1 standard deviation)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Upper bound (1 std. dev) as % of true value:  100.9281\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Upper bound (1 std. dev) as % of true value: \", round(100*sk.get_upper_bound(1) / n, 4))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Estimate as % of true value:  100.0026\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Estimate as % of true value: \", round(100*sk.get_estimate() / n, 4))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Lower bound (1 std. dev) as % of true value:  99.0935\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Lower bound (1 std. dev) as % of true value: \", round(100*sk.get_lower_bound(1) / n, 4))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Finally, we can serialize and deserialize the sketch, which will give us back the same structure."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "2484"
-      ]
-     },
-     "execution_count": 7,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "sk_bytes = sk.serialize()\n",
-    "len(sk_bytes)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "### CPC sketch summary:\n",
-      "   lgK            : 12\n",
-      "   seed hash      : 93cc\n",
-      "   C              : 38212\n",
-      "   flavor         : 4\n",
-      "   merged         : false\n",
-      "   compressed     : false\n",
-      "   intresting col : 5\n",
-      "   HIP estimate   : 2.09721e+06\n",
-      "   kxp            : 11.4725\n",
-      "   offset         : 6\n",
-      "   table          : allocated\n",
-      "   num SV         : 135\n",
-      "   window         : allocated\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "sk2 = cpc_sketch.deserialize(sk_bytes)\n",
-    "print(sk2)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Sketch Union Usage"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Here, we'll create two sketches with partial overlap in values. For good measure, we'll let k be larger in one sketch. For most applications we'd generally create all new data using the same size sketch, allowing differences to creep in when combining new and historica data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "k = 12\n",
-    "n = 1 << 20\n",
-    "offset = int(3 * n / 4)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sk1 = cpc_sketch(k)\n",
-    "sk2 = cpc_sketch(k + 1)\n",
-    "for i in range(0, n):\n",
-    "    sk1.update(i)\n",
-    "    sk2.update(i + offset)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Create a union object and add the sketches to that. To demonstrate smoothly handling multiple sketch sizes, we'll use a size of k+1 here."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "union = cpc_union(k+1)\n",
-    "union.update(sk1)\n",
-    "union.update(sk2)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Note how log config k has automatically adopted the value of the smaller input sketch."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "### CPC sketch summary:\n",
-      "   lgK            : 12\n",
-      "   seed hash      : 93cc\n",
-      "   C              : 37418\n",
-      "   flavor         : 4\n",
-      "   merged         : true\n",
-      "   compressed     : false\n",
-      "   intresting col : 5\n",
-      "   HIP estimate   : 0\n",
-      "   kxp            : 4096\n",
-      "   offset         : 6\n",
-      "   table          : allocated\n",
-      "   num SV         : 123\n",
-      "   window         : allocated\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "result = union.get_result()\n",
-    "print(result)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We can again compare against the exact result, in this case 1.75*n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Estimate as % of true value:  99.6646\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Estimate as % of true value: \", round(100*result.get_estimate() / (7*n/4), 4))"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.0"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}