datasketches 0.3.1 → 0.4.0

data/vendor/datasketches-cpp/python/include/py_serde.hpp

@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include <pybind11/pybind11.h>
-#include <pybind11/functional.h>
-#include <sstream>
-
-#ifndef _PY_SERDE_HPP_
-#define _PY_SERDE_HPP_
-
-namespace py = pybind11;
-
-namespace datasketches {
-
-/**
- * @brief The py_object_serde is an abstract class that implements the
- * datasketches serde interface, and is used to allow custom Python
- * serialization of items wrapped as generic py::object types. The actual
- * Python implementation classes must extend the PyObjectSerDe class.
- */
-struct py_object_serde {
-  /**
-   * @brief Get the serialized size of an object, in bytes
-   * 
-   * @param item A provided item
-   * @return int64_t The serialized size of the item, in bytes
-   */
-  virtual int64_t get_size(const py::object& item) const = 0;
-  
-  /**
-   * @brief Serializes an item to a bytes object
-   * 
-   * @param item A provided item
-   * @return The serialized image of the item as a Python bytes object
-   */
-  virtual py::bytes to_bytes(const py::object& item) const = 0;
-  
-  /**
-   * @brief Constructs an object from a serialized image, reading the
-   * incoming buffer starting at the specified offset.
-   * 
-   * @param bytes A buffer containing items from a serialized sketch
-   * @param offset The starting offset into the bytes buffer
-   * @return A Python tuple of the reconstructed item and the total number of bytes read
-   */
-  virtual py::tuple from_bytes(py::bytes& bytes, size_t offset) const = 0;
-
-  virtual ~py_object_serde() = default;
-
-  // these methods are required by the serde interface; see common/include/serde.hpp for
-  // default implementations for C++ std::string and numeric types.
-  size_t size_of_item(const py::object& item) const;
-  size_t serialize(void* ptr, size_t capacity, const py::object* items, unsigned num) const;
-  size_t deserialize(const void* ptr, size_t capacity, py::object* items, unsigned num) const;
-};
-
-/**
- * @brief The PyObjectSerDe class provides a concrete base class
- * that pybind11 uses as a "trampoline" to pass calls through to
- * the abstract py_object_serde class. Custom Python serde implementations
- * must extend this class.
- */
-struct PyObjectSerDe : public py_object_serde {
-  using py_object_serde::py_object_serde;
-
-  // trampoline definitions -- need one for each virtual function
-  int64_t get_size(const py::object& item) const override {
-    PYBIND11_OVERRIDE_PURE(
-      int64_t,         // Return type
-      py_object_serde, // Parent class
-      get_size,        // Name of function in C++ (must match Python name)
-      item             // Argument(s)
-    );
-  }
-
-  py::bytes to_bytes(const py::object& item) const override {
-    PYBIND11_OVERRIDE_PURE(
-      py::bytes,       // Return type
-      py_object_serde, // Parent class
-      to_bytes,        // Name of function in C++ (must match Python name)
-      item             // Argument(s)
-    );
-  }
-
-  py::tuple from_bytes(py::bytes& bytes, size_t offset) const override {
-    PYBIND11_OVERRIDE_PURE(
-      py::tuple,        // Return type
-      py_object_serde,  // Parent class
-      from_bytes,       // Name of function in C++ (must match Python name)
-      bytes, offset     // Argument(s)
-    );
-  }
-};
-
-}
-
-#endif // _PY_SERDE_HPP_

data/vendor/datasketches-cpp/python/jupyter/CPCSketch.ipynb

@@ -1,345 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## CPC Sketch Examples"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Basic Sketch Usage"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datasketches import cpc_sketch, cpc_union"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We'll create a sketch with log2(k) = 12"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sk = cpc_sketch(12)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Insert ~2 million points. Values are hashed, so using sequential integers is fine for demonstration purposes."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "### CPC sketch summary:\n",
-      "   lgK            : 12\n",
-      "   seed hash      : 93cc\n",
-      "   C              : 38212\n",
-      "   flavor         : 4\n",
-      "   merged         : false\n",
-      "   compressed     : false\n",
-      "   intresting col : 5\n",
-      "   HIP estimate   : 2.09721e+06\n",
-      "   kxp            : 11.4725\n",
-      "   offset         : 6\n",
-      "   table          : allocated\n",
-      "   num SV         : 135\n",
-      "   window         : allocated\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "n = 1 << 21\n",
-    "for i in range(0, n):\n",
-    "    sk.update(i)\n",
-    "print(sk)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Since we know the exact value of n we can look at the estimate and upper/lower bounds as a % of the true value. We'll look at the bounds at 1 standard deviation. In this case, the true value does lie within the bounds, but since these are probabilistic bounds the true value will sometimes be outside them (especially at 1 standard deviation)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Upper bound (1 std. dev) as % of true value:  100.9281\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Upper bound (1 std. dev) as % of true value: \", round(100*sk.get_upper_bound(1) / n, 4))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Estimate as % of true value:  100.0026\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Estimate as % of true value: \", round(100*sk.get_estimate() / n, 4))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Lower bound (1 std. dev) as % of true value:  99.0935\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Lower bound (1 std. dev) as % of true value: \", round(100*sk.get_lower_bound(1) / n, 4))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Finally, we can serialize and deserialize the sketch, which will give us back the same structure."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "2484"
-      ]
-     },
-     "execution_count": 7,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "sk_bytes = sk.serialize()\n",
-    "len(sk_bytes)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "### CPC sketch summary:\n",
-      "   lgK            : 12\n",
-      "   seed hash      : 93cc\n",
-      "   C              : 38212\n",
-      "   flavor         : 4\n",
-      "   merged         : false\n",
-      "   compressed     : false\n",
-      "   intresting col : 5\n",
-      "   HIP estimate   : 2.09721e+06\n",
-      "   kxp            : 11.4725\n",
-      "   offset         : 6\n",
-      "   table          : allocated\n",
-      "   num SV         : 135\n",
-      "   window         : allocated\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "sk2 = cpc_sketch.deserialize(sk_bytes)\n",
-    "print(sk2)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Sketch Union Usage"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Here, we'll create two sketches with partial overlap in values. For good measure, we'll let k be larger in one sketch. For most applications we'd generally create all new data using the same size sketch, allowing differences to creep in when combining new and historica data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "k = 12\n",
-    "n = 1 << 20\n",
-    "offset = int(3 * n / 4)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sk1 = cpc_sketch(k)\n",
-    "sk2 = cpc_sketch(k + 1)\n",
-    "for i in range(0, n):\n",
-    "    sk1.update(i)\n",
-    "    sk2.update(i + offset)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Create a union object and add the sketches to that. To demonstrate smoothly handling multiple sketch sizes, we'll use a size of k+1 here."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "union = cpc_union(k+1)\n",
-    "union.update(sk1)\n",
-    "union.update(sk2)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Note how log config k has automatically adopted the value of the smaller input sketch."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "### CPC sketch summary:\n",
-      "   lgK            : 12\n",
-      "   seed hash      : 93cc\n",
-      "   C              : 37418\n",
-      "   flavor         : 4\n",
-      "   merged         : true\n",
-      "   compressed     : false\n",
-      "   intresting col : 5\n",
-      "   HIP estimate   : 0\n",
-      "   kxp            : 4096\n",
-      "   offset         : 6\n",
-      "   table          : allocated\n",
-      "   num SV         : 123\n",
-      "   window         : allocated\n",
-      "### End sketch summary\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "result = union.get_result()\n",
-    "print(result)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We can again compare against the exact result, in this case 1.75*n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Estimate as % of true value:  99.6646\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(\"Estimate as % of true value: \", round(100*result.get_estimate() / (7*n/4), 4))"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.0"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}