datasketches 0.3.1 → 0.4.0

data/vendor/datasketches-cpp/LICENSE

@@ -205,7 +205,6 @@ APPENDIX A: How to apply the Apache License to your work.
     -------------------------------------------------------------
 
 
-
 APPENDIX B: Additional licenses relevant to this product.
 
     This product includes a number of source files with code that has been 
@@ -215,43 +214,6 @@ APPENDIX B: Additional licenses relevant to this product.
     conditions of the following licenses.
 
 
-
-    =============================================================
-    MIT License
-    =============================================================
-    Original source code:
-      https://github.com/benjaminjack/python_cpp_example
-    -------------------------------------------------------------   
-    Copyright (c) 2017 Benjamin R. Jack
-    
-    MIT License  (https://opensource.org/licenses/MIT):
-    
-    Permission is hereby granted, free of charge, to any person 
-    obtaining a copy of this software and associated documentation 
-    files (the "Software"), to deal in the Software without restriction, 
-    including without limitation the rights to use, copy, modify, merge, 
-    publish, distribute, sublicense, and/or sell copies of the Software, 
-    and to permit persons to whom the Software is furnished to do so, 
-    subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be 
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
-    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
-    OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
-    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 
-    BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 
-    ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 
-    CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 
-    SOFTWARE.
-    -------------------------------------------------------------
-    Code locations:
-      * https://github.com/apache/datasketches-cpp/blob/master/setup.py
-    that is adapted from the above.
-
-
-
     =============================================================
     Boost License  (https://www.boost.org/LICENSE_1_0.txt)
     =============================================================
@@ -287,44 +249,6 @@ APPENDIX B: Additional licenses relevant to this product.
     of CMake configuration if configured to build tests.
 
 
-    =============================================================
-    BSD License
-    =============================================================
-    Original source code:
-       https://github.com/pybind/pybind11/blob/master/LICENSE
-
-    Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch>, All rights reserved.
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are met:
-
-    1. Redistributions of source code must retain the above copyright notice, this
-       list of conditions and the following disclaimer.
-
-    2. Redistributions in binary form must reproduce the above copyright notice,
-       this list of conditions and the following disclaimer in the documentation
-       and/or other materials provided with the distribution.
-
-    3. Neither the name of the copyright holder nor the names of its contributors
-       may be used to endorse or promote products derived from this software
-       without specific prior written permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-    ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-    -------------------------------------------------------------
-    Code Locations:
-    Found only in the convenience binaries distributed from PyPI, which rely
-    on pybind11 code during compilation.
-
-
     =============================================================
     Public Domain
     =============================================================

data/vendor/datasketches-cpp/README.md

@@ -14,9 +14,7 @@ If you are interested in making contributions to this site please see our [Commu
 
 This code requires C++11.
 
-This includes Python bindings. For the Python interface, see the README notes in [the python subdirectory](https://github.com/apache/datasketches-cpp/tree/master/python).
-
-This library is header-only. The build process provided is only for building unit tests and the python library.
+This library is header-only. The build process provided is only for building unit tests.
 
 Building the unit tests requires cmake 3.12.0 or higher.
 

data/vendor/datasketches-cpp/common/CMakeLists.txt

@@ -35,19 +35,20 @@ install(TARGETS common EXPORT ${PROJECT_NAME})
 
 install(FILES 
 			${CMAKE_CURRENT_BINARY_DIR}/include/version.hpp
+      include/binomial_bounds.hpp
+      include/bounds_binomial_proportions.hpp
+      include/ceiling_power_of_2.hpp
 			include/common_defs.hpp
+      include/conditional_back_inserter.hpp
+      include/conditional_forward.hpp
+      include/count_zeros.hpp
+      include/inv_pow2_table.hpp
+      include/kolmogorov_smirnov_impl.hpp
+      include/kolmogorov_smirnov.hpp
 			include/memory_operations.hpp
 			include/MurmurHash3.h
-			include/serde.hpp
-			include/count_zeros.hpp
-			include/inv_pow2_table.hpp
-			include/binomial_bounds.hpp
-			include/conditional_back_inserter.hpp
-			include/conditional_forward.hpp
-			include/ceiling_power_of_2.hpp
-			include/bounds_binomial_proportions.hpp
+      include/optional.hpp
+      include/quantiles_sorted_view_impl.hpp
 			include/quantiles_sorted_view.hpp
-			include/quantiles_sorted_view_impl.hpp
-			include/kolmogorov_smirnov.hpp
-			include/kolmogorov_smirnov_impl.hpp
+      include/serde.hpp
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")

data/vendor/datasketches-cpp/common/include/MurmurHash3.h

@@ -29,14 +29,13 @@ typedef unsigned char uint8_t;
 typedef unsigned int uint32_t;
 typedef unsigned __int64 uint64_t;
 
-#define FORCE_INLINE	__forceinline
+#define MURMUR3_FORCE_INLINE	__forceinline
 
 #include <stdlib.h>
 
-#define ROTL32(x,y)	_rotl(x,y)
-#define ROTL64(x,y)	_rotl64(x,y)
+#define MURMUR3_ROTL64(x,y)	_rotl64(x,y)
 
-#define BIG_CONSTANT(x) (x)
+#define MURMUR3_BIG_CONSTANT(x) (x)
 
 // Other compilers
 
@@ -44,22 +43,16 @@ typedef unsigned __int64 uint64_t;
 
 #include <stdint.h>
 
-#define	FORCE_INLINE inline __attribute__((always_inline))
-
-inline uint32_t rotl32 ( uint32_t x, int8_t r )
-{
-  return (x << r) | (x >> (32 - r));
-}
+#define	MURMUR3_FORCE_INLINE inline __attribute__((always_inline))
 
 inline uint64_t rotl64 ( uint64_t x, int8_t r )
 {
   return (x << r) | (x >> (64 - r));
 }
 
-#define	ROTL32(x,y)	rotl32(x,y)
-#define ROTL64(x,y)	rotl64(x,y)
+#define MURMUR3_ROTL64(x,y)	rotl64(x,y)
 
-#define BIG_CONSTANT(x) (x##LLU)
+#define MURMUR3_BIG_CONSTANT(x) (x##LLU)
 
 #endif // !defined(_MSC_VER)
 
@@ -78,7 +71,7 @@ typedef struct {
 // Block read - if your platform needs to do endian-swapping or can only
 // handle aligned reads, do the conversion here
 
-FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, size_t i )
+MURMUR3_FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, size_t i )
 {
   uint64_t res;
   memcpy(&res, p + i, sizeof(res));
@@ -88,20 +81,21 @@ FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, size_t i )
 //-----------------------------------------------------------------------------
 // Finalization mix - force all bits of a hash block to avalanche
 
-FORCE_INLINE uint64_t fmix64 ( uint64_t k )
+MURMUR3_FORCE_INLINE uint64_t fmix64 ( uint64_t k )
 {
   k ^= k >> 33;
-  k *= BIG_CONSTANT(0xff51afd7ed558ccd);
+  k *= MURMUR3_BIG_CONSTANT(0xff51afd7ed558ccd);
   k ^= k >> 33;
-  k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53);
+  k *= MURMUR3_BIG_CONSTANT(0xc4ceb9fe1a85ec53);
   k ^= k >> 33;
 
   return k;
 }
 
-FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t seed, HashState& out) {
-  static const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5);
-  static const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f);
+MURMUR3_FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes,
+                                              uint64_t seed, HashState& out) {
+  static const uint64_t c1 = MURMUR3_BIG_CONSTANT(0x87c37b91114253d5);
+  static const uint64_t c2 = MURMUR3_BIG_CONSTANT(0x4cf5ad432745937f);
 
   const uint8_t* data = (const uint8_t*)key;
 
@@ -118,13 +112,13 @@ FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t
     uint64_t k1 = getblock64(blocks, i * 2 + 0);
     uint64_t k2 = getblock64(blocks, i * 2 + 1);
 
-    k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; out.h1 ^= k1;
-    out.h1 = ROTL64(out.h1,27);
+    k1 *= c1; k1  = MURMUR3_ROTL64(k1,31); k1 *= c2; out.h1 ^= k1;
+    out.h1 = MURMUR3_ROTL64(out.h1,27);
     out.h1 += out.h2;
     out.h1 = out.h1*5+0x52dce729;
 
-    k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; out.h2 ^= k2;
-    out.h2 = ROTL64(out.h2,31);
+    k2 *= c2; k2  = MURMUR3_ROTL64(k2,33); k2 *= c1; out.h2 ^= k2;
+    out.h2 = MURMUR3_ROTL64(out.h2,31);
     out.h2 += out.h1;
     out.h2 = out.h2*5+0x38495ab5;
   }
@@ -144,7 +138,7 @@ FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t
   case 11: k2 ^= ((uint64_t)tail[10]) << 16; // falls through
   case 10: k2 ^= ((uint64_t)tail[ 9]) << 8;  // falls through
   case  9: k2 ^= ((uint64_t)tail[ 8]) << 0;
-           k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; out.h2 ^= k2;
+           k2 *= c2; k2  = MURMUR3_ROTL64(k2,33); k2 *= c1; out.h2 ^= k2;
            // falls through
   case  8: k1 ^= ((uint64_t)tail[ 7]) << 56; // falls through
   case  7: k1 ^= ((uint64_t)tail[ 6]) << 48; // falls through
@@ -154,7 +148,7 @@ FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t
   case  3: k1 ^= ((uint64_t)tail[ 2]) << 16; // falls through
   case  2: k1 ^= ((uint64_t)tail[ 1]) << 8; // falls through
   case  1: k1 ^= ((uint64_t)tail[ 0]) << 0;
-           k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; out.h1 ^= k1;
+           k1 *= c1; k1  = MURMUR3_ROTL64(k1,31); k1 *= c2; out.h1 ^= k1;
   };
 
   //----------
@@ -175,10 +169,14 @@ FORCE_INLINE void MurmurHash3_x64_128(const void* key, size_t lenBytes, uint64_t
 
 //-----------------------------------------------------------------------------
 
-FORCE_INLINE uint16_t compute_seed_hash(uint64_t seed) {
+MURMUR3_FORCE_INLINE uint16_t compute_seed_hash(uint64_t seed) {
   HashState hashes;
   MurmurHash3_x64_128(&seed, sizeof(seed), 0, hashes);
   return static_cast<uint16_t>(hashes.h1 & 0xffff);
 }
 
+#undef MURMUR3_FORCE_INLINE
+#undef MURMUR3_ROTL64
+#undef MURMUR3_BIG_CONSTANT
+
 #endif // _MURMURHASH3_H_

data/vendor/datasketches-cpp/common/include/common_defs.hpp

@@ -26,27 +26,32 @@
 #include <iostream>
 #include <random>
 #include <chrono>
+#include <thread>
 
+/// DataSketches namespace
 namespace datasketches {
 
 static const uint64_t DEFAULT_SEED = 9001;
 
 enum resize_factor { X1 = 0, X2, X4, X8 };
 
-template<typename A> using AllocChar = typename std::allocator_traits<A>::template rebind_alloc<char>;
-template<typename A> using string = std::basic_string<char, std::char_traits<char>, AllocChar<A>>;
-
-// random bit
-static std::independent_bits_engine<std::mt19937, 1, uint32_t>
-  random_bit(static_cast<uint32_t>(std::chrono::system_clock::now().time_since_epoch().count()));
+template<typename A> using string = std::basic_string<char, std::char_traits<char>, typename std::allocator_traits<A>::template rebind_alloc<char>>;
 
 // common random declarations
 namespace random_utils {
   static std::random_device rd; // possibly unsafe in MinGW with GCC < 9.2
-  static std::mt19937_64 rand(rd());
-  static std::uniform_real_distribution<> next_double(0.0, 1.0);
-}
+  static thread_local std::mt19937_64 rand(rd());
+  static thread_local std::uniform_real_distribution<> next_double(0.0, 1.0);
 
+  // thread-safe random bit
+  static thread_local std::independent_bits_engine<std::mt19937, 1, uint32_t>
+    random_bit(static_cast<uint32_t>(std::chrono::system_clock::now().time_since_epoch().count()
+      + std::hash<std::thread::id>{}(std::this_thread::get_id())));
+
+  inline void override_seed(uint64_t s) {
+    rand.seed(s);
+  }
+}
 
 // utility function to hide unused compiler warning
 // usually has no additional cost
@@ -77,7 +82,7 @@ static inline void read(std::istream& is, T* ptr, size_t size_bytes) {
 }
 
 template<typename T>
-static inline void write(std::ostream& os, T& value) {
+static inline void write(std::ostream& os, T value) {
   os.write(reinterpret_cast<const char*>(&value), sizeof(T));
 }
 

data/vendor/datasketches-cpp/common/include/count_zeros.hpp

@@ -22,8 +22,6 @@
 
 #include <cstdint>
 
-#include <stdio.h>
-
 namespace datasketches {
 
 static const uint8_t byte_leading_zeros_table[256] = {
@@ -91,6 +89,17 @@ static inline uint8_t count_leading_zeros_in_u64(uint64_t input) {
     return 56 + byte_leading_zeros_table[(input      ) & FCLZ_MASK_08];
 }
 
+static inline uint8_t count_leading_zeros_in_u32(uint32_t input) {
+  if (input > FCLZ_MASK_24)
+    return      byte_leading_zeros_table[(input >> 24) & FCLZ_MASK_08];
+  if (input > FCLZ_MASK_16)
+    return 8 + byte_leading_zeros_table[(input >> 16) & FCLZ_MASK_08];
+  if (input > FCLZ_MASK_08)
+    return 16 + byte_leading_zeros_table[(input >>  8) & FCLZ_MASK_08];
+  if (true)
+    return 24 + byte_leading_zeros_table[(input      ) & FCLZ_MASK_08];
+}
+
 static inline uint8_t count_trailing_zeros_in_u32(uint32_t input) {
   for (int i = 0; i < 4; i++) {
     const int byte = input & 0xff;

data/vendor/datasketches-cpp/common/include/kolmogorov_smirnov.hpp

@@ -22,13 +22,16 @@
 
 namespace datasketches {
 
+/**
+ * Kolmogorov-Smirnov test for KLL or Quantiles sketches
+ */
 class kolmogorov_smirnov {
 public:
   /**
    * Computes the raw delta area between two quantile sketches for the Kolmogorov-Smirnov Test.
    * Will work for a type-matched pair of KLL or Quantiles sketches of the same parameterized type T.
-   * @param sketch1 KLL sketch 1
-   * @param sketch2 KLL sketch 2
+   * @param sketch1 sketch 1
+   * @param sketch2 sketch 2
    * @return the raw delta between two KLL quantile sketches
    */
   template<typename Sketch>
@@ -39,8 +42,8 @@ public:
    * Adjusts the computed threshold by the error epsilons of the two given sketches.
    * See <a href="https://en.wikipedia.org/wiki/Kolmogorov-Smirnov_test">Kolmogorov–Smirnov Test</a>
    * Will work for a type-matched pair of KLL or Quantiles sketches of the same parameterized type T.
-   * @param sketch1 KLL sketch 1
-   * @param sketch2 KLL sketch 2
+   * @param sketch1 sketch 1
+   * @param sketch2 sketch 2
    * @param p Target p-value. Typically .001 to .1, e.g., .05.
    * @return the adjusted threshold to be compared with the raw delta
    */
@@ -52,8 +55,8 @@ public:
    * Will work for a type-matched pair of KLL or Quantiles sketches of the same parameterized type T.
    * Note: if the given sketches have insufficient data or if the sketch sizes are too small,
    * this will return false.
-   * @param sketch1 KLL sketch 1
-   * @param sketch2 KLL sketch 2
+   * @param sketch1 sketch 1
+   * @param sketch2 sketch 2
    * @param p Target p-value. Typically .001 to .1, e.g., .05.
    * @return Boolean indicating whether we can reject the null hypothesis (that the sketches
    * reflect the same underlying distribution) using the provided p-value.

data/vendor/datasketches-cpp/common/include/memory_operations.hpp

@@ -24,6 +24,7 @@
 #include <exception>
 #include <iostream>
 #include <string>
+#include <cstring>
 
 namespace datasketches {
 
@@ -54,14 +55,14 @@ static inline size_t copy_to_mem(const void* src, void* dst, size_t size) {
 }
 
 template<typename T>
-static inline size_t copy_to_mem(const T& item, void* dst) {
-  memcpy(dst, &item, sizeof(T));
+static inline size_t copy_from_mem(const void* src, T& item) {
+  memcpy(&item, src, sizeof(T));
   return sizeof(T);
 }
 
 template<typename T>
-static inline size_t copy_from_mem(const void* src, T& item) {
-  memcpy(&item, src, sizeof(T));
+static inline size_t copy_to_mem(T item, void* dst) {
+  memcpy(dst, &item, sizeof(T));
   return sizeof(T);
 }
 

data/vendor/datasketches-cpp/common/include/optional.hpp

@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef _OPTIONAL_HPP_
+#define _OPTIONAL_HPP_
+
+// This is a simplistic substitute for std::optional until we require C++17
+
+#if (__cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L))
+#include <optional>
+using std::optional;
+#else
+
+#include <type_traits>
+
+namespace datasketches {
+
+template<typename T>
+class optional {
+public:
+
+  optional() noexcept: initialized_(false) {}
+
+  optional(const T& value) noexcept(std::is_nothrow_copy_constructible<T>::value) {
+    new (&value_) T(value);
+    initialized_ = true;
+  }
+
+  optional(T&& value) noexcept(std::is_nothrow_move_constructible<T>::value) {
+    new (&value_) T(std::move(value));
+    initialized_ = true;
+  }
+
+  // conversion from compatible types
+  template<typename TT>
+  optional(const optional<TT>& other) noexcept(std::is_nothrow_constructible<T, TT>::value): initialized_(false) {
+    if (other.initialized_) {
+      new (&value_) T(other.value_);
+      initialized_ = true;
+    }
+  }
+
+  optional(const optional& other) noexcept(std::is_nothrow_copy_constructible<T>::value): initialized_(false) {
+    if (other.initialized_) {
+      new (&value_) T(other.value_);
+      initialized_ = true;
+    }
+  }
+
+  optional(optional&& other) noexcept(std::is_nothrow_move_constructible<T>::value): initialized_(false) {
+    if (other.initialized_) {
+      new (&value_) T(std::move(other.value_));
+      initialized_ = true;
+    }
+  }
+
+  ~optional() noexcept(std::is_nothrow_destructible<T>::value) {
+     if (initialized_) value_.~T();
+  }
+
+  explicit operator bool() const noexcept {
+    return initialized_;
+  }
+
+  optional& operator=(const optional& other)
+      noexcept(std::is_nothrow_copy_constructible<T>::value && std::is_nothrow_copy_assignable<T>::value) {
+    if (initialized_) {
+      if (other.initialized_) {
+        value_ = other.value_;
+      } else {
+        reset();
+      }
+    } else {
+      if (other.initialized_) {
+        new (&value_) T(other.value_);
+        initialized_ = true;
+      }
+    }
+    return *this;
+  }
+
+  optional& operator=(optional&& other)
+      noexcept(std::is_nothrow_move_constructible<T>::value && std::is_nothrow_move_assignable<T>::value) {
+    if (initialized_) {
+      if (other.initialized_) {
+        value_ = std::move(other.value_);
+      } else {
+        reset();
+      }
+    } else {
+      if (other.initialized_) {
+        new (&value_) T(std::move(other.value_));
+        initialized_ = true;
+      }
+    }
+    return *this;
+  }
+
+  template<typename... Args>
+  void emplace(Args&&... args) noexcept(std::is_nothrow_constructible<T, Args...>::value) {
+    new (&value_) T(args...);
+    initialized_ = true;
+  }
+
+  T& operator*() & noexcept { return value_; }
+  const T& operator*() const & noexcept { return value_; }
+  T&& operator*() && noexcept { return std::move(value_); }
+  const T&& operator*() const && noexcept { return std::move(value_); }
+
+  T* operator->() noexcept { return &value_; }
+  const T* operator->() const noexcept { return &value_; }
+
+  void reset() noexcept(std::is_nothrow_destructible<T>::value) {
+    if (initialized_) value_.~T();
+    initialized_ = false;
+  }
+
+private:
+  union {
+    T value_;
+  };
+  bool initialized_;
+
+  // for converting constructor
+  template<typename TT> friend class optional;
+};
+
+} // namespace
+
+#endif // C++17
+
+#endif // _OPTIONAL_HPP_