datasketches 0.1.0

data/vendor/datasketches-cpp/hll/include/HllUtil.hpp

@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef _HLLUTIL_HPP_
+#define _HLLUTIL_HPP_
+
+#include "MurmurHash3.h"
+#include "RelativeErrorTables.hpp"
+#include "count_zeros.hpp"
+#include "common_defs.hpp"
+#include "ceiling_power_of_2.hpp"
+
+#include <cmath>
+#include <stdexcept>
+#include <string>
+
+namespace datasketches {
+
+enum hll_mode { LIST = 0, SET, HLL };
+
+// template provides internal consistency and allows static float values
+// but we don't use the template parameter anywhere
+template<typename A = std::allocator<char> >
+class HllUtil final {
+public:
+  // preamble stuff
+  static const int SER_VER = 1;
+  static const int FAMILY_ID = 7;
+
+  static const int EMPTY_FLAG_MASK          = 4;
+  static const int COMPACT_FLAG_MASK        = 8;
+  static const int OUT_OF_ORDER_FLAG_MASK   = 16;
+  static const int FULL_SIZE_FLAG_MASK      = 32;
+
+  static const int PREAMBLE_INTS_BYTE = 0;
+  static const int SER_VER_BYTE       = 1;
+  static const int FAMILY_BYTE        = 2;
+  static const int LG_K_BYTE          = 3;
+  static const int LG_ARR_BYTE        = 4;
+  static const int FLAGS_BYTE         = 5;
+  static const int LIST_COUNT_BYTE    = 6;
+  static const int HLL_CUR_MIN_BYTE   = 6;
+  static const int MODE_BYTE          = 7; // lo2bits = curMode, next 2 bits = tgtHllMode
+
+  // Coupon List
+  static const int LIST_INT_ARR_START = 8;
+  static const int LIST_PREINTS = 2;
+  // Coupon Hash Set
+  static const int HASH_SET_COUNT_INT             = 8;
+  static const int HASH_SET_INT_ARR_START         = 12;
+  static const int HASH_SET_PREINTS         = 3;
+  // HLL
+  static const int HLL_PREINTS = 10;
+  static const int HLL_BYTE_ARR_START = 40;
+  static const int HIP_ACCUM_DOUBLE = 8;
+  static const int KXQ0_DOUBLE = 16;
+  static const int KXQ1_DOUBLE = 24;
+  static const int CUR_MIN_COUNT_INT = 32;
+  static const int AUX_COUNT_INT = 36;
+  
+  static const int EMPTY_SKETCH_SIZE_BYTES = 8;
+
+  // other HllUtil stuff
+  static const int KEY_BITS_26 = 26;
+  static const int VAL_BITS_6 = 6;
+  static const int KEY_MASK_26 = (1 << KEY_BITS_26) - 1;
+  static const int VAL_MASK_6 = (1 << VAL_BITS_6) - 1;
+  static const int EMPTY = 0;
+  static const int MIN_LOG_K = 4;
+  static const int MAX_LOG_K = 21;
+
+  static const double HLL_HIP_RSE_FACTOR; // sqrt(log(2.0)) = 0.8325546
+  static const double HLL_NON_HIP_RSE_FACTOR; // sqrt((3.0 * log(2.0)) - 1.0) = 1.03896
+  static const double COUPON_RSE_FACTOR; // 0.409 at transition point not the asymptote
+  static const double COUPON_RSE; // COUPON_RSE_FACTOR / (1 << 13);
+
+  static const int LG_INIT_LIST_SIZE = 3;
+  static const int LG_INIT_SET_SIZE = 5;
+  static const int RESIZE_NUMER = 3;
+  static const int RESIZE_DENOM = 4;
+
+  static const int loNibbleMask = 0x0f;
+  static const int hiNibbleMask = 0xf0;
+  static const int AUX_TOKEN = 0xf;
+
+  /**
+  * Log2 table sizes for exceptions based on lgK from 0 to 26.
+  * However, only lgK from 4 to 21 are used.
+  */
+  static const int LG_AUX_ARR_INTS[];
+
+  static int coupon(const uint64_t hash[]);
+  static int coupon(const HashState& hashState);
+  static void hash(const void* key, int keyLen, uint64_t seed, HashState& result);
+  static int checkLgK(int lgK);
+  static void checkMemSize(uint64_t minBytes, uint64_t capBytes);
+  static inline void checkNumStdDev(int numStdDev);
+  static int pair(int slotNo, int value);
+  static int getLow26(unsigned int coupon);
+  static int getValue(unsigned int coupon);
+  static double invPow2(int e);
+  static unsigned int ceilingPowerOf2(unsigned int n);
+  static unsigned int simpleIntLog2(unsigned int n); // n must be power of 2
+  static int computeLgArrInts(hll_mode mode, int count, int lgConfigK);
+  static double getRelErr(bool upperBound, bool unioned,
+                          int lgConfigK, int numStdDev);
+};
+
+template<typename A>
+const double HllUtil<A>::HLL_HIP_RSE_FACTOR = sqrt(log(2.0)); // 0.8325546
+template<typename A>
+const double HllUtil<A>::HLL_NON_HIP_RSE_FACTOR = sqrt((3.0 * log(2.0)) - 1.0); // 1.03896
+template<typename A>
+const double HllUtil<A>::COUPON_RSE_FACTOR = 0.409;
+template<typename A>
+const double HllUtil<A>::COUPON_RSE = COUPON_RSE_FACTOR / (1 << 13);
+
+template<typename A>
+const int HllUtil<A>::LG_AUX_ARR_INTS[] = {
+      0, 2, 2, 2, 2, 2, 2, 3, 3, 3,   // 0 - 9
+      4, 4, 5, 5, 6, 7, 8, 9, 10, 11, // 10-19
+      12, 13, 14, 15, 16, 17, 18      // 20-26
+      };
+
+template<typename A>
+inline int HllUtil<A>::coupon(const uint64_t hash[]) {
+  int addr26 = (int) (hash[0] & KEY_MASK_26);
+  int lz = count_leading_zeros_in_u64(hash[1]);
+  int value = ((lz > 62 ? 62 : lz) + 1); 
+  return (value << KEY_BITS_26) | addr26;
+}
+
+template<typename A>
+inline int HllUtil<A>::coupon(const HashState& hashState) {
+  int addr26 = (int) (hashState.h1 & KEY_MASK_26);
+  int lz = count_leading_zeros_in_u64(hashState.h2);  
+  int value = ((lz > 62 ? 62 : lz) + 1); 
+  return (value << KEY_BITS_26) | addr26;
+}
+
+template<typename A>
+inline void HllUtil<A>::hash(const void* key, const int keyLen, const uint64_t seed, HashState& result) {
+  MurmurHash3_x64_128(key, keyLen, seed, result);
+}
+
+template<typename A>
+inline double HllUtil<A>::getRelErr(const bool upperBound, const bool unioned,
+                                    const int lgConfigK, const int numStdDev) {
+  return RelativeErrorTables<A>::getRelErr(upperBound, unioned, lgConfigK, numStdDev);
+}
+
+template<typename A>
+inline int HllUtil<A>::checkLgK(const int lgK) {
+  if ((lgK >= HllUtil<A>::MIN_LOG_K) && (lgK <= HllUtil<A>::MAX_LOG_K)) {
+    return lgK;
+  } else {
+    throw std::invalid_argument("Invalid value of k: " + std::to_string(lgK));
+  }
+}
+
+template<typename A>
+inline void HllUtil<A>::checkMemSize(const uint64_t minBytes, const uint64_t capBytes) {
+  if (capBytes < minBytes) {
+    throw std::invalid_argument("Given destination array is not large enough: " + std::to_string(capBytes));
+  }
+}
+
+template<typename A>
+inline void HllUtil<A>::checkNumStdDev(const int numStdDev) {
+  if ((numStdDev < 1) || (numStdDev > 3)) {
+    throw std::invalid_argument("NumStdDev may not be less than 1 or greater than 3.");
+  }
+}
+
+template<typename A>
+inline int HllUtil<A>::pair(const int slotNo, const int value) {
+  return (value << HllUtil<A>::KEY_BITS_26) | (slotNo & HllUtil<A>::KEY_MASK_26);
+}
+
+template<typename A>
+inline int HllUtil<A>::getLow26(const unsigned int coupon) {
+  return coupon & HllUtil<A>::KEY_MASK_26;
+}
+
+template<typename A>
+inline int HllUtil<A>::getValue(const unsigned int coupon) {
+  return coupon >> HllUtil<A>::KEY_BITS_26;
+}
+
+template<typename A>
+inline double HllUtil<A>::invPow2(const int e) {
+  union {
+    long long longVal;
+    double doubleVal;
+  } conv;
+  conv.longVal = (1023L - e) << 52;
+  return conv.doubleVal;
+}
+
+template<typename A>
+inline uint32_t HllUtil<A>::simpleIntLog2(uint32_t n) {
+  if (n == 0) {
+    throw std::logic_error("cannot take log of 0");
+  }
+  return count_trailing_zeros_in_u32(n);
+}
+
+template<typename A>
+inline int HllUtil<A>::computeLgArrInts(hll_mode mode, int count, int lgConfigK) {
+  // assume value missing and recompute
+  if (mode == LIST) { return HllUtil<A>::LG_INIT_LIST_SIZE; }
+  int ceilPwr2 = ceiling_power_of_2(count);
+  if ((HllUtil<A>::RESIZE_DENOM * count) > (HllUtil<A>::RESIZE_NUMER * ceilPwr2)) { ceilPwr2 <<= 1;}
+  if (mode == SET) {
+    return fmax(HllUtil<A>::LG_INIT_SET_SIZE, HllUtil<A>::simpleIntLog2(ceilPwr2));
+  }
+  //only used for HLL4
+  return fmax(HllUtil<A>::LG_AUX_ARR_INTS[lgConfigK], HllUtil<A>::simpleIntLog2(ceilPwr2));
+}
+
+}
+
+#endif /* _HLLUTIL_HPP_ */

data/vendor/datasketches-cpp/hll/include/RelativeErrorTables-internal.hpp

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef _RELATIVEERRORTABLES_INTERNAL_HPP_
+#define _RELATIVEERRORTABLES_INTERNAL_HPP_
+
+#include "RelativeErrorTables.hpp"
+
+namespace datasketches {
+
+//case 0
+static double HIP_LB[] = //sd 1, 2, 3
+  { //Q(.84134), Q(.97725), Q(.99865) respectively
+    0.207316195, 0.502865572, 0.882303765, //4
+    0.146981579, 0.335426881, 0.557052,    //5
+    0.104026721, 0.227683872, 0.365888317, //6
+    0.073614601, 0.156781585, 0.245740374, //7
+    0.05205248,  0.108783763, 0.168030442, //8
+    0.036770852, 0.075727545, 0.11593785,  //9
+    0.025990219, 0.053145536, 0.080772263, //10
+    0.018373987, 0.037266176, 0.056271814, //11
+    0.012936253, 0.02613829,  0.039387631, //12
+  };
+
+//case 1
+static double HIP_UB[] = //sd 1, 2, 3
+  { //Q(.15866), Q(.02275), Q(.00135) respectively
+    -0.207805347, -0.355574279, -0.475535095, //4
+    -0.146988328, -0.262390832, -0.360864026, //5
+    -0.103877775, -0.191503663, -0.269311582, //6
+    -0.073452978, -0.138513438, -0.198487447, //7
+    -0.051982806, -0.099703123, -0.144128618, //8
+    -0.036768609, -0.07138158,  -0.104430324, //9
+    -0.025991325, -0.050854296, -0.0748143,   //10
+    -0.01834533,  -0.036121138, -0.05327616,  //11
+    -0.012920332, -0.025572893, -0.037896952, //12
+  };
+
+//case 2
+static double NON_HIP_LB[] = //sd 1, 2, 3`
+  { //Q(.84134), Q(.97725), Q(.99865) respectively
+    0.254409839, 0.682266712, 1.304022158, //4
+    0.181817353, 0.443389054, 0.778776219, //5
+    0.129432281, 0.295782195, 0.49252279,  //6
+    0.091640655, 0.201175925, 0.323664385, //7
+    0.064858051, 0.138523393, 0.218805328, //8
+    0.045851855, 0.095925072, 0.148635751, //9
+    0.032454144, 0.067009668, 0.102660669, //10
+    0.022921382, 0.046868565, 0.071307398, //11
+    0.016155679, 0.032825719, 0.049677541  //12
+  };
+
+//case 3
+static double NON_HIP_UB[] = //sd 1, 2, 3
+  { //Q(.15866), Q(.02275), Q(.00135) respectively
+    -0.256980172, -0.411905944, -0.52651057,  //4
+    -0.182332109, -0.310275547, -0.412660505, //5
+    -0.129314228, -0.230142294, -0.315636197, //6
+    -0.091584836, -0.16834013,  -0.236346847, //7
+    -0.06487411,  -0.122045231, -0.174112107, //8
+    -0.04591465,  -0.08784505,  -0.126917615, //9
+    -0.032433119, -0.062897613, -0.091862929, //10
+    -0.022960633, -0.044875401, -0.065736049, //11
+    -0.016186662, -0.031827816, -0.046973459  //12
+  };
+
+template<typename A>
+double RelativeErrorTables<A>::getRelErr(const bool upperBound, const bool oooFlag,
+                                         const int lgK, const int stdDev) {
+  const int idx = ((lgK - 4) * 3) + (stdDev - 1);
+  const int sw = (oooFlag ? 2 : 0) | (upperBound ? 1 : 0);
+  double f = 0;
+  switch (sw) {
+    case 0 : { // HIP, LB
+      f = HIP_LB[idx];
+      break;
+    }
+    case 1 : { // HIP, UB
+      f = HIP_UB[idx];
+      break;
+    }
+    case 2 : { // NON_HIP, LB
+      f = NON_HIP_LB[idx];
+      break;
+    }
+    case 3 : { // NON_HIP, UB
+      f = NON_HIP_UB[idx];
+      break;
+    }
+  }
+  return f;
+}
+
+}
+
+#endif // _RELATIVEERRORTABLES_INTERNAL_HPP_

data/vendor/datasketches-cpp/hll/include/RelativeErrorTables.hpp

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef _RELATIVEERRORTABLES_HPP_
+#define _RELATIVEERRORTABLES_HPP_
+
+#include <memory>
+
+namespace datasketches {
+
+template<typename A = std::allocator<char>>
+class RelativeErrorTables {
+  public:
+    /**
+     * Return Relative Error for UB or LB for HIP or Non-HIP as a function of numStdDev.
+     * @param upperBound true if for upper bound
+     * @param oooFlag true if for Non-HIP
+     * @param lgK must be between 4 and 12 inclusive
+     * @param stdDev must be between 1 and 3 inclusive
+     * @return Relative Error for UB or LB for HIP or Non-HIP as a function of numStdDev.
+     */
+    static double getRelErr(bool upperBound, bool oooFlag,
+                            int lgK, int stdDev);
+};
+
+}
+
+#include "RelativeErrorTables-internal.hpp"
+
+#endif /* _RELATIVEERRORTABLES_HPP_ */

data/vendor/datasketches-cpp/hll/include/coupon_iterator-internal.hpp

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef _INTARRAYPAIRITERATOR_INTERNAL_HPP_
+#define _INTARRAYPAIRITERATOR_INTERNAL_HPP_
+
+#include "HllUtil.hpp"
+
+namespace datasketches {
+
+template<typename A>
+coupon_iterator<A>::coupon_iterator(const int* array, size_t array_size, size_t index, bool all):
+array(array), array_size(array_size), index(index), all(all) {
+  while (this->index < array_size) {
+    if (all || array[this->index] != HllUtil<A>::EMPTY) break;
+    this->index++;
+  }
+}
+
+template<typename A>
+coupon_iterator<A>& coupon_iterator<A>::operator++() {
+  while (++index < array_size) {
+    if (all || array[index] != HllUtil<A>::EMPTY) break;
+  }
+  return *this;
+}
+
+template<typename A>
+bool coupon_iterator<A>::operator!=(const coupon_iterator& other) const {
+  return index != other.index;
+}
+
+template<typename A>
+uint32_t coupon_iterator<A>::operator*() const {
+  return array[index];
+}
+
+}
+
+#endif // _INTARRAYPAIRITERATOR_INTERNAL_HPP_