datasketches 0.1.0

data/vendor/datasketches-cpp/cpc/include/cpc_confidence.hpp

@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// author Kevin Lang, Oath Research
+
+#ifndef CPC_CONFIDENCE_HPP_
+#define CPC_CONFIDENCE_HPP_
+
+#include <cmath>
+
+#include "cpc_sketch.hpp"
+
+namespace datasketches {
+
+// ln 2.0
+static const double ICON_ERROT_CONSTANT = 0.693147180559945286;
+
+//  1,    2,    3, // kappa
+static const int16_t ICON_LOW_SIDE_DATA [33] = {   // Empirically measured at N = 1000 * K.
+ 6037, 5720, 5328, // 4 1000000
+ 6411, 6262, 5682, // 5 1000000
+ 6724, 6403, 6127, // 6 1000000
+ 6665, 6411, 6208, // 7 1000000
+ 6959, 6525, 6427, // 8 1000000
+ 6892, 6665, 6619, // 9 1000000
+ 6792, 6752, 6690, // 10 1000000
+ 6899, 6818, 6708, // 11 1000000
+ 6871, 6845, 6812, // 12 1046369
+ 6909, 6861, 6828, // 13 1043411
+ 6919, 6897, 6842, // 14 1000297
+};                 // lgK numtrials
+
+//  1,    2,    3, // kappa
+static const int16_t ICON_HIGH_SIDE_DATA [33] = {   // Empirically measured at N = 1000 * K.
+ 8031, 8559, 9309, // 4 1000000
+ 7084, 7959, 8660, // 5 1000000
+ 7141, 7514, 7876, // 6 1000000
+ 7458, 7430, 7572, // 7 1000000
+ 6892, 7141, 7497, // 8 1000000
+ 6889, 7132, 7290, // 9 1000000
+ 7075, 7118, 7185, // 10 1000000
+ 7040, 7047, 7085, // 11 1000000
+ 6993, 7019, 7053, // 12 1046369
+ 6953, 7001, 6983, // 13 1043411
+ 6944, 6966, 7004, // 14 1000297
+};                 // lgK numtrials
+
+// sqrt((ln 2.0) / 2.0)
+static const double HIP_ERROR_CONSTANT = 0.588705011257737332;
+
+//  1,    2,    3, // kappa
+static const int16_t HIP_LOW_SIDE_DATA [33] = {   // Empirically measured at N = 1000 * K.
+ 5871, 5247, 4826, // 4 1000000
+ 5877, 5403, 5070, // 5 1000000
+ 5873, 5533, 5304, // 6 1000000
+ 5878, 5632, 5464, // 7 1000000
+ 5874, 5690, 5564, // 8 1000000
+ 5880, 5745, 5619, // 9 1000000
+ 5875, 5784, 5701, // 10 1000000
+ 5866, 5789, 5742, // 11 1000000
+ 5869, 5827, 5784, // 12 1046369
+ 5876, 5860, 5827, // 13 1043411
+ 5881, 5853, 5842, // 14 1000297
+};                 // lgK numtrials
+
+//  1,    2,    3, // kappa
+static const int16_t HIP_HIGH_SIDE_DATA [33] = {   // Empirically measured at N = 1000 * K.
+ 5855, 6688, 7391, // 4 1000000
+ 5886, 6444, 6923, // 5 1000000
+ 5885, 6254, 6594, // 6 1000000
+ 5889, 6134, 6326, // 7 1000000
+ 5900, 6072, 6203, // 8 1000000
+ 5875, 6005, 6089, // 9 1000000
+ 5871, 5980, 6040, // 10 1000000
+ 5889, 5941, 6015, // 11 1000000
+ 5871, 5926, 5973, // 12 1046369
+ 5866, 5901, 5915, // 13 1043411
+ 5880, 5914, 5953, // 14 1000297
+};                 // lgK numtrials
+
+template<typename A>
+double get_icon_confidence_lb(const cpc_sketch_alloc<A>& sketch, int kappa) {
+  if (sketch.get_num_coupons() == 0) return 0.0;
+  const int lg_k = sketch.get_lg_k();
+  const long k = 1 << lg_k;
+  if (lg_k < 4) throw std::logic_error("lgk < 4");
+  if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3");
+  double x = ICON_ERROT_CONSTANT;
+  if (lg_k <= 14) x = ((double) ICON_HIGH_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0;
+  const double rel = x / sqrt(k);
+  const double eps = kappa * rel;
+  const double est = sketch.get_icon_estimate();
+  double result = est / (1.0 + eps);
+  const double check = sketch.get_num_coupons();
+  if (result < check) result = check;
+  return result;
+}
+
+template<typename A>
+double get_icon_confidence_ub(const cpc_sketch_alloc<A>& sketch, int kappa) {
+  if (sketch.get_num_coupons() == 0) return 0.0;
+  const int lg_k = sketch.get_lg_k();
+  const long k = 1 << lg_k;
+  if (lg_k < 4) throw std::logic_error("lgk < 4");
+  if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3");
+  double x = ICON_ERROT_CONSTANT;
+  if (lg_k <= 14) x = ((double) ICON_LOW_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0;
+  const double rel = x / sqrt(k);
+  const double eps = kappa * rel;
+  const double est = sketch.get_icon_estimate();
+  const double result = est / (1.0 - eps);
+  return ceil(result); // widening for coverage
+}
+
+template<typename A>
+double get_hip_confidence_lb(const cpc_sketch_alloc<A>& sketch, int kappa) {
+  if (sketch.get_num_coupons() == 0) return 0.0;
+  const int lg_k = sketch.get_lg_k();
+  const long k = 1 << lg_k;
+  if (lg_k < 4) throw std::logic_error("lgk < 4");
+  if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3");
+  double x = HIP_ERROR_CONSTANT;
+  if (lg_k <= 14) x = ((double) HIP_HIGH_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0;
+  const double rel = x / (sqrt((double) k));
+  const double eps = ((double) kappa) * rel;
+  const double est = sketch.get_hip_estimate();
+  double result = est / (1.0 + eps);
+  const double check = (double) sketch.get_num_coupons();
+  if (result < check) result = check;
+  return result;
+}
+
+template<typename A>
+double get_hip_confidence_ub(const cpc_sketch_alloc<A>& sketch, int kappa) {
+  if (sketch.get_num_coupons() == 0) return 0.0;
+  const int lg_k = sketch.get_lg_k();
+  const long k = 1 << lg_k;
+  if (lg_k < 4) throw std::logic_error("lgk < 4");
+  if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3");
+  double x = HIP_ERROR_CONSTANT;
+  if (lg_k <= 14) x = ((double) HIP_LOW_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0;
+  const double rel = x / sqrt(k);
+  const double eps = kappa * rel;
+  const double est = sketch.get_hip_estimate();
+  const double result = est / (1.0 - eps);
+  return ceil(result); // widening for coverage
+}
+
+} /* namespace datasketches */
+
+#endif

data/vendor/datasketches-cpp/cpc/include/cpc_sketch.hpp

@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef CPC_SKETCH_HPP_
+#define CPC_SKETCH_HPP_
+
+#include <iostream>
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "u32_table.hpp"
+#include "cpc_common.hpp"
+#include "cpc_compressor.hpp"
+#include "cpc_confidence.hpp"
+#include "common_defs.hpp"
+
+namespace datasketches {
+
+/*
+ * High performance C++ implementation of Compressed Probabilistic Counting (CPC) Sketch
+ *
+ * This is a very compact (in serialized form) distinct counting sketch.
+ * The theory is described in the following paper:
+ * https://arxiv.org/abs/1708.06839
+ *
+ * author Kevin Lang
+ * author Alexander Saydakov
+ */
+
+// forward-declarations
+template<typename A> class cpc_sketch_alloc;
+template<typename A> class cpc_union_alloc;
+
+// alias with default allocator for convenience
+typedef cpc_sketch_alloc<std::allocator<void>> cpc_sketch;
+
+// allocation and initialization of global decompression (decoding) tables
+// call this before anything else if you want to control the initialization time
+// for instance, to have this happen outside of a transaction context
+// otherwise initialization happens on the first use (serialization or deserialization)
+// it is safe to call more than once assuming no race conditions
+// this is not thread safe! neither is the rest of the library
+template<typename A> void cpc_init();
+
+template<typename A>
+class cpc_sketch_alloc {
+public:
+  /**
+   * Creates an instance of the sketch given the lg_k parameter and hash seed.
+   * @param lg_k base 2 logarithm of the number of bins in the sketch
+   * @param seed for hash function
+   */
+  explicit cpc_sketch_alloc(uint8_t lg_k = CPC_DEFAULT_LG_K, uint64_t seed = DEFAULT_SEED);
+
+  /**
+   * @return configured lg_k of this sketch
+   */
+  uint8_t get_lg_k() const;
+
+  /**
+   * @return true if this sketch represents an empty set
+   */
+  bool is_empty() const;
+
+  /**
+   * @return estimate of the distinct count of the input stream
+   */
+  double get_estimate() const;
+
+  /**
+   * Returns the approximate lower error bound given a parameter kappa (1, 2 or 3).
+   * This parameter is similar to the number of standard deviations of the normal distribution
+   * and corresponds to approximately 67%, 95% and 99% confidence intervals.
+   * @param kappa parameter to specify confidence interval (1, 2 or 3)
+   * @return the lower bound
+   */
+  double get_lower_bound(unsigned kappa) const;
+
+  /**
+   * Returns the approximate upper error bound given a parameter kappa (1, 2 or 3).
+   * This parameter is similar to the number of standard deviations of the normal distribution
+   * and corresponds to approximately 67%, 95% and 99% confidence intervals.
+   * @param kappa parameter to specify confidence interval (1, 2 or 3)
+   * @return the upper bound
+   */
+  double get_upper_bound(unsigned kappa) const;
+
+  /**
+   * Update this sketch with a given string.
+   * @param value string to update the sketch with
+   */
+  void update(const std::string& value);
+
+  /**
+   * Update this sketch with a given unsigned 64-bit integer.
+   * @param value uint64_t to update the sketch with
+   */
+  void update(uint64_t value);
+
+  /**
+   * Update this sketch with a given signed 64-bit integer.
+   * @param value int64_t to update the sketch with
+   */
+  void update(int64_t value);
+
+  /**
+   * Update this sketch with a given unsigned 32-bit integer.
+   * For compatibility with Java implementation.
+   * @param value uint32_t to update the sketch with
+   */
+  void update(uint32_t value);
+
+  /**
+   * Update this sketch with a given signed 32-bit integer.
+   * For compatibility with Java implementation.
+   * @param value int32_t to update the sketch with
+   */
+  void update(int32_t value);
+
+  /**
+   * Update this sketch with a given unsigned 16-bit integer.
+   * For compatibility with Java implementation.
+   * @param value uint16_t to update the sketch with
+   */
+  void update(uint16_t value);
+
+  /**
+   * Update this sketch with a given signed 16-bit integer.
+   * For compatibility with Java implementation.
+   * @param value int16_t to update the sketch with
+   */
+  void update(int16_t value);
+
+  /**
+   * Update this sketch with a given unsigned 8-bit integer.
+   * For compatibility with Java implementation.
+   * @param value uint8_t to update the sketch with
+   */
+  void update(uint8_t value);
+
+  /**
+   * Update this sketch with a given signed 8-bit integer.
+   * For compatibility with Java implementation.
+   * @param value int8_t to update the sketch with
+   */
+  void update(int8_t value);
+
+  /**
+   * Update this sketch with a given double-precision floating point value.
+   * For compatibility with Java implementation.
+   * @param value double to update the sketch with
+   */
+  void update(double value);
+
+  /**
+   * Update this sketch with a given floating point value.
+   * For compatibility with Java implementation.
+   * @param value float to update the sketch with
+   */
+  void update(float value);
+
+  /**
+   * Update this sketch with given data of any type.
+   * This is a "universal" update that covers all cases above,
+   * but may produce different hashes.
+   * Be very careful to hash input values consistently using the same approach
+   * both over time and on different platforms
+   * and while passing sketches between C++ environment and Java environment.
+   * Otherwise two sketches that should represent overlapping sets will be disjoint
+   * For instance, for signed 32-bit values call update(int32_t) method above,
+   * which does widening conversion to int64_t, if compatibility with Java is expected
+   * @param data pointer to the data
+   * @param length of the data in bytes
+   */
+  void update(const void* value, int size);
+
+  /**
+   * Returns a human-readable summary of this sketch
+   */
+  string<A> to_string() const;
+
+  /**
+   * This method serializes the sketch into a given stream in a binary form
+   * @param os output stream
+   */
+  void serialize(std::ostream& os) const;
+
+  // This is a convenience alias for users
+  // The type returned by the following serialize method
+  typedef vector_u8<A> vector_bytes;
+
+  /**
+   * This method serializes the sketch as a vector of bytes.
+   * An optional header can be reserved in front of the sketch.
+   * It is an uninitialized space of a given size.
+   * This header is used in Datasketches PostgreSQL extension.
+   * @param header_size_bytes space to reserve in front of the sketch
+   */
+  vector_bytes serialize(unsigned header_size_bytes = 0) const;
+
+  /**
+   * This method deserializes a sketch from a given stream.
+   * @param is input stream
+   * @param seed the seed for the hash function that was used to create the sketch
+   * @return an instance of a sketch
+   */
+  static cpc_sketch_alloc<A> deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED);
+
+  /**
+   * This method deserializes a sketch from a given array of bytes.
+   * @param bytes pointer to the array of bytes
+   * @param size the size of the array
+   * @param seed the seed for the hash function that was used to create the sketch
+   * @return an instance of the sketch
+   */
+  static cpc_sketch_alloc<A> deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED);
+
+  // for internal use
+  uint32_t get_num_coupons() const;
+
+  // for debugging
+  // this should catch some forms of corruption during serialization-deserialization
+  bool validate() const;
+
+private:
+  static const uint8_t SERIAL_VERSION = 1;
+  static const uint8_t FAMILY = 16;
+
+  enum flags { IS_BIG_ENDIAN, IS_COMPRESSED, HAS_HIP, HAS_TABLE, HAS_WINDOW };
+
+  // Note: except for brief transitional moments, these sketches always obey
+  // the following strict mapping between the flavor of a sketch and the
+  // number of coupons that it has collected
+  enum flavor {
+    EMPTY,   //     0 == C <     1
+    SPARSE,  //     1 <= C < 3K/32
+    HYBRID,  // 3K/32 <= C <   K/2
+    PINNED,  //   K/2 <= C < 27K/8  [NB: 27/8 = 3 + 3/8]
+    SLIDING  // 27K/8 <= C
+  };
+
+  uint8_t lg_k;
+  uint64_t seed;
+  bool was_merged; // is the sketch the result of merging?
+  uint32_t num_coupons; // the number of coupons collected so far
+
+  u32_table<A> surprising_value_table;
+  vector_u8<A> sliding_window;
+  uint8_t window_offset; // derivable from num_coupons, but made explicit for speed
+  uint8_t first_interesting_column; // This is part of a speed optimization
+
+  double kxp;
+  double hip_est_accum;
+
+  // for deserialization and cpc_union::get_result()
+  cpc_sketch_alloc(uint8_t lg_k, uint32_t num_coupons, uint8_t first_interesting_column, u32_table<A>&& table,
+      vector_u8<A>&& window, bool has_hip, double kxp, double hip_est_accum, uint64_t seed);
+
+  inline void row_col_update(uint32_t row_col);
+  inline void update_sparse(uint32_t row_col);
+  inline void update_windowed(uint32_t row_col);
+  inline void update_hip(uint32_t row_col);
+  void promote_sparse_to_windowed();
+  void move_window();
+  void refresh_kxp(const uint64_t* bit_matrix);
+
+  friend double get_hip_confidence_lb<A>(const cpc_sketch_alloc<A>& sketch, int kappa);
+  friend double get_hip_confidence_ub<A>(const cpc_sketch_alloc<A>& sketch, int kappa);
+  friend double get_icon_confidence_lb<A>(const cpc_sketch_alloc<A>& sketch, int kappa);
+  friend double get_icon_confidence_ub<A>(const cpc_sketch_alloc<A>& sketch, int kappa);
+  double get_hip_estimate() const;
+  double get_icon_estimate() const;
+
+  inline flavor determine_flavor() const;
+  static inline flavor determine_flavor(uint8_t lg_k, uint64_t c);
+
+  static inline uint8_t determine_correct_offset(uint8_t lg_k, uint64_t c);
+
+  // this produces a full-size k-by-64 bit matrix
+  vector_u64<A> build_bit_matrix() const;
+
+  static uint8_t get_preamble_ints(uint32_t num_coupons, bool has_hip, bool has_table, bool has_window);
+  inline void write_hip(std::ostream& os) const;
+  inline size_t copy_hip_to_mem(void* dst) const;
+
+  friend cpc_compressor<A>;
+  friend cpc_union_alloc<A>;
+};
+
+} /* namespace datasketches */
+
+#include "cpc_sketch_impl.hpp"
+
+#endif