datasketches 0.1.0

data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch_impl.hpp

@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename A>
+update_array_of_doubles_sketch_alloc<A>::update_array_of_doubles_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf,
+    uint64_t theta, uint64_t seed, const array_of_doubles_update_policy<A>& policy, const A& allocator):
+Base(lg_cur_size, lg_nom_size, rf, theta, seed, policy, allocator) {}
+
+
+template<typename A>
+uint8_t update_array_of_doubles_sketch_alloc<A>::get_num_values() const {
+  return this->policy_.get_num_values();
+}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A> update_array_of_doubles_sketch_alloc<A>::compact(bool ordered) const {
+  return compact_array_of_doubles_sketch_alloc<A>(*this, ordered);
+}
+
+// builder
+
+template<typename A>
+update_array_of_doubles_sketch_alloc<A>::builder::builder(const array_of_doubles_update_policy<A>& policy, const A& allocator):
+tuple_base_builder<builder, array_of_doubles_update_policy<A>, A>(policy, allocator) {}
+
+template<typename A>
+update_array_of_doubles_sketch_alloc<A> update_array_of_doubles_sketch_alloc<A>::builder::build() const {
+  return update_array_of_doubles_sketch_alloc<A>(this->starting_lg_size(), this->lg_k_, this->rf_, this->starting_theta(), this->seed_, this->policy_, this->allocator_);
+}
+
+// compact sketch
+
+template<typename A>
+template<typename S>
+compact_array_of_doubles_sketch_alloc<A>::compact_array_of_doubles_sketch_alloc(const S& other, bool ordered):
+Base(other, ordered), num_values_(other.get_num_values()) {}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A>::compact_array_of_doubles_sketch_alloc(bool is_empty, bool is_ordered,
+    uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries, uint8_t num_values):
+Base(is_empty, is_ordered, seed_hash, theta, std::move(entries)), num_values_(num_values) {}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A>::compact_array_of_doubles_sketch_alloc(uint8_t num_values, Base&& base):
+Base(std::move(base)), num_values_(num_values) {}
+
+template<typename A>
+uint8_t compact_array_of_doubles_sketch_alloc<A>::get_num_values() const {
+  return num_values_;
+}
+
+template<typename A>
+void compact_array_of_doubles_sketch_alloc<A>::serialize(std::ostream& os) const {
+  const uint8_t preamble_longs = 1;
+  os.write(reinterpret_cast<const char*>(&preamble_longs), sizeof(preamble_longs));
+  const uint8_t serial_version = SERIAL_VERSION;
+  os.write(reinterpret_cast<const char*>(&serial_version), sizeof(serial_version));
+  const uint8_t family = SKETCH_FAMILY;
+  os.write(reinterpret_cast<const char*>(&family), sizeof(family));
+  const uint8_t type = SKETCH_TYPE;
+  os.write(reinterpret_cast<const char*>(&type), sizeof(type));
+  const uint8_t flags_byte(
+    (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+    (this->get_num_retained() > 0 ? 1 << flags::HAS_ENTRIES : 0) |
+    (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+  );
+  os.write(reinterpret_cast<const char*>(&flags_byte), sizeof(flags_byte));
+  os.write(reinterpret_cast<const char*>(&num_values_), sizeof(num_values_));
+  const uint16_t seed_hash = this->get_seed_hash();
+  os.write(reinterpret_cast<const char*>(&seed_hash), sizeof(seed_hash));
+  os.write(reinterpret_cast<const char*>(&(this->theta_)), sizeof(uint64_t));
+  if (this->get_num_retained() > 0) {
+    const uint32_t num_entries = this->entries_.size();
+    os.write(reinterpret_cast<const char*>(&num_entries), sizeof(num_entries));
+    const uint32_t unused32 = 0;
+    os.write(reinterpret_cast<const char*>(&unused32), sizeof(unused32));
+    for (const auto& it: this->entries_) {
+      os.write(reinterpret_cast<const char*>(&it.first), sizeof(uint64_t));
+    }
+    for (const auto& it: this->entries_) {
+      os.write(reinterpret_cast<const char*>(it.second.data()), it.second.size() * sizeof(double));
+    }
+  }
+}
+
+template<typename A>
+auto compact_array_of_doubles_sketch_alloc<A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+  const uint8_t preamble_longs = 1;
+  const size_t size = header_size_bytes + 16 // preamble and theta
+      + (this->entries_.size() > 0 ? 8 : 0)
+      + (sizeof(uint64_t) + sizeof(double) * num_values_) * this->entries_.size();
+  vector_bytes bytes(size, 0, this->entries_.get_allocator());
+  uint8_t* ptr = bytes.data() + header_size_bytes;
+
+  ptr += copy_to_mem(&preamble_longs, ptr, sizeof(preamble_longs));
+  const uint8_t serial_version = SERIAL_VERSION;
+  ptr += copy_to_mem(&serial_version, ptr, sizeof(serial_version));
+  const uint8_t family = SKETCH_FAMILY;
+  ptr += copy_to_mem(&family, ptr, sizeof(family));
+  const uint8_t type = SKETCH_TYPE;
+  ptr += copy_to_mem(&type, ptr, sizeof(type));
+  const uint8_t flags_byte(
+    (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+    (this->get_num_retained() ? 1 << flags::HAS_ENTRIES : 0) |
+    (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+  );
+  ptr += copy_to_mem(&flags_byte, ptr, sizeof(flags_byte));
+  ptr += copy_to_mem(&num_values_, ptr, sizeof(num_values_));
+  const uint16_t seed_hash = this->get_seed_hash();
+  ptr += copy_to_mem(&seed_hash, ptr, sizeof(seed_hash));
+  ptr += copy_to_mem(&(this->theta_), ptr, sizeof(uint64_t));
+  if (this->get_num_retained() > 0) {
+    const uint32_t num_entries = this->entries_.size();
+    ptr += copy_to_mem(&num_entries, ptr, sizeof(num_entries));
+    const uint32_t unused32 = 0;
+    ptr += copy_to_mem(&unused32, ptr, sizeof(unused32));
+    for (const auto& it: this->entries_) {
+      ptr += copy_to_mem(&it.first, ptr, sizeof(uint64_t));
+    }
+    for (const auto& it: this->entries_) {
+      ptr += copy_to_mem(it.second.data(), ptr, it.second.size() * sizeof(double));
+    }
+  }
+  return bytes;
+}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A> compact_array_of_doubles_sketch_alloc<A>::deserialize(std::istream& is, uint64_t seed, const A& allocator) {
+  uint8_t preamble_longs;
+  is.read(reinterpret_cast<char*>(&preamble_longs), sizeof(preamble_longs));
+  uint8_t serial_version;
+  is.read(reinterpret_cast<char*>(&serial_version), sizeof(serial_version));
+  uint8_t family;
+  is.read(reinterpret_cast<char*>(&family), sizeof(family));
+  uint8_t type;
+  is.read(reinterpret_cast<char*>(&type), sizeof(type));
+  uint8_t flags_byte;
+  is.read(reinterpret_cast<char*>(&flags_byte), sizeof(flags_byte));
+  uint8_t num_values;
+  is.read(reinterpret_cast<char*>(&num_values), sizeof(num_values));
+  uint16_t seed_hash;
+  is.read(reinterpret_cast<char*>(&seed_hash), sizeof(seed_hash));
+  checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
+  checker<true>::check_sketch_family(family, SKETCH_FAMILY);
+  checker<true>::check_sketch_type(type, SKETCH_TYPE);
+  const bool has_entries = flags_byte & (1 << flags::HAS_ENTRIES);
+  if (has_entries) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+
+  uint64_t theta;
+  is.read(reinterpret_cast<char*>(&theta), sizeof(theta));
+  std::vector<Entry, AllocEntry> entries(allocator);
+  if (has_entries) {
+    uint32_t num_entries;
+    is.read(reinterpret_cast<char*>(&num_entries), sizeof(num_entries));
+    uint32_t unused32;
+    is.read(reinterpret_cast<char*>(&unused32), sizeof(unused32));
+    entries.reserve(num_entries);
+    std::vector<uint64_t, AllocU64> keys(num_entries, 0, allocator);
+    is.read(reinterpret_cast<char*>(keys.data()), num_entries * sizeof(uint64_t));
+    for (size_t i = 0; i < num_entries; ++i) {
+      aod<A> summary(num_values, allocator);
+      is.read(reinterpret_cast<char*>(summary.data()), num_values * sizeof(double));
+      entries.push_back(Entry(keys[i], std::move(summary)));
+    }
+  }
+  if (!is.good()) throw std::runtime_error("error reading from std::istream");
+  const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+  const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+  return compact_array_of_doubles_sketch_alloc(is_empty, is_ordered, seed_hash, theta, std::move(entries), num_values);
+}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A> compact_array_of_doubles_sketch_alloc<A>::deserialize(const void* bytes, size_t size, uint64_t seed, const A& allocator) {
+  ensure_minimum_memory(size, 16);
+  const char* ptr = static_cast<const char*>(bytes);
+  uint8_t preamble_longs;
+  ptr += copy_from_mem(ptr, &preamble_longs, sizeof(preamble_longs));
+  uint8_t serial_version;
+  ptr += copy_from_mem(ptr, &serial_version, sizeof(serial_version));
+  uint8_t family;
+  ptr += copy_from_mem(ptr, &family, sizeof(family));
+  uint8_t type;
+  ptr += copy_from_mem(ptr, &type, sizeof(type));
+  uint8_t flags_byte;
+  ptr += copy_from_mem(ptr, &flags_byte, sizeof(flags_byte));
+  uint8_t num_values;
+  ptr += copy_from_mem(ptr, &num_values, sizeof(num_values));
+  uint16_t seed_hash;
+  ptr += copy_from_mem(ptr, &seed_hash, sizeof(seed_hash));
+  checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
+  checker<true>::check_sketch_family(family, SKETCH_FAMILY);
+  checker<true>::check_sketch_type(type, SKETCH_TYPE);
+  const bool has_entries = flags_byte & (1 << flags::HAS_ENTRIES);
+  if (has_entries) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+
+  uint64_t theta;
+  ptr += copy_from_mem(ptr, &theta, sizeof(theta));
+  std::vector<Entry, AllocEntry> entries(allocator);
+  if (has_entries) {
+    ensure_minimum_memory(size, 24);
+    uint32_t num_entries;
+    ptr += copy_from_mem(ptr, &num_entries, sizeof(num_entries));
+    uint32_t unused32;
+    ptr += copy_from_mem(ptr, &unused32, sizeof(unused32));
+    ensure_minimum_memory(size, 24 + (sizeof(uint64_t) + sizeof(double) * num_values) * num_entries);
+    entries.reserve(num_entries);
+    std::vector<uint64_t, AllocU64> keys(num_entries, 0, allocator);
+    ptr += copy_from_mem(ptr, keys.data(), sizeof(uint64_t) * num_entries);
+    for (size_t i = 0; i < num_entries; ++i) {
+      aod<A> summary(num_values, allocator);
+      ptr += copy_from_mem(ptr, summary.data(), num_values * sizeof(double));
+      entries.push_back(Entry(keys[i], std::move(summary)));
+    }
+  }
+  const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+  const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+  return compact_array_of_doubles_sketch_alloc(is_empty, is_ordered, seed_hash, theta, std::move(entries), num_values);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/tuple/include/array_of_doubles_union.hpp

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef ARRAY_OF_DOUBLES_UNION_HPP_
+#define ARRAY_OF_DOUBLES_UNION_HPP_
+
+#include <vector>
+#include <memory>
+
+#include "array_of_doubles_sketch.hpp"
+#include "tuple_union.hpp"
+
+namespace datasketches {
+
+template<typename A = std::allocator<double>>
+struct array_of_doubles_union_policy_alloc {
+  array_of_doubles_union_policy_alloc(uint8_t num_values = 1): num_values_(num_values) {}
+
+  void operator()(aod<A>& summary, const aod<A>& other) const {
+    for (size_t i = 0; i < summary.size(); ++i) {
+      summary[i] += other[i];
+    }
+  }
+
+  uint8_t get_num_values() const {
+    return num_values_;
+  }
+private:
+  uint8_t num_values_;
+};
+
+using array_of_doubles_union_policy = array_of_doubles_union_policy_alloc<>;
+
+template<typename Allocator = std::allocator<double>>
+class array_of_doubles_union_alloc: public tuple_union<aod<Allocator>, array_of_doubles_union_policy_alloc<Allocator>, AllocAOD<Allocator>> {
+public:
+  using Policy = array_of_doubles_union_policy_alloc<Allocator>;
+  using Base = tuple_union<aod<Allocator>, Policy, AllocAOD<Allocator>>;
+  using CompactSketch = compact_array_of_doubles_sketch_alloc<Allocator>;
+  using resize_factor = theta_constants::resize_factor;
+
+  class builder;
+
+  CompactSketch get_result(bool ordered = true) const;
+
+private:
+  // for builder
+  array_of_doubles_union_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Policy& policy, const Allocator& allocator);
+};
+
+template<typename Allocator>
+class array_of_doubles_union_alloc<Allocator>::builder: public tuple_base_builder<builder, array_of_doubles_union_policy_alloc<Allocator>, Allocator> {
+public:
+  builder(const array_of_doubles_union_policy_alloc<Allocator>& policy = array_of_doubles_union_policy_alloc<Allocator>(), const Allocator& allocator = Allocator());
+  array_of_doubles_union_alloc<Allocator> build() const;
+};
+
+// alias with default allocator
+using array_of_doubles_union = array_of_doubles_union_alloc<>;
+
+} /* namespace datasketches */
+
+#include "array_of_doubles_union_impl.hpp"
+
+#endif

data/vendor/datasketches-cpp/tuple/include/array_of_doubles_union_impl.hpp

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename A>
+array_of_doubles_union_alloc<A>::array_of_doubles_union_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Policy& policy, const A& allocator):
+Base(lg_cur_size, lg_nom_size, rf, theta, seed, policy, allocator)
+{}
+
+template<typename A>
+auto array_of_doubles_union_alloc<A>::get_result(bool ordered) const -> CompactSketch {
+  return compact_array_of_doubles_sketch_alloc<A>(this->state_.get_policy().get_policy().get_num_values(), Base::get_result(ordered));
+}
+
+// builder
+
+template<typename A>
+array_of_doubles_union_alloc<A>::builder::builder(const Policy& policy, const A& allocator):
+tuple_base_builder<builder, Policy, A>(policy, allocator) {}
+
+template<typename A>
+array_of_doubles_union_alloc<A> array_of_doubles_union_alloc<A>::builder::build() const {
+  return array_of_doubles_union_alloc<A>(this->starting_lg_size(), this->lg_k_, this->rf_, this->starting_theta(), this->seed_, this->policy_, this->allocator_);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/tuple/include/bounds_on_ratios_in_sampled_sets.hpp

@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef BOUNDS_ON_RATIOS_IN_SAMPLED_SETS_HPP_
+#define BOUNDS_ON_RATIOS_IN_SAMPLED_SETS_HPP_
+
+#include <cstdint>
+
+#include <bounds_binomial_proportions.hpp>
+
+namespace datasketches {
+
+/**
+ * This class is used to compute the bounds on the estimate of the ratio <i>|B| / |A|</i>, where:
+ * <ul>
+ * <li><i>|A|</i> is the unknown size of a set <i>A</i> of unique identifiers.</li>
+ * <li><i>|B|</i> is the unknown size of a subset <i>B</i> of <i>A</i>.</li>
+ * <li><i>a</i> = <i>|S<sub>A</sub>|</i> is the observed size of a sample of <i>A</i>
+ * that was obtained by Bernoulli sampling with a known inclusion probability <i>f</i>.</li>
+ * <li><i>b</i> = <i>|S<sub>A</sub> &cap; B|</i> is the observed size of a subset
+ * of <i>S<sub>A</sub></i>.</li>
+ * </ul>
+ */
+class bounds_on_ratios_in_sampled_sets {
+public:
+  static constexpr double NUM_STD_DEVS = 2.0;
+
+  /**
+   * Return the approximate lower bound based on a 95% confidence interval
+   * @param a See class javadoc
+   * @param b See class javadoc
+   * @param f the inclusion probability used to produce the set with size <i>a</i> and should
+   * generally be less than 0.5. Above this value, the results not be reliable.
+   * When <i>f</i> = 1.0 this returns the estimate.
+   * @return the approximate upper bound
+   */
+  static double lower_bound_for_b_over_a(uint64_t a, uint64_t b, double f) {
+    check_inputs(a, b, f);
+    if (a == 0) return 0.0;
+    if (f == 1.0) return static_cast<double>(b) / static_cast<double>(a);
+    return bounds_binomial_proportions::approximate_lower_bound_on_p(a, b, NUM_STD_DEVS * hacky_adjuster(f));
+  }
+
+  /**
+   * Return the approximate upper bound based on a 95% confidence interval
+   * @param a See class javadoc
+   * @param b See class javadoc
+   * @param f the inclusion probability used to produce the set with size <i>a</i>.
+   * @return the approximate lower bound
+   */
+  static double upper_bound_for_b_over_a(uint64_t a, uint64_t b, double f) {
+    check_inputs(a, b, f);
+    if (a == 0) return 1.0;
+    if (f == 1.0) return static_cast<double>(b) / static_cast<double>(a);
+    return bounds_binomial_proportions::approximate_upper_bound_on_p(a, b, NUM_STD_DEVS * hacky_adjuster(f));
+  }
+
+  /**
+   * Return the estimate of b over a
+   * @param a See class javadoc
+   * @param b See class javadoc
+   * @return the estimate of b over a
+   */
+  static double get_estimate_of_b_over_a(uint64_t a, uint64_t b) {
+    check_inputs(a, b, 0.3);
+    if (a == 0) return 0.5;
+    return static_cast<double>(b) / static_cast<double>(a);
+  }
+
+  /**
+   * Return the estimate of A. See class javadoc.
+   * @param a See class javadoc
+   * @param f the inclusion probability used to produce the set with size <i>a</i>.
+   * @return the approximate lower bound
+   */
+  static double estimate_of_a(uint64_t a, uint64_t f) {
+    check_inputs(a, 1, f);
+    return a / f;
+  }
+
+  /**
+   * Return the estimate of B. See class javadoc.
+   * @param b See class javadoc
+   * @param f the inclusion probability used to produce the set with size <i>b</i>.
+   * @return the approximate lower bound
+   */
+  static double estimate_of_b(uint64_t b, double f) {
+    check_inputs(b + 1, b, f);
+    return b / f;
+  }
+
+private:
+  /**
+   * This hackyAdjuster is tightly coupled with the width of the confidence interval normally
+   * specified with number of standard deviations. To simplify this interface the number of
+   * standard deviations has been fixed to 2.0, which corresponds to a confidence interval of
+   * 95%.
+   * @param f the inclusion probability used to produce the set with size <i>a</i>.
+   * @return the hacky Adjuster
+   */
+  static double hacky_adjuster(double f) {
+    const double tmp = sqrt(1.0 - f);
+    return (f <= 0.5) ? tmp : tmp + (0.01 * (f - 0.5));
+  }
+
+  static void check_inputs(uint64_t a, uint64_t b, double f) {
+    if (a < b) {
+      throw std::invalid_argument("a must be >= b: a = " + std::to_string(a) + ", b = " + std::to_string(b));
+    }
+    if ((f > 1.0) || (f <= 0.0)) {
+      throw std::invalid_argument("Required: ((f <= 1.0) && (f > 0.0)): " + std::to_string(f));
+    }
+  }
+
+};
+
+} /* namespace datasketches */
+
+# endif