datasketches 0.1.0

data/vendor/datasketches-cpp/tuple/include/bounds_on_ratios_in_theta_sketched_sets.hpp

@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef BOUNDS_ON_RATIOS_IN_THETA_SKETCHED_SETS_HPP_
+#define BOUNDS_ON_RATIOS_IN_THETA_SKETCHED_SETS_HPP_
+
+#include <cstdint>
+#include <stdexcept>
+
+#include <bounds_on_ratios_in_sampled_sets.hpp>
+
+namespace datasketches {
+
+/**
+ * This is to compute the bounds on the estimate of the ratio <i>B / A</i>, where:
+ * <ul>
+ * <li><i>A</i> is a Theta Sketch of population <i>PopA</i>.</li>
+ * <li><i>B</i> is a Theta Sketch of population <i>PopB</i> that is a subset of <i>A</i>,
+ * obtained by an intersection of <i>A</i> with some other Theta Sketch <i>C</i>,
+ * which acts like a predicate or selection clause.</li>
+ * <li>The estimate of the ratio <i>PopB/PopA</i> is
+ * estimate_of_b_over_a(<i>A, B</i>).</li>
+ * <li>The Upper Bound estimate on the ratio PopB/PopA is
+ * upper_bound_for_b_over_a(<i>A, B</i>).</li>
+ * <li>The Lower Bound estimate on the ratio PopB/PopA is
+ * lower_bound_for_b_over_a(<i>A, B</i>).</li>
+ * </ul>
+ * Note: The theta of <i>A</i> cannot be greater than the theta of <i>B</i>.
+ * If <i>B</i> is formed as an intersection of <i>A</i> and some other set <i>C</i>,
+ * then the theta of <i>B</i> is guaranteed to be less than or equal to the theta of <i>B</i>.
+ */
+template<typename ExtractKey>
+class bounds_on_ratios_in_theta_sketched_sets {
+public:
+  /**
+   * Gets the approximate lower bound for B over A based on a 95% confidence interval
+   * @param sketchA the sketch A
+   * @param sketchB the sketch B
+   * @return the approximate lower bound for B over A
+   */
+  template<typename SketchA, typename SketchB>
+  static double lower_bound_for_b_over_a(const SketchA& sketch_a, const SketchB& sketch_b) {
+    const uint64_t theta64_a = sketch_a.get_theta64();
+    const uint64_t theta64_b = sketch_b.get_theta64();
+    check_thetas(theta64_a, theta64_b);
+
+    const uint64_t count_b = sketch_b.get_num_retained();
+    const uint64_t count_a = theta64_a == theta64_b
+        ? sketch_a.get_num_retained()
+        : count_less_than_theta64(sketch_a, theta64_b);
+
+    if (count_a == 0) return 0;
+    const double f = sketch_b.get_theta();
+    return bounds_on_ratios_in_sampled_sets::lower_bound_for_b_over_a(count_a, count_b, f);
+  }
+
+  /**
+   * Gets the approximate upper bound for B over A based on a 95% confidence interval
+   * @param sketchA the sketch A
+   * @param sketchB the sketch B
+   * @return the approximate upper bound for B over A
+   */
+  template<typename SketchA, typename SketchB>
+  static double upper_bound_for_b_over_a(const SketchA& sketch_a, const SketchB& sketch_b) {
+    const uint64_t theta64_a = sketch_a.get_theta64();
+    const uint64_t theta64_b = sketch_b.get_theta64();
+    check_thetas(theta64_a, theta64_b);
+
+    const uint64_t count_b = sketch_b.get_num_retained();
+    const uint64_t count_a = (theta64_a == theta64_b)
+        ? sketch_a.get_num_retained()
+        : count_less_than_theta64(sketch_a, theta64_b);
+
+    if (count_a == 0) return 1;
+    const double f = sketch_b.get_theta();
+    return bounds_on_ratios_in_sampled_sets::upper_bound_for_b_over_a(count_a, count_b, f);
+  }
+
+  /**
+   * Gets the estimate for B over A
+   * @param sketchA the sketch A
+   * @param sketchB the sketch B
+   * @return the estimate for B over A
+   */
+  template<typename SketchA, typename SketchB>
+  static double estimate_of_b_over_a(const SketchA& sketch_a, const SketchB& sketch_b) {
+    const uint64_t theta64_a = sketch_a.get_theta64();
+    const uint64_t theta64_b = sketch_b.get_theta64();
+    check_thetas(theta64_a, theta64_b);
+
+    const uint64_t count_b = sketch_b.get_num_retained();
+    const uint64_t count_a = (theta64_a == theta64_b)
+        ? sketch_a.get_num_retained()
+        : count_less_than_theta64(sketch_a, theta64_b);
+
+    if (count_a == 0) return 0.5;
+    return static_cast<double>(count_b) / static_cast<double>(count_a);
+  }
+
+private:
+
+  static inline void check_thetas(uint64_t theta_a, uint64_t theta_b) {
+    if (theta_b > theta_a) {
+      throw std::invalid_argument("theta_a must be <= theta_b");
+    }
+  }
+
+  template<typename Sketch>
+  static uint64_t count_less_than_theta64(const Sketch& sketch, uint64_t theta) {
+    uint64_t count = 0;
+    for (const auto& entry: sketch) if (ExtractKey()(entry) < theta) ++count;
+    return count;
+  }
+
+};
+
+} /* namespace datasketches */
+
+# endif

data/vendor/datasketches-cpp/tuple/include/jaccard_similarity.hpp

@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef JACCARD_SIMILARITY_BASE_HPP_
+#define JACCARD_SIMILARITY_BASE_HPP_
+
+#include <memory>
+#include <array>
+
+#include <theta_union_experimental.hpp>
+#include <theta_intersection_experimental.hpp>
+#include <tuple_union.hpp>
+#include <tuple_intersection.hpp>
+#include <bounds_on_ratios_in_theta_sketched_sets.hpp>
+#include <ceiling_power_of_2.hpp>
+#include <common_defs.hpp>
+
+namespace datasketches {
+
+template<typename Union, typename Intersection, typename ExtractKey>
+class jaccard_similarity_base {
+public:
+
+  /**
+   * Computes the Jaccard similarity index with upper and lower bounds. The Jaccard similarity index
+   * <i>J(A,B) = (A ^ B)/(A U B)</i> is used to measure how similar the two sketches are to each
+   * other. If J = 1.0, the sketches are considered equal. If J = 0, the two sketches are
+   * disjoint. A Jaccard of .95 means the overlap between the two
+   * sets is 95% of the union of the two sets.
+   *
+   * <p>Note: For very large pairs of sketches, where the configured nominal entries of the sketches
+   * are 2^25 or 2^26, this method may produce unpredictable results.
+   *
+   * @param sketch_a given sketch A
+   * @param sketch_b given sketch B
+   * @return a double array {LowerBound, Estimate, UpperBound} of the Jaccard index.
+   * The Upper and Lower bounds are for a confidence interval of 95.4% or +/- 2 standard deviations.
+   */
+  template<typename SketchA, typename SketchB>
+  static std::array<double, 3> jaccard(const SketchA& sketch_a, const SketchB& sketch_b) {
+    if (reinterpret_cast<const void*>(&sketch_a) == reinterpret_cast<const void*>(&sketch_b)) return {1, 1, 1};
+    if (sketch_a.is_empty() && sketch_b.is_empty()) return {1, 1, 1};
+    if (sketch_a.is_empty() || sketch_b.is_empty()) return {0, 0, 0};
+
+    auto union_ab = compute_union(sketch_a, sketch_b);
+    if (identical_sets(sketch_a, sketch_b, union_ab)) return {1, 1, 1};
+
+    // intersection
+    Intersection i;
+    i.update(sketch_a);
+    i.update(sketch_b);
+    i.update(union_ab); // ensures that intersection is a subset of the union
+    auto inter_abu = i.get_result(false);
+
+    return {
+      bounds_on_ratios_in_theta_sketched_sets<ExtractKey>::lower_bound_for_b_over_a(union_ab, inter_abu),
+      bounds_on_ratios_in_theta_sketched_sets<ExtractKey>::estimate_of_b_over_a(union_ab, inter_abu),
+      bounds_on_ratios_in_theta_sketched_sets<ExtractKey>::upper_bound_for_b_over_a(union_ab, inter_abu)
+    };
+  }
+
+  /**
+   * Returns true if the two given sketches are equivalent.
+   * @param sketch_a the given sketch A
+   * @param sketch_b the given sketch B
+   * @return true if the two given sketches are exactly equal
+   */
+  template<typename SketchA, typename SketchB>
+  static bool exactly_equal(const SketchA& sketch_a, const SketchB& sketch_b) {
+    if (reinterpret_cast<const void*>(&sketch_a) == reinterpret_cast<const void*>(&sketch_b)) return true;
+    if (sketch_a.is_empty() && sketch_b.is_empty()) return true;
+    if (sketch_a.is_empty() || sketch_b.is_empty()) return false;
+
+    auto union_ab = compute_union(sketch_a, sketch_b);
+    if (identical_sets(sketch_a, sketch_b, union_ab)) return true;
+    return false;
+  }
+
+  /**
+   * Tests similarity of an actual Sketch against an expected Sketch.
+   * Computes the lower bound of the Jaccard index <i>J<sub>LB</sub></i> of the actual and
+   * expected sketches.
+   * if <i>J<sub>LB</sub> &ge; threshold</i>, then the sketches are considered to be
+   * similar with a confidence of 97.7%.
+   *
+   * @param actual the sketch to be tested
+   * @param expected the reference sketch that is considered to be correct
+   * @param threshold a real value between zero and one
+   * @return true if the similarity of the two sketches is greater than the given threshold
+   * with at least 97.7% confidence
+   */
+  template<typename SketchA, typename SketchB>
+  static bool similarity_test(const SketchA& actual, const SketchB& expected, double threshold) {
+    auto jc = jaccard(actual, expected);
+    return jc[0] >= threshold;
+  }
+
+  /**
+   * Tests dissimilarity of an actual Sketch against an expected Sketch.
+   * Computes the upper bound of the Jaccard index <i>J<sub>UB</sub></i> of the actual and
+   * expected sketches.
+   * if <i>J<sub>UB</sub> &le; threshold</i>, then the sketches are considered to be
+   * dissimilar with a confidence of 97.7%.
+   *
+   * @param actual the sketch to be tested
+   * @param expected the reference sketch that is considered to be correct
+   * @param threshold a real value between zero and one
+   * @return true if the dissimilarity of the two sketches is greater than the given threshold
+   * with at least 97.7% confidence
+   */
+  template<typename SketchA, typename SketchB>
+  static bool dissimilarity_test(const SketchA& actual, const SketchB& expected, double threshold) {
+    auto jc = jaccard(actual, expected);
+    return jc[2] <= threshold;
+  }
+
+private:
+
+  template<typename SketchA, typename SketchB>
+  static typename Union::CompactSketch compute_union(const SketchA& sketch_a, const SketchB& sketch_b) {
+    const unsigned count_a = sketch_a.get_num_retained();
+    const unsigned count_b = sketch_b.get_num_retained();
+    const unsigned lg_k = std::min(std::max(log2(ceiling_power_of_2(count_a + count_b)), theta_constants::MIN_LG_K), theta_constants::MAX_LG_K);
+    auto u = typename Union::builder().set_lg_k(lg_k).build();
+    u.update(sketch_a);
+    u.update(sketch_b);
+    return u.get_result(false);
+  }
+
+  template<typename SketchA, typename SketchB, typename UnionAB>
+  static bool identical_sets(const SketchA& sketch_a, const SketchB& sketch_b, const UnionAB& union_ab) {
+    if (union_ab.get_num_retained() == sketch_a.get_num_retained() &&
+        union_ab.get_num_retained() == sketch_b.get_num_retained() &&
+        union_ab.get_theta64() == sketch_a.get_theta64() &&
+        union_ab.get_theta64() == sketch_b.get_theta64()) return true;
+    return false;
+  }
+
+};
+
+template<typename Allocator>
+using theta_jaccard_similarity_alloc = jaccard_similarity_base<theta_union_experimental<Allocator>, theta_intersection_experimental<Allocator>, trivial_extract_key>;
+
+// alias with default allocator for convenience
+using theta_jaccard_similarity = theta_jaccard_similarity_alloc<std::allocator<uint64_t>>;
+
+template<
+  typename Summary,
+  typename IntersectionPolicy,
+  typename UnionPolicy = default_union_policy<Summary>,
+  typename Allocator = std::allocator<Summary>>
+using tuple_jaccard_similarity = jaccard_similarity_base<tuple_union<Summary, UnionPolicy, Allocator>, tuple_intersection<Summary, IntersectionPolicy, Allocator>, pair_extract_key<uint64_t, Summary>>;
+
+} /* namespace datasketches */
+
+# endif

data/vendor/datasketches-cpp/tuple/include/theta_a_not_b_experimental.hpp

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_A_NOT_B_EXPERIMENTAL_HPP_
+#define THETA_A_NOT_B_EXPERIMENTAL_HPP_
+
+#include "theta_sketch_experimental.hpp"
+#include "theta_set_difference_base.hpp"
+
+namespace datasketches {
+
+template<typename Allocator = std::allocator<uint64_t>>
+class theta_a_not_b_experimental {
+public:
+  using Entry = uint64_t;
+  using ExtractKey = trivial_extract_key;
+  using CompactSketch = compact_theta_sketch_experimental<Allocator>;
+  using State = theta_set_difference_base<Entry, ExtractKey, CompactSketch, Allocator>;
+
+  explicit theta_a_not_b_experimental(uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+  /**
+   * Computes the a-not-b set operation given two sketches.
+   * @return the result of a-not-b
+   */
+  template<typename FwdSketch, typename Sketch>
+  CompactSketch compute(FwdSketch&& a, const Sketch& b, bool ordered = true) const;
+
+private:
+  State state_;
+};
+
+} /* namespace datasketches */
+
+#include "theta_a_not_b_experimental_impl.hpp"
+
+#endif

data/vendor/datasketches-cpp/tuple/include/theta_a_not_b_experimental_impl.hpp

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename A>
+theta_a_not_b_experimental<A>::theta_a_not_b_experimental(uint64_t seed, const A& allocator):
+state_(seed, allocator)
+{}
+
+template<typename A>
+template<typename FwdSketch, typename Sketch>
+auto theta_a_not_b_experimental<A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const -> CompactSketch {
+  return state_.compute(std::forward<FwdSketch>(a), b, ordered);
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/tuple/include/theta_comparators.hpp

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_COMPARATORS_HPP_
+#define THETA_COMPARATORS_HPP_
+
+namespace datasketches {
+
+template<typename ExtractKey>
+struct compare_by_key {
+  template<typename Entry1, typename Entry2>
+  bool operator()(Entry1&& a, Entry2&& b) const {
+    return ExtractKey()(std::forward<Entry1>(a)) < ExtractKey()(std::forward<Entry2>(b));
+  }
+};
+
+// less than
+
+template<typename Key, typename Entry, typename ExtractKey>
+class key_less_than {
+public:
+  explicit key_less_than(const Key& key): key(key) {}
+  bool operator()(const Entry& entry) const {
+    return ExtractKey()(entry) < this->key;
+  }
+private:
+  Key key;
+};
+
+} /* namespace datasketches */
+
+#endif

data/vendor/datasketches-cpp/tuple/include/theta_constants.hpp

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_CONSTANTS_HPP_
+#define THETA_CONSTANTS_HPP_
+
+namespace datasketches {
+
+namespace theta_constants {
+  enum resize_factor { X1, X2, X4, X8 };
+  static const uint64_t MAX_THETA = LLONG_MAX; // signed max for compatibility with Java
+  static const uint8_t MIN_LG_K = 5;
+  static const uint8_t MAX_LG_K = 26;
+}
+
+} /* namespace datasketches */
+
+#endif