datasketches 0.1.0

data/vendor/datasketches-cpp/theta/include/theta_union.hpp

@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_UNION_HPP_
+#define THETA_UNION_HPP_
+
+#include <memory>
+#include <functional>
+#include <climits>
+
+#include "theta_sketch.hpp"
+
+namespace datasketches {
+
+/*
+ * author Alexander Saydakov
+ * author Lee Rhodes
+ * author Kevin Lang
+ */
+
+template<typename A>
+class theta_union_alloc {
+public:
+  class builder;
+
+  // No constructor here. Use builder instead.
+
+  /**
+   * This method is to update the union with a given sketch
+   * @param sketch to update the union with
+   */
+  void update(const theta_sketch_alloc<A>& sketch);
+
+  /**
+   * This method produces a copy of the current state of the union as a compact sketch.
+   * @param ordered optional flag to specify if ordered sketch should be produced
+   * @return the result of the union
+   */
+  compact_theta_sketch_alloc<A> get_result(bool ordered = true) const;
+
+private:
+  bool is_empty_;
+  uint64_t theta_;
+  update_theta_sketch_alloc<A> state_;
+
+  // for builder
+  theta_union_alloc(uint64_t theta, update_theta_sketch_alloc<A>&& state);
+};
+
+// builder
+
+template<typename A>
+class theta_union_alloc<A>::builder {
+public:
+  typedef typename update_theta_sketch_alloc<A>::resize_factor resize_factor;
+
+  /**
+   * Set log2(k), where k is a nominal number of entries in the sketch
+   * @param lg_k base 2 logarithm of nominal number of entries
+   * @return this builder
+   */
+  builder& set_lg_k(uint8_t lg_k);
+
+  /**
+   * Set resize factor for the internal hash table (defaults to 8)
+   * @param rf resize factor
+   * @return this builder
+   */
+  builder& set_resize_factor(resize_factor rf);
+
+  /**
+   * Set sampling probability (initial theta). The default is 1, so the sketch retains
+   * all entries until it reaches the limit, at which point it goes into the estimation mode
+   * and reduces the effective sampling probability (theta) as necessary.
+   * @param p sampling probability
+   * @return this builder
+   */
+  builder& set_p(float p);
+
+  /**
+   * Set the seed for the hash function. Should be used carefully if needed.
+   * Sketches produced with different seed are not compatible
+   * and cannot be mixed in set operations.
+   * @param seed hash seed
+   * @return this builder
+   */
+  builder& set_seed(uint64_t seed);
+
+  /**
+   * This is to create an instance of the union with predefined parameters.
+   * @return and instance of the union
+   */
+  theta_union_alloc<A> build() const;
+
+private:
+  typename update_theta_sketch_alloc<A>::builder sketch_builder;
+};
+
+// alias with default allocator for convenience
+typedef theta_union_alloc<std::allocator<void>> theta_union;
+
+} /* namespace datasketches */
+
+#include "theta_union_impl.hpp"
+
+# endif

data/vendor/datasketches-cpp/theta/include/theta_union_impl.hpp

@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_UNION_IMPL_HPP_
+#define THETA_UNION_IMPL_HPP_
+
+namespace datasketches {
+
+/*
+ * author Alexander Saydakov
+ * author Lee Rhodes
+ * author Kevin Lang
+ */
+
+template<typename A>
+theta_union_alloc<A>::theta_union_alloc(uint64_t theta, update_theta_sketch_alloc<A>&& state):
+is_empty_(true), theta_(theta), state_(std::move(state)) {}
+
+template<typename A>
+void theta_union_alloc<A>::update(const theta_sketch_alloc<A>& sketch) {
+  if (sketch.is_empty()) return;
+  if (sketch.get_seed_hash() != state_.get_seed_hash()) throw std::invalid_argument("seed hash mismatch");
+  is_empty_ = false;
+  if (sketch.get_theta64() < theta_) theta_ = sketch.get_theta64();
+  if (sketch.is_ordered()) {
+    for (auto hash: sketch) {
+      if (hash >= theta_) break; // early stop
+      state_.internal_update(hash);
+    }
+  } else {
+    for (auto hash: sketch) if (hash < theta_) state_.internal_update(hash);
+  }
+  if (state_.get_theta64() < theta_) theta_ = state_.get_theta64();
+}
+
+template<typename A>
+compact_theta_sketch_alloc<A> theta_union_alloc<A>::get_result(bool ordered) const {
+  if (is_empty_) return state_.compact(ordered);
+  const uint32_t nom_num_keys = 1 << state_.lg_nom_size_;
+  if (theta_ >= state_.theta_ && state_.get_num_retained() <= nom_num_keys) return state_.compact(ordered);
+  uint64_t theta = std::min(theta_, state_.get_theta64());
+  vector_u64<A> keys(state_.get_num_retained());
+  uint32_t num_keys = 0;
+  for (auto key: state_) {
+    if (key < theta) keys[num_keys++] = key;
+  }
+  if (num_keys > nom_num_keys) {
+    std::nth_element(keys.begin(), keys.begin() + nom_num_keys, keys.begin() + num_keys);
+    theta = keys[nom_num_keys];
+    num_keys = nom_num_keys;
+  }
+  if (num_keys != state_.get_num_retained()) {
+    keys.resize(num_keys);
+  }
+  if (ordered) std::sort(keys.begin(), keys.end());
+  return compact_theta_sketch_alloc<A>(false, theta, std::move(keys), state_.get_seed_hash(), ordered);
+}
+
+// builder
+
+template<typename A>
+typename theta_union_alloc<A>::builder& theta_union_alloc<A>::builder::set_lg_k(uint8_t lg_k) {
+  sketch_builder.set_lg_k(lg_k);
+  return *this;
+}
+
+template<typename A>
+typename theta_union_alloc<A>::builder& theta_union_alloc<A>::builder::set_resize_factor(resize_factor rf) {
+  sketch_builder.set_resize_factor(rf);
+  return *this;
+}
+
+template<typename A>
+typename theta_union_alloc<A>::builder& theta_union_alloc<A>::builder::set_p(float p) {
+  sketch_builder.set_p(p);
+  return *this;
+}
+
+template<typename A>
+typename theta_union_alloc<A>::builder& theta_union_alloc<A>::builder::set_seed(uint64_t seed) {
+  sketch_builder.set_seed(seed);
+  return *this;
+}
+
+template<typename A>
+theta_union_alloc<A> theta_union_alloc<A>::builder::build() const {
+  update_theta_sketch_alloc<A> sketch = sketch_builder.build();
+  return theta_union_alloc(sketch.get_theta64(), std::move(sketch));
+}
+
+} /* namespace datasketches */
+
+# endif

data/vendor/datasketches-cpp/theta/test/CMakeLists.txt

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+add_executable(theta_test)
+
+target_link_libraries(theta_test theta common_test)
+
+set_target_properties(theta_test PROPERTIES
+  CXX_STANDARD 11
+  CXX_STANDARD_REQUIRED YES
+)
+
+file(TO_CMAKE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" THETA_TEST_BINARY_PATH)
+string(APPEND THETA_TEST_BINARY_PATH "/")
+target_compile_definitions(theta_test
+  PRIVATE
+    TEST_BINARY_INPUT_PATH="${THETA_TEST_BINARY_PATH}"
+)
+
+add_test(
+  NAME theta_test
+  COMMAND theta_test
+)
+
+target_sources(theta_test
+  PRIVATE
+    theta_sketch_test.cpp
+    theta_union_test.cpp
+    theta_intersection_test.cpp
+    theta_a_not_b_test.cpp
+)

data/vendor/datasketches-cpp/theta/test/theta_a_not_b_test.cpp

@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch.hpp>
+
+#include <theta_a_not_b.hpp>
+
+namespace datasketches {
+
+TEST_CASE("theta a-not-b: empty", "[theta_a_not_b]") {
+  theta_a_not_b a_not_b;
+  update_theta_sketch a = update_theta_sketch::builder().build();
+  update_theta_sketch b = update_theta_sketch::builder().build();
+  compact_theta_sketch result = a_not_b.compute(a, b);
+  REQUIRE(result.get_num_retained() == 0);
+  REQUIRE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta a-not-b: non empty no retained keys", "[theta_a_not_b]") {
+  update_theta_sketch a = update_theta_sketch::builder().build();
+  a.update(1);
+  update_theta_sketch b = update_theta_sketch::builder().set_p(0.001).build();
+  theta_a_not_b a_not_b;
+
+  // B is still empty
+  compact_theta_sketch result = a_not_b.compute(a, b);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE(result.get_num_retained() == 1);
+  REQUIRE(result.get_theta() == Approx(1).margin(1e-10));
+  REQUIRE(result.get_estimate() == 1.0);
+
+  // B is not empty in estimation mode and no entries
+  b.update(1);
+  REQUIRE(b.get_num_retained() == 0U);
+
+  result = a_not_b.compute(a, b);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.is_estimation_mode());
+  REQUIRE(result.get_num_retained() == 0);
+  REQUIRE(result.get_theta() == Approx(0.001).margin(1e-10));
+  REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta a-not-b: exact mode half overlap", "[theta_a_not_b]") {
+  update_theta_sketch a = update_theta_sketch::builder().build();
+  int value = 0;
+  for (int i = 0; i < 1000; i++) a.update(value++);
+
+  update_theta_sketch b = update_theta_sketch::builder().build();
+  value = 500;
+  for (int i = 0; i < 1000; i++) b.update(value++);
+
+  theta_a_not_b a_not_b;
+
+  // unordered inputs, ordered result
+  compact_theta_sketch result = a_not_b.compute(a, b);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE(result.is_ordered());
+  REQUIRE(result.get_estimate() == 500.0);
+
+  // unordered inputs, unordered result
+  result = a_not_b.compute(a, b, false);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE_FALSE(result.is_ordered());
+  REQUIRE(result.get_estimate() == 500.0);
+
+  // ordered inputs
+  result = a_not_b.compute(a.compact(), b.compact());
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE(result.is_ordered());
+  REQUIRE(result.get_estimate() == 500.0);
+
+  // A is ordered, so the result is ordered regardless
+  result = a_not_b.compute(a.compact(), b, false);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE(result.is_ordered());
+  REQUIRE(result.get_estimate() == 500.0);
+}
+
+TEST_CASE("theta a-not-b: exact mode disjoint", "[theta_a_not_b]") {
+  update_theta_sketch a = update_theta_sketch::builder().build();
+  int value = 0;
+  for (int i = 0; i < 1000; i++) a.update(value++);
+
+  update_theta_sketch b = update_theta_sketch::builder().build();
+  for (int i = 0; i < 1000; i++) b.update(value++);
+
+  theta_a_not_b a_not_b;
+
+  // unordered inputs
+  compact_theta_sketch result = a_not_b.compute(a, b);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == 1000.0);
+
+  // ordered inputs
+  result = a_not_b.compute(a.compact(), b.compact());
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == 1000.0);
+}
+
+TEST_CASE("theta a-not-b: exact mode full overlap", "[theta_a_not_b]") {
+  update_theta_sketch sketch = update_theta_sketch::builder().build();
+  int value = 0;
+  for (int i = 0; i < 1000; i++) sketch.update(value++);
+
+  theta_a_not_b a_not_b;
+
+  // unordered inputs
+  compact_theta_sketch result = a_not_b.compute(sketch, sketch);
+  REQUIRE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == 0.0);
+
+  // ordered inputs
+  result = a_not_b.compute(sketch.compact(), sketch.compact());
+  REQUIRE(result.is_empty());
+  REQUIRE_FALSE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta a-not-b: estimation mode half overlap", "[theta_a_not_b]") {
+  update_theta_sketch a = update_theta_sketch::builder().build();
+  int value = 0;
+  for (int i = 0; i < 10000; i++) a.update(value++);
+
+  update_theta_sketch b = update_theta_sketch::builder().build();
+  value = 5000;
+  for (int i = 0; i < 10000; i++) b.update(value++);
+
+  theta_a_not_b a_not_b;
+
+  // unordered inputs
+  compact_theta_sketch result = a_not_b.compute(a, b);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+
+  // ordered inputs
+  result = a_not_b.compute(a.compact(), b.compact());
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+}
+
+TEST_CASE("theta a-not-b: estimation mode disjoint", "[theta_a_not_b]") {
+  update_theta_sketch a = update_theta_sketch::builder().build();
+  int value = 0;
+  for (int i = 0; i < 10000; i++) a.update(value++);
+
+  update_theta_sketch b = update_theta_sketch::builder().build();
+  for (int i = 0; i < 10000; i++) b.update(value++);
+
+  theta_a_not_b a_not_b;
+
+  // unordered inputs
+  compact_theta_sketch result = a_not_b.compute(a, b);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == Approx(10000).margin(10000 * 0.02));
+
+  // ordered inputs
+  result = a_not_b.compute(a.compact(), b.compact());
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == Approx(10000).margin(10000 * 0.02));
+}
+
+TEST_CASE("theta a-not-b: estimation mode full overlap", "[theta_a_not_b]") {
+  update_theta_sketch sketch = update_theta_sketch::builder().build();
+  int value = 0;
+  for (int i = 0; i < 10000; i++) sketch.update(value++);
+
+  theta_a_not_b a_not_b;
+
+  // unordered inputs
+  compact_theta_sketch result = a_not_b.compute(sketch, sketch);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == 0.0);
+
+  // ordered inputs
+  result = a_not_b.compute(sketch.compact(), sketch.compact());
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta a-not-b: seed mismatch", "[theta_a_not_b]") {
+  update_theta_sketch sketch = update_theta_sketch::builder().build();
+  sketch.update(1); // non-empty should not be ignored
+  theta_a_not_b a_not_b(123);
+  REQUIRE_THROWS_AS(a_not_b.compute(sketch, sketch), std::invalid_argument);
+}
+
+TEST_CASE("theta a-not-b: issue #152", "[theta_a_not_b]") {
+  update_theta_sketch a = update_theta_sketch::builder().build();
+  int value = 0;
+  for (int i = 0; i < 10000; i++) a.update(value++);
+
+  update_theta_sketch b = update_theta_sketch::builder().build();
+  value = 5000;
+  for (int i = 0; i < 25000; i++) b.update(value++);
+
+  theta_a_not_b a_not_b;
+
+  // unordered inputs
+  compact_theta_sketch result = a_not_b.compute(a, b);
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.03));
+
+  // ordered inputs
+  result = a_not_b.compute(a.compact(), b.compact());
+  REQUIRE_FALSE(result.is_empty());
+  REQUIRE(result.is_estimation_mode());
+  REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.03));
+}
+
+} /* namespace datasketches */