datasketches 0.1.2 → 0.2.0

data/vendor/datasketches-cpp/sampling/include/var_opt_union.hpp

@@ -51,7 +51,7 @@ class var_opt_union {
 public:
   static const uint32_t MAX_K = ((uint32_t) 1 << 31) - 2;
 
-  explicit var_opt_union(uint32_t max_k);
+  explicit var_opt_union(uint32_t max_k, const A& allocator = A());
   var_opt_union(const var_opt_union& other);
   var_opt_union(var_opt_union&& other) noexcept;
     
@@ -119,16 +119,16 @@ public:
    * @param is input stream
    * @return an instance of a union
    */
-  static var_opt_union deserialize(std::istream& is);
+  static var_opt_union deserialize(std::istream& is, const A& allocator = A());
 
   /**
    * NOTE: This method may be deprecated in a future version.
-   * This method deserializes a skeuniontch from a given array of bytes.
+   * This method deserializes a union from a given array of bytes.
    * @param bytes pointer to the array of bytes
    * @param size the size of the array
    * @return an instance of a union
    */
-  static var_opt_union deserialize(const void* bytes, size_t size);
+  static var_opt_union deserialize(const void* bytes, size_t size, const A& allocator = A());
 
   /**
    * Prints a summary of the union as a string.
@@ -236,4 +236,4 @@ private:
 
 #include "var_opt_union_impl.hpp"
 
-#endif // _VAR_OPT_UNION_HPP_
+#endif // _VAR_OPT_UNION_HPP_

data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp

@@ -28,12 +28,12 @@
 namespace datasketches {
 
 template<typename T, typename S, typename A>
-var_opt_union<T,S,A>::var_opt_union(uint32_t max_k) :
+var_opt_union<T,S,A>::var_opt_union(uint32_t max_k, const A& allocator) :
   n_(0),
   outer_tau_numer_(0),
   outer_tau_denom_(0.0),
   max_k_(max_k),
-  gadget_(max_k, var_opt_sketch<T,S,A>::DEFAULT_RESIZE_FACTOR, true)
+  gadget_(max_k, var_opt_sketch<T,S,A>::DEFAULT_RESIZE_FACTOR, true, allocator)
 {}
 
 template<typename T, typename S, typename A>
@@ -128,7 +128,7 @@ var_opt_union<T,S,A>& var_opt_union<T,S,A>::operator=(var_opt_union&& other) {
  */
 
 template<typename T, typename S, typename A>
-var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(std::istream& is) {
+var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(std::istream& is, const A& allocator) {
   uint8_t preamble_longs;
   is.read((char*)&preamble_longs, sizeof(preamble_longs));
   uint8_t serial_version;
@@ -163,7 +163,7 @@ var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(std::istream& is) {
   uint64_t outer_tau_denom;
   is.read((char*)&outer_tau_denom, sizeof(outer_tau_denom));
 
-  var_opt_sketch<T,S,A> gadget = var_opt_sketch<T,S,A>::deserialize(is);
+  var_opt_sketch<T,S,A> gadget = var_opt_sketch<T,S,A>::deserialize(is, allocator);
 
   if (!is.good())
     throw std::runtime_error("error reading from std::istream"); 
@@ -172,7 +172,7 @@ var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(std::istream& is) {
 }
 
 template<typename T, typename S, typename A>
-var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(const void* bytes, size_t size) {
+var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(const void* bytes, size_t size, const A& allocator) {
   ensure_minimum_memory(size, 8);
   const char* ptr = static_cast<const char*>(bytes);
   uint8_t preamble_longs;
@@ -207,7 +207,7 @@ var_opt_union<T,S,A> var_opt_union<T,S,A>::deserialize(const void* bytes, size_t
   ptr += copy_from_mem(ptr, &outer_tau_denom, sizeof(outer_tau_denom));
 
   const size_t gadget_size = size - (PREAMBLE_LONGS_NON_EMPTY << 3);
-  var_opt_sketch<T,S,A> gadget = var_opt_sketch<T,S,A>::deserialize(ptr, gadget_size);
+  var_opt_sketch<T,S,A> gadget = var_opt_sketch<T,S,A>::deserialize(ptr, gadget_size, allocator);
 
   return var_opt_union<T,S,A>(items_seen, outer_tau_numer, outer_tau_denom, max_k, std::move(gadget));
 }
@@ -255,7 +255,7 @@ void var_opt_union<T,S,A>::serialize(std::ostream& os) const {
 template<typename T, typename S, typename A>
 std::vector<uint8_t, AllocU8<A>> var_opt_union<T,S,A>::serialize(unsigned header_size_bytes) const {
   const size_t size = header_size_bytes + get_serialized_size_bytes();
-  std::vector<uint8_t, AllocU8<A>> bytes(size);
+  std::vector<uint8_t, AllocU8<A>> bytes(size, 0, gadget_.allocator_);
   uint8_t* ptr = bytes.data() + header_size_bytes;
 
   const bool empty = n_ == 0;

data/vendor/datasketches-cpp/sampling/test/CMakeLists.txt

@@ -40,4 +40,5 @@ target_sources(sampling_test
   PRIVATE
   var_opt_sketch_test.cpp
   var_opt_union_test.cpp
+  var_opt_allocation_test.cpp
 )

data/vendor/datasketches-cpp/sampling/test/var_opt_allocation_test.cpp

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <var_opt_sketch.hpp>
+#include <var_opt_union.hpp>
+#include <test_type.hpp>
+#include <test_allocator.hpp>
+
+#include <catch.hpp>
+
+#include <sstream>
+
+namespace datasketches {
+
+using var_opt_test_sketch = var_opt_sketch<test_type, test_type_serde, test_allocator<test_type>>;
+using var_opt_test_union = var_opt_union<test_type, test_type_serde, test_allocator<test_type>>;
+using alloc = test_allocator<test_type>;
+
+TEST_CASE("varopt allocation test", "[var_opt_sketch]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    var_opt_test_sketch sk1(10, var_opt_test_sketch::DEFAULT_RESIZE_FACTOR, 0);
+    for (int i = 0; i < 100; ++i) sk1.update(i);
+    auto bytes1 = sk1.serialize();
+    auto sk2 = var_opt_test_sketch::deserialize(bytes1.data(), bytes1.size(), 0);
+
+    std::stringstream ss;
+    sk1.serialize(ss);
+    auto sk3 = var_opt_test_sketch::deserialize(ss, alloc(0));
+
+    var_opt_test_union u1(10, 0);
+    u1.update(sk1);
+    u1.update(sk2);
+    u1.update(sk3);
+
+    auto bytes2 = u1.serialize();
+    auto u2 = var_opt_test_union::deserialize(bytes2.data(), bytes2.size(), 0);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+TEST_CASE( "varopt union: move", "[var_opt_union][test_type]") {
+  test_allocator_total_bytes = 0;
+  test_allocator_net_allocations = 0;
+  {
+    uint32_t n = 20;
+    uint32_t k = 5;
+    var_opt_test_union u(k, 0);
+    var_opt_test_sketch sk1(k, var_opt_test_sketch::DEFAULT_RESIZE_FACTOR, 0);
+    var_opt_test_sketch sk2(k, var_opt_test_sketch::DEFAULT_RESIZE_FACTOR, 0);
+
+    // move udpates
+    for (int i = 0; i < (int) n; ++i) {
+      sk1.update(i);
+      sk2.update(-i);
+    }
+    REQUIRE(sk1.get_n() == n);
+    REQUIRE(sk2.get_n() == n);
+
+    // move unions
+    u.update(std::move(sk2));
+    u.update(std::move(sk1));
+    REQUIRE(u.get_result().get_n() == 2 * n);
+
+    // move constructor
+    var_opt_test_union u2(std::move(u));
+    REQUIRE(u2.get_result().get_n() == 2 * n);
+
+    // move assignment
+    var_opt_test_union u3(k, 0);
+    u3 = std::move(u2);
+    REQUIRE(u3.get_result().get_n() == 2 * n);
+  }
+  REQUIRE(test_allocator_total_bytes == 0);
+  REQUIRE(test_allocator_net_allocations == 0);
+}
+
+}

data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp

@@ -18,7 +18,6 @@
  */
 
 #include <var_opt_union.hpp>
-#include "test_type.hpp"
 
 #include <catch.hpp>
 
@@ -325,34 +324,4 @@ TEST_CASE("varopt union: deserialize from java", "[var_opt_union]") {
   REQUIRE(result.get_k() < 128);
 }
 
-TEST_CASE( "varopt union: move", "[var_opt_union][test_type]") {
-  uint32_t n = 20;
-  uint32_t k = 5;
-  var_opt_union<test_type> u(k);
-  var_opt_sketch<test_type> sk1(k);
-  var_opt_sketch<test_type> sk2(k);
-
-  // move udpates
-  for (int i = 0; i < (int) n; ++i) {
-    sk1.update(i);
-    sk2.update(-i);
-  }
-  REQUIRE(sk1.get_n() == n);
-  REQUIRE(sk2.get_n() == n);
-
-  // move unions
-  u.update(std::move(sk2));
-  u.update(std::move(sk1));
-  REQUIRE(u.get_result().get_n() == 2 * n);
-
-  // move constructor
-  var_opt_union<test_type> u2(std::move(u));
-  REQUIRE(u2.get_result().get_n() == 2 * n);
-
-  // move assignment
-  var_opt_union<test_type> u3(k);
-  u3 = std::move(u2);
-  REQUIRE(u3.get_result().get_n() == 2 * n);
-}
-
 }

data/vendor/datasketches-cpp/setup.py

@@ -49,6 +49,8 @@ class CMakeBuild(build_ext):
             os.path.dirname(self.get_ext_fullpath(ext.name)))
         cmake_args =  ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir]
         cmake_args += ['-DWITH_PYTHON=True']
+        # ensure we use a consistent python version
+        cmake_args += ['-DPYTHON_EXECUTABLE=' + sys.executable]
         cfg = 'Debug' if self.debug else 'Release'
         build_args = ['--config', cfg]
 
@@ -77,10 +79,10 @@ class CMakeBuild(build_ext):
 
 setup(
     name='datasketches',
-    version='2.2.0-SNAPSHOT',
-    author='Datasketches Developers',
+    version='3.0.0',
+    author='Apache DataSketches Developers',
     author_email='dev@datasketches.apache.org',
-    description='A wrapper for the C++ Datasketches library',
+    description='A wrapper for the C++ Apache DataSketches library',
     license='Apache License 2.0',
     url='http://datasketches.apache.org',
     long_description=open('python/README.md').read(),

data/vendor/datasketches-cpp/theta/CMakeLists.txt

@@ -33,9 +33,21 @@ target_link_libraries(theta INTERFACE common)
 target_compile_features(theta INTERFACE cxx_std_11)
 
 set(theta_HEADERS "")
-list(APPEND theta_HEADERS "include/theta_sketch.hpp;include/theta_union.hpp;include/theta_intersection.hpp")
-list(APPEND theta_HEADERS "include/theta_a_not_b.hpp;include/theta_sketch_impl.hpp")
-list(APPEND theta_HEADERS "include/theta_union_impl.hpp;include/theta_intersection_impl.hpp;include/theta_a_not_b_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_sketch.hpp;include/theta_sketch_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_union.hpp;include/theta_union_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_intersection.hpp;include/theta_intersection_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_a_not_b.hpp;include/theta_a_not_b_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_jaccard_similarity.hpp")
+list(APPEND theta_HEADERS "include/theta_comparators.hpp")
+list(APPEND theta_HEADERS "include/theta_constants.hpp")
+list(APPEND theta_HEADERS "include/theta_helpers.hpp")
+list(APPEND theta_HEADERS "include/theta_update_sketch_base.hpp;include/theta_update_sketch_base_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_union_base.hpp;include/theta_union_base_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_intersection_base.hpp;include/theta_intersection_base_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_set_difference_base.hpp;include/theta_set_difference_base_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_jaccard_similarity_base.hpp")
+list(APPEND theta_HEADERS "include/bounds_on_ratios_in_sampled_sets.hpp")
+list(APPEND theta_HEADERS "include/bounds_on_ratios_in_theta_sketched_sets.hpp")
 
 install(TARGETS theta
   EXPORT ${PROJECT_NAME}
@@ -54,4 +66,19 @@ target_sources(theta
     ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_impl.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_impl.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_a_not_b_impl.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_jaccard_similarity.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_comparators.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_constants.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_helpers.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_update_sketch_base.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_update_sketch_base_impl.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_base.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_base_impl.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_base.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_base_impl.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_set_difference_base.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_set_difference_base_impl.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_jaccard_similarity_base.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/bounds_on_ratios_in_sampled_sets.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/bounds_on_ratios_in_theta_sketched_sets.hpp
 )

data/vendor/datasketches-cpp/{tuple → theta}/include/bounds_on_ratios_in_sampled_sets.hpp

@@ -21,8 +21,9 @@
 #define BOUNDS_ON_RATIOS_IN_SAMPLED_SETS_HPP_
 
 #include <cstdint>
+#include <string>
 
-#include <bounds_binomial_proportions.hpp>
+#include "bounds_binomial_proportions.hpp"
 
 namespace datasketches {
 

data/vendor/datasketches-cpp/{tuple → theta}/include/bounds_on_ratios_in_theta_sketched_sets.hpp

@@ -23,7 +23,7 @@
 #include <cstdint>
 #include <stdexcept>
 
-#include <bounds_on_ratios_in_sampled_sets.hpp>
+#include "bounds_on_ratios_in_sampled_sets.hpp"
 
 namespace datasketches {
 

data/vendor/datasketches-cpp/theta/include/theta_a_not_b.hpp

@@ -20,51 +20,34 @@
 #ifndef THETA_A_NOT_B_HPP_
 #define THETA_A_NOT_B_HPP_
 
-#include <memory>
-#include <functional>
-#include <climits>
-
 #include "theta_sketch.hpp"
-#include "common_defs.hpp"
+#include "theta_set_difference_base.hpp"
 
 namespace datasketches {
 
-/*
- * author Alexander Saydakov
- * author Lee Rhodes
- * author Kevin Lang
- */
-
-template<typename A>
+template<typename Allocator = std::allocator<uint64_t>>
 class theta_a_not_b_alloc {
 public:
-  /**
-   * Creates an instance of the a-not-b operation (set difference) with a given has seed.
-   * @param seed hash seed
-   */
-  explicit theta_a_not_b_alloc(uint64_t seed = DEFAULT_SEED);
+  using Entry = uint64_t;
+  using ExtractKey = trivial_extract_key;
+  using CompactSketch = compact_theta_sketch_alloc<Allocator>;
+  using State = theta_set_difference_base<Entry, ExtractKey, CompactSketch, Allocator>;
+
+  explicit theta_a_not_b_alloc(uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
 
   /**
    * Computes the a-not-b set operation given two sketches.
    * @return the result of a-not-b
    */
-  compact_theta_sketch_alloc<A> compute(const theta_sketch_alloc<A>& a, const theta_sketch_alloc<A>& b, bool ordered = true) const;
+  template<typename FwdSketch, typename Sketch>
+  CompactSketch compute(FwdSketch&& a, const Sketch& b, bool ordered = true) const;
 
 private:
-  typedef typename std::allocator_traits<A>::template rebind_alloc<uint64_t> AllocU64;
-  uint16_t seed_hash_;
-
-  class less_than {
-  public:
-    explicit less_than(uint64_t value): value(value) {}
-    bool operator()(uint64_t value) const { return value < this->value; }
-  private:
-    uint64_t value;
-  };
+  State state_;
 };
 
 // alias with default allocator for convenience
-typedef theta_a_not_b_alloc<std::allocator<void>> theta_a_not_b;
+using theta_a_not_b = theta_a_not_b_alloc<std::allocator<uint64_t>>;
 
 } /* namespace datasketches */
 

data/vendor/datasketches-cpp/theta/include/theta_a_not_b_impl.hpp

@@ -26,56 +26,15 @@
 
 namespace datasketches {
 
-/*
- * author Alexander Saydakov
- * author Lee Rhodes
- * author Kevin Lang
- */
-
 template<typename A>
-theta_a_not_b_alloc<A>::theta_a_not_b_alloc(uint64_t seed):
-seed_hash_(theta_sketch_alloc<A>::get_seed_hash(seed))
+theta_a_not_b_alloc<A>::theta_a_not_b_alloc(uint64_t seed, const A& allocator):
+state_(seed, allocator)
 {}
 
 template<typename A>
-compact_theta_sketch_alloc<A> theta_a_not_b_alloc<A>::compute(const theta_sketch_alloc<A>& a, const theta_sketch_alloc<A>& b, bool ordered) const {
-  if (a.is_empty() || a.get_num_retained() == 0 || b.is_empty()) return compact_theta_sketch_alloc<A>(a, ordered);
-  if (a.get_seed_hash() != seed_hash_) throw std::invalid_argument("A seed hash mismatch");
-  if (b.get_seed_hash() != seed_hash_) throw std::invalid_argument("B seed hash mismatch");
-
-  const uint64_t theta = std::min(a.get_theta64(), b.get_theta64());
-  vector_u64<A> keys;
-  bool is_empty = a.is_empty();
-
-  if (b.get_num_retained() == 0) {
-    std::copy_if(a.begin(), a.end(), std::back_inserter(keys), less_than(theta));
-  } else {
-    if (a.is_ordered() && b.is_ordered()) { // sort-based
-      std::set_difference(a.begin(), a.end(), b.begin(), b.end(), conditional_back_inserter(keys, less_than(theta)));
-    } else { // hash-based
-      const uint8_t lg_size = lg_size_from_count(b.get_num_retained(), update_theta_sketch_alloc<A>::REBUILD_THRESHOLD);
-      vector_u64<A> b_hash_table(1 << lg_size, 0);
-      for (auto key: b) {
-        if (key < theta) {
-          update_theta_sketch_alloc<A>::hash_search_or_insert(key, b_hash_table.data(), lg_size);
-        } else if (b.is_ordered()) {
-          break; // early stop
-        }
-      }
-
-      // scan A lookup B
-      for (auto key: a) {
-        if (key < theta) {
-          if (!update_theta_sketch_alloc<A>::hash_search(key, b_hash_table.data(), lg_size)) keys.push_back(key);
-        } else if (a.is_ordered()) {
-          break; // early stop
-        }
-      }
-    }
-  }
-  if (keys.empty() && theta == theta_sketch_alloc<A>::MAX_THETA) is_empty = true;
-  if (ordered && !a.is_ordered()) std::sort(keys.begin(), keys.end());
-  return compact_theta_sketch_alloc<A>(is_empty, theta, std::move(keys), seed_hash_, a.is_ordered() || ordered);
+template<typename FwdSketch, typename Sketch>
+auto theta_a_not_b_alloc<A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const -> CompactSketch {
+  return state_.compute(std::forward<FwdSketch>(a), b, ordered);
 }
 
 } /* namespace datasketches */