datasketches 0.3.0 → 0.3.2

data/vendor/datasketches-cpp/theta/include/theta_union_base_impl.hpp

@@ -41,7 +41,7 @@ void theta_union_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
   if (sketch.is_empty()) return;
   if (sketch.get_seed_hash() != compute_seed_hash(table_.seed_)) throw std::invalid_argument("seed hash mismatch");
   table_.is_empty_ = false;
-  if (sketch.get_theta64() < union_theta_) union_theta_ = sketch.get_theta64();
+  union_theta_ = std::min(union_theta_, sketch.get_theta64());
   for (auto& entry: sketch) {
     const uint64_t hash = EK()(entry);
     if (hash < union_theta_ && hash < table_.theta_) {
@@ -55,7 +55,7 @@ void theta_union_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
       if (sketch.is_ordered()) break; // early stop
     }
   }
-  if (table_.theta_ < union_theta_) union_theta_ = table_.theta_;
+  union_theta_ = std::min(union_theta_, table_.theta_);
 }
 
 template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
@@ -65,16 +65,16 @@ CS theta_union_base<EN, EK, P, S, CS, A>::get_result(bool ordered) const {
   entries.reserve(table_.num_entries_);
   uint64_t theta = std::min(union_theta_, table_.theta_);
   const uint32_t nominal_num = 1 << table_.lg_nom_size_;
-  if (union_theta_ >= theta && table_.num_entries_ <= nominal_num) {
+  if (union_theta_ >= table_.theta_) {
     std::copy_if(table_.begin(), table_.end(), std::back_inserter(entries), key_not_zero<EN, EK>());
   } else {
     std::copy_if(table_.begin(), table_.end(), std::back_inserter(entries), key_not_zero_less_than<uint64_t, EN, EK>(theta));
-    if (entries.size() > nominal_num) {
-      std::nth_element(entries.begin(), entries.begin() + nominal_num, entries.end(), comparator());
-      theta = EK()(entries[nominal_num]);
-      entries.erase(entries.begin() + nominal_num, entries.end());
-      entries.shrink_to_fit();
-    }
+  }
+  if (entries.size() > nominal_num) {
+    std::nth_element(entries.begin(), entries.begin() + nominal_num, entries.end(), comparator());
+    theta = EK()(entries[nominal_num]);
+    entries.erase(entries.begin() + nominal_num, entries.end());
+    entries.shrink_to_fit();
   }
   if (ordered) std::sort(entries.begin(), entries.end(), comparator());
   return CS(table_.is_empty_, ordered, compute_seed_hash(table_.seed_), theta, std::move(entries));

data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp

@@ -23,8 +23,8 @@
 #include <vector>
 #include <climits>
 #include <cmath>
+#include <iterator>
 
-#include "common_defs.hpp"
 #include "MurmurHash3.h"
 #include "theta_comparators.hpp"
 #include "theta_constants.hpp"
@@ -185,8 +185,14 @@ static inline uint64_t compute_hash(const void* data, size_t length, uint64_t se
 // iterators
 
 template<typename Entry, typename ExtractKey>
-class theta_iterator: public std::iterator<std::input_iterator_tag, Entry> {
+class theta_iterator {
 public:
+  using iterator_category = std::input_iterator_tag;
+  using value_type = Entry;
+  using difference_type = std::ptrdiff_t;
+  using pointer = Entry*;
+  using reference = Entry&;
+
   theta_iterator(Entry* entries, uint32_t size, uint32_t index);
   theta_iterator& operator++();
   theta_iterator operator++(int);
@@ -201,14 +207,20 @@ private:
 };
 
 template<typename Entry, typename ExtractKey>
-class theta_const_iterator: public std::iterator<std::input_iterator_tag, Entry> {
+class theta_const_iterator {
 public:
+  using iterator_category = std::input_iterator_tag;
+  using value_type = const Entry;
+  using difference_type = std::ptrdiff_t;
+  using pointer = const Entry*;
+  using reference = const Entry&;
+
   theta_const_iterator(const Entry* entries, uint32_t size, uint32_t index);
   theta_const_iterator& operator++();
   theta_const_iterator operator++(int);
   bool operator==(const theta_const_iterator& other) const;
   bool operator!=(const theta_const_iterator& other) const;
-  const Entry& operator*() const;
+  reference operator*() const;
 
 private:
   const Entry* entries_;

data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp

@@ -188,7 +188,7 @@ auto theta_update_sketch_base<EN, EK, A>::begin() const -> iterator {
 
 template<typename EN, typename EK, typename A>
 auto theta_update_sketch_base<EN, EK, A>::end() const -> iterator {
-  return &entries_[1ULL << lg_cur_size_];
+  return entries_ + (1ULL << lg_cur_size_);
 }
 
 template<typename EN, typename EK, typename A>
@@ -382,7 +382,7 @@ bool theta_iterator<Entry, ExtractKey>::operator==(const theta_iterator& other)
 }
 
 template<typename Entry, typename ExtractKey>
-auto theta_iterator<Entry, ExtractKey>::operator*() const -> Entry& {
+auto theta_iterator<Entry, ExtractKey>::operator*() const -> reference {
   return entries_[index_];
 }
 

data/vendor/datasketches-cpp/theta/test/CMakeLists.txt

@@ -44,4 +44,5 @@ target_sources(theta_test
     theta_a_not_b_test.cpp
     theta_jaccard_similarity_test.cpp
     theta_setop_test.cpp
+    bit_packing_test.cpp
 )

data/vendor/datasketches-cpp/theta/test/bit_packing_test.cpp

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <bit_packing.hpp>
+
+namespace datasketches {
+
+// for every number of bits from 1 to 63
+// generate pseudo-random data, pack, unpack and compare
+
+// inverse golden ratio (0.618.. of max uint64_t)
+static const uint64_t IGOLDEN64 = 0x9e3779b97f4a7c13ULL;
+
+TEST_CASE("pack unpack bits") {
+  for (uint8_t bits = 1; bits <= 63; ++bits) {
+    const uint64_t mask = (1ULL << bits) - 1;
+    std::vector<uint64_t> input(8, 0);
+    const uint64_t igolden64 = IGOLDEN64;
+    uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value
+    for (int i = 0; i < 8; ++i) {
+      input[i] = value & mask;
+      value += igolden64;
+    }
+    std::vector<uint8_t> bytes(8 * sizeof(uint64_t), 0);
+    uint8_t offset = 0;
+    uint8_t* ptr = bytes.data();
+    for (int i = 0; i < 8; ++i) {
+      offset = pack_bits(input[i], bits, ptr, offset);
+    }
+
+    std::vector<uint64_t> output(8, 0);
+    offset = 0;
+    const uint8_t* cptr = bytes.data();
+    for (int i = 0; i < 8; ++i) {
+      offset = unpack_bits(output[i], bits, cptr, offset);
+    }
+    for (int i = 0; i < 8; ++i) {
+      REQUIRE((input[i] & mask) == output[i]);
+    }
+  }
+}
+
+TEST_CASE("pack unpack blocks") {
+  for (uint8_t bits = 1; bits <= 63; ++bits) {
+    const uint64_t mask = (1ULL << bits) - 1;
+    std::vector<uint64_t> input(8, 0);
+    const uint64_t igolden64 = IGOLDEN64;
+    uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value
+    for (int i = 0; i < 8; ++i) {
+      input[i] = value & mask;
+      value += igolden64;
+    }
+    std::vector<uint8_t> bytes(8 * sizeof(uint64_t), 0);
+    pack_bits_block8(input.data(), bytes.data(), bits);
+    std::vector<uint64_t> output(8, 0);
+    unpack_bits_block8(output.data(), bytes.data(), bits);
+    for (int i = 0; i < 8; ++i) {
+      REQUIRE((input[i] & mask) == output[i]);
+    }
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp

@@ -607,7 +607,7 @@ TEST_CASE("theta sketch: wrap compact estimation from java", "[theta_sketch]") {
   compact_theta_sketch compact_sketch = update_sketch.compact();
   // the sketches are ordered, so the iteration sequence must match exactly
   auto iter = sketch.begin();
-  for (const auto& key: compact_sketch) {
+  for (const auto key: compact_sketch) {
     REQUIRE(*iter == key);
     ++iter;
   }
@@ -652,7 +652,7 @@ TEST_CASE("theta sketch: wrap compact v1 estimation from java", "[theta_sketch]"
   compact_theta_sketch compact_sketch = update_sketch.compact();
   // the sketches are ordered, so the iteration sequence must match exactly
   auto iter = sketch.begin();
-  for (const auto& key: compact_sketch) {
+  for (const auto key: compact_sketch) {
     REQUIRE(*iter == key);
     ++iter;
   }
@@ -697,7 +697,46 @@ TEST_CASE("theta sketch: wrap compact v2 estimation from java", "[theta_sketch]"
   compact_theta_sketch compact_sketch = update_sketch.compact();
   // the sketches are ordered, so the iteration sequence must match exactly
   auto iter = sketch.begin();
-  for (const auto& key: compact_sketch) {
+  for (const auto key: compact_sketch) {
+    REQUIRE(*iter == key);
+    ++iter;
+  }
+}
+
+TEST_CASE("theta sketch: serialize deserialize compressed", "[theta_sketch]") {
+  auto update_sketch = update_theta_sketch::builder().build();
+  for (int i = 0; i < 10000; i++) update_sketch.update(i);
+  auto compact_sketch = update_sketch.compact();
+
+  auto bytes = compact_sketch.serialize_compressed();
+  { // deserialize bytes
+    auto deserialized_sketch = compact_theta_sketch::deserialize(bytes.data(), bytes.size());
+    REQUIRE(deserialized_sketch.get_num_retained() == compact_sketch.get_num_retained());
+    REQUIRE(deserialized_sketch.get_theta() == compact_sketch.get_theta());
+    auto iter = deserialized_sketch.begin();
+    for (const auto key: compact_sketch) {
+      REQUIRE(*iter == key);
+      ++iter;
+    }
+  }
+  { // wrap bytes
+    auto wrapped_sketch = wrapped_compact_theta_sketch::wrap(bytes.data(), bytes.size());
+    REQUIRE(wrapped_sketch.get_num_retained() == compact_sketch.get_num_retained());
+    REQUIRE(wrapped_sketch.get_theta() == compact_sketch.get_theta());
+    auto iter = wrapped_sketch.begin();
+    for (const auto key: compact_sketch) {
+      REQUIRE(*iter == key);
+      ++iter;
+    }
+  }
+
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  compact_sketch.serialize_compressed(s);
+  auto deserialized_sketch = compact_theta_sketch::deserialize(s);
+  REQUIRE(deserialized_sketch.get_num_retained() == compact_sketch.get_num_retained());
+  REQUIRE(deserialized_sketch.get_theta() == compact_sketch.get_theta());
+  auto iter = deserialized_sketch.begin();
+  for (const auto key: compact_sketch) {
     REQUIRE(*iter == key);
     ++iter;
   }

data/vendor/datasketches-cpp/theta/test/theta_union_test.cpp

@@ -128,4 +128,29 @@ TEST_CASE("theta union: seed mismatch", "[theta_union]") {
   REQUIRE_THROWS_AS(u.update(sketch), std::invalid_argument);
 }
 
+TEST_CASE("theta union: larger K", "[theta_union]") {
+  auto update_sketch1 = datasketches::update_theta_sketch::builder().set_lg_k(14).build();
+  for(int i = 0; i < 16384; ++i) update_sketch1.update(i);
+
+  auto update_sketch2 = datasketches::update_theta_sketch::builder().set_lg_k(14).build();
+  for(int i = 0; i < 26384; ++i) update_sketch2.update(i);
+
+  auto update_sketch3 = datasketches::update_theta_sketch::builder().set_lg_k(14).build();
+  for(int i = 0; i < 86384; ++i) update_sketch3.update(i);
+
+  auto union1 = datasketches::theta_union::builder().set_lg_k(16).build();
+  union1.update(update_sketch2);
+  union1.update(update_sketch1);
+  union1.update(update_sketch3);
+  auto result1 = union1.get_result();
+  REQUIRE(result1.get_estimate() == update_sketch3.get_estimate());
+
+  auto union2 = datasketches::theta_union::builder().set_lg_k(16).build();
+  union2.update(update_sketch1);
+  union2.update(update_sketch3);
+  union2.update(update_sketch2);
+  auto result2 = union2.get_result();
+  REQUIRE(result2.get_estimate() == update_sketch3.get_estimate());
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp

@@ -32,7 +32,8 @@ bool tuple_sketch<S, A>::is_estimation_mode() const {
 
 template<typename S, typename A>
 double tuple_sketch<S, A>::get_theta() const {
-  return static_cast<double>(get_theta64()) / theta_constants::MAX_THETA;
+  return static_cast<double>(get_theta64()) /
+         static_cast<double>(theta_constants::MAX_THETA);
 }
 
 template<typename S, typename A>

data/vendor/datasketches-cpp/version.cfg.in

@@ -1 +1 @@
-4.0.0
+4.1.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datasketches
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-09 00:00:00.000000000 Z
+date: 2023-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -76,6 +76,12 @@ files:
 - vendor/datasketches-cpp/common/test/test_allocator.cpp
 - vendor/datasketches-cpp/common/test/test_allocator.hpp
 - vendor/datasketches-cpp/common/test/test_type.hpp
+- vendor/datasketches-cpp/count/CMakeLists.txt
+- vendor/datasketches-cpp/count/include/count_min.hpp
+- vendor/datasketches-cpp/count/include/count_min_impl.hpp
+- vendor/datasketches-cpp/count/test/CMakeLists.txt
+- vendor/datasketches-cpp/count/test/count_min_allocation_test.cpp
+- vendor/datasketches-cpp/count/test/count_min_test.cpp
 - vendor/datasketches-cpp/cpc/CMakeLists.txt
 - vendor/datasketches-cpp/cpc/include/compression_data.hpp
 - vendor/datasketches-cpp/cpc/include/cpc_common.hpp
@@ -96,6 +102,11 @@ files:
 - vendor/datasketches-cpp/cpc/test/cpc_sketch_allocation_test.cpp
 - vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp
 - vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp
+- vendor/datasketches-cpp/density/CMakeLists.txt
+- vendor/datasketches-cpp/density/include/density_sketch.hpp
+- vendor/datasketches-cpp/density/include/density_sketch_impl.hpp
+- vendor/datasketches-cpp/density/test/CMakeLists.txt
+- vendor/datasketches-cpp/density/test/density_sketch_test.cpp
 - vendor/datasketches-cpp/fi/CMakeLists.txt
 - vendor/datasketches-cpp/fi/include/frequent_items_sketch.hpp
 - vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp
@@ -173,9 +184,18 @@ files:
 - vendor/datasketches-cpp/pyproject.toml
 - vendor/datasketches-cpp/python/CMakeLists.txt
 - vendor/datasketches-cpp/python/README.md
+- vendor/datasketches-cpp/python/datasketches/DensityWrapper.py
+- vendor/datasketches-cpp/python/datasketches/KernelFunction.py
 - vendor/datasketches-cpp/python/datasketches/PySerDe.py
+- vendor/datasketches-cpp/python/datasketches/TuplePolicy.py
+- vendor/datasketches-cpp/python/datasketches/TupleWrapper.py
 - vendor/datasketches-cpp/python/datasketches/__init__.py
+- vendor/datasketches-cpp/python/include/kernel_function.hpp
+- vendor/datasketches-cpp/python/include/py_object_lt.hpp
+- vendor/datasketches-cpp/python/include/py_object_ostream.hpp
 - vendor/datasketches-cpp/python/include/py_serde.hpp
+- vendor/datasketches-cpp/python/include/quantile_conditional.hpp
+- vendor/datasketches-cpp/python/include/tuple_policy.hpp
 - vendor/datasketches-cpp/python/jupyter/CPCSketch.ipynb
 - vendor/datasketches-cpp/python/jupyter/FrequentItemsSketch.ipynb
 - vendor/datasketches-cpp/python/jupyter/HLLSketch.ipynb
@@ -183,8 +203,10 @@ files:
 - vendor/datasketches-cpp/python/jupyter/ThetaSketchNotebook.ipynb
 - vendor/datasketches-cpp/python/pybind11Path.cmd
 - vendor/datasketches-cpp/python/src/__init__.py
+- vendor/datasketches-cpp/python/src/count_wrapper.cpp
 - vendor/datasketches-cpp/python/src/cpc_wrapper.cpp
 - vendor/datasketches-cpp/python/src/datasketches.cpp
+- vendor/datasketches-cpp/python/src/density_wrapper.cpp
 - vendor/datasketches-cpp/python/src/fi_wrapper.cpp
 - vendor/datasketches-cpp/python/src/hll_wrapper.cpp
 - vendor/datasketches-cpp/python/src/kll_wrapper.cpp
@@ -193,16 +215,20 @@ files:
 - vendor/datasketches-cpp/python/src/quantiles_wrapper.cpp
 - vendor/datasketches-cpp/python/src/req_wrapper.cpp
 - vendor/datasketches-cpp/python/src/theta_wrapper.cpp
+- vendor/datasketches-cpp/python/src/tuple_wrapper.cpp
 - vendor/datasketches-cpp/python/src/vector_of_kll.cpp
 - vendor/datasketches-cpp/python/src/vo_wrapper.cpp
 - vendor/datasketches-cpp/python/tests/__init__.py
+- vendor/datasketches-cpp/python/tests/count_min_test.py
 - vendor/datasketches-cpp/python/tests/cpc_test.py
+- vendor/datasketches-cpp/python/tests/density_test.py
 - vendor/datasketches-cpp/python/tests/fi_test.py
 - vendor/datasketches-cpp/python/tests/hll_test.py
 - vendor/datasketches-cpp/python/tests/kll_test.py
 - vendor/datasketches-cpp/python/tests/quantiles_test.py
 - vendor/datasketches-cpp/python/tests/req_test.py
 - vendor/datasketches-cpp/python/tests/theta_test.py
+- vendor/datasketches-cpp/python/tests/tuple_test.py
 - vendor/datasketches-cpp/python/tests/vector_of_kll_test.py
 - vendor/datasketches-cpp/python/tests/vo_test.py
 - vendor/datasketches-cpp/quantiles/CMakeLists.txt
@@ -249,6 +275,7 @@ files:
 - vendor/datasketches-cpp/sampling/test/varopt_union_double_sampling.sk
 - vendor/datasketches-cpp/setup.py
 - vendor/datasketches-cpp/theta/CMakeLists.txt
+- vendor/datasketches-cpp/theta/include/bit_packing.hpp
 - vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_sampled_sets.hpp
 - vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_theta_sketched_sets.hpp
 - vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser.hpp
@@ -275,6 +302,7 @@ files:
 - vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp
 - vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp
 - vendor/datasketches-cpp/theta/test/CMakeLists.txt
+- vendor/datasketches-cpp/theta/test/bit_packing_test.cpp
 - vendor/datasketches-cpp/theta/test/theta_a_not_b_test.cpp
 - vendor/datasketches-cpp/theta/test/theta_compact_empty_from_java.sk
 - vendor/datasketches-cpp/theta/test/theta_compact_empty_from_java_v1.sk
@@ -342,7 +370,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Sketch data structures for Ruby