datasketches 0.3.2 → 0.4.0

data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp

@@ -37,7 +37,7 @@ kll_sketch<T, C, A>::kll_sketch(uint16_t k, const C& comparator, const A& alloca
 comparator_(comparator),
 allocator_(allocator),
 k_(k),
-m_(DEFAULT_M),
+m_(kll_constants::DEFAULT_M),
 min_k_(k),
 num_levels_(1),
 is_level_zero_sorted_(false),
@@ -45,12 +45,13 @@ n_(0),
 levels_(2, 0, allocator),
 items_(nullptr),
 items_size_(k_),
-min_item_(nullptr),
-max_item_(nullptr),
+min_item_(),
+max_item_(),
 sorted_view_(nullptr)
 {
-  if (k < MIN_K || k > MAX_K) {
-    throw std::invalid_argument("K must be >= " + std::to_string(MIN_K) + " and <= " + std::to_string(MAX_K) + ": " + std::to_string(k));
+  if (k < kll_constants::MIN_K || k > kll_constants::MAX_K) {
+    throw std::invalid_argument("K must be >= " + std::to_string(kll_constants::MIN_K) + " and <= "
+        + std::to_string(kll_constants::MAX_K) + ": " + std::to_string(k));
   }
   levels_[0] = levels_[1] = k;
   items_ = allocator_.allocate(items_size_);
@@ -69,14 +70,12 @@ n_(other.n_),
 levels_(other.levels_),
 items_(nullptr),
 items_size_(other.items_size_),
-min_item_(nullptr),
-max_item_(nullptr),
+min_item_(other.min_item_),
+max_item_(other.max_item_),
 sorted_view_(nullptr)
 {
   items_ = allocator_.allocate(items_size_);
   for (auto i = levels_[0]; i < levels_[num_levels_]; ++i) new (&items_[i]) T(other.items_[i]);
-  if (other.min_item_ != nullptr) min_item_ = new (allocator_.allocate(1)) T(*other.min_item_);
-  if (other.max_item_ != nullptr) max_item_ = new (allocator_.allocate(1)) T(*other.max_item_);
 }
 
 template<typename T, typename C, typename A>
@@ -92,13 +91,11 @@ n_(other.n_),
 levels_(std::move(other.levels_)),
 items_(other.items_),
 items_size_(other.items_size_),
-min_item_(other.min_item_),
-max_item_(other.max_item_),
+min_item_(std::move(other.min_item_)),
+max_item_(std::move(other.max_item_)),
 sorted_view_(nullptr)
 {
   other.items_ = nullptr;
-  other.min_item_ = nullptr;
-  other.max_item_ = nullptr;
 }
 
 template<typename T, typename C, typename A>
@@ -148,14 +145,6 @@ kll_sketch<T, C, A>::~kll_sketch() {
     for (uint32_t i = begin; i < end; i++) items_[i].~T();
     allocator_.deallocate(items_, items_size_);
   }
-  if (min_item_ != nullptr) {
-    min_item_->~T();
-    allocator_.deallocate(min_item_, 1);
-  }
-  if (max_item_ != nullptr) {
-    max_item_->~T();
-    allocator_.deallocate(max_item_, 1);
-  }
   reset_sorted_view();
 }
 
@@ -173,8 +162,8 @@ n_(other.n_),
 levels_(other.levels_, allocator_),
 items_(nullptr),
 items_size_(other.items_size_),
-min_item_(nullptr),
-max_item_(nullptr),
+min_item_(other.min_item_),
+max_item_(other.max_item_),
 sorted_view_(nullptr)
 {
   static_assert(
@@ -183,8 +172,6 @@ sorted_view_(nullptr)
   );
   items_ = allocator_.allocate(items_size_);
   for (auto i = levels_[0]; i < levels_[num_levels_]; ++i) new (&items_[i]) T(other.items_[i]);
-  if (other.min_item_ != nullptr) min_item_ = new (allocator_.allocate(1)) T(*other.min_item_);
-  if (other.max_item_ != nullptr) max_item_ = new (allocator_.allocate(1)) T(*other.max_item_);
   check_sorting();
 }
 
@@ -192,7 +179,7 @@ template<typename T, typename C, typename A>
 template<typename FwdT>
 void kll_sketch<T, C, A>::update(FwdT&& item) {
   if (!check_update_item(item)) { return; }
-  update_min_max(item);
+  update_min_max(static_cast<const T&>(item)); // min and max are always copies
   const uint32_t index = internal_update();
   new (&items_[index]) T(std::forward<FwdT>(item));
   reset_sorted_view();
@@ -201,8 +188,8 @@ void kll_sketch<T, C, A>::update(FwdT&& item) {
 template<typename T, typename C, typename A>
 void kll_sketch<T, C, A>::update_min_max(const T& item) {
   if (is_empty()) {
-    min_item_ = new (allocator_.allocate(1)) T(item);
-    max_item_ = new (allocator_.allocate(1)) T(item);
+    min_item_.emplace(item);
+    max_item_.emplace(item);
   } else {
     if (comparator_(item, *min_item_)) *min_item_ = item;
     if (comparator_(*max_item_, item)) *max_item_ = item;
@@ -225,8 +212,8 @@ void kll_sketch<T, C, A>::merge(FwdSk&& other) {
     throw std::invalid_argument("incompatible M: " + std::to_string(m_) + " and " + std::to_string(other.m_));
   }
   if (is_empty()) {
-    min_item_ = new (allocator_.allocate(1)) T(conditional_forward<FwdSk>(*other.min_item_));
-    max_item_ = new (allocator_.allocate(1)) T(conditional_forward<FwdSk>(*other.max_item_));
+    min_item_.emplace(conditional_forward<FwdSk>(*other.min_item_));
+    max_item_.emplace(conditional_forward<FwdSk>(*other.max_item_));
   } else {
     if (comparator_(*other.min_item_, *min_item_)) *min_item_ = conditional_forward<FwdSk>(*other.min_item_);
     if (comparator_(*max_item_, *other.max_item_)) *max_item_ = conditional_forward<FwdSk>(*other.max_item_);
@@ -322,42 +309,6 @@ auto kll_sketch<T, C, A>::get_quantile(double rank, bool inclusive) const -> qua
   return sorted_view_->get_quantile(rank, inclusive);
 }
 
-template<typename T, typename C, typename A>
-std::vector<T, A> kll_sketch<T, C, A>::get_quantiles(const double* ranks, uint32_t size, bool inclusive) const {
-  if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch");
-  std::vector<T, A> quantiles(allocator_);
-  quantiles.reserve(size);
-
-  // may have a side effect of sorting level zero if needed
-  setup_sorted_view();
-
-  for (uint32_t i = 0; i < size; i++) {
-    const double rank = ranks[i];
-    if ((rank < 0.0) || (rank > 1.0)) {
-      throw std::invalid_argument("normalized rank cannot be less than 0 or greater than 1");
-    }
-    quantiles.push_back(sorted_view_->get_quantile(rank, inclusive));
-  }
-  return quantiles;
-}
-
-template<typename T, typename C, typename A>
-std::vector<T, A> kll_sketch<T, C, A>::get_quantiles(uint32_t num, bool inclusive) const {
-  if (is_empty()) throw std::runtime_error("operation is undefined for an empty sketch");
-  if (num == 0) {
-    throw std::invalid_argument("num must be > 0");
-  }
-  vector_double ranks(num, 0, allocator_);
-  ranks[0] = 0.0;
-  for (size_t i = 1; i < num; i++) {
-    ranks[i] = static_cast<double>(i) / (num - 1);
-  }
-  if (num > 1) {
-    ranks[num - 1] = 1.0;
-  }
-  return get_quantiles(ranks.data(), num, inclusive);
-}
-
 template<typename T, typename C, typename A>
 double kll_sketch<T, C, A>::get_normalized_rank_error(bool pmf) const {
   return get_normalized_rank_error(min_k_, pmf);
@@ -396,7 +347,7 @@ template<typename T, typename C, typename A>
 template<typename TT, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type>
 size_t kll_sketch<T, C, A>::get_max_serialized_size_bytes(uint16_t k, uint64_t n) {
   const uint8_t num_levels = kll_helper::ub_on_num_levels(n);
-  const uint32_t max_num_retained = kll_helper::compute_total_capacity(k, DEFAULT_M, num_levels);
+  const uint32_t max_num_retained = kll_helper::compute_total_capacity(k, kll_constants::DEFAULT_M, num_levels);
   // the last integer in the levels_ array is not serialized because it can be derived
   return DATA_START + num_levels * sizeof(uint32_t) + (max_num_retained + 2) * sizeof(TT);
 }
@@ -406,7 +357,7 @@ template<typename T, typename C, typename A>
 template<typename TT, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type>
 size_t kll_sketch<T, C, A>::get_max_serialized_size_bytes(uint16_t k, uint64_t n, size_t max_item_size_bytes) {
   const uint8_t num_levels = kll_helper::ub_on_num_levels(n);
-  const uint32_t max_num_retained = kll_helper::compute_total_capacity(k, DEFAULT_M, num_levels);
+  const uint32_t max_num_retained = kll_helper::compute_total_capacity(k, kll_constants::DEFAULT_M, num_levels);
   // the last integer in the levels_ array is not serialized because it can be derived
   return DATA_START + num_levels * sizeof(uint32_t) + (max_num_retained + 2) * max_item_size_bytes;
 }
@@ -438,8 +389,8 @@ void kll_sketch<T, C, A>::serialize(std::ostream& os, const SerDe& sd) const {
     write(os, num_levels_);
     write(os, unused);
     write(os, levels_.data(), sizeof(levels_[0]) * num_levels_);
-    sd.serialize(os, min_item_, 1);
-    sd.serialize(os, max_item_, 1);
+    sd.serialize(os, &*min_item_, 1);
+    sd.serialize(os, &*max_item_, 1);
   }
   sd.serialize(os, &items_[levels_[0]], get_num_retained());
 }
@@ -474,8 +425,8 @@ auto kll_sketch<T, C, A>::serialize(unsigned header_size_bytes, const SerDe& sd)
       ptr += copy_to_mem(num_levels_, ptr);
       ptr += sizeof(uint8_t); // unused
       ptr += copy_to_mem(levels_.data(), ptr, sizeof(levels_[0]) * num_levels_);
-      ptr += sd.serialize(ptr, end_ptr - ptr, min_item_, 1);
-      ptr += sd.serialize(ptr, end_ptr - ptr, max_item_, 1);
+      ptr += sd.serialize(ptr, end_ptr - ptr, &*min_item_, 1);
+      ptr += sd.serialize(ptr, end_ptr - ptr, &*max_item_, 1);
     }
     const size_t bytes_remaining = end_ptr - ptr;
     ptr += sd.serialize(ptr, bytes_remaining, &items_[levels_[0]], get_num_retained());
@@ -530,20 +481,20 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(std::istream& is, const Ser
     read(is, levels.data(), sizeof(levels[0]) * num_levels);
   }
   levels[num_levels] = capacity;
-  A alloc(allocator);
-  auto item_buffer_deleter = [&alloc](T* ptr) { alloc.deallocate(ptr, 1); };
-  std::unique_ptr<T, decltype(item_buffer_deleter)> min_item_buffer(alloc.allocate(1), item_buffer_deleter);
-  std::unique_ptr<T, decltype(item_buffer_deleter)> max_item_buffer(alloc.allocate(1), item_buffer_deleter);
-  std::unique_ptr<T, item_deleter> min_item(nullptr, item_deleter(allocator));
-  std::unique_ptr<T, item_deleter> max_item(nullptr, item_deleter(allocator));
+  optional<T> tmp; // space to deserialize min and max
+  optional<T> min_item;
+  optional<T> max_item;
   if (!is_single_item) {
-    sd.deserialize(is, min_item_buffer.get(), 1);
-    // serde call did not throw, repackage with destrtuctor
-    min_item = std::unique_ptr<T, item_deleter>(min_item_buffer.release(), item_deleter(allocator));
-    sd.deserialize(is, max_item_buffer.get(), 1);
-    // serde call did not throw, repackage with destrtuctor
-    max_item = std::unique_ptr<T, item_deleter>(max_item_buffer.release(), item_deleter(allocator));
+    sd.deserialize(is, &*tmp, 1);
+    // serde call did not throw, repackage and cleanup
+    min_item.emplace(*tmp);
+    (*tmp).~T();
+    sd.deserialize(is, &*tmp, 1);
+    // serde call did not throw, repackage and cleanup
+    max_item.emplace(*tmp);
+    (*tmp).~T();
   }
+  A alloc(allocator);
   auto items_buffer_deleter = [capacity, &alloc](T* ptr) { alloc.deallocate(ptr, capacity); };
   std::unique_ptr<T, decltype(items_buffer_deleter)> items_buffer(alloc.allocate(capacity), items_buffer_deleter);
   const auto num_items = levels[num_levels] - levels[0];
@@ -552,12 +503,8 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(std::istream& is, const Ser
   std::unique_ptr<T, items_deleter> items(items_buffer.release(), items_deleter(levels[0], capacity, allocator));
   const bool is_level_zero_sorted = (flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED)) > 0;
   if (is_single_item) {
-    new (min_item_buffer.get()) T(items.get()[levels[0]]);
-    // copy did not throw, repackage with destrtuctor
-    min_item = std::unique_ptr<T, item_deleter>(min_item_buffer.release(), item_deleter(allocator));
-    new (max_item_buffer.get()) T(items.get()[levels[0]]);
-    // copy did not throw, repackage with destrtuctor
-    max_item = std::unique_ptr<T, item_deleter>(max_item_buffer.release(), item_deleter(allocator));
+    min_item.emplace(items.get()[levels[0]]);
+    max_item.emplace(items.get()[levels[0]]);
   }
   if (!is.good())
     throw std::runtime_error("error reading from std::istream");
@@ -618,20 +565,20 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(const void* bytes, size_t s
     ptr += copy_from_mem(ptr, levels.data(), sizeof(levels[0]) * num_levels);
   }
   levels[num_levels] = capacity;
-  A alloc(allocator);
-  auto item_buffer_deleter = [&alloc](T* ptr) { alloc.deallocate(ptr, 1); };
-  std::unique_ptr<T, decltype(item_buffer_deleter)> min_item_buffer(alloc.allocate(1), item_buffer_deleter);
-  std::unique_ptr<T, decltype(item_buffer_deleter)> max_item_buffer(alloc.allocate(1), item_buffer_deleter);
-  std::unique_ptr<T, item_deleter> min_item(nullptr, item_deleter(allocator));
-  std::unique_ptr<T, item_deleter> max_item(nullptr, item_deleter(allocator));
+  optional<T> tmp; // space to deserialize min and max
+  optional<T> min_item;
+  optional<T> max_item;
   if (!is_single_item) {
-    ptr += sd.deserialize(ptr, end_ptr - ptr, min_item_buffer.get(), 1);
-    // serde call did not throw, repackage with destrtuctor
-    min_item = std::unique_ptr<T, item_deleter>(min_item_buffer.release(), item_deleter(allocator));
-    ptr += sd.deserialize(ptr, end_ptr - ptr, max_item_buffer.get(), 1);
-    // serde call did not throw, repackage with destrtuctor
-    max_item = std::unique_ptr<T, item_deleter>(max_item_buffer.release(), item_deleter(allocator));
+    ptr += sd.deserialize(ptr, end_ptr - ptr, &*tmp, 1);
+    // serde call did not throw, repackage and cleanup
+    min_item.emplace(*tmp);
+    (*tmp).~T();
+    ptr += sd.deserialize(ptr, end_ptr - ptr, &*tmp, 1);
+    // serde call did not throw, repackage and cleanup
+    max_item.emplace(*tmp);
+    (*tmp).~T();
   }
+  A alloc(allocator);
   auto items_buffer_deleter = [capacity, &alloc](T* ptr) { alloc.deallocate(ptr, capacity); };
   std::unique_ptr<T, decltype(items_buffer_deleter)> items_buffer(alloc.allocate(capacity), items_buffer_deleter);
   const auto num_items = levels[num_levels] - levels[0];
@@ -642,12 +589,8 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(const void* bytes, size_t s
   if (delta != size) throw std::logic_error("deserialized size mismatch: " + std::to_string(delta) + " != " + std::to_string(size));
   const bool is_level_zero_sorted = (flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED)) > 0;
   if (is_single_item) {
-    new (min_item_buffer.get()) T(items.get()[levels[0]]);
-    // copy did not throw, repackage with destrtuctor
-    min_item = std::unique_ptr<T, item_deleter>(min_item_buffer.release(), item_deleter(allocator));
-    new (max_item_buffer.get()) T(items.get()[levels[0]]);
-    // copy did not throw, repackage with destrtuctor
-    max_item = std::unique_ptr<T, item_deleter>(max_item_buffer.release(), item_deleter(allocator));
+    min_item.emplace(items.get()[levels[0]]);
+    max_item.emplace(items.get()[levels[0]]);
   }
   return kll_sketch(k, min_k, n, num_levels, std::move(levels), std::move(items), capacity,
       std::move(min_item), std::move(max_item), is_level_zero_sorted, comparator);
@@ -670,12 +613,12 @@ double kll_sketch<T, C, A>::get_normalized_rank_error(uint16_t k, bool pmf) {
 // for deserialization
 template<typename T, typename C, typename A>
 kll_sketch<T, C, A>::kll_sketch(uint16_t k, uint16_t min_k, uint64_t n, uint8_t num_levels, vector_u32&& levels,
-    std::unique_ptr<T, items_deleter> items, uint32_t items_size, std::unique_ptr<T, item_deleter> min_item,
-    std::unique_ptr<T, item_deleter> max_item, bool is_level_zero_sorted, const C& comparator):
+    std::unique_ptr<T, items_deleter> items, uint32_t items_size, optional<T>&& min_item,
+    optional<T>&& max_item, bool is_level_zero_sorted, const C& comparator):
 comparator_(comparator),
 allocator_(levels.get_allocator()),
 k_(k),
-m_(DEFAULT_M),
+m_(kll_constants::DEFAULT_M),
 min_k_(min_k),
 num_levels_(num_levels),
 is_level_zero_sorted_(is_level_zero_sorted),
@@ -683,8 +626,8 @@ n_(n),
 levels_(std::move(levels)),
 items_(items.release()),
 items_size_(items_size),
-min_item_(min_item.release()),
-max_item_(max_item.release()),
+min_item_(std::move(min_item)),
+max_item_(std::move(max_item)),
 sorted_view_(nullptr)
 {}
 
@@ -820,7 +763,7 @@ quantiles_sorted_view<T, C, A> kll_sketch<T, C, A>::get_sorted_view() const {
   for (uint8_t level = 0; level < num_levels_; ++level) {
     const auto from = items_ + levels_[level];
     const auto to = items_ + levels_[level + 1]; // exclusive
-    view.add(from, to, 1 << level);
+    view.add(from, to, 1ULL << level);
   }
   view.convert_to_cummulative();
   return view;
@@ -917,8 +860,8 @@ uint32_t kll_sketch<T, C, A>::get_num_retained_above_level_zero() const {
 
 template<typename T, typename C, typename A>
 void kll_sketch<T, C, A>::check_m(uint8_t m) {
-  if (m != DEFAULT_M) {
-    throw std::invalid_argument("Possible corruption: M must be " + std::to_string(DEFAULT_M)
+  if (m != kll_constants::DEFAULT_M) {
+    throw std::invalid_argument("Possible corruption: M must be " + std::to_string(kll_constants::DEFAULT_M)
         + ": " + std::to_string(m));
   }
 }
@@ -1019,20 +962,6 @@ typename kll_sketch<T, C, A>::const_iterator kll_sketch<T, C, A>::end() const {
   return kll_sketch<T, C, A>::const_iterator(nullptr, levels_.data(), num_levels_);
 }
 
-template<typename T, typename C, typename A>
-class kll_sketch<T, C, A>::item_deleter {
-  public:
-  item_deleter(const A& allocator): allocator_(allocator) {}
-  void operator() (T* ptr) {
-    if (ptr != nullptr) {
-      ptr->~T();
-      allocator_.deallocate(ptr, 1);
-    }
-  }
-  private:
-  A allocator_;
-};
-
 template<typename T, typename C, typename A>
 class kll_sketch<T, C, A>::items_deleter {
   public:

data/vendor/datasketches-cpp/kll/test/CMakeLists.txt

@@ -20,7 +20,6 @@ add_executable(kll_test)
 target_link_libraries(kll_test kll common_test_lib)
 
 set_target_properties(kll_test PROPERTIES
-  CXX_STANDARD 11
   CXX_STANDARD_REQUIRED YES
 )
 
@@ -43,3 +42,17 @@ target_sources(kll_test
     kll_sketch_validation.cpp
     kolmogorov_smirnov_test.cpp
 )
+
+if (SERDE_COMPAT)
+target_sources(kll_test
+  PRIVATE
+    kll_sketch_deserialize_from_java_test.cpp
+)
+endif()
+
+if (GENERATE)
+target_sources(kll_test
+  PRIVATE
+    kll_sketch_serialize_for_java.cpp
+)
+endif()

data/vendor/datasketches-cpp/kll/test/kll_sketch_deserialize_from_java_test.cpp

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <kll_sketch.hpp>
+
+namespace datasketches {
+
+// assume the binary sketches for this test have been generated by datasketches-java code
+// in the subdirectory called "java" in the root directory of this project
+static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../java/";
+
+TEST_CASE("kll float", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "kll_float_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = kll_sketch<float>::deserialize(is);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.is_estimation_mode() == (n > kll_constants::DEFAULT_K));
+    REQUIRE(sketch.get_n() == n);
+    if (n > 0) {
+      REQUIRE(sketch.get_min_item() == 1.0f);
+      REQUIRE(sketch.get_max_item() == static_cast<float>(n));
+      uint64_t weight = 0;
+      for (const auto pair: sketch) {
+        REQUIRE(pair.first >= sketch.get_min_item());
+        REQUIRE(pair.first <= sketch.get_max_item());
+        weight += pair.second;
+      }
+      REQUIRE(weight == sketch.get_n());
+    }
+  }
+}
+
+TEST_CASE("kll double", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "kll_double_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = kll_sketch<double>::deserialize(is);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.is_estimation_mode() == (n > kll_constants::DEFAULT_K));
+    REQUIRE(sketch.get_n() == n);
+    if (n > 0) {
+      REQUIRE(sketch.get_min_item() == 1.0);
+      REQUIRE(sketch.get_max_item() == static_cast<double>(n));
+      uint64_t weight = 0;
+      for (const auto pair: sketch) {
+        REQUIRE(pair.first >= sketch.get_min_item());
+        REQUIRE(pair.first <= sketch.get_max_item());
+        weight += pair.second;
+      }
+      REQUIRE(weight == sketch.get_n());
+    }
+  }
+}
+
+// numbers are padded with leading spaces so that natural order works
+TEST_CASE("kll string", "[serde_compat]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    std::ifstream is;
+    is.exceptions(std::ios::failbit | std::ios::badbit);
+    is.open(testBinaryInputPath + "kll_string_n" + std::to_string(n) + "_java.sk", std::ios::binary);
+    const auto sketch = kll_sketch<std::string>::deserialize(is);
+    REQUIRE(sketch.is_empty() == (n == 0));
+    REQUIRE(sketch.is_estimation_mode() == (n > kll_constants::DEFAULT_K));
+    REQUIRE(sketch.get_n() == n);
+    if (n > 0) {
+      REQUIRE(std::stoul(sketch.get_min_item()) == 1);
+      REQUIRE(std::stoul(sketch.get_max_item()) == n);
+      uint64_t weight = 0;
+      for (const auto pair: sketch) {
+        REQUIRE(pair.first >= sketch.get_min_item());
+        REQUIRE(pair.first <= sketch.get_max_item());
+        weight += pair.second;
+      }
+      REQUIRE(weight == sketch.get_n());
+    }
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/kll/test/kll_sketch_serialize_for_java.cpp

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch2/catch.hpp>
+#include <fstream>
+#include <kll_sketch.hpp>
+
+namespace datasketches {
+
+TEST_CASE("kll sketch float generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    kll_sketch<float> sketch;
+    for (unsigned i = 1; i <= n; ++i) sketch.update(i);
+    std::ofstream os("kll_float_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.serialize(os);
+  }
+}
+
+TEST_CASE("kll sketch double generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    kll_sketch<double> sketch;
+    for (unsigned i = 1; i <= n; ++i) sketch.update(i);
+    std::ofstream os("kll_double_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.serialize(os);
+  }
+}
+
+struct compare_as_number {
+  bool operator()(const std::string& a, const std::string& b) const {
+    return std::stoi(a) < std::stoi(b);
+  }
+};
+
+TEST_CASE("kll sketch string generate", "[serialize_for_java]") {
+  const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+  for (const unsigned n: n_arr) {
+    kll_sketch<std::string, compare_as_number> sketch;
+    for (unsigned i = 1; i <= n; ++i) sketch.update(std::to_string(i));
+    std::ofstream os("kll_string_n" + std::to_string(n) + "_cpp.sk", std::ios::binary);
+    sketch.serialize(os);
+  }
+}
+
+} /* namespace datasketches */

data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp

@@ -49,9 +49,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
   test_allocator_total_bytes = 0;
 
   SECTION("k limits") {
-    kll_float_sketch sketch1(kll_float_sketch::MIN_K, std::less<float>(), 0); // this should work
-    kll_float_sketch sketch2(kll_float_sketch::MAX_K, std::less<float>(), 0); // this should work
-    REQUIRE_THROWS_AS(new kll_float_sketch(kll_float_sketch::MIN_K - 1, std::less<float>(), 0), std::invalid_argument);
+    kll_float_sketch sketch1(kll_constants::MIN_K, std::less<float>(), 0); // this should work
+    kll_float_sketch sketch2(kll_constants::MAX_K, std::less<float>(), 0); // this should work
+    REQUIRE_THROWS_AS(new kll_float_sketch(kll_constants::MIN_K - 1, std::less<float>(), 0), std::invalid_argument);
     // MAX_K + 1 makes no sense because k is uint16_t
     //std::cout << "sizeof(kll_sketch<float>)=" << sizeof(kll_sketch<float>) << "\n";
     //std::cout << "sizeof(kll_sketch<double>)=" << sizeof(kll_sketch<double>) << "\n";
@@ -67,8 +67,6 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE_THROWS_AS(sketch.get_max_item(), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_rank(0), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_quantile(0.5), std::runtime_error);
-    const double ranks[3] {0, 0.5, 1};
-    REQUIRE_THROWS_AS(sketch.get_quantiles(ranks, 3), std::runtime_error);
     const float split_points[1] {0};
     REQUIRE_THROWS_AS(sketch.get_PMF(split_points, 1), std::runtime_error);
     REQUIRE_THROWS_AS(sketch.get_CDF(split_points, 1), std::runtime_error);
@@ -99,12 +97,6 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch.get_min_item() == 1.0);
     REQUIRE(sketch.get_max_item() == 1.0);
     REQUIRE(sketch.get_quantile(0.5) == 1.0);
-    const double ranks[3] {0, 0.5, 1};
-    auto quantiles = sketch.get_quantiles(ranks, 3);
-    REQUIRE(quantiles.size() == 3);
-    REQUIRE(quantiles[0] == 1.0);
-    REQUIRE(quantiles[1] == 1.0);
-    REQUIRE(quantiles[2] == 1.0);
 
     int count = 0;
     for (auto pair: sketch) {
@@ -144,20 +136,6 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     REQUIRE(sketch.get_max_item() == n);
     REQUIRE(sketch.get_quantile(1) == n);
 
-    const double ranks[3] {0, 0.5, 1};
-    auto quantiles = sketch.get_quantiles(ranks, 3);
-    REQUIRE(quantiles.size() == 3);
-    REQUIRE(quantiles[0] == 1);
-    REQUIRE(quantiles[1] == n / 2);
-    REQUIRE(quantiles[2] == n);
-
-    // alternative method must produce the same result
-    auto quantiles2 = sketch.get_quantiles(3);
-    REQUIRE(quantiles2.size() == 3);
-    REQUIRE(quantiles[0] == quantiles2[0]);
-    REQUIRE(quantiles[1] == quantiles2[1]);
-    REQUIRE(quantiles[2] == quantiles2[2]);
-
     for (uint32_t i = 1; i <= n; i++) {
       const double true_rank_inclusive = static_cast<double>(i) / n;
       REQUIRE(sketch.get_rank(static_cast<float>(i)) == true_rank_inclusive);
@@ -264,19 +242,6 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
     }
   }
 
-  SECTION("deserialize from java") {
-    std::ifstream is;
-    is.exceptions(std::ios::failbit | std::ios::badbit);
-    is.open(testBinaryInputPath + "kll_sketch_from_java.sk", std::ios::binary);
-    auto sketch = kll_float_sketch::deserialize(is, serde<float>(), std::less<float>(), 0);
-    REQUIRE_FALSE(sketch.is_empty());
-    REQUIRE(sketch.is_estimation_mode());
-    REQUIRE(sketch.get_n() == 1000000);
-    REQUIRE(sketch.get_num_retained() == 614);
-    REQUIRE(sketch.get_min_item() == 0.0);
-    REQUIRE(sketch.get_max_item() == 999999.0);
-  }
-
   SECTION("stream serialize deserialize empty") {
     kll_float_sketch sketch(200, std::less<float>(), 0);
     std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);