datasketches 0.5.1 → 0.5.3

data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp

@@ -22,6 +22,8 @@
 #include <sstream>
 #include <vector>
 #include <stdexcept>
+#include <algorithm>
+#include <cstring>
 
 #include <catch2/catch.hpp>
 #include <theta_sketch.hpp>
@@ -57,6 +59,48 @@ TEST_CASE("theta sketch: empty", "[theta_sketch]") {
   REQUIRE(update_sketch.compact(false).is_ordered());
 }
 
+TEST_CASE("theta sketch: min lg_k", "[theta_sketch]") {
+  // lg_k = 4 (nominal 16) is the smallest allowed nominal size, matching Java's
+  // ThetaUtil.MIN_LG_NOM_LONGS. Anything below it must throw.
+  REQUIRE(theta_constants::MIN_LG_K == 4);
+  REQUIRE_THROWS_AS(update_theta_sketch::builder().set_lg_k(theta_constants::MIN_LG_K - 1),
+      std::invalid_argument);
+  update_theta_sketch min_sketch = update_theta_sketch::builder().set_lg_k(theta_constants::MIN_LG_K).build();
+  REQUIRE(min_sketch.get_lg_k() == theta_constants::MIN_LG_K);
+
+  // update well past the nominal size to force estimation mode and exercise the rebuild path,
+  // tracking the peak number of retained entries seen between rebuilds.
+  const int n = 10000;
+  uint32_t max_retained = 0;
+  for (int i = 0; i < n; ++i) {
+    min_sketch.update(i);
+    max_retained = std::max(max_retained, min_sketch.get_num_retained());
+  }
+  REQUIRE(min_sketch.is_estimation_mode());
+  REQUIRE(min_sketch.get_theta() < 1.0);
+
+  // The internal hash table is floored at MIN_LG_ARR (5, i.e. 32 slots), one lg above the
+  // nominal size. This is exactly what MIN_LG_ARR guarantees: between rebuilds the sketch holds
+  // more than the nominal 2^MIN_LG_K (16) entries, but never more than the 2^MIN_LG_ARR (32)
+  // slots of the table. Were the table sized to the nominal 16, it could not retain more than 16.
+  REQUIRE(max_retained > (1 << theta_constants::MIN_LG_K));
+  REQUIRE(max_retained <= (1 << theta_constants::MIN_LG_ARR));
+
+  // the true count is bracketed by the 2-standard-deviation confidence bounds
+  REQUIRE(min_sketch.get_lower_bound(2) <= n);
+  REQUIRE(min_sketch.get_upper_bound(2) >= n);
+
+  // trimming reduces the sketch to exactly the nominal number of entries (2^4 = 16)
+  min_sketch.trim();
+  REQUIRE(min_sketch.get_num_retained() == (1 << theta_constants::MIN_LG_K));
+
+  // a compacted min-size sketch round trips through serialization
+  auto bytes = min_sketch.compact().serialize();
+  compact_theta_sketch deserialized = compact_theta_sketch::deserialize(bytes.data(), bytes.size());
+  REQUIRE(deserialized.get_num_retained() == min_sketch.get_num_retained());
+  REQUIRE(deserialized.get_estimate() == min_sketch.get_estimate());
+}
+
 TEST_CASE("theta sketch: non empty no retained keys", "[theta_sketch]") {
   update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001f).build();
   update_sketch.update(1);
@@ -109,6 +153,196 @@ TEST_CASE("theta sketch: single item", "[theta_sketch]") {
   REQUIRE(update_sketch.compact(false).is_ordered());
 }
 
+// Java writes exactly these images (EmptyCompactSketch.EMPTY_COMPACT_SKETCH_ARR and SingleItemSketch),
+// and the cross-language tests compare the two languages byte for byte.
+// An empty sketch carries no hashes, so its seed hash bytes are zero; a single item keeps its seed hash.
+static std::vector<uint8_t> serialized_bytes(const compact_theta_sketch& sketch) {
+  auto bytes = sketch.serialize();
+  return std::vector<uint8_t>(bytes.begin(), bytes.end());
+}
+
+static std::vector<uint8_t> serialized_stream(const compact_theta_sketch& sketch) {
+  std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+  sketch.serialize(s);
+  const std::string str = s.str();
+  return std::vector<uint8_t>(str.begin(), str.end());
+}
+
+static std::vector<uint8_t> serialized_compressed(const compact_theta_sketch& sketch) {
+  auto bytes = sketch.serialize_compressed();
+  return std::vector<uint8_t>(bytes.begin(), bytes.end());
+}
+
+TEST_CASE("theta sketch: empty serialized image", "[theta_sketch]") {
+  const uint64_t seed = 12345; // not the default, to prove the seed hash is not written
+  auto sketch = update_theta_sketch::builder().set_seed(seed).build().compact();
+  const std::vector<uint8_t> expected = {1, 3, 3, 0, 0, 0x1E, 0, 0};
+  REQUIRE(serialized_bytes(sketch) == expected);
+  REQUIRE(serialized_stream(sketch) == expected);
+  REQUIRE(serialized_compressed(sketch) == expected); // no compressed form of an empty sketch
+}
+
+TEST_CASE("theta sketch: single item serialized image", "[theta_sketch]") {
+  const uint64_t seed = 12345;
+  const uint16_t seed_hash = compute_seed_hash(seed);
+  update_theta_sketch update_sketch = update_theta_sketch::builder().set_seed(seed).build();
+  update_sketch.update(1);
+  auto sketch = update_sketch.compact();
+  const uint64_t hash = *sketch.begin();
+
+  // flags: single item | ordered | compact | read-only, then seed hash and the one hash, little-endian
+  std::vector<uint8_t> expected = {1, 3, 3, 0, 0, 0x3A,
+      static_cast<uint8_t>(seed_hash), static_cast<uint8_t>(seed_hash >> 8)};
+  for (unsigned i = 0; i < 8; ++i) expected.push_back(static_cast<uint8_t>(hash >> (i * 8)));
+  REQUIRE(serialized_bytes(sketch) == expected);
+  REQUIRE(serialized_stream(sketch) == expected);
+  REQUIRE(serialized_compressed(sketch) == expected); // no compressed form of a single item
+
+  // the single item flag is informational: the reader does not depend on it
+  auto deserialized = compact_theta_sketch::deserialize(expected.data(), expected.size(), seed);
+  REQUIRE(deserialized.get_num_retained() == 1);
+  REQUIRE(*deserialized.begin() == hash);
+  expected[5] = 0x1A; // as written before this flag existed
+  deserialized = compact_theta_sketch::deserialize(expected.data(), expected.size(), seed);
+  REQUIRE(deserialized.get_num_retained() == 1);
+  REQUIRE(*deserialized.begin() == hash);
+}
+
+TEST_CASE("theta sketch: one entry in estimation mode is not a single item", "[theta_sketch]") {
+  update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.5f).build();
+  for (uint64_t i = 0; update_sketch.get_num_retained() < 1; ++i) update_sketch.update(i);
+  auto sketch = update_sketch.compact();
+  REQUIRE(sketch.get_num_retained() == 1);
+  REQUIRE(sketch.is_estimation_mode());
+  const auto bytes = serialized_bytes(sketch);
+  REQUIRE(bytes[0] == 3); // theta is present, so this is the 3-long preamble
+  REQUIRE(bytes[5] == 0x1A); // ordered | compact | read-only, no single item flag
+}
+
+TEST_CASE("theta sketch: compact with trim, all four cases", "[theta_sketch]") {
+  update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+  for (int i = 0; i < 8000; i++) update_sketch.update(i);
+  const uint32_t k = 1 << theta_constants::DEFAULT_LG_K;
+  // an update sketch retains more than k between rebuilds, so trimming has work to do
+  REQUIRE(update_sketch.get_num_retained() > k);
+  const uint32_t retained_before = update_sketch.get_num_retained();
+  const uint64_t theta_before = update_sketch.get_theta64();
+
+  // the trimmed result is what trim() + compact() would produce
+  update_theta_sketch trimmed = update_sketch;
+  trimmed.trim();
+  compact_theta_sketch expected = trimmed.compact(true);
+  const std::vector<uint64_t> expected_entries(expected.begin(), expected.end());
+
+  // case 1: ordered, not trimmed (the default) keeps every retained entry
+  compact_theta_sketch c1 = update_sketch.compact(true, false);
+  REQUIRE(c1.is_ordered());
+  REQUIRE(c1.get_num_retained() == retained_before);
+  REQUIRE(c1.get_theta64() == theta_before);
+  REQUIRE(std::is_sorted(c1.begin(), c1.end()));
+
+  // case 2: unordered, not trimmed
+  compact_theta_sketch c2 = update_sketch.compact(false, false);
+  REQUIRE_FALSE(c2.is_ordered());
+  REQUIRE(c2.get_num_retained() == retained_before);
+  REQUIRE(c2.get_theta64() == theta_before);
+
+  // case 3: ordered and trimmed
+  compact_theta_sketch c3 = update_sketch.compact(true, true);
+  REQUIRE(c3.is_ordered());
+  REQUIRE(c3.get_num_retained() == k);
+  REQUIRE(c3.get_theta64() < theta_before); // new theta is the k-th smallest hash
+  REQUIRE(c3.get_theta64() == expected.get_theta64());
+  REQUIRE(std::is_sorted(c3.begin(), c3.end()));
+  REQUIRE(std::vector<uint64_t>(c3.begin(), c3.end()) == expected_entries);
+  // every retained hash is strictly below the new theta
+  for (auto h: c3) REQUIRE(h < c3.get_theta64());
+
+  // case 4: unordered and trimmed: same set and theta, no sort
+  compact_theta_sketch c4 = update_sketch.compact(false, true);
+  REQUIRE_FALSE(c4.is_ordered());
+  REQUIRE(c4.get_num_retained() == k);
+  REQUIRE(c4.get_theta64() == expected.get_theta64());
+  std::vector<uint64_t> c4_entries(c4.begin(), c4.end());
+  std::sort(c4_entries.begin(), c4_entries.end());
+  REQUIRE(c4_entries == expected_entries);
+
+  // the source sketch must be untouched by any of the four
+  REQUIRE(update_sketch.get_num_retained() == retained_before);
+  REQUIRE(update_sketch.get_theta64() == theta_before);
+}
+
+TEST_CASE("theta sketch: compact with trim widens the bounds in estimation mode", "[theta_sketch]") {
+  // trimming is lossy even when the source is already estimating: it discards
+  // retained entries, and the relative error scales with 1 / sqrt(retained)
+  const uint32_t k = 1 << theta_constants::DEFAULT_LG_K;
+  update_theta_sketch sketch = update_theta_sketch::builder().build();
+  for (int i = 0; i < 40000; i++) sketch.update(i);
+  REQUIRE(sketch.is_estimation_mode());
+  REQUIRE(sketch.get_num_retained() > k);
+
+  compact_theta_sketch plain = sketch.compact(true, false);
+  compact_theta_sketch trimmed = sketch.compact(true, true);
+  REQUIRE(trimmed.get_num_retained() == k);
+  REQUIRE(plain.get_num_retained() > trimmed.get_num_retained());
+
+  // fewer retained entries -> strictly wider confidence interval
+  const double plain_width = plain.get_upper_bound(2) - plain.get_lower_bound(2);
+  const double trimmed_width = trimmed.get_upper_bound(2) - trimmed.get_lower_bound(2);
+  REQUIRE(trimmed_width > plain_width);
+}
+
+TEST_CASE("theta sketch: compact with trim converts exact mode to estimation", "[theta_sketch]") {
+  // an update sketch can retain far more than k entries while still in exact mode:
+  // nothing has been evicted yet, so theta is still 1.0 and the count is exact
+  const uint32_t k = 1 << theta_constants::DEFAULT_LG_K;
+  update_theta_sketch sketch = update_theta_sketch::builder().build();
+  const int n = 5000;
+  for (int i = 0; i < n; i++) sketch.update(i);
+  REQUIRE(sketch.get_num_retained() > k);
+  REQUIRE_FALSE(sketch.is_estimation_mode());
+  REQUIRE(sketch.get_theta() == 1.0);
+
+  // not trimming keeps every entry and the exact count
+  compact_theta_sketch exact = sketch.compact(true, false);
+  REQUIRE_FALSE(exact.is_estimation_mode());
+  REQUIRE(exact.get_num_retained() == static_cast<uint32_t>(n));
+  REQUIRE(exact.get_estimate() == Approx(n));
+
+  // trimming is lossy: it discards real data, lowers theta below 1.0 and the
+  // result is an estimate carrying error where the source held an exact count
+  compact_theta_sketch trimmed = sketch.compact(true, true);
+  REQUIRE(trimmed.is_estimation_mode());
+  REQUIRE(trimmed.get_num_retained() == k);
+  REQUIRE(trimmed.get_theta() < 1.0);
+  REQUIRE(trimmed.get_estimate() != Approx(n));
+  // the estimate is still sound: n must lie inside the 3-sigma bounds
+  REQUIRE(trimmed.get_lower_bound(3) <= n);
+  REQUIRE(trimmed.get_upper_bound(3) >= n);
+}
+
+TEST_CASE("theta sketch: compact with trim, empty and below k", "[theta_sketch]") {
+  // empty: trimming changes nothing
+  update_theta_sketch empty_sketch = update_theta_sketch::builder().build();
+  compact_theta_sketch empty_result = empty_sketch.compact(true, true);
+  REQUIRE(empty_result.is_empty());
+  REQUIRE(empty_result.get_num_retained() == 0);
+  REQUIRE(empty_result.is_ordered());
+
+  // below k: nothing to trim, theta and entries are preserved
+  update_theta_sketch small = update_theta_sketch::builder().build();
+  for (int i = 0; i < 100; i++) small.update(i);
+  REQUIRE_FALSE(small.is_estimation_mode());
+
+  compact_theta_sketch small_result = small.compact(true, true);
+  REQUIRE_FALSE(small_result.is_estimation_mode());
+  REQUIRE(small_result.get_num_retained() == 100);
+  REQUIRE(small_result.get_theta64() == small.get_theta64());
+  REQUIRE(small_result.get_estimate() == Approx(100.0));
+  REQUIRE(std::is_sorted(small_result.begin(), small_result.end()));
+  REQUIRE_FALSE(small.compact(false, true).is_ordered());
+}
+
 TEST_CASE("theta sketch: resize exact", "[theta_sketch]") {
   update_theta_sketch update_sketch = update_theta_sketch::builder().build();
   for (int i = 0; i < 2000; i++) update_sketch.update(i);
@@ -337,6 +571,50 @@ TEST_CASE("theta sketch: deserialize estimation mode buffer overrun", "[theta_sk
   REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(bytes.data(), bytes.size() - 1), std::out_of_range);
 }
 
+// The header fields of serial versions 1 to 3 that follow the first 8 bytes must not be read
+// before the size of the buffer is checked. Each buffer below holds exactly the preamble bytes,
+// so that reading beyond them would be an out-of-bounds read, and is followed by the bytes
+// that would make the sketch look empty if they were read.
+TEST_CASE("theta sketch: deserialize truncated preamble", "[theta_sketch]") {
+  const uint16_t seed_hash = compute_seed_hash(DEFAULT_SEED);
+  const uint8_t seed_hash_lo = seed_hash & 0xff;
+  const uint8_t seed_hash_hi = seed_hash >> 8;
+  const uint8_t max_theta[8] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f};
+
+  // serial version 1: three preamble longs, num_entries = 0 and theta = max would be empty
+  std::vector<uint8_t> v1 = {3, 1, 3, 0, 0, 0x1a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  v1.insert(v1.end(), max_theta, max_theta + 8);
+  for (size_t size: {8, 16}) {
+    const std::vector<uint8_t> bytes(v1.begin(), v1.begin() + size);
+    REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(bytes.data(), bytes.size()), std::out_of_range);
+    REQUIRE_THROWS_AS(wrapped_compact_theta_sketch::wrap(bytes.data(), bytes.size()), std::out_of_range);
+  }
+
+  // serial version 2: two preamble longs, num_entries = 0 would be empty
+  std::vector<uint8_t> v2_exact = {2, 2, 3, 0, 0, 0x1a, seed_hash_lo, seed_hash_hi, 0, 0, 0, 0, 0, 0, 0, 0};
+  {
+    const std::vector<uint8_t> bytes(v2_exact.begin(), v2_exact.begin() + 8);
+    REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(bytes.data(), bytes.size()), std::out_of_range);
+    REQUIRE_THROWS_AS(wrapped_compact_theta_sketch::wrap(bytes.data(), bytes.size()), std::out_of_range);
+  }
+  REQUIRE(compact_theta_sketch::deserialize(v2_exact.data(), v2_exact.size()).is_empty());
+
+  // serial version 2: three preamble longs, num_entries = 0 and theta = max would be empty
+  std::vector<uint8_t> v2_estimation = {3, 2, 3, 0, 0, 0x1a, seed_hash_lo, seed_hash_hi, 0, 0, 0, 0, 0, 0, 0, 0};
+  v2_estimation.insert(v2_estimation.end(), max_theta, max_theta + 8);
+  for (size_t size: {8, 16}) {
+    const std::vector<uint8_t> bytes(v2_estimation.begin(), v2_estimation.begin() + size);
+    REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(bytes.data(), bytes.size()), std::out_of_range);
+    REQUIRE_THROWS_AS(wrapped_compact_theta_sketch::wrap(bytes.data(), bytes.size()), std::out_of_range);
+  }
+  REQUIRE(compact_theta_sketch::deserialize(v2_estimation.data(), v2_estimation.size()).is_empty());
+
+  // serial version 3: two preamble longs, not empty
+  const std::vector<uint8_t> v3 = {2, 3, 3, 0, 0, 0x1a, seed_hash_lo, seed_hash_hi};
+  REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(v3.data(), v3.size()), std::out_of_range);
+  REQUIRE_THROWS_AS(wrapped_compact_theta_sketch::wrap(v3.data(), v3.size()), std::out_of_range);
+}
+
 TEST_CASE("theta sketch: conversion constructor and wrapped compact", "[theta_sketch]") {
   update_theta_sketch update_sketch = update_theta_sketch::builder().build();
   const int n = 8192;
@@ -625,4 +903,41 @@ TEST_CASE("max serialized size", "[theta_sketch]") {
   REQUIRE(max_size_bytes == compact_theta_sketch::get_max_serialized_size_bytes(lg_k));
 }
 
+TEST_CASE("theta sketch: deserialize v4 corrupt header", "[theta_sketch]") {
+  update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+  for (int i = 0; i < 100; ++i) update_sketch.update(i);
+  const auto bytes = update_sketch.compact().serialize_compressed();
+  REQUIRE(bytes[1] == 4);
+  for (uint8_t entry_bits: {0, 64, 255}) {
+    auto corrupt = bytes;
+    corrupt[3] = entry_bits;
+    REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(corrupt.data(), corrupt.size()), std::invalid_argument);
+    std::stringstream s;
+    s.write(reinterpret_cast<const char*>(corrupt.data()), corrupt.size());
+    REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(s), std::invalid_argument);
+  }
+  for (uint8_t num_entries_bytes: {0, 5, 255}) {
+    auto corrupt = bytes;
+    corrupt[4] = num_entries_bytes;
+    REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(corrupt.data(), corrupt.size()), std::invalid_argument);
+    std::stringstream s;
+    s.write(reinterpret_cast<const char*>(corrupt.data()), corrupt.size());
+    REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(s), std::invalid_argument);
+  }
+}
+
+TEST_CASE("theta sketch: deserialize v4 entry bits overflow", "[theta_sketch]") {
+  update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+  for (int i = 0; i < 100; ++i) update_sketch.update(i);
+  auto bytes = update_sketch.compact().serialize_compressed();
+  REQUIRE(bytes[0] == 1); // exact mode, num_entries follows the first preamble long
+  // 8 bits * 2^29 entries = 2^32 bits wraps to 0 in 32-bit arithmetic
+  bytes.resize(12);
+  bytes[3] = 8;
+  bytes[4] = 4;
+  const uint32_t num_entries = 1U << 29;
+  std::memcpy(bytes.data() + 8, &num_entries, sizeof(num_entries));
+  REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(bytes.data(), bytes.size()), std::out_of_range);
+}
+
 } /* namespace datasketches */

data/vendor/datasketches-cpp/tools/rat-check.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Runs the Apache Release Audit Tool over the tracked files of this repository
+# and fails if any file lacks an approved license header.
+#
+# Only tracked files are examined: the tree is exported with `git archive`, so a
+# local build directory or other untracked scratch cannot affect the result.
+#
+# Usage:  tools/rat-check.sh [git-ref]      (default: HEAD)
+
+set -euo pipefail
+
+RAT_VERSION="${RAT_VERSION:-0.16.1}"
+RAT_JAR="${RAT_JAR:-}"
+REF="${1:-HEAD}"
+
+repo_root="$(git rev-parse --show-toplevel)"
+cd "$repo_root"
+
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+
+if [[ -z "$RAT_JAR" ]]; then
+  RAT_JAR="$work/apache-rat.jar"
+  url="https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar"
+  echo "Downloading Apache RAT ${RAT_VERSION}"
+  curl -fsSL -o "$RAT_JAR" "$url"
+fi
+
+mkdir -p "$work/src"
+git archive "$REF" | tar -x -C "$work/src"
+cp .rat-excludes "$work/src/.rat-excludes"
+
+report="$work/rat-report.txt"
+java -jar "$RAT_JAR" -E "$work/src/.rat-excludes" -d "$work/src" > "$report" 2>/dev/null
+
+unknown="$(sed -n 's/^\([0-9][0-9]*\) Unknown Licenses$/\1/p' "$report" | head -1)"
+unknown="${unknown:-0}"
+
+if [[ "$unknown" -ne 0 ]]; then
+  echo
+  echo "Apache RAT found ${unknown} file(s) without an approved license header:"
+  awk '/^Files with unapproved licenses:/{f=1;next} f&&/^\*\*\*\*/{exit} f&&NF{print "  " $0}' "$report" \
+    | sed "s|$work/src/||"
+  echo
+  echo "Add the ASF header to each file. If a file is third-party source that must"
+  echo "keep its own notice, add it to .rat-excludes AND record it in LICENSE."
+  exit 1
+fi
+
+echo "Apache RAT: all tracked files carry an approved license header."

data/vendor/datasketches-cpp/tuple/include/array_tuple_sketch.hpp

@@ -22,6 +22,8 @@
 
 #include <vector>
 #include <memory>
+#include <type_traits>
+#include <algorithm>
 
 #include "serde.hpp"
 #include "tuple_sketch.hpp"
@@ -34,17 +36,18 @@ class array {
 public:
   using value_type = T;
   using allocator_type = Allocator;
+  using alloc_traits = std::allocator_traits<Allocator>;
 
-  explicit array(uint8_t size, T value, const Allocator& allocator = Allocator()):
+  explicit array(uint8_t size, const T& value, const Allocator& allocator = Allocator()):
   allocator_(allocator), size_(size), array_(allocator_.allocate(size_)) {
-    std::fill(array_, array_ + size_, value);
+    init_values(value, std::is_trivially_copyable<T>());
   }
   array(const array& other):
     allocator_(other.allocator_),
     size_(other.size_),
     array_(allocator_.allocate(size_))
   {
-    std::copy(other.array_, other.array_ + size_, array_);
+    copy_from(other, std::is_trivially_copyable<T>());
   }
   array(array&& other) noexcept:
     allocator_(std::move(other.allocator_)),
@@ -52,9 +55,13 @@ public:
     array_(other.array_)
   {
     other.array_ = nullptr;
+    other.size_ = 0;
   }
   ~array() {
-    if (array_ != nullptr) allocator_.deallocate(array_, size_);
+    if (array_ != nullptr) {
+      destroy_values(std::is_trivially_destructible<T>());
+      allocator_.deallocate(array_, size_);
+    }
   }
   array& operator=(const array& other) {
     array copy(other);
@@ -75,10 +82,34 @@ public:
   T* data() { return array_; }
   const T* data() const { return array_; }
   bool operator==(const array& other) const {
+    if (size_ != other.size_) return false;
     for (uint8_t i = 0; i < size_; ++i) if (array_[i] != other.array_[i]) return false;
     return true;
   }
 private:
+  void init_values(const T& value, std::true_type) {
+    std::fill(array_, array_ + size_, value);
+  }
+  void init_values(const T& value, std::false_type) {
+    for (uint8_t i = 0; i < size_; ++i) {
+      alloc_traits::construct(allocator_, array_ + i, value);
+    }
+  }
+  void copy_from(const array& other, std::true_type) {
+    std::copy(other.array_, other.array_ + size_, array_);
+  }
+  void copy_from(const array& other, std::false_type) {
+    for (uint8_t i = 0; i < size_; ++i) {
+      alloc_traits::construct(allocator_, array_ + i, other.array_[i]);
+    }
+  }
+  void destroy_values(std::true_type) {}
+  void destroy_values(std::false_type) {
+    for (uint8_t i = 0; i < size_; ++i) {
+      alloc_traits::destroy(allocator_, array_ + i);
+    }
+  }
+
   Allocator allocator_;
   uint8_t size_;
   T* array_;

data/vendor/datasketches-cpp/tuple/include/array_tuple_sketch_impl.hpp

@@ -166,7 +166,7 @@ compact_array_tuple_sketch<Array, Allocator> compact_array_tuple_sketch<Array, A
     for (size_t i = 0; i < num_entries; ++i) {
       Array summary(num_values, 0, allocator);
       read(is, summary.data(), num_values * sizeof(typename Array::value_type));
-      entries.push_back(Entry(keys[i], std::move(summary)));
+      entries.emplace_back(keys[i], std::move(summary));
     }
   }
   if (!is.good()) throw std::runtime_error("error reading from std::istream");
@@ -213,7 +213,7 @@ compact_array_tuple_sketch<Array, Allocator> compact_array_tuple_sketch<Array, A
     for (size_t i = 0; i < num_entries; ++i) {
       Array summary(num_values, 0, allocator);
       ptr += copy_from_mem(ptr, summary.data(), num_values * sizeof(typename Array::value_type));
-      entries.push_back(Entry(keys[i], std::move(summary)));
+      entries.emplace_back(keys[i], std::move(summary));
     }
   }
   const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);

data/vendor/datasketches-cpp/tuple/include/tuple_sketch.hpp

@@ -46,6 +46,11 @@ struct pair_extract_key {
 /**
  * Base class for Tuple sketch.
  * This is an extension of Theta sketch that allows keeping arbitrary Summary associated with each retained key.
+ *
+ * Summary that may retain string values.
+ * For Summary containing strings, cross-language portability depends on
+ * using compatible string encodings. This class does not by itself enforce
+ * UTF-8 validity for all string inputs.
  */
 template<
   typename Summary,
@@ -253,6 +258,9 @@ public:
 
   /**
    * Update this sketch with a given string.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * @param key string to update the sketch with
    * @param value to update the sketch with
    */
@@ -261,6 +269,9 @@ public:
 
   /**
    * Update this sketch with a given unsigned 64-bit integer.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * @param key uint64_t to update the sketch with
    * @param value to update the sketch with
    */
@@ -269,6 +280,9 @@ public:
 
   /**
    * Update this sketch with a given signed 64-bit integer.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * @param key int64_t to update the sketch with
    * @param value to update the sketch with
    */
@@ -277,6 +291,9 @@ public:
 
   /**
    * Update this sketch with a given unsigned 32-bit integer.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * For compatibility with Java implementation.
    * @param key uint32_t to update the sketch with
    * @param value to update the sketch with
@@ -286,6 +303,9 @@ public:
 
   /**
    * Update this sketch with a given signed 32-bit integer.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * For compatibility with Java implementation.
    * @param key int32_t to update the sketch with
    * @param value to update the sketch with
@@ -295,6 +315,9 @@ public:
 
   /**
    * Update this sketch with a given unsigned 16-bit integer.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * For compatibility with Java implementation.
    * @param key uint16_t to update the sketch with
    * @param value to update the sketch with
@@ -304,6 +327,9 @@ public:
 
   /**
    * Update this sketch with a given signed 16-bit integer.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * For compatibility with Java implementation.
    * @param key int16_t to update the sketch with
    * @param value to update the sketch with
@@ -313,6 +339,9 @@ public:
 
   /**
    * Update this sketch with a given unsigned 8-bit integer.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * For compatibility with Java implementation.
    * @param key uint8_t to update the sketch with
    * @param value to update the sketch with
@@ -322,6 +351,9 @@ public:
 
   /**
    * Update this sketch with a given signed 8-bit integer.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * For compatibility with Java implementation.
    * @param key int8_t to update the sketch with
    * @param value to update the sketch with
@@ -331,6 +363,9 @@ public:
 
   /**
    * Update this sketch with a given double-precision floating point value.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * For compatibility with Java implementation.
    * @param key double to update the sketch with
    * @param value to update the sketch with
@@ -340,6 +375,9 @@ public:
 
   /**
    * Update this sketch with a given floating point value.
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * For compatibility with Java implementation.
    * @param key float to update the sketch with
    * @param value to update the sketch with
@@ -357,6 +395,9 @@ public:
    * Otherwise two sketches that should represent overlapping sets will be disjoint
    * For instance, for signed 32-bit values call update(int32_t) method above,
    * which does widening conversion to int64_t, if compatibility with Java is expected
+   * If the summary contains strings and cross-language portability is required,
+   * callers should ensure that any strings in the summary
+   * use a compatible encoding (valid UTF-8).
    * @param key pointer to the data
    * @param length of the data in bytes
    * @param value to update the sketch with

data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp

@@ -315,7 +315,7 @@ entries_(other.get_allocator())
 {
   entries_.reserve(other.get_num_retained());
   for (uint64_t hash: other) {
-    entries_.push_back(Entry(hash, summary));
+    entries_.emplace_back(hash, summary);
   }
   if (ordered && !other.is_ordered()) std::sort(entries_.begin(), entries_.end(), comparator());
 }
@@ -347,7 +347,7 @@ uint32_t compact_tuple_sketch<S, A>::get_num_retained() const {
 
 template<typename S, typename A>
 uint16_t compact_tuple_sketch<S, A>::get_seed_hash() const {
-  return seed_hash_;
+  return is_empty_ ? 0 : seed_hash_;
 }
 
 template<typename S, typename A>
@@ -518,7 +518,7 @@ compact_tuple_sketch<S, A> compact_tuple_sketch<S, A>::deserialize(std::istream&
     for (size_t i = 0; i < num_entries; ++i) {
       const auto key = read<uint64_t>(is);
       sd.deserialize(is, summary.get(), 1);
-      entries.push_back(Entry(key, std::move(*summary)));
+      entries.emplace_back(key, std::move(*summary));
       (*summary).~S();
     }
   }
@@ -583,9 +583,10 @@ compact_tuple_sketch<S, A> compact_tuple_sketch<S, A>::deserialize(const void* b
     std::unique_ptr<S, deleter_of_summaries> summary(alloc.allocate(1), deleter_of_summaries(1, false, allocator));
     for (size_t i = 0; i < num_entries; ++i) {
       uint64_t key;
+      ensure_minimum_memory(base + size - ptr, sizeof(uint64_t));
       ptr += copy_from_mem(ptr, key);
       ptr += sd.deserialize(ptr, base + size - ptr, summary.get(), 1);
-      entries.push_back(Entry(key, std::move(*summary)));
+      entries.emplace_back(key, std::move(*summary));
       (*summary).~S();
     }
   }