datasketches 0.5.1 → 0.5.3

data/vendor/datasketches-cpp/benchmarks/benchmark_count_min_sketch.cpp

@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <benchmark/benchmark.h>
+#include <count_min.hpp>
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace
+{
+
+using Sketch = datasketches::count_min_sketch<uint64_t>;
+
+// Roughly 99.9% confidence and 0.1% relative error:
+// suggest_num_hashes(0.999) == 7 and suggest_num_buckets(0.001) == 2719.
+constexpr uint8_t NUM_HASHES = 7;
+constexpr uint32_t NUM_BUCKETS = 2719;
+
+std::vector<uint64_t> makeUInt64Keys(size_t size)
+{
+    std::vector<uint64_t> keys;
+    keys.reserve(size);
+    for (size_t i = 0; i < size; ++i)
+        keys.push_back(static_cast<uint64_t>(i * 0x9e3779b97f4a7c15ULL));
+    return keys;
+}
+
+std::vector<std::string> makeStringKeys(size_t size)
+{
+    std::vector<std::string> keys;
+    keys.reserve(size);
+    for (size_t i = 0; i < size; ++i)
+        keys.push_back("countmin-key-" + std::to_string(i * 2654435761ULL));
+    return keys;
+}
+
+int64_t totalStringBytes(const std::vector<std::string> & keys)
+{
+    int64_t bytes = 0;
+    for (const auto & key : keys)
+        bytes += static_cast<int64_t>(key.size());
+    return bytes;
+}
+
+void BM_CountMinUpdateUInt64(benchmark::State & state)
+{
+    const auto keys = makeUInt64Keys(static_cast<size_t>(state.range(0)));
+    for (auto _ : state)
+    {
+        state.PauseTiming();
+        Sketch sketch(NUM_HASHES, NUM_BUCKETS);
+        state.ResumeTiming();
+
+        for (const auto key : keys)
+            sketch.update(&key, sizeof(key), 1);
+
+        benchmark::ClobberMemory();
+        benchmark::DoNotOptimize(sketch.get_total_weight());
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(keys.size()));
+    state.SetBytesProcessed(state.iterations() * static_cast<int64_t>(keys.size() * sizeof(uint64_t)));
+}
+
+void BM_CountMinUpdateStringBytes(benchmark::State & state)
+{
+    const auto keys = makeStringKeys(static_cast<size_t>(state.range(0)));
+    const auto bytes_per_iteration = totalStringBytes(keys);
+
+    for (auto _ : state)
+    {
+        state.PauseTiming();
+        Sketch sketch(NUM_HASHES, NUM_BUCKETS);
+        state.ResumeTiming();
+
+        for (const auto & key : keys)
+            sketch.update(key.data(), key.size(), 1);
+
+        benchmark::ClobberMemory();
+        benchmark::DoNotOptimize(sketch.get_total_weight());
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(keys.size()));
+    state.SetBytesProcessed(state.iterations() * bytes_per_iteration);
+}
+
+void BM_CountMinEstimateUInt64(benchmark::State & state)
+{
+    const auto keys = makeUInt64Keys(static_cast<size_t>(state.range(0)));
+    Sketch sketch(NUM_HASHES, NUM_BUCKETS);
+    for (const auto key : keys)
+        sketch.update(&key, sizeof(key), 1);
+
+    uint64_t sum = 0;
+    for (auto _ : state)
+    {
+        for (const auto key : keys)
+            sum += sketch.get_estimate(&key, sizeof(key));
+
+        benchmark::DoNotOptimize(sum);
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(keys.size()));
+    state.SetBytesProcessed(state.iterations() * static_cast<int64_t>(keys.size() * sizeof(uint64_t)));
+}
+
+void BM_CountMinEstimateStringBytes(benchmark::State & state)
+{
+    const auto keys = makeStringKeys(static_cast<size_t>(state.range(0)));
+    const auto bytes_per_iteration = totalStringBytes(keys);
+
+    Sketch sketch(NUM_HASHES, NUM_BUCKETS);
+    for (const auto & key : keys)
+        sketch.update(key.data(), key.size(), 1);
+
+    uint64_t sum = 0;
+    for (auto _ : state)
+    {
+        for (const auto & key : keys)
+            sum += sketch.get_estimate(key.data(), key.size());
+
+        benchmark::DoNotOptimize(sum);
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(keys.size()));
+    state.SetBytesProcessed(state.iterations() * bytes_per_iteration);
+}
+
+}
+
+BENCHMARK(BM_CountMinUpdateUInt64)->RangeMultiplier(8)->Range(1024, 65536);
+BENCHMARK(BM_CountMinUpdateStringBytes)->RangeMultiplier(8)->Range(1024, 65536);
+BENCHMARK(BM_CountMinEstimateUInt64)->RangeMultiplier(8)->Range(1024, 65536);
+BENCHMARK(BM_CountMinEstimateStringBytes)->RangeMultiplier(8)->Range(1024, 65536);

data/vendor/datasketches-cpp/benchmarks/benchmark_count_min_sketch_serialization.cpp

@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <benchmark/benchmark.h>
+#include <count_min.hpp>
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <ostream>
+#include <streambuf>
+#include <vector>
+
+namespace
+{
+
+using Sketch = datasketches::count_min_sketch<uint64_t>;
+
+// Roughly 99.9% confidence. Serialization cost then scales with the bucket count.
+constexpr uint8_t NUM_HASHES = 7;
+
+Sketch makeSketch(uint32_t num_buckets)
+{
+    Sketch sketch(NUM_HASHES, num_buckets);
+    for (uint64_t i = 0; i < NUM_HASHES; ++i)
+        sketch.update(i, i + 1);
+    return sketch;
+}
+
+class fixed_buffer_streambuf final: public std::streambuf
+{
+public:
+    explicit fixed_buffer_streambuf(std::vector<uint8_t>& buffer): buffer_(buffer), position_(0) {}
+
+    void reset()
+    {
+        position_ = 0;
+    }
+
+    size_t bytes_written() const
+    {
+        return position_;
+    }
+
+protected:
+    std::streamsize xsputn(const char* data, std::streamsize size) override
+    {
+        const size_t requested = static_cast<size_t>(size);
+        const size_t available = buffer_.size() - position_;
+        const size_t bytes_to_write = std::min(requested, available);
+        if (bytes_to_write > 0) {
+            std::memcpy(buffer_.data() + position_, data, bytes_to_write);
+            position_ += bytes_to_write;
+        }
+        return static_cast<std::streamsize>(bytes_to_write);
+    }
+
+    int_type overflow(int_type ch) override
+    {
+        if (traits_type::eq_int_type(ch, traits_type::eof()))
+            return traits_type::not_eof(ch);
+        if (position_ == buffer_.size())
+            return traits_type::eof();
+        buffer_[position_++] = static_cast<uint8_t>(traits_type::to_char_type(ch));
+        return ch;
+    }
+
+private:
+    std::vector<uint8_t>& buffer_;
+    size_t position_;
+};
+
+void BM_CountMinGetSerializedSizeBytes(benchmark::State& state)
+{
+    const auto sketch = makeSketch(static_cast<uint32_t>(state.range(0)));
+
+    for (auto _ : state) {
+        const auto size = sketch.get_serialized_size_bytes();
+        benchmark::DoNotOptimize(size);
+    }
+
+    state.SetItemsProcessed(state.iterations());
+}
+
+void BM_CountMinSerializeVector(benchmark::State& state)
+{
+    const auto sketch = makeSketch(static_cast<uint32_t>(state.range(0)));
+    const auto serialized_size = sketch.get_serialized_size_bytes();
+
+    for (auto _ : state) {
+        auto bytes = sketch.serialize();
+        benchmark::DoNotOptimize(bytes.data());
+        benchmark::DoNotOptimize(bytes.size());
+        benchmark::ClobberMemory();
+    }
+
+    state.SetBytesProcessed(state.iterations() * static_cast<int64_t>(serialized_size));
+}
+
+void BM_CountMinSerializeOStream(benchmark::State& state)
+{
+    const auto sketch = makeSketch(static_cast<uint32_t>(state.range(0)));
+    const auto serialized_size = sketch.get_serialized_size_bytes();
+    std::vector<uint8_t> bytes(serialized_size);
+    fixed_buffer_streambuf stream_buffer(bytes);
+    std::ostream os(&stream_buffer);
+
+    for (auto _ : state) {
+        stream_buffer.reset();
+        os.clear();
+        sketch.serialize(os);
+        benchmark::DoNotOptimize(stream_buffer.bytes_written());
+        benchmark::ClobberMemory();
+    }
+
+    state.SetBytesProcessed(state.iterations() * static_cast<int64_t>(serialized_size));
+}
+
+void BM_CountMinSerializeToSink(benchmark::State& state)
+{
+    const auto sketch = makeSketch(static_cast<uint32_t>(state.range(0)));
+    const auto serialized_size = sketch.get_serialized_size_bytes();
+    std::vector<uint8_t> bytes(serialized_size);
+
+    for (auto _ : state) {
+        uint8_t* ptr = bytes.data();
+        const auto bytes_written = sketch.serialize_to([&ptr](const void* data, size_t size) {
+            std::memcpy(ptr, data, size);
+            ptr += size;
+        });
+        benchmark::DoNotOptimize(ptr);
+        benchmark::DoNotOptimize(bytes_written);
+        benchmark::ClobberMemory();
+    }
+
+    state.SetBytesProcessed(state.iterations() * static_cast<int64_t>(serialized_size));
+}
+
+}
+
+BENCHMARK(BM_CountMinGetSerializedSizeBytes)->RangeMultiplier(8)->Range(1024, 65536);
+BENCHMARK(BM_CountMinSerializeVector)->RangeMultiplier(8)->Range(1024, 65536);
+BENCHMARK(BM_CountMinSerializeOStream)->RangeMultiplier(8)->Range(1024, 65536);
+BENCHMARK(BM_CountMinSerializeToSink)->RangeMultiplier(8)->Range(1024, 65536);

data/vendor/datasketches-cpp/common/CMakeLists.txt

@@ -40,6 +40,7 @@ install(FILES
       include/conditional_back_inserter.hpp
       include/conditional_forward.hpp
       include/count_zeros.hpp
+      include/fdlibm_log.hpp
       include/inv_pow2_table.hpp
       include/kolmogorov_smirnov_impl.hpp
       include/kolmogorov_smirnov.hpp

data/vendor/datasketches-cpp/common/include/binomial_bounds.hpp

@@ -441,8 +441,8 @@ private:
   }
 
   static void check_theta(double theta) {
-    if (theta < 0 || theta > 1) {
-      throw std::invalid_argument("theta must be in [0, 1]");
+    if (theta <= 0 || theta > 1) {
+      throw std::invalid_argument("theta must be in (0, 1]");
     }
   }
 

data/vendor/datasketches-cpp/common/include/fdlibm_log.hpp

@@ -0,0 +1,101 @@
+// fdlibm __ieee754_log, used by Java's StrictMath.log (and by Math.log on most JVMs).
+// Derived from FDLIBM 5.3, Copyright (C) 1993 by Sun Microsystems, Inc.
+// "Permission to use, copy, modify, and distribute this software is freely granted,
+//  provided that this notice is preserved."
+#ifndef FDLIBM_LOG_HPP
+#define FDLIBM_LOG_HPP
+#include <cstdint>
+#include <cstring>
+#include <limits>
+
+namespace fdlibm {
+
+inline int32_t hi_word(double x){ uint64_t u; std::memcpy(&u,&x,8); return (int32_t)(uint32_t)(u>>32); }
+inline uint32_t lo_word(double x){ uint64_t u; std::memcpy(&u,&x,8); return (uint32_t)u; }
+inline void set_hi_word(double& x, uint32_t hi){ uint64_t u; std::memcpy(&u,&x,8);
+  u = (u & 0x00000000ffffffffULL) | ((uint64_t)hi<<32); std::memcpy(&x,&u,8); }
+
+// Forces a value to be rounded to a double before it is used again. fdlibm needs strict
+// IEEE-754 evaluation: a fused multiply-add anywhere in the polynomial below changes the
+// result. Pragmas are overridden by an explicit -ffp-contract=fast, so use a volatile
+// round-trip, which the standard requires the compiler to honour.
+inline double rnd(double v) { volatile double t = v; return t; }
+
+inline double log(double x) {
+// fdlibm depends on strict IEEE-754 evaluation: a fused multiply-add would change the result
+// of the polynomial evaluation below, so contraction must be off for this function.
+#if defined(__clang__)
+#pragma clang fp contract(off)
+#endif
+  static const double
+    ln2_hi = 6.93147180369123816490e-01,  /* 3fe62e42 fee00000 */
+    ln2_lo = 1.90821492927058770002e-10,  /* 3dea39ef 35793c76 */
+    two54  = 1.80143985094819840000e+16,  /* 43500000 00000000 */
+    Lg1 = 6.666666666666735130e-01,       /* 3FE55555 55555593 */
+    Lg2 = 3.999999999940941908e-01,       /* 3FD99999 9997FA04 */
+    Lg3 = 2.857142874366239149e-01,       /* 3FD24924 94229359 */
+    Lg4 = 2.222219843214978396e-01,       /* 3FCC71C5 1D8E78AF */
+    Lg5 = 1.818357216161805012e-01,       /* 3FC74664 96CB03DE */
+    Lg6 = 1.531383769920937332e-01,       /* 3FC39A09 D078C69F */
+    Lg7 = 1.479819860511658591e-01,       /* 3FC2F112 DF3E5244 */
+    zero = 0.0;
+
+  double hfsq,f,s,z,R,w,t1,t2,dk;
+  int32_t k,hx,i,j;
+  uint32_t lx;
+
+  hx = hi_word(x);
+  lx = lo_word(x);
+
+  k = 0;
+  if (hx < 0x00100000) {                       /* x < 2**-1022 */
+    // fdlibm writes these as -two54/zero and (x-x)/zero, which also raise the divide-by-zero
+    // and invalid flags. MSVC rejects a compile-time division by a zero constant (C2124), so
+    // return the same values directly. The estimators never call log() with these inputs.
+    if (((hx & 0x7fffffff) | lx) == 0) {                          /* log(+-0) = -inf */
+      return -std::numeric_limits<double>::infinity();
+    }
+    if (hx < 0) {                                                 /* log(-#) = NaN */
+      return std::numeric_limits<double>::quiet_NaN();
+    }
+    k -= 54; x *= two54;                       /* subnormal: scale up */
+    hx = hi_word(x);
+  }
+  if (hx >= 0x7ff00000) { return x+x; }
+  k += (hx>>20) - 1023;
+  hx &= 0x000fffff;
+  i = (hx + 0x95f64) & 0x100000;
+  set_hi_word(x, (uint32_t)(hx | (i ^ 0x3ff00000)));   /* normalize x or x/2 */
+  k += (i>>20);
+  f = x - 1.0;
+  if ((0x000fffff & (2+hx)) < 3) {             /* |f| < 2**-20 */
+    if (f == zero) {
+      if (k == 0) { return zero; }
+      dk = (double)k; return rnd(dk*ln2_hi) + rnd(dk*ln2_lo);
+    }
+    R = rnd(rnd(f*f)*rnd(0.5 - rnd(0.33333333333333333*f)));
+    if (k == 0) { return f-R; }
+    dk = (double)k; return rnd(dk*ln2_hi) - (rnd(R - rnd(dk*ln2_lo)) - f);
+  }
+  s = f/(2.0+f);
+  dk = (double)k;
+  z = s*s;
+  i = hx - 0x6147a;
+  w = z*z;
+  j = 0x6b851 - hx;
+  t1 = rnd(w*rnd(Lg2 + rnd(w*rnd(Lg4 + rnd(w*Lg6)))));
+  t2 = rnd(z*rnd(Lg1 + rnd(w*rnd(Lg3 + rnd(w*rnd(Lg5 + rnd(w*Lg7)))))));
+  i |= j;
+  R = t2 + t1;
+  if (i > 0) {
+    hfsq = rnd(0.5*f)*f;
+    if (k == 0) { return f - rnd(hfsq - rnd(s*(hfsq+R))); }
+    return rnd(dk*ln2_hi) - (rnd(hfsq - rnd(rnd(s*(hfsq+R)) + rnd(dk*ln2_lo))) - f);
+  } else {
+    if (k == 0) { return f - rnd(s*(f-R)); }
+    return rnd(dk*ln2_hi) - (rnd(rnd(s*(f-R)) - rnd(dk*ln2_lo)) - f);
+  }
+}
+
+} // namespace fdlibm
+#endif

data/vendor/datasketches-cpp/common/include/serde.hpp

@@ -20,6 +20,7 @@
 #ifndef DATASKETCHES_SERDE_HPP_
 #define DATASKETCHES_SERDE_HPP_
 
+#include <cstdint>
 #include <cstring>
 #include <iostream>
 #include <memory>
@@ -132,6 +133,11 @@ struct serde<T, typename std::enable_if<std::is_arithmetic<T>::value>::type> {
 /// ItemsSketch<String> with ArrayOfStringsSerDe in Java.
 /// The length of each string is stored as a 32-bit integer (historically),
 /// which may be too wasteful. Treat this as an example.
+///
+/// This implementation treats std::string as an arbitrary byte container.
+/// It does not check whether string contents are valid UTF-8.
+///
+/// Use a UTF-8-validating SerDe when cross-language portability is required.
 template<>
 struct serde<std::string> {
   /// @copydoc serde::serialize

data/vendor/datasketches-cpp/common/test/CMakeLists.txt

@@ -69,15 +69,21 @@ target_sources(common_test
   PRIVATE
     quantiles_sorted_view_test.cpp
     optional_test.cpp
+    binomial_bounds_test.cpp
 )
 
 # now the integration test part
 add_executable(integration_test)
 
-target_link_libraries(integration_test count cpc density fi hll kll req sampling theta tuple common_test_lib)
+target_link_libraries(integration_test count cpc density fi hll kll req sampling theta tuple quantiles common_test_lib)
 
+# Use CMAKE_CXX_STANDARD if defined, otherwise C++11
+set(_integration_cxx_standard 11)
+if(DEFINED CMAKE_CXX_STANDARD)
+  set(_integration_cxx_standard ${CMAKE_CXX_STANDARD})
+endif()
 set_target_properties(integration_test PROPERTIES
-  CXX_STANDARD 11
+  CXX_STANDARD ${_integration_cxx_standard}
   CXX_STANDARD_REQUIRED YES
 )
 
@@ -90,3 +96,39 @@ target_sources(integration_test
   PRIVATE
     integration_test.cpp
 )
+
+# Separate hardening test executable (header-only, no pre-compiled libs)
+# This ensures the sketch code is compiled with C++17 + hardening
+# Always build this target - it will use CMAKE_CXX_STANDARD if set (and >= 17), otherwise C++17
+
+add_executable(hardening_test)
+target_link_libraries(hardening_test common common_test_lib)
+
+# Include directories for header-only sketch implementations
+target_include_directories(hardening_test PRIVATE
+  ${CMAKE_SOURCE_DIR}/quantiles/include
+  ${CMAKE_SOURCE_DIR}/kll/include
+  ${CMAKE_SOURCE_DIR}/req/include
+  ${CMAKE_SOURCE_DIR}/common/include
+)
+
+# Use C++17 minimum for hardening tests
+set(_hardening_cxx_standard 17)
+if(DEFINED CMAKE_CXX_STANDARD AND CMAKE_CXX_STANDARD GREATER_EQUAL 17)
+  set(_hardening_cxx_standard ${CMAKE_CXX_STANDARD})
+endif()
+set_target_properties(hardening_test PROPERTIES
+  CXX_STANDARD ${_hardening_cxx_standard}
+  CXX_STANDARD_REQUIRED YES
+)
+message(STATUS "hardening_test will use C++${_hardening_cxx_standard}")
+
+add_test(
+  NAME hardening_test
+  COMMAND hardening_test "[deserialize_hardening]"
+)
+
+target_sources(hardening_test
+  PRIVATE
+    deserialize_hardening_test.cpp
+)