datasketches 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/datasketches/version.rb +1 -1
- data/vendor/datasketches-cpp/LICENSE +0 -34
- data/vendor/datasketches-cpp/NOTICE +2 -2
- data/vendor/datasketches-cpp/README.md +1 -1
- data/vendor/datasketches-cpp/common/include/memory_operations.hpp +2 -2
- data/vendor/datasketches-cpp/count/include/count_min.hpp +1 -1
- data/vendor/datasketches-cpp/count/include/count_min_impl.hpp +2 -1
- data/vendor/datasketches-cpp/count/test/count_min_allocation_test.cpp +17 -0
- data/vendor/datasketches-cpp/density/include/density_sketch.hpp +3 -2
- data/vendor/datasketches-cpp/density/test/density_sketch_test.cpp +30 -0
- data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +18 -9
- data/vendor/datasketches-cpp/hll/include/hll.hpp +1 -1
- data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +73 -0
- data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +3 -0
- data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +35 -0
- data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +70 -0
- data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch_impl.hpp +1 -0
- data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_test.cpp +13 -0
- data/vendor/datasketches-cpp/req/include/req_common.hpp +2 -0
- data/vendor/datasketches-cpp/req/include/req_compactor.hpp +3 -2
- data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +31 -3
- data/vendor/datasketches-cpp/req/include/req_sketch.hpp +5 -1
- data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +31 -3
- data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +176 -0
- data/vendor/datasketches-cpp/sampling/include/ebpps_sketch_impl.hpp +1 -1
- data/vendor/datasketches-cpp/sampling/test/ebpps_allocation_test.cpp +33 -0
- data/vendor/datasketches-cpp/tdigest/include/tdigest_impl.hpp +1 -1
- data/vendor/datasketches-cpp/tdigest/test/tdigest_test.cpp +22 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection.hpp +9 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection_impl.hpp +14 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_union.hpp +9 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_union_impl.hpp +14 -0
- data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +47 -0
- data/vendor/datasketches-cpp/version.cfg.in +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 355d6f2a9ee22a1d96b01f87a655cc5c3f9421910eeb32fe24af92a23717ef54
|
|
4
|
+
data.tar.gz: c9ab5403080ced096259eb620dd41051a8ee62d5311568a14f2ff580c6ea53ed
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ce3dc862138b6e1fe9dafc30828dc8a4e79b5ec9891c3a3516d38abc75091b13478efd8a3992eb2366bacce582ad6784f5455adc9dd1e2bc3c9770fd7e493c5a
|
|
7
|
+
data.tar.gz: a4d6f0ee542ed5b7db7792d06e5a8e447dd60a95abea1e1dba7b1ce557de1dfef7349f2c4e3b814df0a56ca08c5d66bedc84405a4ffc666f003a9dfe4028913a
|
data/CHANGELOG.md
CHANGED
data/lib/datasketches/version.rb
CHANGED
|
@@ -214,40 +214,6 @@ APPENDIX B: Additional licenses relevant to this product.
|
|
|
214
214
|
conditions of the following licenses.
|
|
215
215
|
|
|
216
216
|
|
|
217
|
-
=============================================================
|
|
218
|
-
Boost License (https://www.boost.org/LICENSE_1_0.txt)
|
|
219
|
-
=============================================================
|
|
220
|
-
Original source:
|
|
221
|
-
https://github.com/catchorg/Catch2/blob/v2.x/LICENSE.txt
|
|
222
|
-
|
|
223
|
-
Boost Software License - Version 1.0 - August 17th, 2003
|
|
224
|
-
|
|
225
|
-
Permission is hereby granted, free of charge, to any person or organization
|
|
226
|
-
obtaining a copy of the software and accompanying documentation covered by
|
|
227
|
-
this license (the "Software") to use, reproduce, display, distribute,
|
|
228
|
-
execute, and transmit the Software, and to prepare derivative works of the
|
|
229
|
-
Software, and to permit third-parties to whom the Software is furnished to
|
|
230
|
-
do so, all subject to the following:
|
|
231
|
-
|
|
232
|
-
The copyright notices in the Software and this entire statement, including
|
|
233
|
-
the above license grant, this restriction and the following disclaimer,
|
|
234
|
-
must be included in all copies of the Software, in whole or in part, and
|
|
235
|
-
all derivative works of the Software, unless such copies or derivative
|
|
236
|
-
works are solely in the form of machine-executable object code generated by
|
|
237
|
-
a source language processor.
|
|
238
|
-
|
|
239
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
240
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
241
|
-
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
|
242
|
-
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
|
243
|
-
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
|
244
|
-
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
245
|
-
DEALINGS IN THE SOFTWARE.
|
|
246
|
-
-------------------------------------------------------------
|
|
247
|
-
Code Locations:
|
|
248
|
-
Found in the Catch2 unit test code that is downloaded from github.com as part
|
|
249
|
-
of CMake configuration if configured to build tests.
|
|
250
|
-
|
|
251
217
|
=============================================================
|
|
252
218
|
MIT License
|
|
253
219
|
=============================================================
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Apache DataSketches C++
|
|
2
|
-
Copyright
|
|
2
|
+
Copyright 2019-2026 The Apache Software Foundation
|
|
3
3
|
|
|
4
4
|
Copyright 2015-2018 Yahoo Inc.
|
|
5
5
|
Copyright 2019-2020 Verizon Media
|
|
6
|
-
Copyright 2021- Yahoo Inc.
|
|
6
|
+
Copyright 2021-2025 Yahoo Inc.
|
|
7
7
|
|
|
8
8
|
This product includes software developed at
|
|
9
9
|
The Apache Software Foundation (http://www.apache.org/).
|
|
@@ -21,7 +21,7 @@ This code requires C++11.
|
|
|
21
21
|
|
|
22
22
|
This library is header-only. The provided build process is only for unit tests.
|
|
23
23
|
|
|
24
|
-
Building the unit tests requires CMake 3.
|
|
24
|
+
Building the unit tests requires CMake 3.16.0 or higher.
|
|
25
25
|
|
|
26
26
|
Installing the latest CMake on OSX: `brew install cmake`.
|
|
27
27
|
|
|
@@ -44,13 +44,13 @@ static inline void check_memory_size(size_t requested_index, size_t capacity) {
|
|
|
44
44
|
|
|
45
45
|
// note: size is in bytes, not items
|
|
46
46
|
static inline size_t copy_from_mem(const void* src, void* dst, size_t size) {
|
|
47
|
-
memcpy(dst, src, size);
|
|
47
|
+
if (size > 0) memcpy(dst, src, size);
|
|
48
48
|
return size;
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
// note: size is in bytes, not items
|
|
52
52
|
static inline size_t copy_to_mem(const void* src, void* dst, size_t size) {
|
|
53
|
-
memcpy(dst, src, size);
|
|
53
|
+
if (size > 0) memcpy(dst, src, size);
|
|
54
54
|
return size;
|
|
55
55
|
}
|
|
56
56
|
|
|
@@ -382,7 +382,7 @@ private:
|
|
|
382
382
|
std::vector<W, Allocator> _sketch_array; // the array stored by the sketch
|
|
383
383
|
uint64_t _seed;
|
|
384
384
|
W _total_weight;
|
|
385
|
-
std::vector<uint64_t
|
|
385
|
+
std::vector<uint64_t, typename std::allocator_traits<Allocator>::template rebind_alloc<uint64_t>> hash_seeds;
|
|
386
386
|
|
|
387
387
|
enum flags {IS_EMPTY};
|
|
388
388
|
static const uint8_t PREAMBLE_LONGS_SHORT = 2; // Empty -> need second byte for sketch parameters
|
|
@@ -38,7 +38,8 @@ _num_hashes(num_hashes),
|
|
|
38
38
|
_num_buckets(num_buckets),
|
|
39
39
|
_sketch_array((static_cast<uint64_t>(num_hashes) * num_buckets < 1<<30) ? static_cast<size_t>(num_hashes) * num_buckets : 0, 0, _allocator),
|
|
40
40
|
_seed(seed),
|
|
41
|
-
_total_weight(0)
|
|
41
|
+
_total_weight(0),
|
|
42
|
+
hash_seeds(_allocator) {
|
|
42
43
|
if (num_buckets < 3) {
|
|
43
44
|
throw std::invalid_argument("Using fewer than 3 buckets incurs relative error greater than 1.");
|
|
44
45
|
}
|
|
@@ -152,4 +152,21 @@ TEST_CASE("CountMin sketch test allocator: bytes serialize-deserialize non-empty
|
|
|
152
152
|
REQUIRE(test_allocator_net_allocations == 0);
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
+
TEST_CASE("CountMin sketch test allocator: hash seeds use the sketch's allocator", "[cm_sketch_alloc]") {
|
|
156
|
+
test_allocator_total_bytes = 0;
|
|
157
|
+
test_allocator_net_allocations = 0;
|
|
158
|
+
{
|
|
159
|
+
const uint8_t n_hashes = 5;
|
|
160
|
+
const uint32_t n_buckets = 100;
|
|
161
|
+
const long long sketch_bytes = n_hashes * n_buckets * sizeof(uint64_t) + n_hashes * sizeof(uint64_t);
|
|
162
|
+
count_min_sketch_test_alloc c(n_hashes, n_buckets, DEFAULT_SEED, alloc(0));
|
|
163
|
+
// the counters and the hash seeds, all through the sketch's allocator
|
|
164
|
+
REQUIRE(test_allocator_total_bytes == sketch_bytes);
|
|
165
|
+
count_min_sketch_test_alloc copy(c);
|
|
166
|
+
REQUIRE(test_allocator_total_bytes == 2 * sketch_bytes);
|
|
167
|
+
}
|
|
168
|
+
REQUIRE(test_allocator_total_bytes == 0);
|
|
169
|
+
REQUIRE(test_allocator_net_allocations == 0);
|
|
170
|
+
}
|
|
171
|
+
|
|
155
172
|
} // namespace datasketches
|
|
@@ -32,7 +32,9 @@ namespace datasketches {
|
|
|
32
32
|
|
|
33
33
|
template<typename T>
|
|
34
34
|
struct gaussian_kernel {
|
|
35
|
-
|
|
35
|
+
// any two vectors of T, so the sketch's points can use its allocator
|
|
36
|
+
template<typename V1, typename V2>
|
|
37
|
+
T operator()(const V1& v1, const V2& v2) const {
|
|
36
38
|
return exp(-std::inner_product(v1.begin(), v1.end(), v2.begin(), 0.0, std::plus<T>(), [](T a, T b){return (a-b)*(a-b);}));
|
|
37
39
|
}
|
|
38
40
|
};
|
|
@@ -203,7 +205,6 @@ private:
|
|
|
203
205
|
static const uint8_t SERIAL_VERSION = 1;
|
|
204
206
|
static const size_t LEVELS_ARRAY_START = 5;
|
|
205
207
|
|
|
206
|
-
Allocator allocator_;
|
|
207
208
|
Kernel kernel_;
|
|
208
209
|
uint16_t k_;
|
|
209
210
|
uint32_t dim_;
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
#include <catch2/catch.hpp>
|
|
22
22
|
|
|
23
23
|
#include <density_sketch.hpp>
|
|
24
|
+
#include <test_allocator.hpp>
|
|
24
25
|
|
|
25
26
|
namespace datasketches {
|
|
26
27
|
|
|
@@ -241,4 +242,33 @@ TEST_CASE("serialize stream", "[density_sketch]") {
|
|
|
241
242
|
}
|
|
242
243
|
}
|
|
243
244
|
|
|
245
|
+
TEST_CASE("density sketch: custom allocator", "[density_sketch]") {
|
|
246
|
+
using alloc = test_allocator<double>;
|
|
247
|
+
using sketch = density_sketch<double, gaussian_kernel<double>, alloc>;
|
|
248
|
+
using point = std::vector<double, alloc>;
|
|
249
|
+
test_allocator_total_bytes = 0;
|
|
250
|
+
test_allocator_net_allocations = 0;
|
|
251
|
+
{
|
|
252
|
+
sketch s(10, 3, gaussian_kernel<double>(), alloc(0));
|
|
253
|
+
// Keep every retained point close to the query so the Gaussian kernel cannot underflow to zero.
|
|
254
|
+
for (int i = 0; i < 1000; ++i) s.update(point({i / 1000.0, 1.0, 2.0}, alloc(0)));
|
|
255
|
+
REQUIRE(s.get_n() == 1000);
|
|
256
|
+
REQUIRE(s.is_estimation_mode());
|
|
257
|
+
REQUIRE(s.get_estimate({0.5, 1.0, 2.0}) > 0);
|
|
258
|
+
REQUIRE(test_allocator_total_bytes > 0);
|
|
259
|
+
|
|
260
|
+
sketch other(10, 3, gaussian_kernel<double>(), alloc(0));
|
|
261
|
+
for (int i = 0; i < 100; ++i) other.update(point({i / 1000.0, 1.0, 2.0}, alloc(0)));
|
|
262
|
+
s.merge(other);
|
|
263
|
+
REQUIRE(s.get_n() == 1100);
|
|
264
|
+
|
|
265
|
+
auto bytes = s.serialize();
|
|
266
|
+
auto copy = sketch::deserialize(bytes.data(), bytes.size(), gaussian_kernel<double>(), alloc(0));
|
|
267
|
+
REQUIRE(copy.get_n() == s.get_n());
|
|
268
|
+
REQUIRE(copy.get_estimate({0.5, 1.0, 2.0}) == s.get_estimate({0.5, 1.0, 2.0}));
|
|
269
|
+
}
|
|
270
|
+
REQUIRE(test_allocator_total_bytes == 0);
|
|
271
|
+
REQUIRE(test_allocator_net_allocations == 0);
|
|
272
|
+
}
|
|
273
|
+
|
|
244
274
|
} /* namespace datasketches */
|
|
@@ -54,7 +54,9 @@ void hll_union_alloc<A>::update(const hll_sketch_alloc<A>& sketch) {
|
|
|
54
54
|
template<typename A>
|
|
55
55
|
void hll_union_alloc<A>::update(hll_sketch_alloc<A>&& sketch) {
|
|
56
56
|
if (sketch.is_empty()) { return; }
|
|
57
|
-
|
|
57
|
+
// taking over the sketch is only possible if its memory can be released with the union's allocator
|
|
58
|
+
if (gadget_.is_empty() && sketch.get_target_type() == HLL_8 && sketch.get_lg_config_k() <= lg_max_k_
|
|
59
|
+
&& sketch.sketch_impl->getAllocator() == gadget_.sketch_impl->getAllocator()) {
|
|
58
60
|
if (sketch.get_current_mode() == HLL || sketch.get_lg_config_k() == lg_max_k_) {
|
|
59
61
|
gadget_ = std::move(sketch);
|
|
60
62
|
}
|
|
@@ -208,17 +210,20 @@ double hll_union_alloc<A>::get_rel_err(bool upper_bound, bool unioned,
|
|
|
208
210
|
}
|
|
209
211
|
|
|
210
212
|
template<typename A>
|
|
211
|
-
HllSketchImpl<A>* hll_union_alloc<A>::copy_or_downsample(const HllSketchImpl<A>* src_impl, uint8_t tgt_lg_k
|
|
213
|
+
HllSketchImpl<A>* hll_union_alloc<A>::copy_or_downsample(const HllSketchImpl<A>* src_impl, uint8_t tgt_lg_k,
|
|
214
|
+
const A& allocator) {
|
|
212
215
|
if (src_impl->getCurMode() != HLL) {
|
|
213
216
|
throw std::logic_error("Attempt to downsample non-HLL sketch");
|
|
214
217
|
}
|
|
215
218
|
const HllArray<A>* src = static_cast<const HllArray<A>*>(src_impl);
|
|
216
|
-
const
|
|
217
|
-
if
|
|
219
|
+
const uint8_t src_lg_k = src->getLgConfigK();
|
|
220
|
+
// a copy keeps the source's allocator, so it is only made if that is the union's allocator
|
|
221
|
+
if (src_lg_k <= tgt_lg_k && src->getAllocator() == allocator) {
|
|
218
222
|
return src->copyAs(HLL_8);
|
|
219
223
|
}
|
|
220
224
|
typedef typename std::allocator_traits<A>::template rebind_alloc<Hll8Array<A>> hll8Alloc;
|
|
221
|
-
|
|
225
|
+
const uint8_t lg_k = std::min(src_lg_k, tgt_lg_k);
|
|
226
|
+
Hll8Array<A>* tgtHllArr = new (hll8Alloc(allocator).allocate(1)) Hll8Array<A>(lg_k, allocator);
|
|
222
227
|
tgtHllArr->mergeHll(*src);
|
|
223
228
|
//both of these are required for isomorphism
|
|
224
229
|
tgtHllArr->putHipAccum(src->getHipAccum());
|
|
@@ -239,8 +244,12 @@ template<typename A>
|
|
|
239
244
|
void hll_union_alloc<A>::union_impl(const hll_sketch_alloc<A>& sketch, uint8_t lg_max_k) {
|
|
240
245
|
const HllSketchImpl<A>* src_impl = sketch.sketch_impl; //default
|
|
241
246
|
HllSketchImpl<A>* dst_impl = gadget_.sketch_impl; //default
|
|
247
|
+
// the gadget allocates with the union's allocator: the input sketch may use another allocator
|
|
248
|
+
// instance (e.g. a stateful one), and copies of it would keep that allocator
|
|
249
|
+
const A allocator = dst_impl->getAllocator();
|
|
242
250
|
if (src_impl->getCurMode() == LIST || src_impl->getCurMode() == SET) {
|
|
243
|
-
if (dst_impl->isEmpty() && src_impl->getLgConfigK() == dst_impl->getLgConfigK()
|
|
251
|
+
if (dst_impl->isEmpty() && src_impl->getLgConfigK() == dst_impl->getLgConfigK()
|
|
252
|
+
&& src_impl->getAllocator() == allocator) {
|
|
244
253
|
dst_impl = src_impl->copyAs(HLL_8);
|
|
245
254
|
gadget_.sketch_impl->get_deleter()(gadget_.sketch_impl); // gadget to be replaced
|
|
246
255
|
} else {
|
|
@@ -254,12 +263,12 @@ void hll_union_alloc<A>::union_impl(const hll_sketch_alloc<A>& sketch, uint8_t l
|
|
|
254
263
|
// swap so that src is LIST or SET, tgt is HLL
|
|
255
264
|
// use lg_max_k because LIST has effective K of 2^26
|
|
256
265
|
const CouponList<A>* src = static_cast<const CouponList<A>*>(dst_impl);
|
|
257
|
-
dst_impl = copy_or_downsample(src_impl, lg_max_k);
|
|
266
|
+
dst_impl = copy_or_downsample(src_impl, lg_max_k, allocator);
|
|
258
267
|
static_cast<Hll8Array<A>*>(dst_impl)->mergeList(*src);
|
|
259
268
|
gadget_.sketch_impl->get_deleter()(gadget_.sketch_impl); // gadget to be replaced
|
|
260
269
|
} else { // gadget is HLL
|
|
261
270
|
if (src_impl->getLgConfigK() < dst_impl->getLgConfigK()) {
|
|
262
|
-
dst_impl = copy_or_downsample(dst_impl, sketch.get_lg_config_k());
|
|
271
|
+
dst_impl = copy_or_downsample(dst_impl, sketch.get_lg_config_k(), allocator);
|
|
263
272
|
gadget_.sketch_impl->get_deleter()(gadget_.sketch_impl); // gadget to be replaced
|
|
264
273
|
}
|
|
265
274
|
const HllArray<A>* src = static_cast<const HllArray<A>*>(src_impl);
|
|
@@ -268,7 +277,7 @@ void hll_union_alloc<A>::union_impl(const hll_sketch_alloc<A>& sketch, uint8_t l
|
|
|
268
277
|
static_cast<Hll8Array<A>*>(dst_impl)->putHipAccum(0);
|
|
269
278
|
}
|
|
270
279
|
} else { // src is HLL, gadget is empty
|
|
271
|
-
dst_impl = copy_or_downsample(src_impl, lg_max_k);
|
|
280
|
+
dst_impl = copy_or_downsample(src_impl, lg_max_k, allocator);
|
|
272
281
|
gadget_.sketch_impl->get_deleter()(gadget_.sketch_impl); // gadget to be replaced
|
|
273
282
|
}
|
|
274
283
|
gadget_.sketch_impl = dst_impl; // gadget replaced
|
|
@@ -640,7 +640,7 @@ class hll_union_alloc {
|
|
|
640
640
|
*/
|
|
641
641
|
inline void union_impl(const hll_sketch_alloc<A>& sketch, uint8_t lg_max_k);
|
|
642
642
|
|
|
643
|
-
static HllSketchImpl<A>* copy_or_downsample(const HllSketchImpl<A>* src_impl, uint8_t tgt_lg_k);
|
|
643
|
+
static HllSketchImpl<A>* copy_or_downsample(const HllSketchImpl<A>* src_impl, uint8_t tgt_lg_k, const A& allocator);
|
|
644
644
|
|
|
645
645
|
void coupon_update(uint32_t coupon);
|
|
646
646
|
|
|
@@ -384,4 +384,77 @@ TEST_CASE("hll union: serialization is grouping independent", "[hll_union]") {
|
|
|
384
384
|
REQUIRE(serialize_nested_union(a, b, c) == serialize_flat_union(a, b, c));
|
|
385
385
|
}
|
|
386
386
|
|
|
387
|
+
// A stateful allocator: instances with different ids are different allocators, and the bytes each one
|
|
388
|
+
// has outstanding are counted.
|
|
389
|
+
static long long pool_bytes[2] = {0, 0};
|
|
390
|
+
|
|
391
|
+
template<typename T>
|
|
392
|
+
struct pool_allocator {
|
|
393
|
+
using value_type = T;
|
|
394
|
+
int id;
|
|
395
|
+
explicit pool_allocator(int id): id(id) {}
|
|
396
|
+
template<typename U>
|
|
397
|
+
pool_allocator(const pool_allocator<U>& other): id(other.id) {}
|
|
398
|
+
T* allocate(std::size_t n) {
|
|
399
|
+
pool_bytes[id] += n * sizeof(T);
|
|
400
|
+
return static_cast<T*>(::operator new(n * sizeof(T)));
|
|
401
|
+
}
|
|
402
|
+
void deallocate(T* p, std::size_t n) {
|
|
403
|
+
pool_bytes[id] -= n * sizeof(T);
|
|
404
|
+
::operator delete(p);
|
|
405
|
+
}
|
|
406
|
+
};
|
|
407
|
+
template<typename T, typename U>
|
|
408
|
+
bool operator==(const pool_allocator<T>& a, const pool_allocator<U>& b) { return a.id == b.id; }
|
|
409
|
+
template<typename T, typename U>
|
|
410
|
+
bool operator!=(const pool_allocator<T>& a, const pool_allocator<U>& b) { return a.id != b.id; }
|
|
411
|
+
|
|
412
|
+
using pool_hll_sketch = hll_sketch_alloc<pool_allocator<uint8_t>>;
|
|
413
|
+
using pool_hll_union = hll_union_alloc<pool_allocator<uint8_t>>;
|
|
414
|
+
|
|
415
|
+
static pool_hll_sketch make_pool_sketch(int id, uint8_t lg_k, target_hll_type type, uint64_t n) {
|
|
416
|
+
pool_hll_sketch sketch(lg_k, type, false, pool_allocator<uint8_t>(id));
|
|
417
|
+
for (uint64_t i = 0; i < n; ++i) sketch.update(i);
|
|
418
|
+
return sketch;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
TEST_CASE("hll union: keeps no memory of an input sketch's allocator", "[hll_union]") {
|
|
422
|
+
const uint8_t lg_max_k = 10;
|
|
423
|
+
// LIST, SET and HLL inputs, with a smaller, the same and a larger lg_k than the union
|
|
424
|
+
const uint64_t sizes[] = {5, 100, 10000};
|
|
425
|
+
const uint8_t lg_ks[] = {8, 10, 12};
|
|
426
|
+
for (uint64_t n: sizes) {
|
|
427
|
+
for (uint8_t lg_k: lg_ks) {
|
|
428
|
+
for (target_hll_type type: {HLL_4, HLL_6, HLL_8}) {
|
|
429
|
+
for (bool gadget_empty: {true, false}) {
|
|
430
|
+
for (bool move: {false, true}) {
|
|
431
|
+
hll_union u(lg_max_k);
|
|
432
|
+
pool_hll_union pool_union(lg_max_k, pool_allocator<uint8_t>(0));
|
|
433
|
+
if (!gadget_empty) {
|
|
434
|
+
for (uint64_t i = 0; i < 3 * n; ++i) {
|
|
435
|
+
u.update(i + n);
|
|
436
|
+
pool_union.update(i + n);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
{
|
|
440
|
+
hll_sketch control(lg_k, type);
|
|
441
|
+
for (uint64_t i = 0; i < n; ++i) control.update(i);
|
|
442
|
+
u.update(control);
|
|
443
|
+
pool_hll_sketch input = make_pool_sketch(1, lg_k, type, n);
|
|
444
|
+
if (move) {
|
|
445
|
+
pool_union.update(std::move(input));
|
|
446
|
+
} else {
|
|
447
|
+
pool_union.update(input);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
REQUIRE(pool_bytes[1] == 0);
|
|
451
|
+
REQUIRE(pool_union.get_estimate() == u.get_estimate());
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
REQUIRE(pool_bytes[0] == 0);
|
|
458
|
+
}
|
|
459
|
+
|
|
387
460
|
} /* namespace datasketches */
|
|
@@ -591,10 +591,13 @@ class kll_sketch {
|
|
|
591
591
|
uint32_t safe_level_size(uint8_t level) const;
|
|
592
592
|
uint32_t get_num_retained_above_level_zero() const;
|
|
593
593
|
|
|
594
|
+
static void check_k(uint16_t k);
|
|
594
595
|
static void check_m(uint8_t m);
|
|
595
596
|
static void check_preamble_ints(uint8_t preamble_ints, uint8_t flags_byte);
|
|
596
597
|
static void check_serial_version(uint8_t serial_version);
|
|
597
598
|
static void check_family_id(uint8_t family_id);
|
|
599
|
+
static void check_num_levels(uint8_t num_levels);
|
|
600
|
+
static void check_levels(const vector_u32& levels);
|
|
598
601
|
|
|
599
602
|
void check_sorting() const;
|
|
600
603
|
|
|
@@ -450,6 +450,7 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(std::istream& is, const Ser
|
|
|
450
450
|
const auto m = read<uint8_t>(is);
|
|
451
451
|
read<uint8_t>(is); // skip unused byte
|
|
452
452
|
|
|
453
|
+
check_k(k);
|
|
453
454
|
check_m(m);
|
|
454
455
|
check_preamble_ints(preamble_ints, flags_byte);
|
|
455
456
|
check_serial_version(serial_version);
|
|
@@ -472,6 +473,7 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(std::istream& is, const Ser
|
|
|
472
473
|
min_k = read<uint16_t>(is);
|
|
473
474
|
num_levels = read<uint8_t>(is);
|
|
474
475
|
read<uint8_t>(is); // skip unused byte
|
|
476
|
+
check_num_levels(num_levels);
|
|
475
477
|
}
|
|
476
478
|
vector_u32 levels(num_levels + 1, 0, allocator);
|
|
477
479
|
const uint32_t capacity(kll_helper::compute_total_capacity(k, m, num_levels));
|
|
@@ -480,8 +482,10 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(std::istream& is, const Ser
|
|
|
480
482
|
} else {
|
|
481
483
|
// the last integer in levels_ is not serialized because it can be derived
|
|
482
484
|
read(is, levels.data(), sizeof(levels[0]) * num_levels);
|
|
485
|
+
if (!is.good()) throw std::runtime_error("error reading from std::istream");
|
|
483
486
|
}
|
|
484
487
|
levels[num_levels] = capacity;
|
|
488
|
+
check_levels(levels);
|
|
485
489
|
optional<T> min_item;
|
|
486
490
|
optional<T> max_item;
|
|
487
491
|
if (!is_single_item) {
|
|
@@ -537,6 +541,7 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(const void* bytes, size_t s
|
|
|
537
541
|
ptr += copy_from_mem(ptr, m);
|
|
538
542
|
ptr += sizeof(uint8_t); // skip unused byte
|
|
539
543
|
|
|
544
|
+
check_k(k);
|
|
540
545
|
check_m(m);
|
|
541
546
|
check_preamble_ints(preamble_ints, flags_byte);
|
|
542
547
|
check_serial_version(serial_version);
|
|
@@ -560,6 +565,7 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(const void* bytes, size_t s
|
|
|
560
565
|
ptr += copy_from_mem(ptr, min_k);
|
|
561
566
|
ptr += copy_from_mem(ptr, num_levels);
|
|
562
567
|
ptr += sizeof(uint8_t); // skip unused byte
|
|
568
|
+
check_num_levels(num_levels);
|
|
563
569
|
}
|
|
564
570
|
vector_u32 levels(num_levels + 1, 0, allocator);
|
|
565
571
|
const uint32_t capacity(kll_helper::compute_total_capacity(k, m, num_levels));
|
|
@@ -567,9 +573,11 @@ kll_sketch<T, C, A> kll_sketch<T, C, A>::deserialize(const void* bytes, size_t s
|
|
|
567
573
|
levels[0] = capacity - 1;
|
|
568
574
|
} else {
|
|
569
575
|
// the last integer in levels_ is not serialized because it can be derived
|
|
576
|
+
ensure_minimum_memory(size, (ptr - static_cast<const char*>(bytes)) + sizeof(levels[0]) * num_levels);
|
|
570
577
|
ptr += copy_from_mem(ptr, levels.data(), sizeof(levels[0]) * num_levels);
|
|
571
578
|
}
|
|
572
579
|
levels[num_levels] = capacity;
|
|
580
|
+
check_levels(levels);
|
|
573
581
|
optional<T> min_item;
|
|
574
582
|
optional<T> max_item;
|
|
575
583
|
if (!is_single_item) {
|
|
@@ -867,6 +875,14 @@ uint32_t kll_sketch<T, C, A>::get_num_retained_above_level_zero() const {
|
|
|
867
875
|
return levels_[num_levels_] - levels_[1];
|
|
868
876
|
}
|
|
869
877
|
|
|
878
|
+
template<typename T, typename C, typename A>
|
|
879
|
+
void kll_sketch<T, C, A>::check_k(uint16_t k) {
|
|
880
|
+
if (k < kll_constants::MIN_K) {
|
|
881
|
+
throw std::invalid_argument("Possible corruption: K must be >= " + std::to_string(kll_constants::MIN_K)
|
|
882
|
+
+ ": " + std::to_string(k));
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
|
|
870
886
|
template<typename T, typename C, typename A>
|
|
871
887
|
void kll_sketch<T, C, A>::check_m(uint8_t m) {
|
|
872
888
|
if (m != kll_constants::DEFAULT_M) {
|
|
@@ -909,6 +925,25 @@ void kll_sketch<T, C, A>::check_family_id(uint8_t family_id) {
|
|
|
909
925
|
}
|
|
910
926
|
}
|
|
911
927
|
|
|
928
|
+
template<typename T, typename C, typename A>
|
|
929
|
+
void kll_sketch<T, C, A>::check_num_levels(uint8_t num_levels) {
|
|
930
|
+
if (num_levels == 0) {
|
|
931
|
+
throw std::invalid_argument("Possible corruption: number of levels must be > 0");
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
// The last entry is the capacity. Each level must start at or after the previous one,
|
|
936
|
+
// so every level lies within the items buffer.
|
|
937
|
+
template<typename T, typename C, typename A>
|
|
938
|
+
void kll_sketch<T, C, A>::check_levels(const vector_u32& levels) {
|
|
939
|
+
for (size_t i = 1; i < levels.size(); ++i) {
|
|
940
|
+
if (levels[i - 1] > levels[i]) {
|
|
941
|
+
throw std::invalid_argument("Possible corruption: levels[" + std::to_string(i - 1) + "] = "
|
|
942
|
+
+ std::to_string(levels[i - 1]) + " > levels[" + std::to_string(i) + "] = " + std::to_string(levels[i]));
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
|
|
912
947
|
template <typename T, typename C, typename A>
|
|
913
948
|
string<A> kll_sketch<T, C, A>::to_string(bool print_levels, bool print_items) const {
|
|
914
949
|
// Using a temporary stream for implementation here does not comply with AllocatorAwareContainer requirements.
|
|
@@ -835,4 +835,74 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
|
|
|
835
835
|
REQUIRE(test_allocator_total_bytes == 0);
|
|
836
836
|
}
|
|
837
837
|
|
|
838
|
+
|
|
839
|
+
TEST_CASE("kll sketch: deserialize rejects inconsistent levels", "[kll_sketch]") {
|
|
840
|
+
using vector_bytes = kll_sketch<int>::vector_bytes;
|
|
841
|
+
const size_t k_offset = 4;
|
|
842
|
+
const size_t num_levels_offset = 18;
|
|
843
|
+
const size_t levels_offset = 20;
|
|
844
|
+
auto put_u32 = [](vector_bytes& bytes, size_t offset, uint32_t value) {
|
|
845
|
+
std::memcpy(bytes.data() + offset, &value, sizeof(value));
|
|
846
|
+
};
|
|
847
|
+
auto get_u32 = [](const vector_bytes& bytes, size_t offset) {
|
|
848
|
+
uint32_t value;
|
|
849
|
+
std::memcpy(&value, bytes.data() + offset, sizeof(value));
|
|
850
|
+
return value;
|
|
851
|
+
};
|
|
852
|
+
auto check_rejected = [](const vector_bytes& bytes) {
|
|
853
|
+
REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
|
|
854
|
+
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
|
|
855
|
+
s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
|
|
856
|
+
REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(s), std::invalid_argument);
|
|
857
|
+
};
|
|
858
|
+
|
|
859
|
+
kll_sketch<int> one_level(8);
|
|
860
|
+
for (int i = 0; i < 3; ++i) one_level.update(i);
|
|
861
|
+
kll_sketch<int> three_levels(8);
|
|
862
|
+
for (int i = 0; i < 40; ++i) three_levels.update(i);
|
|
863
|
+
REQUIRE(three_levels.serialize()[num_levels_offset] == 3);
|
|
864
|
+
|
|
865
|
+
SECTION("first level beyond capacity") {
|
|
866
|
+
auto bytes = one_level.serialize();
|
|
867
|
+
put_u32(bytes, levels_offset, 1009);
|
|
868
|
+
check_rejected(bytes);
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
SECTION("level beyond capacity") {
|
|
872
|
+
auto bytes = three_levels.serialize();
|
|
873
|
+
put_u32(bytes, levels_offset + sizeof(uint32_t), 100000);
|
|
874
|
+
check_rejected(bytes);
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
SECTION("decreasing levels") {
|
|
878
|
+
auto bytes = three_levels.serialize();
|
|
879
|
+
REQUIRE(get_u32(bytes, levels_offset) > 0);
|
|
880
|
+
put_u32(bytes, levels_offset + sizeof(uint32_t), 0);
|
|
881
|
+
check_rejected(bytes);
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
SECTION("zero levels") {
|
|
885
|
+
auto bytes = three_levels.serialize();
|
|
886
|
+
bytes[num_levels_offset] = 0;
|
|
887
|
+
check_rejected(bytes);
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
SECTION("k below minimum") {
|
|
891
|
+
auto bytes = three_levels.serialize();
|
|
892
|
+
const uint16_t k = 4;
|
|
893
|
+
std::memcpy(bytes.data() + k_offset, &k, sizeof(k));
|
|
894
|
+
check_rejected(bytes);
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
SECTION("levels beyond the input") {
|
|
898
|
+
auto bytes = three_levels.serialize();
|
|
899
|
+
bytes[num_levels_offset] = 40;
|
|
900
|
+
bytes.resize(levels_offset);
|
|
901
|
+
REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(bytes.data(), bytes.size()), std::out_of_range);
|
|
902
|
+
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
|
|
903
|
+
s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
|
|
904
|
+
REQUIRE_THROWS_AS(kll_sketch<int>::deserialize(s), std::runtime_error);
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
|
|
838
908
|
} /* namespace datasketches */
|
|
@@ -531,6 +531,7 @@ auto quantiles_sketch<T, C, A>::deserialize(const void* bytes, size_t size, cons
|
|
|
531
531
|
|
|
532
532
|
if (serial_version == 1) {
|
|
533
533
|
uint64_t unused_long;
|
|
534
|
+
ensure_minimum_memory(end_ptr - ptr, sizeof(unused_long));
|
|
534
535
|
ptr += copy_from_mem(ptr, unused_long); // no longer used
|
|
535
536
|
}
|
|
536
537
|
|
|
@@ -956,4 +956,17 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
|
|
|
956
956
|
}
|
|
957
957
|
}
|
|
958
958
|
|
|
959
|
+
|
|
960
|
+
TEST_CASE("quantiles sketch: deserialize serial version 1 truncated after min and max", "[quantiles_sketch]") {
|
|
961
|
+
quantiles_sketch<int> sketch(16);
|
|
962
|
+
for (int i = 0; i < 1000; ++i) sketch.update(i);
|
|
963
|
+
auto bytes = sketch.serialize();
|
|
964
|
+
// serial version 1 has an unused 8-byte field after min and max
|
|
965
|
+
bytes[0] = 5; // preamble longs
|
|
966
|
+
bytes[1] = 1; // serial version
|
|
967
|
+
bytes[3] &= ~(1 << 3); // not compact
|
|
968
|
+
bytes.resize(8 + sizeof(uint64_t) + 2 * sizeof(int)); // preamble, n, min and max
|
|
969
|
+
REQUIRE_THROWS_AS(quantiles_sketch<int>::deserialize(bytes.data(), bytes.size()), std::out_of_range);
|
|
970
|
+
}
|
|
971
|
+
|
|
959
972
|
} /* namespace datasketches */
|
|
@@ -31,6 +31,8 @@ namespace datasketches {
|
|
|
31
31
|
namespace req_constants {
|
|
32
32
|
/// minimum value of parameter K
|
|
33
33
|
const uint16_t MIN_K = 4;
|
|
34
|
+
/// maximum value of parameter K
|
|
35
|
+
const uint16_t MAX_K = 1024;
|
|
34
36
|
/// initial number of sections
|
|
35
37
|
const uint8_t INIT_NUM_SECTIONS = 3;
|
|
36
38
|
/// multiplier for nominal capacity
|
|
@@ -87,11 +87,11 @@ public:
|
|
|
87
87
|
|
|
88
88
|
template<typename S>
|
|
89
89
|
static req_compactor deserialize(std::istream& is, const S& serde, const Comparator& comparator,
|
|
90
|
-
const Allocator& allocator, bool sorted, bool hra);
|
|
90
|
+
const Allocator& allocator, bool sorted, bool hra, uint16_t k, uint8_t level);
|
|
91
91
|
|
|
92
92
|
template<typename S>
|
|
93
93
|
static std::pair<req_compactor, size_t> deserialize(const void* bytes, size_t size, const S& serde,
|
|
94
|
-
const Comparator& comparator, const Allocator& allocator, bool sorted, bool hra);
|
|
94
|
+
const Comparator& comparator, const Allocator& allocator, bool sorted, bool hra, uint16_t k, uint8_t level);
|
|
95
95
|
|
|
96
96
|
template<typename S>
|
|
97
97
|
static req_compactor deserialize(std::istream& is, const S& serde, const Comparator& comparator,
|
|
@@ -122,6 +122,7 @@ private:
|
|
|
122
122
|
void ensure_space(uint32_t num);
|
|
123
123
|
|
|
124
124
|
static uint32_t nearest_even(float value);
|
|
125
|
+
static void check_header(uint16_t k, uint8_t level, uint8_t lg_weight, float section_size_raw, uint8_t num_sections);
|
|
125
126
|
|
|
126
127
|
template<typename InIter, typename OutIter>
|
|
127
128
|
static void promote_evens_or_odds(InIter from, InIter to, bool flag, OutIter dst);
|