datasketches 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/datasketches/version.rb +1 -1
- data/vendor/datasketches-cpp/LICENSE +0 -34
- data/vendor/datasketches-cpp/NOTICE +2 -2
- data/vendor/datasketches-cpp/README.md +1 -1
- data/vendor/datasketches-cpp/common/include/memory_operations.hpp +2 -2
- data/vendor/datasketches-cpp/count/include/count_min.hpp +1 -1
- data/vendor/datasketches-cpp/count/include/count_min_impl.hpp +2 -1
- data/vendor/datasketches-cpp/count/test/count_min_allocation_test.cpp +17 -0
- data/vendor/datasketches-cpp/density/include/density_sketch.hpp +3 -2
- data/vendor/datasketches-cpp/density/test/density_sketch_test.cpp +30 -0
- data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +18 -9
- data/vendor/datasketches-cpp/hll/include/hll.hpp +1 -1
- data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +73 -0
- data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +3 -0
- data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +35 -0
- data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +70 -0
- data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch_impl.hpp +1 -0
- data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_test.cpp +13 -0
- data/vendor/datasketches-cpp/req/include/req_common.hpp +2 -0
- data/vendor/datasketches-cpp/req/include/req_compactor.hpp +3 -2
- data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +31 -3
- data/vendor/datasketches-cpp/req/include/req_sketch.hpp +5 -1
- data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +31 -3
- data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +176 -0
- data/vendor/datasketches-cpp/sampling/include/ebpps_sketch_impl.hpp +1 -1
- data/vendor/datasketches-cpp/sampling/test/ebpps_allocation_test.cpp +33 -0
- data/vendor/datasketches-cpp/tdigest/include/tdigest_impl.hpp +1 -1
- data/vendor/datasketches-cpp/tdigest/test/tdigest_test.cpp +22 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection.hpp +9 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection_impl.hpp +14 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_union.hpp +9 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_union_impl.hpp +14 -0
- data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +47 -0
- data/vendor/datasketches-cpp/version.cfg.in +1 -1
- metadata +1 -1
|
@@ -326,6 +326,30 @@ uint32_t req_compactor<T, C, A>::nearest_even(float value) {
|
|
|
326
326
|
return static_cast<uint32_t>(round(value / 2)) << 1;
|
|
327
327
|
}
|
|
328
328
|
|
|
329
|
+
template<typename T, typename C, typename A>
|
|
330
|
+
void req_compactor<T, C, A>::check_header(uint16_t k, uint8_t level, uint8_t lg_weight, float section_size_raw,
|
|
331
|
+
uint8_t num_sections) {
|
|
332
|
+
// the sketch creates the compactor at level h with lg_weight h
|
|
333
|
+
if (lg_weight != level) {
|
|
334
|
+
throw std::invalid_argument("Possible corruption: compactor " + std::to_string(level) + " has lg_weight "
|
|
335
|
+
+ std::to_string(lg_weight));
|
|
336
|
+
}
|
|
337
|
+
// the section size starts at k and shrinks to no less than MIN_K
|
|
338
|
+
// (the first check is false for NaN and keeps nearest_even() in range)
|
|
339
|
+
const uint32_t section_size = (section_size_raw > 0 && section_size_raw <= k + 1) ? nearest_even(section_size_raw) : 0;
|
|
340
|
+
if (section_size < req_constants::MIN_K || section_size > k) {
|
|
341
|
+
throw std::invalid_argument("Possible corruption: section size " + std::to_string(section_size_raw)
|
|
342
|
+
+ " is out of range for k " + std::to_string(k));
|
|
343
|
+
}
|
|
344
|
+
// the number of sections starts at 3 and doubles, and must keep 1 << (num_sections - 1) defined
|
|
345
|
+
// the limit uses the raw section size: when that has shrunk to 3, the even section size is 4 and can exceed it
|
|
346
|
+
const uint32_t rounded_section_size = static_cast<uint32_t>(round(section_size_raw));
|
|
347
|
+
if (num_sections < 1 || num_sections > 64 || rounded_section_size * num_sections > static_cast<uint32_t>(k) * k) {
|
|
348
|
+
throw std::invalid_argument("Possible corruption: number of sections " + std::to_string(num_sections)
|
|
349
|
+
+ " is out of range for section size " + std::to_string(section_size_raw) + " and k " + std::to_string(k));
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
329
353
|
template<typename T, typename C, typename A>
|
|
330
354
|
template<typename InIter, typename OutIter>
|
|
331
355
|
void req_compactor<T, C, A>::promote_evens_or_odds(InIter from, InIter to, bool odds, OutIter dst) {
|
|
@@ -394,13 +418,15 @@ size_t req_compactor<T, C, A>::serialize(void* dst, size_t capacity, const S& se
|
|
|
394
418
|
template<typename T, typename C, typename A>
|
|
395
419
|
template<typename S>
|
|
396
420
|
req_compactor<T, C, A> req_compactor<T, C, A>::deserialize(std::istream& is, const S& serde,
|
|
397
|
-
const C& comparator, const A& allocator, bool sorted, bool hra) {
|
|
421
|
+
const C& comparator, const A& allocator, bool sorted, bool hra, uint16_t k, uint8_t level) {
|
|
398
422
|
auto state = read<decltype(state_)>(is);
|
|
399
423
|
auto section_size_raw = read<decltype(section_size_raw_)>(is);
|
|
400
424
|
auto lg_weight = read<decltype(lg_weight_)>(is);
|
|
401
425
|
auto num_sections = read<decltype(num_sections_)>(is);
|
|
402
426
|
read<uint16_t>(is); // padding
|
|
403
427
|
auto num_items = read<uint32_t>(is);
|
|
428
|
+
if (!is.good()) throw std::runtime_error("error reading from std::istream");
|
|
429
|
+
check_header(k, level, lg_weight, section_size_raw, num_sections);
|
|
404
430
|
auto items = deserialize_items(is, serde, allocator, num_items);
|
|
405
431
|
return req_compactor(hra, lg_weight, sorted, section_size_raw, num_sections, state, std::move(items), num_items,
|
|
406
432
|
comparator, allocator);
|
|
@@ -431,8 +457,9 @@ auto req_compactor<T, C, A>::deserialize_items(std::istream& is, const S& serde,
|
|
|
431
457
|
template<typename T, typename C, typename A>
|
|
432
458
|
template<typename S>
|
|
433
459
|
std::pair<req_compactor<T, C, A>, size_t> req_compactor<T, C, A>::deserialize(const void* bytes, size_t size,
|
|
434
|
-
const S& serde, const C& comparator, const A& allocator, bool sorted, bool hra) {
|
|
435
|
-
ensure_minimum_memory(size,
|
|
460
|
+
const S& serde, const C& comparator, const A& allocator, bool sorted, bool hra, uint16_t k, uint8_t level) {
|
|
461
|
+
ensure_minimum_memory(size, sizeof(state_) + sizeof(section_size_raw_) + sizeof(lg_weight_) + sizeof(num_sections_)
|
|
462
|
+
+ sizeof(uint16_t) + sizeof(num_items_));
|
|
436
463
|
const char* ptr = static_cast<const char*>(bytes);
|
|
437
464
|
const char* end_ptr = static_cast<const char*>(bytes) + size;
|
|
438
465
|
|
|
@@ -447,6 +474,7 @@ std::pair<req_compactor<T, C, A>, size_t> req_compactor<T, C, A>::deserialize(co
|
|
|
447
474
|
ptr += 2; // padding
|
|
448
475
|
uint32_t num_items;
|
|
449
476
|
ptr += copy_from_mem(ptr, num_items);
|
|
477
|
+
check_header(k, level, lg_weight, section_size_raw, num_sections);
|
|
450
478
|
auto pair = deserialize_items(ptr, end_ptr - ptr, serde, allocator, num_items);
|
|
451
479
|
ptr += pair.second;
|
|
452
480
|
return std::pair<req_compactor, size_t>(
|
|
@@ -98,12 +98,14 @@ public:
|
|
|
98
98
|
|
|
99
99
|
/**
|
|
100
100
|
* Constructor
|
|
101
|
-
* @param k Controls the size and error of the sketch. It must be
|
|
101
|
+
* @param k Controls the size and error of the sketch. It must be no greater than 1024.
|
|
102
|
+
* Values below 4 are raised to 4, and odd values are rounded down to the nearest even value.
|
|
102
103
|
* Value of 12 roughly corresponds to 1% relative error guarantee at 95% confidence.
|
|
103
104
|
* @param hra if true, the default, the high ranks are prioritized for better
|
|
104
105
|
* accuracy. Otherwise the low ranks are prioritized for better accuracy.
|
|
105
106
|
* @param comparator strict weak ordering function (see C++ named requirements: Compare)
|
|
106
107
|
* @param allocator used by this sketch to allocate memory
|
|
108
|
+
* @throws std::invalid_argument if k is greater than 1024
|
|
107
109
|
*/
|
|
108
110
|
explicit req_sketch(uint16_t k, bool hra = true, const Comparator& comparator = Comparator(),
|
|
109
111
|
const Allocator& allocator = Allocator());
|
|
@@ -463,7 +465,9 @@ private:
|
|
|
463
465
|
optional<T>&& min_item, optional<T>&& max_item,
|
|
464
466
|
std::vector<Compactor, AllocCompactor>&& compactors, const Comparator& comparator);
|
|
465
467
|
|
|
468
|
+
static void check_k(uint16_t k);
|
|
466
469
|
static void check_preamble_ints(uint8_t preamble_ints, uint8_t num_levels);
|
|
470
|
+
static void check_num_levels(uint8_t num_levels, bool raw_items);
|
|
467
471
|
static void check_serial_version(uint8_t serial_version);
|
|
468
472
|
static void check_family_id(uint8_t family_id);
|
|
469
473
|
|
|
@@ -30,7 +30,7 @@ template<typename T, typename C, typename A>
|
|
|
30
30
|
req_sketch<T, C, A>::req_sketch(uint16_t k, bool hra, const C& comparator, const A& allocator):
|
|
31
31
|
comparator_(comparator),
|
|
32
32
|
allocator_(allocator),
|
|
33
|
-
k_(std::max<
|
|
33
|
+
k_(std::max<uint16_t>(static_cast<int>(k) & -2, static_cast<int>(req_constants::MIN_K))), //rounds down one if odd
|
|
34
34
|
hra_(hra),
|
|
35
35
|
max_nom_size_(0),
|
|
36
36
|
num_retained_(0),
|
|
@@ -40,6 +40,9 @@ min_item_(),
|
|
|
40
40
|
max_item_(),
|
|
41
41
|
sorted_view_(nullptr)
|
|
42
42
|
{
|
|
43
|
+
if (k > req_constants::MAX_K) {
|
|
44
|
+
throw std::invalid_argument("K must be <= " + std::to_string(req_constants::MAX_K) + ": " + std::to_string(k));
|
|
45
|
+
}
|
|
43
46
|
grow();
|
|
44
47
|
}
|
|
45
48
|
|
|
@@ -453,6 +456,7 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(std::istream& is, const Ser
|
|
|
453
456
|
const auto num_levels = read<uint8_t>(is);
|
|
454
457
|
const auto num_raw_items = read<uint8_t>(is);
|
|
455
458
|
|
|
459
|
+
check_k(k);
|
|
456
460
|
check_preamble_ints(preamble_ints, num_levels);
|
|
457
461
|
check_serial_version(serial_version);
|
|
458
462
|
check_family_id(family_id);
|
|
@@ -466,6 +470,7 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(std::istream& is, const Ser
|
|
|
466
470
|
optional<T> max_item;
|
|
467
471
|
|
|
468
472
|
const bool raw_items = flags_byte & (1 << flags::RAW_ITEMS);
|
|
473
|
+
check_num_levels(num_levels, raw_items);
|
|
469
474
|
const bool is_level_0_sorted = flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED);
|
|
470
475
|
std::vector<Compactor, AllocCompactor> compactors(allocator);
|
|
471
476
|
|
|
@@ -491,10 +496,12 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(std::istream& is, const Ser
|
|
|
491
496
|
compactors.push_back(Compactor::deserialize(is, sd, comparator, allocator, is_level_0_sorted, k, num_raw_items, hra));
|
|
492
497
|
} else {
|
|
493
498
|
for (size_t i = 0; i < num_levels; ++i) {
|
|
494
|
-
compactors.push_back(Compactor::deserialize(is, sd, comparator, allocator, i == 0 ? is_level_0_sorted : true, hra
|
|
499
|
+
compactors.push_back(Compactor::deserialize(is, sd, comparator, allocator, i == 0 ? is_level_0_sorted : true, hra,
|
|
500
|
+
k, static_cast<uint8_t>(i)));
|
|
495
501
|
}
|
|
496
502
|
}
|
|
497
503
|
if (num_levels == 1) {
|
|
504
|
+
if (compactors[0].get_num_items() == 0) throw std::invalid_argument("Possible corruption: non-empty sketch with no items");
|
|
498
505
|
const auto begin = compactors[0].begin();
|
|
499
506
|
const auto end = compactors[0].end();
|
|
500
507
|
n = compactors[0].get_num_items();
|
|
@@ -534,6 +541,7 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(const void* bytes, size_t s
|
|
|
534
541
|
uint8_t num_raw_items;
|
|
535
542
|
ptr += copy_from_mem(ptr, num_raw_items);
|
|
536
543
|
|
|
544
|
+
check_k(k);
|
|
537
545
|
check_preamble_ints(preamble_ints, num_levels);
|
|
538
546
|
check_serial_version(serial_version);
|
|
539
547
|
check_family_id(family_id);
|
|
@@ -546,6 +554,7 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(const void* bytes, size_t s
|
|
|
546
554
|
optional<T> max_item;
|
|
547
555
|
|
|
548
556
|
const bool raw_items = flags_byte & (1 << flags::RAW_ITEMS);
|
|
557
|
+
check_num_levels(num_levels, raw_items);
|
|
549
558
|
const bool is_level_0_sorted = flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED);
|
|
550
559
|
std::vector<Compactor, AllocCompactor> compactors(allocator);
|
|
551
560
|
|
|
@@ -574,12 +583,14 @@ req_sketch<T, C, A> req_sketch<T, C, A>::deserialize(const void* bytes, size_t s
|
|
|
574
583
|
ptr += pair.second;
|
|
575
584
|
} else {
|
|
576
585
|
for (size_t i = 0; i < num_levels; ++i) {
|
|
577
|
-
auto pair = Compactor::deserialize(ptr, end_ptr - ptr, sd, comparator, allocator, i == 0 ? is_level_0_sorted : true, hra
|
|
586
|
+
auto pair = Compactor::deserialize(ptr, end_ptr - ptr, sd, comparator, allocator, i == 0 ? is_level_0_sorted : true, hra,
|
|
587
|
+
k, static_cast<uint8_t>(i));
|
|
578
588
|
compactors.push_back(std::move(pair.first));
|
|
579
589
|
ptr += pair.second;
|
|
580
590
|
}
|
|
581
591
|
}
|
|
582
592
|
if (num_levels == 1) {
|
|
593
|
+
if (compactors[0].get_num_items() == 0) throw std::invalid_argument("Possible corruption: non-empty sketch with no items");
|
|
583
594
|
const auto begin = compactors[0].begin();
|
|
584
595
|
const auto end = compactors[0].end();
|
|
585
596
|
n = compactors[0].get_num_items();
|
|
@@ -703,6 +714,23 @@ sorted_view_(nullptr)
|
|
|
703
714
|
update_num_retained();
|
|
704
715
|
}
|
|
705
716
|
|
|
717
|
+
template<typename T, typename C, typename A>
|
|
718
|
+
void req_sketch<T, C, A>::check_k(uint16_t k) {
|
|
719
|
+
if (k < req_constants::MIN_K || k > req_constants::MAX_K || (k & 1) != 0) {
|
|
720
|
+
throw std::invalid_argument("Possible corruption: k must be even and in the range ["
|
|
721
|
+
+ std::to_string(req_constants::MIN_K) + ", " + std::to_string(req_constants::MAX_K) + "]: " + std::to_string(k));
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
template<typename T, typename C, typename A>
|
|
726
|
+
void req_sketch<T, C, A>::check_num_levels(uint8_t num_levels, bool raw_items) {
|
|
727
|
+
// a non-empty sketch has at least one compactor, and raw items fit in one
|
|
728
|
+
if (num_levels == 0 || (raw_items && num_levels != 1)) {
|
|
729
|
+
throw std::invalid_argument("Possible corruption: number of levels " + std::to_string(num_levels)
|
|
730
|
+
+ (raw_items ? " with raw items" : ""));
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
|
|
706
734
|
template<typename T, typename C, typename A>
|
|
707
735
|
void req_sketch<T, C, A>::check_preamble_ints(uint8_t preamble_ints, uint8_t num_levels) {
|
|
708
736
|
const uint8_t expected_preamble_ints = num_levels > 1 ? 4 : 2;
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
#include <fstream>
|
|
25
25
|
#include <sstream>
|
|
26
|
+
#include <cstring>
|
|
26
27
|
#include <limits>
|
|
27
28
|
#include <stdexcept>
|
|
28
29
|
|
|
@@ -54,6 +55,39 @@ TEST_CASE("req sketch: empty", "[req_sketch]") {
|
|
|
54
55
|
REQUIRE_THROWS_AS(sketch.get_PMF(split_points, 1), std::runtime_error);
|
|
55
56
|
}
|
|
56
57
|
|
|
58
|
+
TEST_CASE("req sketch: k above 255", "[req_sketch]") {
|
|
59
|
+
// k used to be truncated to 8 bits: 256 became 4 and 500 became 244
|
|
60
|
+
REQUIRE(req_sketch<float>(256).get_k() == 256);
|
|
61
|
+
REQUIRE(req_sketch<float>(500).get_k() == 500);
|
|
62
|
+
REQUIRE(req_sketch<float>(1024).get_k() == 1024);
|
|
63
|
+
REQUIRE(req_sketch<float>(1001).get_k() == 1000); // rounds down if odd
|
|
64
|
+
|
|
65
|
+
req_sketch<float> sketch(1024);
|
|
66
|
+
const size_t n = 100000;
|
|
67
|
+
for (size_t i = 0; i < n; i++) sketch.update(static_cast<float>(i));
|
|
68
|
+
REQUIRE(sketch.is_estimation_mode());
|
|
69
|
+
auto bytes = sketch.serialize();
|
|
70
|
+
auto sketch2 = req_sketch<float>::deserialize(bytes.data(), bytes.size());
|
|
71
|
+
REQUIRE(sketch2.get_k() == 1024);
|
|
72
|
+
REQUIRE(sketch2.get_n() == n);
|
|
73
|
+
REQUIRE(sketch2.get_num_retained() == sketch.get_num_retained());
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
TEST_CASE("req sketch: k bounds", "[req_sketch]") {
|
|
77
|
+
for (bool hra: {true, false}) {
|
|
78
|
+
REQUIRE(req_sketch<float>(0, hra).get_k() == 4);
|
|
79
|
+
REQUIRE(req_sketch<float>(3, hra).get_k() == 4);
|
|
80
|
+
REQUIRE(req_sketch<float>(4, hra).get_k() == 4);
|
|
81
|
+
REQUIRE(req_sketch<float>(5, hra).get_k() == 4);
|
|
82
|
+
REQUIRE(req_sketch<float>(1023, hra).get_k() == 1022);
|
|
83
|
+
REQUIRE(req_sketch<float>(1024, hra).get_k() == 1024);
|
|
84
|
+
// Reject the original k, even if rounding down would put it in range.
|
|
85
|
+
REQUIRE_THROWS_AS(req_sketch<float>(1025, hra), std::invalid_argument);
|
|
86
|
+
REQUIRE_THROWS_AS(req_sketch<float>(1026, hra), std::invalid_argument);
|
|
87
|
+
REQUIRE_THROWS_AS(req_sketch<float>(std::numeric_limits<uint16_t>::max(), hra), std::invalid_argument);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
57
91
|
TEST_CASE("req sketch: single value, lra", "[req_sketch]") {
|
|
58
92
|
req_sketch<float> sketch(12, false);
|
|
59
93
|
sketch.update(1.0f);
|
|
@@ -559,4 +593,146 @@ TEST_CASE("get_rank equivalence") {
|
|
|
559
593
|
// sketch.serialize(os);
|
|
560
594
|
//}
|
|
561
595
|
|
|
596
|
+
|
|
597
|
+
template<typename Bytes, typename V>
|
|
598
|
+
static void put_value(Bytes& bytes, size_t offset, V value) {
|
|
599
|
+
std::memcpy(bytes.data() + offset, &value, sizeof(value));
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
TEST_CASE("req sketch: deserialize rejects inconsistent compactors", "[req_sketch]") {
|
|
603
|
+
using vector_bytes = req_sketch<float>::vector_bytes;
|
|
604
|
+
// preamble: k at 4, number of levels at 6, number of raw items at 7
|
|
605
|
+
// compactor: state(8), section size(4), lg_weight(1), number of sections(1), padding(2), number of items(4), items
|
|
606
|
+
const size_t k_offset = 4;
|
|
607
|
+
const size_t num_levels_offset = 6;
|
|
608
|
+
const size_t num_raw_items_offset = 7;
|
|
609
|
+
const size_t estimation_compactor = 24; // after the preamble, n, min and max
|
|
610
|
+
const size_t exact_compactor = 8; // after the preamble
|
|
611
|
+
const size_t section_size = 8;
|
|
612
|
+
const size_t lg_weight = 12;
|
|
613
|
+
const size_t num_sections = 13;
|
|
614
|
+
const size_t num_items = 16;
|
|
615
|
+
const size_t items = 20;
|
|
616
|
+
auto check_rejected = [](const vector_bytes& bytes) {
|
|
617
|
+
REQUIRE_THROWS_AS(req_sketch<float>::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
|
|
618
|
+
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
|
|
619
|
+
s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
|
|
620
|
+
REQUIRE_THROWS_AS(req_sketch<float>::deserialize(s), std::invalid_argument);
|
|
621
|
+
};
|
|
622
|
+
|
|
623
|
+
req_sketch<float> raw(12);
|
|
624
|
+
for (int i = 0; i < 3; ++i) raw.update(static_cast<float>(i));
|
|
625
|
+
req_sketch<float> exact(12);
|
|
626
|
+
for (int i = 0; i < 50; ++i) exact.update(static_cast<float>(i));
|
|
627
|
+
req_sketch<float> estimation(12);
|
|
628
|
+
for (int i = 0; i < 1000; ++i) estimation.update(static_cast<float>(i));
|
|
629
|
+
REQUIRE(estimation.serialize()[num_levels_offset] > 2);
|
|
630
|
+
|
|
631
|
+
SECTION("valid images") {
|
|
632
|
+
// sketches deep enough for the sections to have grown, in both rank accuracy modes
|
|
633
|
+
for (bool hra: {true, false}) {
|
|
634
|
+
req_sketch<float> sketch(12, hra);
|
|
635
|
+
for (int i = 0; i < 1000000; ++i) sketch.update(static_cast<float>(i));
|
|
636
|
+
auto bytes = sketch.serialize();
|
|
637
|
+
REQUIRE(req_sketch<float>::deserialize(bytes.data(), bytes.size()).get_n() == sketch.get_n());
|
|
638
|
+
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
|
|
639
|
+
sketch.serialize(s);
|
|
640
|
+
REQUIRE(req_sketch<float>::deserialize(s).get_n() == sketch.get_n());
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
SECTION("valid images with the smallest section size") {
|
|
645
|
+
// the raw section size has shrunk to 3, which the section size rounds up to 4
|
|
646
|
+
req_sketch<float> sketch(6);
|
|
647
|
+
for (int i = 0; i < 100000; ++i) sketch.update(static_cast<float>(i));
|
|
648
|
+
auto bytes = sketch.serialize();
|
|
649
|
+
float raw_section_size;
|
|
650
|
+
std::memcpy(&raw_section_size, bytes.data() + estimation_compactor + section_size, sizeof(raw_section_size));
|
|
651
|
+
REQUIRE(raw_section_size == Approx(3).margin(0.001));
|
|
652
|
+
REQUIRE(bytes[estimation_compactor + num_sections] == 12);
|
|
653
|
+
REQUIRE(req_sketch<float>::deserialize(bytes.data(), bytes.size()).get_n() == sketch.get_n());
|
|
654
|
+
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
|
|
655
|
+
sketch.serialize(s);
|
|
656
|
+
REQUIRE(req_sketch<float>::deserialize(s).get_n() == sketch.get_n());
|
|
657
|
+
|
|
658
|
+
// k = 12 needs about 2 billion items to get here, so set the header it would have
|
|
659
|
+
bytes = estimation.serialize();
|
|
660
|
+
put_value(bytes, estimation_compactor + section_size, 3.0f);
|
|
661
|
+
put_value(bytes, estimation_compactor + num_sections, static_cast<uint8_t>(48));
|
|
662
|
+
REQUIRE(req_sketch<float>::deserialize(bytes.data(), bytes.size()).get_n() == estimation.get_n());
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
SECTION("invalid k") {
|
|
666
|
+
for (uint16_t k: {2, 13, 2000}) {
|
|
667
|
+
auto bytes = estimation.serialize();
|
|
668
|
+
put_value(bytes, k_offset, k);
|
|
669
|
+
check_rejected(bytes);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
SECTION("section size out of range") {
|
|
674
|
+
for (float size: {0.0f, 2.0f, 14.0f, 1e12f, std::numeric_limits<float>::quiet_NaN()}) {
|
|
675
|
+
auto bytes = estimation.serialize();
|
|
676
|
+
put_value(bytes, estimation_compactor + section_size, size);
|
|
677
|
+
check_rejected(bytes);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
SECTION("number of sections out of range") {
|
|
682
|
+
for (uint8_t sections: {0, 65}) {
|
|
683
|
+
auto bytes = estimation.serialize();
|
|
684
|
+
put_value(bytes, estimation_compactor + num_sections, sections);
|
|
685
|
+
check_rejected(bytes);
|
|
686
|
+
}
|
|
687
|
+
// in range, but too many for this section size and k
|
|
688
|
+
auto bytes = estimation.serialize();
|
|
689
|
+
put_value(bytes, estimation_compactor + section_size, 3.0f);
|
|
690
|
+
put_value(bytes, estimation_compactor + num_sections, static_cast<uint8_t>(64));
|
|
691
|
+
check_rejected(bytes);
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
SECTION("wrong lg_weight") {
|
|
695
|
+
for (uint8_t weight: {1, 70}) {
|
|
696
|
+
auto bytes = estimation.serialize();
|
|
697
|
+
put_value(bytes, estimation_compactor + lg_weight, weight);
|
|
698
|
+
check_rejected(bytes);
|
|
699
|
+
}
|
|
700
|
+
auto bytes = exact.serialize();
|
|
701
|
+
put_value(bytes, exact_compactor + lg_weight, static_cast<uint8_t>(1));
|
|
702
|
+
check_rejected(bytes);
|
|
703
|
+
// the second compactor repeats the first one's lg_weight
|
|
704
|
+
bytes = estimation.serialize();
|
|
705
|
+
uint32_t count;
|
|
706
|
+
std::memcpy(&count, bytes.data() + estimation_compactor + num_items, sizeof(count));
|
|
707
|
+
const size_t second_compactor = estimation_compactor + items + count * sizeof(float);
|
|
708
|
+
REQUIRE(bytes[second_compactor + lg_weight] == 1);
|
|
709
|
+
put_value(bytes, second_compactor + lg_weight, static_cast<uint8_t>(0));
|
|
710
|
+
check_rejected(bytes);
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
SECTION("no levels") {
|
|
714
|
+
auto bytes = exact.serialize();
|
|
715
|
+
bytes[num_levels_offset] = 0;
|
|
716
|
+
check_rejected(bytes);
|
|
717
|
+
bytes = raw.serialize();
|
|
718
|
+
bytes[num_levels_offset] = 0;
|
|
719
|
+
check_rejected(bytes);
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
SECTION("no items") {
|
|
723
|
+
auto bytes = exact.serialize();
|
|
724
|
+
put_value(bytes, exact_compactor + num_items, static_cast<uint32_t>(0));
|
|
725
|
+
check_rejected(bytes);
|
|
726
|
+
bytes = raw.serialize();
|
|
727
|
+
bytes[num_raw_items_offset] = 0;
|
|
728
|
+
check_rejected(bytes);
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
SECTION("compactor header beyond the input") {
|
|
732
|
+
auto bytes = estimation.serialize();
|
|
733
|
+
bytes.resize(estimation_compactor + 10);
|
|
734
|
+
REQUIRE_THROWS_AS(req_sketch<float>::deserialize(bytes.data(), bytes.size()), std::out_of_range);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
|
|
562
738
|
} /* namespace datasketches */
|
|
@@ -202,7 +202,7 @@ void ebpps_sketch<T, A>::merge(const ebpps_sketch<T, A>& sk) {
|
|
|
202
202
|
// need to swap this with sk to merge, so make a copy, swap,
|
|
203
203
|
// and use that to merge
|
|
204
204
|
ebpps_sketch sk_copy(sk);
|
|
205
|
-
swap(*this, sk_copy);
|
|
205
|
+
std::swap(*this, sk_copy);
|
|
206
206
|
internal_merge(sk_copy);
|
|
207
207
|
} else {
|
|
208
208
|
internal_merge(sk);
|
|
@@ -25,6 +25,26 @@
|
|
|
25
25
|
|
|
26
26
|
#include <sstream>
|
|
27
27
|
|
|
28
|
+
// An allocator outside the datasketches namespace, as an application's would be: argument-dependent
|
|
29
|
+
// lookup finds no swap() in its namespace.
|
|
30
|
+
namespace ebpps_test_alloc {
|
|
31
|
+
|
|
32
|
+
template<typename T>
|
|
33
|
+
struct plain_allocator {
|
|
34
|
+
using value_type = T;
|
|
35
|
+
plain_allocator() = default;
|
|
36
|
+
template<typename U>
|
|
37
|
+
plain_allocator(const plain_allocator<U>&) {}
|
|
38
|
+
T* allocate(std::size_t n) { return static_cast<T*>(::operator new(n * sizeof(T))); }
|
|
39
|
+
void deallocate(T* p, std::size_t) { ::operator delete(p); }
|
|
40
|
+
};
|
|
41
|
+
template<typename T, typename U>
|
|
42
|
+
bool operator==(const plain_allocator<T>&, const plain_allocator<U>&) { return true; }
|
|
43
|
+
template<typename T, typename U>
|
|
44
|
+
bool operator!=(const plain_allocator<T>&, const plain_allocator<U>&) { return false; }
|
|
45
|
+
|
|
46
|
+
} // namespace ebpps_test_alloc
|
|
47
|
+
|
|
28
48
|
namespace datasketches {
|
|
29
49
|
|
|
30
50
|
using ebpps_test_sketch = ebpps_sketch<test_type, test_allocator<test_type>>;
|
|
@@ -93,4 +113,17 @@ TEST_CASE( "ebpps merge", "[ebpps_sketch][test_type]") {
|
|
|
93
113
|
REQUIRE(test_allocator_net_allocations == 0);
|
|
94
114
|
}
|
|
95
115
|
|
|
116
|
+
TEST_CASE("ebpps merge heavier sketch by const reference with an application allocator", "[ebpps_sketch]") {
|
|
117
|
+
using sketch = ebpps_sketch<int, ebpps_test_alloc::plain_allocator<int>>;
|
|
118
|
+
sketch light(5);
|
|
119
|
+
sketch heavy(5);
|
|
120
|
+
for (int i = 0; i < 10; ++i) light.update(i);
|
|
121
|
+
for (int i = 0; i < 100; ++i) heavy.update(i, 2.0);
|
|
122
|
+
const sketch& heavy_ref = heavy;
|
|
123
|
+
light.merge(heavy_ref);
|
|
124
|
+
REQUIRE(light.get_n() == 110);
|
|
125
|
+
REQUIRE(light.get_cumulative_weight() == Approx(210.0));
|
|
126
|
+
REQUIRE(heavy.get_n() == 100);
|
|
96
127
|
}
|
|
128
|
+
|
|
129
|
+
} // namespace datasketches
|
|
@@ -378,7 +378,7 @@ size_t tdigest<T, A>::get_serialized_size_bytes(bool with_buffer) const {
|
|
|
378
378
|
template<typename T, typename A>
|
|
379
379
|
auto tdigest<T, A>::serialize(unsigned header_size_bytes, bool with_buffer) const -> vector_bytes {
|
|
380
380
|
if (!with_buffer) const_cast<tdigest*>(this)->compress(); // side effect
|
|
381
|
-
vector_bytes bytes(get_serialized_size_bytes(with_buffer), 0, buffer_.get_allocator());
|
|
381
|
+
vector_bytes bytes(header_size_bytes + get_serialized_size_bytes(with_buffer), 0, buffer_.get_allocator());
|
|
382
382
|
uint8_t* ptr = bytes.data() + header_size_bytes;
|
|
383
383
|
*ptr++ = get_preamble_longs();
|
|
384
384
|
*ptr++ = SERIAL_VERSION;
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
20
|
#include <catch2/catch.hpp>
|
|
21
|
+
#include <algorithm>
|
|
21
22
|
#include <cmath>
|
|
22
23
|
#include <cstdint>
|
|
23
24
|
#include <cstring>
|
|
@@ -347,6 +348,27 @@ TEST_CASE("serialize deserialize bytes many values with buffer", "[tdigest]") {
|
|
|
347
348
|
REQUIRE(td.get_quantile(0.5) == deserialized_td.get_quantile(0.5));
|
|
348
349
|
}
|
|
349
350
|
|
|
351
|
+
TEST_CASE("serialize bytes with header", "[tdigest]") {
|
|
352
|
+
const unsigned header_size_bytes = 7;
|
|
353
|
+
auto check = [header_size_bytes](const tdigest<double>& td, bool with_buffer) {
|
|
354
|
+
auto bytes = td.serialize(header_size_bytes, with_buffer);
|
|
355
|
+
auto bytes_no_header = td.serialize(0, with_buffer);
|
|
356
|
+
REQUIRE(bytes.size() == header_size_bytes + bytes_no_header.size());
|
|
357
|
+
REQUIRE(bytes.size() == header_size_bytes + td.get_serialized_size_bytes(with_buffer));
|
|
358
|
+
REQUIRE(std::equal(bytes_no_header.begin(), bytes_no_header.end(), bytes.begin() + header_size_bytes));
|
|
359
|
+
auto deserialized_td = tdigest<double>::deserialize(bytes.data() + header_size_bytes, bytes.size() - header_size_bytes);
|
|
360
|
+
REQUIRE(td.get_total_weight() == deserialized_td.get_total_weight());
|
|
361
|
+
};
|
|
362
|
+
tdigest<double> td(100);
|
|
363
|
+
check(td, false);
|
|
364
|
+
td.update(123);
|
|
365
|
+
check(td, false);
|
|
366
|
+
check(td, true);
|
|
367
|
+
for (int i = 0; i < 10000; ++i) td.update(i);
|
|
368
|
+
check(td, true);
|
|
369
|
+
check(td, false);
|
|
370
|
+
}
|
|
371
|
+
|
|
350
372
|
TEST_CASE("serialize deserialize steam and bytes equivalence empty", "[tdigest]") {
|
|
351
373
|
tdigest<double> td(100);
|
|
352
374
|
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
|
|
@@ -48,6 +48,15 @@ public:
|
|
|
48
48
|
*/
|
|
49
49
|
explicit array_tuple_intersection(uint64_t seed = DEFAULT_SEED, const Policy& policy = Policy(), const Allocator& allocator = Allocator());
|
|
50
50
|
|
|
51
|
+
/**
|
|
52
|
+
* Updates the intersection with a given sketch.
|
|
53
|
+
* The sketch must have the same number of values as the intersection policy.
|
|
54
|
+
* @param sketch to intersect with
|
|
55
|
+
* @throw std::invalid_argument if the number of values does not match
|
|
56
|
+
*/
|
|
57
|
+
template<typename FwdSketch>
|
|
58
|
+
void update(FwdSketch&& sketch);
|
|
59
|
+
|
|
51
60
|
/**
|
|
52
61
|
* Produces a copy of the current state of the intersection.
|
|
53
62
|
* If update() was not called, the state is the infinite "universe",
|
|
@@ -17,12 +17,26 @@
|
|
|
17
17
|
* under the License.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
+
#include <stdexcept>
|
|
21
|
+
#include <string>
|
|
22
|
+
|
|
20
23
|
namespace datasketches {
|
|
21
24
|
|
|
22
25
|
template<typename Array, typename Policy, typename Allocator>
|
|
23
26
|
array_tuple_intersection<Array, Policy, Allocator>::array_tuple_intersection(uint64_t seed, const Policy& policy, const Allocator& allocator):
|
|
24
27
|
Base(seed, policy, allocator) {}
|
|
25
28
|
|
|
29
|
+
template<typename Array, typename Policy, typename Allocator>
|
|
30
|
+
template<typename FwdSketch>
|
|
31
|
+
void array_tuple_intersection<Array, Policy, Allocator>::update(FwdSketch&& sketch) {
|
|
32
|
+
const uint8_t num_values = this->state_.get_policy().get_external_policy().get_num_values();
|
|
33
|
+
if (sketch.get_num_values() != num_values) {
|
|
34
|
+
throw std::invalid_argument("number of values mismatch: intersection has " + std::to_string(num_values)
|
|
35
|
+
+ ", sketch has " + std::to_string(sketch.get_num_values()));
|
|
36
|
+
}
|
|
37
|
+
Base::update(std::forward<FwdSketch>(sketch));
|
|
38
|
+
}
|
|
39
|
+
|
|
26
40
|
template<typename Array, typename Policy, typename Allocator>
|
|
27
41
|
auto array_tuple_intersection<Array, Policy, Allocator>::get_result(bool ordered) const -> CompactSketch {
|
|
28
42
|
return CompactSketch(this->state_.get_policy().get_external_policy().get_num_values(), Base::get_result(ordered));
|
|
@@ -60,6 +60,15 @@ public:
|
|
|
60
60
|
|
|
61
61
|
class builder;
|
|
62
62
|
|
|
63
|
+
/**
|
|
64
|
+
* Update the union with a given sketch.
|
|
65
|
+
* The sketch must have the same number of values as the union policy.
|
|
66
|
+
* @param sketch to update the union with
|
|
67
|
+
* @throw std::invalid_argument if the number of values does not match
|
|
68
|
+
*/
|
|
69
|
+
template<typename FwdSketch>
|
|
70
|
+
void update(FwdSketch&& sketch);
|
|
71
|
+
|
|
63
72
|
CompactSketch get_result(bool ordered = true) const;
|
|
64
73
|
|
|
65
74
|
private:
|
|
@@ -17,6 +17,9 @@
|
|
|
17
17
|
* under the License.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
+
#include <stdexcept>
|
|
21
|
+
#include <string>
|
|
22
|
+
|
|
20
23
|
namespace datasketches {
|
|
21
24
|
|
|
22
25
|
template<typename Array, typename Policy, typename Allocator>
|
|
@@ -24,6 +27,17 @@ array_tuple_union<Array, Policy, Allocator>::array_tuple_union(uint8_t lg_cur_si
|
|
|
24
27
|
Base(lg_cur_size, lg_nom_size, rf, p, theta, seed, policy, allocator)
|
|
25
28
|
{}
|
|
26
29
|
|
|
30
|
+
template<typename Array, typename Policy, typename Allocator>
|
|
31
|
+
template<typename FwdSketch>
|
|
32
|
+
void array_tuple_union<Array, Policy, Allocator>::update(FwdSketch&& sketch) {
|
|
33
|
+
const uint8_t num_values = this->state_.get_policy().get_external_policy().get_num_values();
|
|
34
|
+
if (sketch.get_num_values() != num_values) {
|
|
35
|
+
throw std::invalid_argument("number of values mismatch: union has " + std::to_string(num_values)
|
|
36
|
+
+ ", sketch has " + std::to_string(sketch.get_num_values()));
|
|
37
|
+
}
|
|
38
|
+
Base::update(std::forward<FwdSketch>(sketch));
|
|
39
|
+
}
|
|
40
|
+
|
|
27
41
|
template<typename Array, typename Policy, typename Allocator>
|
|
28
42
|
auto array_tuple_union<Array, Policy, Allocator>::get_result(bool ordered) const -> CompactSketch {
|
|
29
43
|
return CompactSketch(this->state_.get_policy().get_external_policy().get_num_values(), Base::get_result(ordered));
|
|
@@ -173,6 +173,53 @@ TEST_CASE("aod intersection: half overlap", "[tuple_sketch]") {
|
|
|
173
173
|
REQUIRE(result.get_estimate() == Approx(500).margin(0.01));
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
+
TEST_CASE("aod union: number of values", "[tuple_sketch]") {
|
|
177
|
+
auto sketch1 = update_array_of_doubles_sketch::builder(1).build();
|
|
178
|
+
sketch1.update(1, std::vector<double>{1});
|
|
179
|
+
auto sketch3 = update_array_of_doubles_sketch::builder(3).build();
|
|
180
|
+
sketch3.update(1, std::vector<double>{1, 2, 3});
|
|
181
|
+
sketch3.update(2, std::vector<double>{4, 5, 6});
|
|
182
|
+
|
|
183
|
+
// matching number of values: entries for the same key are summed
|
|
184
|
+
auto u = array_of_doubles_union::builder(3).build();
|
|
185
|
+
u.update(sketch3);
|
|
186
|
+
u.update(sketch3.compact());
|
|
187
|
+
auto result = u.get_result();
|
|
188
|
+
REQUIRE(result.get_num_values() == 3);
|
|
189
|
+
REQUIRE(result.get_num_retained() == 2);
|
|
190
|
+
for (const auto& entry: result) {
|
|
191
|
+
REQUIRE(entry.second.size() == 3);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// mismatch is rejected whether or not keys overlap, and for update and compact sketches
|
|
195
|
+
REQUIRE_THROWS_AS(u.update(sketch1), std::invalid_argument);
|
|
196
|
+
REQUIRE_THROWS_AS(u.update(sketch1.compact()), std::invalid_argument);
|
|
197
|
+
auto u1 = array_of_doubles_union::builder(1).build();
|
|
198
|
+
REQUIRE_THROWS_AS(u1.update(sketch3), std::invalid_argument);
|
|
199
|
+
auto empty3 = update_array_of_doubles_sketch::builder(3).build();
|
|
200
|
+
REQUIRE_THROWS_AS(u1.update(empty3), std::invalid_argument);
|
|
201
|
+
|
|
202
|
+
// a rejected update leaves the union usable
|
|
203
|
+
u1.update(sketch1);
|
|
204
|
+
REQUIRE(u1.get_result().get_num_retained() == 1);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
TEST_CASE("aod intersection: number of values", "[tuple_sketch]") {
|
|
208
|
+
auto sketch1 = update_array_of_doubles_sketch::builder(1).build();
|
|
209
|
+
sketch1.update(1, std::vector<double>{1});
|
|
210
|
+
auto sketch3 = update_array_of_doubles_sketch::builder(3).build();
|
|
211
|
+
sketch3.update(1, std::vector<double>{1, 2, 3});
|
|
212
|
+
|
|
213
|
+
array_of_doubles_intersection<default_array_of_doubles_union_policy> intersection(DEFAULT_SEED, 3);
|
|
214
|
+
REQUIRE_THROWS_AS(intersection.update(sketch1), std::invalid_argument);
|
|
215
|
+
intersection.update(sketch3);
|
|
216
|
+
REQUIRE_THROWS_AS(intersection.update(sketch1.compact()), std::invalid_argument);
|
|
217
|
+
intersection.update(sketch3.compact());
|
|
218
|
+
auto result = intersection.get_result();
|
|
219
|
+
REQUIRE(result.get_num_values() == 3);
|
|
220
|
+
REQUIRE(result.get_num_retained() == 1);
|
|
221
|
+
}
|
|
222
|
+
|
|
176
223
|
TEST_CASE("aod a-not-b: half overlap", "[tuple_sketch]") {
|
|
177
224
|
double a[1] = {1};
|
|
178
225
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
5.3.
|
|
1
|
+
5.3.1
|