datasketches 0.1.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/LICENSE +40 -3
- data/NOTICE +1 -1
- data/ext/datasketches/cpc_wrapper.cpp +12 -13
- data/ext/datasketches/ext.cpp +1 -1
- data/ext/datasketches/ext.h +4 -0
- data/ext/datasketches/extconf.rb +1 -1
- data/ext/datasketches/fi_wrapper.cpp +6 -8
- data/ext/datasketches/hll_wrapper.cpp +13 -14
- data/ext/datasketches/kll_wrapper.cpp +28 -76
- data/ext/datasketches/theta_wrapper.cpp +27 -41
- data/ext/datasketches/vo_wrapper.cpp +4 -6
- data/lib/datasketches/version.rb +1 -1
- data/vendor/datasketches-cpp/CMakeLists.txt +10 -0
- data/vendor/datasketches-cpp/LICENSE +40 -3
- data/vendor/datasketches-cpp/NOTICE +1 -1
- data/vendor/datasketches-cpp/README.md +4 -4
- data/vendor/datasketches-cpp/common/include/MurmurHash3.h +18 -7
- data/vendor/datasketches-cpp/common/include/binomial_bounds.hpp +8 -8
- data/vendor/datasketches-cpp/common/include/bounds_binomial_proportions.hpp +12 -15
- data/vendor/datasketches-cpp/common/include/common_defs.hpp +26 -0
- data/vendor/datasketches-cpp/common/include/conditional_forward.hpp +20 -8
- data/vendor/datasketches-cpp/common/include/count_zeros.hpp +2 -2
- data/vendor/datasketches-cpp/common/include/memory_operations.hpp +12 -0
- data/vendor/datasketches-cpp/common/include/serde.hpp +7 -7
- data/vendor/datasketches-cpp/common/test/CMakeLists.txt +24 -0
- data/vendor/datasketches-cpp/common/test/integration_test.cpp +77 -0
- data/vendor/datasketches-cpp/common/test/test_allocator.hpp +9 -1
- data/vendor/datasketches-cpp/cpc/include/cpc_common.hpp +13 -3
- data/vendor/datasketches-cpp/cpc/include/cpc_compressor.hpp +20 -20
- data/vendor/datasketches-cpp/cpc/include/cpc_compressor_impl.hpp +116 -105
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch.hpp +22 -6
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch_impl.hpp +140 -101
- data/vendor/datasketches-cpp/cpc/include/cpc_union.hpp +2 -2
- data/vendor/datasketches-cpp/cpc/include/cpc_union_impl.hpp +20 -20
- data/vendor/datasketches-cpp/cpc/include/cpc_util.hpp +10 -16
- data/vendor/datasketches-cpp/cpc/include/icon_estimator.hpp +6 -6
- data/vendor/datasketches-cpp/cpc/include/u32_table.hpp +10 -10
- data/vendor/datasketches-cpp/cpc/include/u32_table_impl.hpp +21 -21
- data/vendor/datasketches-cpp/cpc/test/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/cpc/test/compression_test.cpp +10 -10
- data/vendor/datasketches-cpp/cpc/test/cpc_sketch_allocation_test.cpp +237 -0
- data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp +25 -0
- data/vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp +1 -1
- data/vendor/datasketches-cpp/fi/include/frequent_items_sketch.hpp +15 -10
- data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp +102 -105
- data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map.hpp +19 -13
- data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map_impl.hpp +141 -125
- data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_custom_type_test.cpp +15 -12
- data/vendor/datasketches-cpp/fi/test/reverse_purge_hash_map_test.cpp +5 -5
- data/vendor/datasketches-cpp/hll/CMakeLists.txt +3 -0
- data/vendor/datasketches-cpp/hll/include/AuxHashMap-internal.hpp +81 -109
- data/vendor/datasketches-cpp/hll/include/AuxHashMap.hpp +25 -24
- data/vendor/datasketches-cpp/hll/include/CompositeInterpolationXTable-internal.hpp +15 -15
- data/vendor/datasketches-cpp/hll/include/CompositeInterpolationXTable.hpp +5 -5
- data/vendor/datasketches-cpp/hll/include/CouponHashSet-internal.hpp +89 -105
- data/vendor/datasketches-cpp/hll/include/CouponHashSet.hpp +13 -13
- data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp +130 -165
- data/vendor/datasketches-cpp/hll/include/CouponList.hpp +21 -22
- data/vendor/datasketches-cpp/hll/include/CubicInterpolation-internal.hpp +2 -4
- data/vendor/datasketches-cpp/hll/include/CubicInterpolation.hpp +2 -2
- data/vendor/datasketches-cpp/hll/include/HarmonicNumbers-internal.hpp +1 -1
- data/vendor/datasketches-cpp/hll/include/HarmonicNumbers.hpp +2 -2
- data/vendor/datasketches-cpp/hll/include/Hll4Array-internal.hpp +88 -83
- data/vendor/datasketches-cpp/hll/include/Hll4Array.hpp +9 -9
- data/vendor/datasketches-cpp/hll/include/Hll6Array-internal.hpp +34 -45
- data/vendor/datasketches-cpp/hll/include/Hll6Array.hpp +7 -8
- data/vendor/datasketches-cpp/hll/include/Hll8Array-internal.hpp +41 -52
- data/vendor/datasketches-cpp/hll/include/Hll8Array.hpp +7 -8
- data/vendor/datasketches-cpp/hll/include/HllArray-internal.hpp +220 -251
- data/vendor/datasketches-cpp/hll/include/HllArray.hpp +42 -42
- data/vendor/datasketches-cpp/hll/include/HllSketch-internal.hpp +36 -38
- data/vendor/datasketches-cpp/hll/include/HllSketchImpl-internal.hpp +22 -22
- data/vendor/datasketches-cpp/hll/include/HllSketchImpl.hpp +15 -14
- data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp +47 -44
- data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +62 -87
- data/vendor/datasketches-cpp/hll/include/HllUtil.hpp +121 -128
- data/vendor/datasketches-cpp/hll/include/RelativeErrorTables.hpp +1 -1
- data/vendor/datasketches-cpp/hll/include/coupon_iterator-internal.hpp +9 -9
- data/vendor/datasketches-cpp/hll/include/coupon_iterator.hpp +5 -5
- data/vendor/datasketches-cpp/hll/include/hll.hpp +25 -53
- data/vendor/datasketches-cpp/hll/test/AuxHashMapTest.cpp +8 -8
- data/vendor/datasketches-cpp/hll/test/CouponHashSetTest.cpp +36 -36
- data/vendor/datasketches-cpp/hll/test/CouponListTest.cpp +28 -28
- data/vendor/datasketches-cpp/hll/test/CrossCountingTest.cpp +2 -2
- data/vendor/datasketches-cpp/hll/test/HllArrayTest.cpp +37 -37
- data/vendor/datasketches-cpp/hll/test/HllSketchTest.cpp +57 -61
- data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +10 -14
- data/vendor/datasketches-cpp/hll/test/IsomorphicTest.cpp +3 -3
- data/vendor/datasketches-cpp/hll/test/ToFromByteArrayTest.cpp +4 -4
- data/vendor/datasketches-cpp/kll/include/kll_helper.hpp +5 -4
- data/vendor/datasketches-cpp/kll/include/kll_helper_impl.hpp +6 -6
- data/vendor/datasketches-cpp/kll/include/kll_quantile_calculator.hpp +14 -6
- data/vendor/datasketches-cpp/kll/include/kll_quantile_calculator_impl.hpp +40 -25
- data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +50 -6
- data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +164 -136
- data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov.hpp +67 -0
- data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov_impl.hpp +78 -0
- data/vendor/datasketches-cpp/kll/test/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/kll/test/kll_sketch_custom_type_test.cpp +11 -10
- data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +178 -88
- data/vendor/datasketches-cpp/kll/test/kolmogorov_smirnov_test.cpp +111 -0
- data/vendor/datasketches-cpp/pyproject.toml +4 -2
- data/vendor/datasketches-cpp/python/CMakeLists.txt +12 -6
- data/vendor/datasketches-cpp/python/README.md +52 -49
- data/vendor/datasketches-cpp/python/pybind11Path.cmd +3 -0
- data/vendor/datasketches-cpp/python/src/cpc_wrapper.cpp +1 -1
- data/vendor/datasketches-cpp/python/src/datasketches.cpp +2 -0
- data/vendor/datasketches-cpp/python/src/hll_wrapper.cpp +4 -6
- data/vendor/datasketches-cpp/python/src/kll_wrapper.cpp +4 -2
- data/vendor/datasketches-cpp/python/src/req_wrapper.cpp +246 -0
- data/vendor/datasketches-cpp/python/src/theta_wrapper.cpp +38 -28
- data/vendor/datasketches-cpp/python/src/vector_of_kll.cpp +11 -5
- data/vendor/datasketches-cpp/python/src/vo_wrapper.cpp +2 -2
- data/vendor/datasketches-cpp/python/tests/hll_test.py +1 -2
- data/vendor/datasketches-cpp/python/tests/kll_test.py +5 -5
- data/vendor/datasketches-cpp/python/tests/req_test.py +126 -0
- data/vendor/datasketches-cpp/python/tests/theta_test.py +28 -3
- data/vendor/datasketches-cpp/python/tests/vector_of_kll_test.py +4 -4
- data/vendor/datasketches-cpp/python/tests/vo_test.py +3 -3
- data/vendor/datasketches-cpp/req/CMakeLists.txt +60 -0
- data/vendor/datasketches-cpp/{tuple/include/theta_a_not_b_experimental_impl.hpp → req/include/req_common.hpp} +18 -8
- data/vendor/datasketches-cpp/req/include/req_compactor.hpp +137 -0
- data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +488 -0
- data/vendor/datasketches-cpp/req/include/req_quantile_calculator.hpp +69 -0
- data/vendor/datasketches-cpp/req/include/req_quantile_calculator_impl.hpp +60 -0
- data/vendor/datasketches-cpp/req/include/req_sketch.hpp +395 -0
- data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +810 -0
- data/vendor/datasketches-cpp/req/test/CMakeLists.txt +43 -0
- data/vendor/datasketches-cpp/req/test/req_float_empty_from_java.sk +0 -0
- data/vendor/datasketches-cpp/req/test/req_float_estimation_from_java.sk +0 -0
- data/vendor/datasketches-cpp/req/test/req_float_exact_from_java.sk +0 -0
- data/vendor/datasketches-cpp/req/test/req_float_raw_items_from_java.sk +0 -0
- data/vendor/datasketches-cpp/req/test/req_float_single_item_from_java.sk +0 -0
- data/vendor/datasketches-cpp/req/test/req_sketch_custom_type_test.cpp +128 -0
- data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +494 -0
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch.hpp +19 -13
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp +130 -127
- data/vendor/datasketches-cpp/sampling/include/var_opt_union.hpp +5 -5
- data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp +41 -49
- data/vendor/datasketches-cpp/sampling/test/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/sampling/test/var_opt_allocation_test.cpp +96 -0
- data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp +6 -6
- data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp +13 -44
- data/vendor/datasketches-cpp/setup.py +11 -6
- data/vendor/datasketches-cpp/theta/CMakeLists.txt +30 -3
- data/vendor/datasketches-cpp/{tuple → theta}/include/bounds_on_ratios_in_sampled_sets.hpp +3 -2
- data/vendor/datasketches-cpp/{tuple → theta}/include/bounds_on_ratios_in_theta_sketched_sets.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser.hpp +67 -0
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser_impl.hpp +70 -0
- data/vendor/datasketches-cpp/theta/include/theta_a_not_b.hpp +12 -29
- data/vendor/datasketches-cpp/theta/include/theta_a_not_b_impl.hpp +5 -46
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_comparators.hpp +0 -0
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_constants.hpp +11 -4
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_helpers.hpp +0 -0
- data/vendor/datasketches-cpp/theta/include/theta_intersection.hpp +26 -28
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_intersection_base.hpp +0 -0
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_intersection_base_impl.hpp +0 -0
- data/vendor/datasketches-cpp/theta/include/theta_intersection_impl.hpp +8 -90
- data/vendor/datasketches-cpp/{tuple/test/theta_union_experimental_test.cpp → theta/include/theta_jaccard_similarity.hpp} +11 -18
- data/vendor/datasketches-cpp/{tuple/include/jaccard_similarity.hpp → theta/include/theta_jaccard_similarity_base.hpp} +24 -36
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_set_difference_base.hpp +0 -0
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_set_difference_base_impl.hpp +5 -0
- data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp +163 -256
- data/vendor/datasketches-cpp/theta/include/theta_sketch_impl.hpp +250 -651
- data/vendor/datasketches-cpp/theta/include/theta_union.hpp +27 -60
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_union_base.hpp +1 -1
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_union_base_impl.hpp +6 -1
- data/vendor/datasketches-cpp/theta/include/theta_union_impl.hpp +13 -69
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_update_sketch_base.hpp +10 -21
- data/vendor/datasketches-cpp/{tuple → theta}/include/theta_update_sketch_base_impl.hpp +44 -30
- data/vendor/datasketches-cpp/theta/test/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/theta/test/theta_a_not_b_test.cpp +23 -1
- data/vendor/datasketches-cpp/theta/test/theta_intersection_test.cpp +21 -1
- data/vendor/datasketches-cpp/{tuple → theta}/test/theta_jaccard_similarity_test.cpp +60 -5
- data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp +74 -235
- data/vendor/datasketches-cpp/theta/test/theta_union_test.cpp +22 -2
- data/vendor/datasketches-cpp/tuple/CMakeLists.txt +3 -35
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch_impl.hpp +47 -60
- data/vendor/datasketches-cpp/tuple/include/tuple_jaccard_similarity.hpp +38 -0
- data/vendor/datasketches-cpp/tuple/include/tuple_sketch.hpp +28 -13
- data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp +57 -70
- data/vendor/datasketches-cpp/tuple/test/CMakeLists.txt +1 -6
- data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +1 -1
- data/vendor/datasketches-cpp/tuple/test/tuple_a_not_b_test.cpp +18 -21
- data/vendor/datasketches-cpp/tuple/test/tuple_intersection_test.cpp +13 -16
- data/vendor/datasketches-cpp/tuple/test/tuple_jaccard_similarity_test.cpp +7 -6
- data/vendor/datasketches-cpp/tuple/test/tuple_sketch_allocation_test.cpp +3 -3
- data/vendor/datasketches-cpp/tuple/test/tuple_sketch_test.cpp +20 -20
- data/vendor/datasketches-cpp/tuple/test/tuple_union_test.cpp +13 -16
- metadata +51 -36
- data/vendor/datasketches-cpp/tuple/include/theta_a_not_b_experimental.hpp +0 -53
- data/vendor/datasketches-cpp/tuple/include/theta_intersection_experimental.hpp +0 -78
- data/vendor/datasketches-cpp/tuple/include/theta_intersection_experimental_impl.hpp +0 -43
- data/vendor/datasketches-cpp/tuple/include/theta_sketch_experimental.hpp +0 -393
- data/vendor/datasketches-cpp/tuple/include/theta_sketch_experimental_impl.hpp +0 -481
- data/vendor/datasketches-cpp/tuple/include/theta_union_experimental.hpp +0 -88
- data/vendor/datasketches-cpp/tuple/include/theta_union_experimental_impl.hpp +0 -47
- data/vendor/datasketches-cpp/tuple/test/theta_a_not_b_experimental_test.cpp +0 -250
- data/vendor/datasketches-cpp/tuple/test/theta_compact_empty_from_java.sk +0 -0
- data/vendor/datasketches-cpp/tuple/test/theta_compact_estimation_from_java.sk +0 -0
- data/vendor/datasketches-cpp/tuple/test/theta_compact_single_item_from_java.sk +0 -0
- data/vendor/datasketches-cpp/tuple/test/theta_intersection_experimental_test.cpp +0 -224
- data/vendor/datasketches-cpp/tuple/test/theta_sketch_experimental_test.cpp +0 -247
@@ -30,7 +30,7 @@
|
|
30
30
|
namespace datasketches {
|
31
31
|
|
32
32
|
TEST_CASE("coupon hash set: check corrupt bytearray", "[coupon_hash_set]") {
|
33
|
-
|
33
|
+
uint8_t lgK = 8;
|
34
34
|
hll_sketch sk1(lgK);
|
35
35
|
for (int i = 0; i < 24; ++i) {
|
36
36
|
sk1.update(i);
|
@@ -39,42 +39,42 @@ TEST_CASE("coupon hash set: check corrupt bytearray", "[coupon_hash_set]") {
|
|
39
39
|
uint8_t* bytes = sketchBytes.data();
|
40
40
|
const size_t size = sketchBytes.size();
|
41
41
|
|
42
|
-
bytes[
|
42
|
+
bytes[hll_constants::PREAMBLE_INTS_BYTE] = 0;
|
43
43
|
// fail in HllSketchImpl
|
44
44
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
45
45
|
// fail in CouponHashSet
|
46
|
-
REQUIRE_THROWS_AS(CouponHashSet
|
47
|
-
bytes[
|
46
|
+
REQUIRE_THROWS_AS(CouponHashSet<std::allocator<uint8_t>>::newSet(bytes, size, std::allocator<uint8_t>()), std::invalid_argument);
|
47
|
+
bytes[hll_constants::PREAMBLE_INTS_BYTE] = hll_constants::HASH_SET_PREINTS;
|
48
48
|
|
49
|
-
bytes[
|
49
|
+
bytes[hll_constants::SER_VER_BYTE] = 0;
|
50
50
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
51
|
-
bytes[
|
51
|
+
bytes[hll_constants::SER_VER_BYTE] = hll_constants::SER_VER;
|
52
52
|
|
53
|
-
bytes[
|
53
|
+
bytes[hll_constants::FAMILY_BYTE] = 0;
|
54
54
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
55
|
-
bytes[
|
55
|
+
bytes[hll_constants::FAMILY_BYTE] = hll_constants::FAMILY_ID;
|
56
56
|
|
57
|
-
bytes[
|
57
|
+
bytes[hll_constants::LG_K_BYTE] = 6;
|
58
58
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
59
|
-
bytes[
|
59
|
+
bytes[hll_constants::LG_K_BYTE] = lgK;
|
60
60
|
|
61
|
-
uint8_t tmp = bytes[
|
62
|
-
bytes[
|
61
|
+
uint8_t tmp = bytes[hll_constants::MODE_BYTE];
|
62
|
+
bytes[hll_constants::MODE_BYTE] = 0x10; // HLL_6, LIST
|
63
63
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
64
|
-
bytes[
|
64
|
+
bytes[hll_constants::MODE_BYTE] = tmp;
|
65
65
|
|
66
|
-
tmp = bytes[
|
67
|
-
bytes[
|
66
|
+
tmp = bytes[hll_constants::LG_ARR_BYTE];
|
67
|
+
bytes[hll_constants::LG_ARR_BYTE] = 0;
|
68
68
|
hll_sketch::deserialize(bytes, size);
|
69
69
|
// should work fine despite the corruption
|
70
|
-
bytes[
|
70
|
+
bytes[hll_constants::LG_ARR_BYTE] = tmp;
|
71
71
|
|
72
72
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size - 1), std::out_of_range);
|
73
73
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, 3), std::out_of_range);
|
74
74
|
}
|
75
75
|
|
76
76
|
TEST_CASE("coupon hash set: check corrupt stream", "[coupon_hash_set]") {
|
77
|
-
|
77
|
+
uint8_t lgK = 9;
|
78
78
|
hll_sketch sk1(lgK);
|
79
79
|
for (int i = 0; i < 24; ++i) {
|
80
80
|
sk1.update(i);
|
@@ -82,48 +82,48 @@ TEST_CASE("coupon hash set: check corrupt stream", "[coupon_hash_set]") {
|
|
82
82
|
std::stringstream ss;
|
83
83
|
sk1.serialize_compact(ss);
|
84
84
|
|
85
|
-
ss.seekp(
|
85
|
+
ss.seekp(hll_constants::PREAMBLE_INTS_BYTE);
|
86
86
|
ss.put(0);
|
87
87
|
ss.seekg(0);
|
88
88
|
// fail in HllSketchImpl
|
89
89
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
90
90
|
// fail in CouponHashSet
|
91
|
-
REQUIRE_THROWS_AS(CouponHashSet
|
92
|
-
ss.seekp(
|
93
|
-
ss.put(
|
91
|
+
REQUIRE_THROWS_AS(CouponHashSet<std::allocator<uint8_t>>::newSet(ss, std::allocator<uint8_t>()), std::invalid_argument);
|
92
|
+
ss.seekp(hll_constants::PREAMBLE_INTS_BYTE);
|
93
|
+
ss.put(hll_constants::HASH_SET_PREINTS);
|
94
94
|
|
95
|
-
ss.seekp(
|
95
|
+
ss.seekp(hll_constants::SER_VER_BYTE);
|
96
96
|
ss.put(0);
|
97
97
|
ss.seekg(0);
|
98
98
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
99
|
-
ss.seekp(
|
100
|
-
ss.put(
|
99
|
+
ss.seekp(hll_constants::SER_VER_BYTE);
|
100
|
+
ss.put(hll_constants::SER_VER);
|
101
101
|
|
102
|
-
ss.seekp(
|
102
|
+
ss.seekp(hll_constants::FAMILY_BYTE);
|
103
103
|
ss.put(0);
|
104
104
|
ss.seekg(0);
|
105
105
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
106
|
-
ss.seekp(
|
107
|
-
ss.put(
|
106
|
+
ss.seekp(hll_constants::FAMILY_BYTE);
|
107
|
+
ss.put(hll_constants::FAMILY_ID);
|
108
108
|
|
109
|
-
ss.seekg(
|
110
|
-
|
111
|
-
ss.seekp(
|
109
|
+
ss.seekg(hll_constants::MODE_BYTE);
|
110
|
+
auto tmp = ss.get();
|
111
|
+
ss.seekp(hll_constants::MODE_BYTE);
|
112
112
|
ss.put(0x22); // HLL_8, HLL
|
113
113
|
ss.seekg(0);
|
114
114
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
115
|
-
ss.seekp(
|
116
|
-
ss.put(tmp);
|
115
|
+
ss.seekp(hll_constants::MODE_BYTE);
|
116
|
+
ss.put((char)tmp);
|
117
117
|
|
118
|
-
ss.seekg(
|
118
|
+
ss.seekg(hll_constants::LG_ARR_BYTE);
|
119
119
|
tmp = ss.get();
|
120
|
-
ss.seekp(
|
120
|
+
ss.seekp(hll_constants::LG_ARR_BYTE);
|
121
121
|
ss.put(0);
|
122
122
|
ss.seekg(0);
|
123
123
|
hll_sketch::deserialize(ss);
|
124
124
|
// should work fine despite the corruption
|
125
|
-
ss.seekp(
|
126
|
-
ss.put(tmp);
|
125
|
+
ss.seekp(hll_constants::LG_ARR_BYTE);
|
126
|
+
ss.put((char)tmp);
|
127
127
|
}
|
128
128
|
|
129
129
|
|
@@ -35,9 +35,9 @@ void println_string(std::string str) {
|
|
35
35
|
}
|
36
36
|
|
37
37
|
TEST_CASE("coupon list: check iterator", "[coupon_list]") {
|
38
|
-
|
39
|
-
CouponList
|
40
|
-
for (
|
38
|
+
uint8_t lgConfigK = 8;
|
39
|
+
CouponList<std::allocator<uint8_t>> cl(lgConfigK, HLL_4, LIST, std::allocator<uint8_t>());
|
40
|
+
for (uint8_t i = 1; i <= 7; ++i) { cl.couponUpdate(HllUtil<>::pair(i, i)); } // not hashes but distinct values
|
41
41
|
const int mask = (1 << lgConfigK) - 1;
|
42
42
|
int idx = 0;
|
43
43
|
auto itr = cl.begin(false);
|
@@ -56,7 +56,7 @@ TEST_CASE("coupon list: check iterator", "[coupon_list]") {
|
|
56
56
|
}
|
57
57
|
|
58
58
|
TEST_CASE("coupon list: check duplicates and misc", "[coupon_list]") {
|
59
|
-
|
59
|
+
uint8_t lgConfigK = 8;
|
60
60
|
hll_sketch sk(lgConfigK);
|
61
61
|
|
62
62
|
for (int i = 1; i <= 7; ++i) {
|
@@ -79,7 +79,7 @@ TEST_CASE("coupon list: check duplicates and misc", "[coupon_list]") {
|
|
79
79
|
REQUIRE(relErr < 0.0);
|
80
80
|
}
|
81
81
|
|
82
|
-
static void serializeDeserialize(
|
82
|
+
static void serializeDeserialize(uint8_t lgK) {
|
83
83
|
hll_sketch sk1(lgK);
|
84
84
|
|
85
85
|
int u = (lgK < 8) ? 7 : (((1 << (lgK - 3))/ 4) * 3);
|
@@ -110,7 +110,7 @@ TEST_CASE("coupon list: check serialize deserialize", "[coupon_list]") {
|
|
110
110
|
}
|
111
111
|
|
112
112
|
TEST_CASE("coupon list: check corrupt bytearray data", "[coupon_list]") {
|
113
|
-
|
113
|
+
uint8_t lgK = 6;
|
114
114
|
hll_sketch sk1(lgK);
|
115
115
|
sk1.update(1);
|
116
116
|
sk1.update(2);
|
@@ -118,24 +118,24 @@ TEST_CASE("coupon list: check corrupt bytearray data", "[coupon_list]") {
|
|
118
118
|
uint8_t* bytes = sketchBytes.data();
|
119
119
|
const size_t size = sketchBytes.size();
|
120
120
|
|
121
|
-
bytes[
|
121
|
+
bytes[hll_constants::PREAMBLE_INTS_BYTE] = 0;
|
122
122
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
123
|
-
REQUIRE_THROWS_AS(CouponList
|
123
|
+
REQUIRE_THROWS_AS(CouponList<std::allocator<uint8_t>>::newList(bytes, size, std::allocator<uint8_t>()), std::invalid_argument);
|
124
124
|
|
125
|
-
bytes[
|
125
|
+
bytes[hll_constants::PREAMBLE_INTS_BYTE] = hll_constants::LIST_PREINTS;
|
126
126
|
|
127
|
-
bytes[
|
127
|
+
bytes[hll_constants::SER_VER_BYTE] = 0;
|
128
128
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
129
|
-
bytes[
|
129
|
+
bytes[hll_constants::SER_VER_BYTE] = hll_constants::SER_VER;
|
130
130
|
|
131
|
-
bytes[
|
131
|
+
bytes[hll_constants::FAMILY_BYTE] = 0;
|
132
132
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
133
|
-
bytes[
|
133
|
+
bytes[hll_constants::FAMILY_BYTE] = hll_constants::FAMILY_ID;
|
134
134
|
|
135
|
-
uint8_t tmp = bytes[
|
136
|
-
bytes[
|
135
|
+
uint8_t tmp = bytes[hll_constants::MODE_BYTE];
|
136
|
+
bytes[hll_constants::MODE_BYTE] = 0x01; // HLL_4, SET
|
137
137
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
138
|
-
bytes[
|
138
|
+
bytes[hll_constants::MODE_BYTE] = tmp;
|
139
139
|
|
140
140
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size - 1), std::out_of_range);
|
141
141
|
|
@@ -143,36 +143,36 @@ TEST_CASE("coupon list: check corrupt bytearray data", "[coupon_list]") {
|
|
143
143
|
}
|
144
144
|
|
145
145
|
TEST_CASE("coupon list: check corrupt stream data", "[coupon_list]") {
|
146
|
-
|
146
|
+
uint8_t lgK = 6;
|
147
147
|
hll_sketch sk1(lgK);
|
148
148
|
sk1.update(1);
|
149
149
|
sk1.update(2);
|
150
150
|
std::stringstream ss;
|
151
151
|
sk1.serialize_compact(ss);
|
152
152
|
|
153
|
-
ss.seekp(
|
153
|
+
ss.seekp(hll_constants::PREAMBLE_INTS_BYTE);
|
154
154
|
ss.put(0);
|
155
155
|
ss.seekg(0);
|
156
156
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
157
|
-
REQUIRE_THROWS_AS(CouponList
|
158
|
-
ss.seekp(
|
159
|
-
ss.put(
|
157
|
+
REQUIRE_THROWS_AS(CouponList<std::allocator<uint8_t>>::newList(ss, std::allocator<uint8_t>()), std::invalid_argument);
|
158
|
+
ss.seekp(hll_constants::PREAMBLE_INTS_BYTE);
|
159
|
+
ss.put(hll_constants::LIST_PREINTS);
|
160
160
|
|
161
|
-
ss.seekp(
|
161
|
+
ss.seekp(hll_constants::SER_VER_BYTE);
|
162
162
|
ss.put(0);
|
163
163
|
ss.seekg(0);
|
164
164
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
165
|
-
ss.seekp(
|
166
|
-
ss.put(
|
165
|
+
ss.seekp(hll_constants::SER_VER_BYTE);
|
166
|
+
ss.put(hll_constants::SER_VER);
|
167
167
|
|
168
|
-
ss.seekp(
|
168
|
+
ss.seekp(hll_constants::FAMILY_BYTE);
|
169
169
|
ss.put(0);
|
170
170
|
ss.seekg(0);
|
171
171
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
172
|
-
ss.seekp(
|
173
|
-
ss.put(
|
172
|
+
ss.seekp(hll_constants::FAMILY_BYTE);
|
173
|
+
ss.put(hll_constants::FAMILY_ID);
|
174
174
|
|
175
|
-
ss.seekp(
|
175
|
+
ss.seekp(hll_constants::MODE_BYTE);
|
176
176
|
ss.put(0x22); // HLL_8, HLL
|
177
177
|
ss.seekg(0);
|
178
178
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
@@ -26,7 +26,7 @@
|
|
26
26
|
|
27
27
|
namespace datasketches {
|
28
28
|
|
29
|
-
static hll_sketch buildSketch(const int n, const
|
29
|
+
static hll_sketch buildSketch(const int n, const uint8_t lgK, const target_hll_type tgtHllType) {
|
30
30
|
hll_sketch sketch(lgK, tgtHllType);
|
31
31
|
for (int i = 0; i < n; ++i) {
|
32
32
|
sketch.update(i);
|
@@ -34,7 +34,7 @@ static hll_sketch buildSketch(const int n, const int lgK, const target_hll_type
|
|
34
34
|
return sketch;
|
35
35
|
}
|
36
36
|
|
37
|
-
static void crossCountingCheck(const
|
37
|
+
static void crossCountingCheck(const uint8_t lgK, const int n) {
|
38
38
|
hll_sketch sk4 = buildSketch(n, lgK, HLL_4);
|
39
39
|
const double est = sk4.get_estimate();
|
40
40
|
const double lb = sk4.get_lower_bound(1);
|
@@ -25,7 +25,7 @@
|
|
25
25
|
|
26
26
|
namespace datasketches {
|
27
27
|
|
28
|
-
static void testComposite(
|
28
|
+
static void testComposite(uint8_t lgK, const target_hll_type tgtHllType, const int n) {
|
29
29
|
hll_union u(lgK);
|
30
30
|
hll_sketch sk(lgK, tgtHllType);
|
31
31
|
for (int i = 0; i < n; ++i) {
|
@@ -45,7 +45,7 @@ TEST_CASE("hll array: check composite estimate", "[hll_array]") {
|
|
45
45
|
testComposite(13, target_hll_type::HLL_8, 10000);
|
46
46
|
}
|
47
47
|
|
48
|
-
static void serializeDeserialize(
|
48
|
+
static void serializeDeserialize(uint8_t lgK, target_hll_type tgtHllType, const int n) {
|
49
49
|
hll_sketch sk1(lgK, tgtHllType);
|
50
50
|
|
51
51
|
for (int i = 0; i < n; ++i) {
|
@@ -72,7 +72,7 @@ static void serializeDeserialize(const int lgK, target_hll_type tgtHllType, cons
|
|
72
72
|
}
|
73
73
|
|
74
74
|
TEST_CASE("hll array: check serialize deserialize", "[hll_array]") {
|
75
|
-
|
75
|
+
uint8_t lgK = 4;
|
76
76
|
int n = 8;
|
77
77
|
serializeDeserialize(lgK, HLL_4, n);
|
78
78
|
serializeDeserialize(lgK, HLL_6, n);
|
@@ -100,7 +100,7 @@ TEST_CASE("hll array: check is compact", "[hll_array]") {
|
|
100
100
|
}
|
101
101
|
|
102
102
|
TEST_CASE("hll array: check corrupt bytearray", "[hll_array]") {
|
103
|
-
|
103
|
+
uint8_t lgK = 8;
|
104
104
|
hll_sketch sk1(lgK, HLL_8);
|
105
105
|
for (int i = 0; i < 50; ++i) {
|
106
106
|
sk1.update(i);
|
@@ -109,36 +109,36 @@ TEST_CASE("hll array: check corrupt bytearray", "[hll_array]") {
|
|
109
109
|
uint8_t* bytes = sketchBytes.data();
|
110
110
|
const size_t size = sketchBytes.size();
|
111
111
|
|
112
|
-
bytes[
|
112
|
+
bytes[hll_constants::PREAMBLE_INTS_BYTE] = 0;
|
113
113
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
114
|
-
REQUIRE_THROWS_AS(HllArray
|
115
|
-
bytes[
|
114
|
+
REQUIRE_THROWS_AS(HllArray<std::allocator<uint8_t>>::newHll(bytes, size, std::allocator<uint8_t>()), std::invalid_argument);
|
115
|
+
bytes[hll_constants::PREAMBLE_INTS_BYTE] = hll_constants::HLL_PREINTS;
|
116
116
|
|
117
|
-
bytes[
|
117
|
+
bytes[hll_constants::SER_VER_BYTE] = 0;
|
118
118
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
119
|
-
bytes[
|
119
|
+
bytes[hll_constants::SER_VER_BYTE] = hll_constants::SER_VER;
|
120
120
|
|
121
|
-
bytes[
|
121
|
+
bytes[hll_constants::FAMILY_BYTE] = 0;
|
122
122
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
123
|
-
bytes[
|
123
|
+
bytes[hll_constants::FAMILY_BYTE] = hll_constants::FAMILY_ID;
|
124
124
|
|
125
|
-
uint8_t tmp = bytes[
|
126
|
-
bytes[
|
125
|
+
uint8_t tmp = bytes[hll_constants::MODE_BYTE];
|
126
|
+
bytes[hll_constants::MODE_BYTE] = 0x10; // HLL_6, LIST
|
127
127
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size), std::invalid_argument);
|
128
|
-
bytes[
|
128
|
+
bytes[hll_constants::MODE_BYTE] = tmp;
|
129
129
|
|
130
|
-
tmp = bytes[
|
131
|
-
bytes[
|
130
|
+
tmp = bytes[hll_constants::LG_ARR_BYTE];
|
131
|
+
bytes[hll_constants::LG_ARR_BYTE] = 0;
|
132
132
|
hll_sketch::deserialize(bytes, size);
|
133
133
|
// should work fine despite the corruption
|
134
|
-
bytes[
|
134
|
+
bytes[hll_constants::LG_ARR_BYTE] = tmp;
|
135
135
|
|
136
136
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, size - 1), std::out_of_range);
|
137
137
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(bytes, 3), std::out_of_range);
|
138
138
|
}
|
139
139
|
|
140
140
|
TEST_CASE("hll array: check corrupt stream", "[hll_array]") {
|
141
|
-
|
141
|
+
uint8_t lgK = 6;
|
142
142
|
hll_sketch sk1(lgK);
|
143
143
|
for (int i = 0; i < 50; ++i) {
|
144
144
|
sk1.update(i);
|
@@ -146,46 +146,46 @@ TEST_CASE("hll array: check corrupt stream", "[hll_array]") {
|
|
146
146
|
std::stringstream ss;
|
147
147
|
sk1.serialize_compact(ss);
|
148
148
|
|
149
|
-
ss.seekp(
|
149
|
+
ss.seekp(hll_constants::PREAMBLE_INTS_BYTE);
|
150
150
|
ss.put(0);
|
151
151
|
ss.seekg(0);
|
152
152
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
153
|
-
REQUIRE_THROWS_AS(HllArray
|
154
|
-
ss.seekp(
|
155
|
-
ss.put(
|
153
|
+
REQUIRE_THROWS_AS(HllArray<std::allocator<uint8_t>>::newHll(ss, std::allocator<uint8_t>()), std::invalid_argument);
|
154
|
+
ss.seekp(hll_constants::PREAMBLE_INTS_BYTE);
|
155
|
+
ss.put(hll_constants::HLL_PREINTS);
|
156
156
|
|
157
|
-
ss.seekp(
|
157
|
+
ss.seekp(hll_constants::SER_VER_BYTE);
|
158
158
|
ss.put(0);
|
159
159
|
ss.seekg(0);
|
160
160
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
161
|
-
ss.seekp(
|
162
|
-
ss.put(
|
161
|
+
ss.seekp(hll_constants::SER_VER_BYTE);
|
162
|
+
ss.put(hll_constants::SER_VER);
|
163
163
|
|
164
|
-
ss.seekp(
|
164
|
+
ss.seekp(hll_constants::FAMILY_BYTE);
|
165
165
|
ss.put(0);
|
166
166
|
ss.seekg(0);
|
167
167
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
168
|
-
ss.seekp(
|
169
|
-
ss.put(
|
168
|
+
ss.seekp(hll_constants::FAMILY_BYTE);
|
169
|
+
ss.put(hll_constants::FAMILY_ID);
|
170
170
|
|
171
|
-
ss.seekg(
|
172
|
-
|
173
|
-
ss.seekp(
|
171
|
+
ss.seekg(hll_constants::MODE_BYTE);
|
172
|
+
auto tmp = ss.get();
|
173
|
+
ss.seekp(hll_constants::MODE_BYTE);
|
174
174
|
ss.put(0x11); // HLL_6, SET
|
175
175
|
ss.seekg(0);
|
176
176
|
REQUIRE_THROWS_AS(hll_sketch::deserialize(ss), std::invalid_argument);
|
177
|
-
ss.seekp(
|
178
|
-
ss.put(tmp);
|
177
|
+
ss.seekp(hll_constants::MODE_BYTE);
|
178
|
+
ss.put((char)tmp);
|
179
179
|
|
180
|
-
ss.seekg(
|
180
|
+
ss.seekg(hll_constants::LG_ARR_BYTE);
|
181
181
|
tmp = ss.get();
|
182
|
-
ss.seekp(
|
182
|
+
ss.seekp(hll_constants::LG_ARR_BYTE);
|
183
183
|
ss.put(0);
|
184
184
|
ss.seekg(0);
|
185
185
|
hll_sketch::deserialize(ss);
|
186
186
|
// should work fine despite the corruption
|
187
|
-
ss.seekp(
|
188
|
-
ss.put(tmp);
|
187
|
+
ss.seekp(hll_constants::LG_ARR_BYTE);
|
188
|
+
ss.put((char)tmp);
|
189
189
|
}
|
190
190
|
|
191
191
|
} /* namespace datasketches */
|