datasketches 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/datasketches/version.rb +1 -1
- data/vendor/datasketches-cpp/CMakeLists.txt +7 -0
- data/vendor/datasketches-cpp/common/include/MurmurHash3.h +11 -7
- data/vendor/datasketches-cpp/common/include/binomial_bounds.hpp +8 -8
- data/vendor/datasketches-cpp/common/include/bounds_binomial_proportions.hpp +12 -15
- data/vendor/datasketches-cpp/common/include/common_defs.hpp +24 -0
- data/vendor/datasketches-cpp/common/include/conditional_forward.hpp +20 -8
- data/vendor/datasketches-cpp/common/include/count_zeros.hpp +2 -2
- data/vendor/datasketches-cpp/common/include/serde.hpp +7 -7
- data/vendor/datasketches-cpp/cpc/include/cpc_compressor.hpp +19 -19
- data/vendor/datasketches-cpp/cpc/include/cpc_compressor_impl.hpp +91 -89
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch.hpp +14 -1
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch_impl.hpp +121 -87
- data/vendor/datasketches-cpp/cpc/include/cpc_union_impl.hpp +14 -14
- data/vendor/datasketches-cpp/cpc/include/cpc_util.hpp +10 -10
- data/vendor/datasketches-cpp/cpc/include/icon_estimator.hpp +4 -4
- data/vendor/datasketches-cpp/cpc/include/u32_table.hpp +8 -8
- data/vendor/datasketches-cpp/cpc/include/u32_table_impl.hpp +14 -14
- data/vendor/datasketches-cpp/cpc/test/compression_test.cpp +10 -10
- data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp +25 -0
- data/vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp +1 -1
- data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp +65 -80
- data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map_impl.hpp +10 -10
- data/vendor/datasketches-cpp/fi/test/reverse_purge_hash_map_test.cpp +2 -2
- data/vendor/datasketches-cpp/hll/include/AuxHashMap-internal.hpp +60 -63
- data/vendor/datasketches-cpp/hll/include/AuxHashMap.hpp +19 -19
- data/vendor/datasketches-cpp/hll/include/CompositeInterpolationXTable-internal.hpp +15 -15
- data/vendor/datasketches-cpp/hll/include/CompositeInterpolationXTable.hpp +3 -3
- data/vendor/datasketches-cpp/hll/include/CouponHashSet-internal.hpp +74 -76
- data/vendor/datasketches-cpp/hll/include/CouponHashSet.hpp +6 -6
- data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp +110 -113
- data/vendor/datasketches-cpp/hll/include/CouponList.hpp +13 -13
- data/vendor/datasketches-cpp/hll/include/CubicInterpolation-internal.hpp +2 -4
- data/vendor/datasketches-cpp/hll/include/HarmonicNumbers-internal.hpp +1 -1
- data/vendor/datasketches-cpp/hll/include/Hll4Array-internal.hpp +80 -76
- data/vendor/datasketches-cpp/hll/include/Hll4Array.hpp +9 -9
- data/vendor/datasketches-cpp/hll/include/Hll6Array-internal.hpp +26 -26
- data/vendor/datasketches-cpp/hll/include/Hll6Array.hpp +6 -6
- data/vendor/datasketches-cpp/hll/include/Hll8Array-internal.hpp +33 -33
- data/vendor/datasketches-cpp/hll/include/Hll8Array.hpp +6 -6
- data/vendor/datasketches-cpp/hll/include/HllArray-internal.hpp +205 -209
- data/vendor/datasketches-cpp/hll/include/HllArray.hpp +36 -36
- data/vendor/datasketches-cpp/hll/include/HllSketch-internal.hpp +28 -28
- data/vendor/datasketches-cpp/hll/include/HllSketchImpl-internal.hpp +22 -22
- data/vendor/datasketches-cpp/hll/include/HllSketchImpl.hpp +13 -13
- data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp +15 -15
- data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +61 -61
- data/vendor/datasketches-cpp/hll/include/HllUtil.hpp +120 -127
- data/vendor/datasketches-cpp/hll/include/coupon_iterator-internal.hpp +9 -9
- data/vendor/datasketches-cpp/hll/include/coupon_iterator.hpp +5 -5
- data/vendor/datasketches-cpp/hll/include/hll.hpp +21 -21
- data/vendor/datasketches-cpp/hll/test/AuxHashMapTest.cpp +1 -1
- data/vendor/datasketches-cpp/hll/test/CouponHashSetTest.cpp +34 -34
- data/vendor/datasketches-cpp/hll/test/CouponListTest.cpp +25 -25
- data/vendor/datasketches-cpp/hll/test/CrossCountingTest.cpp +2 -2
- data/vendor/datasketches-cpp/hll/test/HllArrayTest.cpp +35 -35
- data/vendor/datasketches-cpp/hll/test/HllSketchTest.cpp +15 -15
- data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +10 -14
- data/vendor/datasketches-cpp/hll/test/IsomorphicTest.cpp +3 -3
- data/vendor/datasketches-cpp/hll/test/ToFromByteArrayTest.cpp +4 -4
- data/vendor/datasketches-cpp/kll/include/kll_helper.hpp +5 -4
- data/vendor/datasketches-cpp/kll/include/kll_helper_impl.hpp +6 -6
- data/vendor/datasketches-cpp/kll/include/kll_quantile_calculator.hpp +14 -6
- data/vendor/datasketches-cpp/kll/include/kll_quantile_calculator_impl.hpp +39 -24
- data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +34 -2
- data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +72 -62
- data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov.hpp +67 -0
- data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov_impl.hpp +78 -0
- data/vendor/datasketches-cpp/kll/test/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +68 -45
- data/vendor/datasketches-cpp/kll/test/kolmogorov_smirnov_test.cpp +111 -0
- data/vendor/datasketches-cpp/python/src/hll_wrapper.cpp +4 -4
- data/vendor/datasketches-cpp/python/src/theta_wrapper.cpp +6 -6
- data/vendor/datasketches-cpp/python/src/vo_wrapper.cpp +2 -2
- data/vendor/datasketches-cpp/python/tests/hll_test.py +1 -1
- data/vendor/datasketches-cpp/python/tests/vo_test.py +3 -3
- data/vendor/datasketches-cpp/req/include/req_common.hpp +2 -1
- data/vendor/datasketches-cpp/req/include/req_compactor.hpp +4 -4
- data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +26 -39
- data/vendor/datasketches-cpp/req/include/req_sketch.hpp +1 -1
- data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +9 -9
- data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +52 -52
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp +47 -56
- data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp +34 -42
- data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp +6 -6
- data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp +13 -13
- data/vendor/datasketches-cpp/setup.py +1 -1
- data/vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_sampled_sets.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser.hpp +67 -0
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser_impl.hpp +70 -0
- data/vendor/datasketches-cpp/theta/include/theta_intersection.hpp +9 -4
- data/vendor/datasketches-cpp/theta/include/theta_intersection_impl.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/theta_jaccard_similarity_base.hpp +18 -14
- data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp +42 -1
- data/vendor/datasketches-cpp/theta/include/theta_sketch_impl.hpp +107 -58
- data/vendor/datasketches-cpp/theta/include/theta_union.hpp +4 -4
- data/vendor/datasketches-cpp/theta/include/theta_union_base_impl.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/theta_union_impl.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp +2 -0
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp +33 -28
- data/vendor/datasketches-cpp/theta/test/theta_a_not_b_test.cpp +23 -1
- data/vendor/datasketches-cpp/theta/test/theta_intersection_test.cpp +21 -1
- data/vendor/datasketches-cpp/theta/test/theta_jaccard_similarity_test.cpp +58 -2
- data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp +37 -1
- data/vendor/datasketches-cpp/theta/test/theta_union_test.cpp +22 -2
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch_impl.hpp +47 -60
- data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp +51 -64
- data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +1 -1
- data/vendor/datasketches-cpp/tuple/test/tuple_a_not_b_test.cpp +17 -17
- data/vendor/datasketches-cpp/tuple/test/tuple_intersection_test.cpp +12 -12
- data/vendor/datasketches-cpp/tuple/test/tuple_jaccard_similarity_test.cpp +5 -5
- data/vendor/datasketches-cpp/tuple/test/tuple_sketch_allocation_test.cpp +1 -1
- data/vendor/datasketches-cpp/tuple/test/tuple_sketch_test.cpp +20 -20
- data/vendor/datasketches-cpp/tuple/test/tuple_union_test.cpp +12 -12
- metadata +8 -3
|
@@ -27,10 +27,10 @@ namespace datasketches {
|
|
|
27
27
|
template<typename A = std::allocator<uint8_t>>
|
|
28
28
|
class CompositeInterpolationXTable {
|
|
29
29
|
public:
|
|
30
|
-
static
|
|
30
|
+
static uint32_t get_y_stride(uint8_t logK);
|
|
31
31
|
|
|
32
|
-
static const double* get_x_arr(
|
|
33
|
-
static
|
|
32
|
+
static const double* get_x_arr(uint8_t logK);
|
|
33
|
+
static uint32_t get_x_arr_length();
|
|
34
34
|
};
|
|
35
35
|
|
|
36
36
|
}
|
|
@@ -28,10 +28,10 @@
|
|
|
28
28
|
namespace datasketches {
|
|
29
29
|
|
|
30
30
|
template<typename A>
|
|
31
|
-
static
|
|
31
|
+
static int32_t find(const uint32_t* array, uint8_t lgArrInts, uint32_t coupon);
|
|
32
32
|
|
|
33
33
|
template<typename A>
|
|
34
|
-
CouponHashSet<A>::CouponHashSet(
|
|
34
|
+
CouponHashSet<A>::CouponHashSet(uint8_t lgConfigK, target_hll_type tgtHllType, const A& allocator)
|
|
35
35
|
: CouponList<A>(lgConfigK, tgtHllType, hll_mode::SET, allocator)
|
|
36
36
|
{
|
|
37
37
|
if (lgConfigK <= 7) {
|
|
@@ -56,45 +56,45 @@ std::function<void(HllSketchImpl<A>*)> CouponHashSet<A>::get_deleter() const {
|
|
|
56
56
|
|
|
57
57
|
template<typename A>
|
|
58
58
|
CouponHashSet<A>* CouponHashSet<A>::newSet(const void* bytes, size_t len, const A& allocator) {
|
|
59
|
-
if (len <
|
|
59
|
+
if (len < hll_constants::HASH_SET_INT_ARR_START) { // hard-coded
|
|
60
60
|
throw std::out_of_range("Input data length insufficient to hold CouponHashSet");
|
|
61
61
|
}
|
|
62
62
|
|
|
63
63
|
const uint8_t* data = static_cast<const uint8_t*>(bytes);
|
|
64
|
-
if (data[
|
|
64
|
+
if (data[hll_constants::PREAMBLE_INTS_BYTE] != hll_constants::HASH_SET_PREINTS) {
|
|
65
65
|
throw std::invalid_argument("Incorrect number of preInts in input stream");
|
|
66
66
|
}
|
|
67
|
-
if (data[
|
|
67
|
+
if (data[hll_constants::SER_VER_BYTE] != hll_constants::SER_VER) {
|
|
68
68
|
throw std::invalid_argument("Wrong ser ver in input stream");
|
|
69
69
|
}
|
|
70
|
-
if (data[
|
|
70
|
+
if (data[hll_constants::FAMILY_BYTE] != hll_constants::FAMILY_ID) {
|
|
71
71
|
throw std::invalid_argument("Input stream is not an HLL sketch");
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
-
const hll_mode mode = HllSketchImpl<A>::extractCurMode(data[
|
|
74
|
+
const hll_mode mode = HllSketchImpl<A>::extractCurMode(data[hll_constants::MODE_BYTE]);
|
|
75
75
|
if (mode != SET) {
|
|
76
76
|
throw std::invalid_argument("Calling set constructor with non-set mode data");
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
-
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(data[
|
|
79
|
+
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(data[hll_constants::MODE_BYTE]);
|
|
80
80
|
|
|
81
|
-
const
|
|
81
|
+
const uint8_t lgK = data[hll_constants::LG_K_BYTE];
|
|
82
82
|
if (lgK <= 7) {
|
|
83
83
|
throw std::invalid_argument("Attempt to deserialize invalid CouponHashSet with lgConfigK <= 7. Found: "
|
|
84
84
|
+ std::to_string(lgK));
|
|
85
85
|
}
|
|
86
|
-
|
|
87
|
-
const bool compactFlag = ((data[
|
|
86
|
+
uint8_t lgArrInts = data[hll_constants::LG_ARR_BYTE];
|
|
87
|
+
const bool compactFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
|
|
88
88
|
|
|
89
|
-
|
|
90
|
-
std::memcpy(&couponCount, data +
|
|
91
|
-
if (lgArrInts <
|
|
92
|
-
lgArrInts = HllUtil
|
|
89
|
+
uint32_t couponCount;
|
|
90
|
+
std::memcpy(&couponCount, data + hll_constants::HASH_SET_COUNT_INT, sizeof(couponCount));
|
|
91
|
+
if (lgArrInts < hll_constants::LG_INIT_SET_SIZE) {
|
|
92
|
+
lgArrInts = HllUtil<>::computeLgArrInts(SET, couponCount, lgK);
|
|
93
93
|
}
|
|
94
94
|
// Don't set couponCount in sketch here;
|
|
95
95
|
// we'll set later if updatable, and increment with updates if compact
|
|
96
|
-
const
|
|
97
|
-
const size_t expectedLength =
|
|
96
|
+
const uint32_t couponsInArray = (compactFlag ? couponCount : (1 << lgArrInts));
|
|
97
|
+
const size_t expectedLength = hll_constants::HASH_SET_INT_ARR_START + (couponsInArray * sizeof(uint32_t));
|
|
98
98
|
if (len < expectedLength) {
|
|
99
99
|
throw std::out_of_range("Byte array too short for sketch. Expected " + std::to_string(expectedLength)
|
|
100
100
|
+ ", found: " + std::to_string(len));
|
|
@@ -104,19 +104,19 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(const void* bytes, size_t len, const
|
|
|
104
104
|
CouponHashSet<A>* sketch = new (chsa.allocate(1)) CouponHashSet<A>(lgK, tgtHllType, allocator);
|
|
105
105
|
|
|
106
106
|
if (compactFlag) {
|
|
107
|
-
const uint8_t* curPos = data +
|
|
108
|
-
|
|
109
|
-
for (
|
|
107
|
+
const uint8_t* curPos = data + hll_constants::HASH_SET_INT_ARR_START;
|
|
108
|
+
uint32_t coupon;
|
|
109
|
+
for (uint32_t i = 0; i < couponCount; ++i, curPos += sizeof(coupon)) {
|
|
110
110
|
std::memcpy(&coupon, curPos, sizeof(coupon));
|
|
111
111
|
sketch->couponUpdate(coupon);
|
|
112
112
|
}
|
|
113
113
|
} else {
|
|
114
|
-
sketch->
|
|
115
|
-
sketch->
|
|
114
|
+
sketch->coupons_.resize(1ULL << lgArrInts);
|
|
115
|
+
sketch->couponCount_ = couponCount;
|
|
116
116
|
// only need to read valid coupons, unlike in stream case
|
|
117
|
-
std::memcpy(sketch->
|
|
118
|
-
data +
|
|
119
|
-
couponCount * sizeof(
|
|
117
|
+
std::memcpy(sketch->coupons_.data(),
|
|
118
|
+
data + hll_constants::HASH_SET_INT_ARR_START,
|
|
119
|
+
couponCount * sizeof(uint32_t));
|
|
120
120
|
}
|
|
121
121
|
|
|
122
122
|
return sketch;
|
|
@@ -125,37 +125,36 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(const void* bytes, size_t len, const
|
|
|
125
125
|
template<typename A>
|
|
126
126
|
CouponHashSet<A>* CouponHashSet<A>::newSet(std::istream& is, const A& allocator) {
|
|
127
127
|
uint8_t listHeader[8];
|
|
128
|
-
|
|
128
|
+
read(is, listHeader, 8 * sizeof(uint8_t));
|
|
129
129
|
|
|
130
|
-
if (listHeader[
|
|
130
|
+
if (listHeader[hll_constants::PREAMBLE_INTS_BYTE] != hll_constants::HASH_SET_PREINTS) {
|
|
131
131
|
throw std::invalid_argument("Incorrect number of preInts in input stream");
|
|
132
132
|
}
|
|
133
|
-
if (listHeader[
|
|
133
|
+
if (listHeader[hll_constants::SER_VER_BYTE] != hll_constants::SER_VER) {
|
|
134
134
|
throw std::invalid_argument("Wrong ser ver in input stream");
|
|
135
135
|
}
|
|
136
|
-
if (listHeader[
|
|
136
|
+
if (listHeader[hll_constants::FAMILY_BYTE] != hll_constants::FAMILY_ID) {
|
|
137
137
|
throw std::invalid_argument("Input stream is not an HLL sketch");
|
|
138
138
|
}
|
|
139
139
|
|
|
140
|
-
hll_mode mode = HllSketchImpl<A>::extractCurMode(listHeader[
|
|
140
|
+
hll_mode mode = HllSketchImpl<A>::extractCurMode(listHeader[hll_constants::MODE_BYTE]);
|
|
141
141
|
if (mode != SET) {
|
|
142
142
|
throw std::invalid_argument("Calling set constructor with non-set mode data");
|
|
143
143
|
}
|
|
144
144
|
|
|
145
|
-
target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(listHeader[
|
|
145
|
+
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(listHeader[hll_constants::MODE_BYTE]);
|
|
146
146
|
|
|
147
|
-
const
|
|
147
|
+
const uint8_t lgK = listHeader[hll_constants::LG_K_BYTE];
|
|
148
148
|
if (lgK <= 7) {
|
|
149
149
|
throw std::invalid_argument("Attempt to deserialize invalid CouponHashSet with lgConfigK <= 7. Found: "
|
|
150
150
|
+ std::to_string(lgK));
|
|
151
151
|
}
|
|
152
|
-
|
|
153
|
-
const bool compactFlag = ((listHeader[
|
|
152
|
+
uint8_t lgArrInts = listHeader[hll_constants::LG_ARR_BYTE];
|
|
153
|
+
const bool compactFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
|
|
154
154
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
lgArrInts = HllUtil<A>::computeLgArrInts(SET, couponCount, lgK);
|
|
155
|
+
const auto couponCount = read<uint32_t>(is);
|
|
156
|
+
if (lgArrInts < hll_constants::LG_INIT_SET_SIZE) {
|
|
157
|
+
lgArrInts = HllUtil<>::computeLgArrInts(SET, couponCount, lgK);
|
|
159
158
|
}
|
|
160
159
|
|
|
161
160
|
ChsAlloc chsa(allocator);
|
|
@@ -166,16 +165,15 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(std::istream& is, const A& allocator)
|
|
|
166
165
|
// Don't set couponCount here;
|
|
167
166
|
// we'll set later if updatable, and increment with updates if compact
|
|
168
167
|
if (compactFlag) {
|
|
169
|
-
for (
|
|
170
|
-
|
|
171
|
-
is.read((char*)&coupon, sizeof(coupon));
|
|
168
|
+
for (uint32_t i = 0; i < couponCount; ++i) {
|
|
169
|
+
const auto coupon = read<uint32_t>(is);
|
|
172
170
|
sketch->couponUpdate(coupon);
|
|
173
171
|
}
|
|
174
172
|
} else {
|
|
175
|
-
sketch->
|
|
176
|
-
sketch->
|
|
173
|
+
sketch->coupons_.resize(1ULL << lgArrInts);
|
|
174
|
+
sketch->couponCount_ = couponCount;
|
|
177
175
|
// for stream processing, read entire list so read pointer ends up set correctly
|
|
178
|
-
|
|
176
|
+
read(is, sketch->coupons_.data(), sketch->coupons_.size() * sizeof(uint32_t));
|
|
179
177
|
}
|
|
180
178
|
|
|
181
179
|
if (!is.good())
|
|
@@ -186,25 +184,25 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(std::istream& is, const A& allocator)
|
|
|
186
184
|
|
|
187
185
|
template<typename A>
|
|
188
186
|
CouponHashSet<A>* CouponHashSet<A>::copy() const {
|
|
189
|
-
ChsAlloc chsa(this->
|
|
187
|
+
ChsAlloc chsa(this->coupons_.get_allocator());
|
|
190
188
|
return new (chsa.allocate(1)) CouponHashSet<A>(*this);
|
|
191
189
|
}
|
|
192
190
|
|
|
193
191
|
template<typename A>
|
|
194
|
-
CouponHashSet<A>* CouponHashSet<A>::copyAs(
|
|
195
|
-
ChsAlloc chsa(this->
|
|
192
|
+
CouponHashSet<A>* CouponHashSet<A>::copyAs(target_hll_type tgtHllType) const {
|
|
193
|
+
ChsAlloc chsa(this->coupons_.get_allocator());
|
|
196
194
|
return new (chsa.allocate(1)) CouponHashSet<A>(*this, tgtHllType);
|
|
197
195
|
}
|
|
198
196
|
|
|
199
197
|
template<typename A>
|
|
200
|
-
HllSketchImpl<A>* CouponHashSet<A>::couponUpdate(
|
|
201
|
-
const uint8_t lgCouponArrInts = count_trailing_zeros_in_u32(this->
|
|
202
|
-
const
|
|
198
|
+
HllSketchImpl<A>* CouponHashSet<A>::couponUpdate(uint32_t coupon) {
|
|
199
|
+
const uint8_t lgCouponArrInts = count_trailing_zeros_in_u32(static_cast<uint32_t>(this->coupons_.size()));
|
|
200
|
+
const int32_t index = find<A>(this->coupons_.data(), lgCouponArrInts, coupon);
|
|
203
201
|
if (index >= 0) {
|
|
204
202
|
return this; // found duplicate, ignore
|
|
205
203
|
}
|
|
206
|
-
this->
|
|
207
|
-
++this->
|
|
204
|
+
this->coupons_[~index] = coupon; // found empty
|
|
205
|
+
++this->couponCount_;
|
|
208
206
|
if (checkGrowOrPromote()) {
|
|
209
207
|
return this->promoteHeapListOrSetToHll(*this);
|
|
210
208
|
}
|
|
@@ -212,20 +210,20 @@ HllSketchImpl<A>* CouponHashSet<A>::couponUpdate(int coupon) {
|
|
|
212
210
|
}
|
|
213
211
|
|
|
214
212
|
template<typename A>
|
|
215
|
-
|
|
216
|
-
return
|
|
213
|
+
uint32_t CouponHashSet<A>::getMemDataStart() const {
|
|
214
|
+
return hll_constants::HASH_SET_INT_ARR_START;
|
|
217
215
|
}
|
|
218
216
|
|
|
219
217
|
template<typename A>
|
|
220
|
-
|
|
221
|
-
return
|
|
218
|
+
uint8_t CouponHashSet<A>::getPreInts() const {
|
|
219
|
+
return hll_constants::HASH_SET_PREINTS;
|
|
222
220
|
}
|
|
223
221
|
|
|
224
222
|
template<typename A>
|
|
225
223
|
bool CouponHashSet<A>::checkGrowOrPromote() {
|
|
226
|
-
if (static_cast<size_t>(
|
|
227
|
-
const uint8_t lgCouponArrInts = count_trailing_zeros_in_u32(this->
|
|
228
|
-
if (lgCouponArrInts == (this->
|
|
224
|
+
if (static_cast<size_t>(hll_constants::RESIZE_DENOM * this->couponCount_) > (hll_constants::RESIZE_NUMER * this->coupons_.size())) {
|
|
225
|
+
const uint8_t lgCouponArrInts = count_trailing_zeros_in_u32(static_cast<uint32_t>(this->coupons_.size()));
|
|
226
|
+
if (lgCouponArrInts == (this->lgConfigK_ - 3)) { // at max size
|
|
229
227
|
return true; // promote to HLL
|
|
230
228
|
}
|
|
231
229
|
growHashSet(lgCouponArrInts + 1);
|
|
@@ -234,15 +232,15 @@ bool CouponHashSet<A>::checkGrowOrPromote() {
|
|
|
234
232
|
}
|
|
235
233
|
|
|
236
234
|
template<typename A>
|
|
237
|
-
void CouponHashSet<A>::growHashSet(
|
|
238
|
-
const
|
|
239
|
-
vector_int coupons_new(tgtLen, 0, this->
|
|
240
|
-
|
|
241
|
-
const
|
|
242
|
-
for (
|
|
243
|
-
const
|
|
244
|
-
if (fetched !=
|
|
245
|
-
const
|
|
235
|
+
void CouponHashSet<A>::growHashSet(uint8_t tgtLgCoupArrSize) {
|
|
236
|
+
const uint32_t tgtLen = 1 << tgtLgCoupArrSize;
|
|
237
|
+
vector_int coupons_new(tgtLen, 0, this->coupons_.get_allocator());
|
|
238
|
+
|
|
239
|
+
const uint32_t srcLen = static_cast<uint32_t>(this->coupons_.size());
|
|
240
|
+
for (uint32_t i = 0; i < srcLen; ++i) { // scan existing array for non-zero values
|
|
241
|
+
const uint32_t fetched = this->coupons_[i];
|
|
242
|
+
if (fetched != hll_constants::EMPTY) {
|
|
243
|
+
const int32_t idx = find<A>(coupons_new.data(), tgtLgCoupArrSize, fetched); // search TGT array
|
|
246
244
|
if (idx < 0) { // found EMPTY
|
|
247
245
|
coupons_new[~idx] = fetched; // insert
|
|
248
246
|
continue;
|
|
@@ -250,23 +248,23 @@ void CouponHashSet<A>::growHashSet(int tgtLgCoupArrSize) {
|
|
|
250
248
|
throw std::runtime_error("Error: Found duplicate coupon");
|
|
251
249
|
}
|
|
252
250
|
}
|
|
253
|
-
this->
|
|
251
|
+
this->coupons_ = std::move(coupons_new);
|
|
254
252
|
}
|
|
255
253
|
|
|
256
254
|
template<typename A>
|
|
257
|
-
static
|
|
258
|
-
const
|
|
259
|
-
|
|
260
|
-
const
|
|
255
|
+
static int32_t find(const uint32_t* array, uint8_t lgArrInts, uint32_t coupon) {
|
|
256
|
+
const uint32_t arrMask = (1 << lgArrInts) - 1;
|
|
257
|
+
uint32_t probe = coupon & arrMask;
|
|
258
|
+
const uint32_t loopIndex = probe;
|
|
261
259
|
do {
|
|
262
|
-
const
|
|
263
|
-
if (couponAtIdx ==
|
|
260
|
+
const uint32_t couponAtIdx = array[probe];
|
|
261
|
+
if (couponAtIdx == hll_constants::EMPTY) {
|
|
264
262
|
return ~probe; //empty
|
|
265
263
|
}
|
|
266
264
|
else if (coupon == couponAtIdx) {
|
|
267
265
|
return probe; //duplicate
|
|
268
266
|
}
|
|
269
|
-
const
|
|
267
|
+
const uint32_t stride = ((coupon & hll_constants::KEY_MASK_26) >> lgArrInts) | 1;
|
|
270
268
|
probe = (probe + stride) & arrMask;
|
|
271
269
|
} while (probe != loopIndex);
|
|
272
270
|
throw std::invalid_argument("Key not found and no empty slots!");
|
|
@@ -29,29 +29,29 @@ class CouponHashSet : public CouponList<A> {
|
|
|
29
29
|
public:
|
|
30
30
|
static CouponHashSet* newSet(const void* bytes, size_t len, const A& allocator);
|
|
31
31
|
static CouponHashSet* newSet(std::istream& is, const A& allocator);
|
|
32
|
-
CouponHashSet(
|
|
32
|
+
CouponHashSet(uint8_t lgConfigK, target_hll_type tgtHllType, const A& allocator);
|
|
33
33
|
CouponHashSet(const CouponHashSet& that, target_hll_type tgtHllType);
|
|
34
34
|
|
|
35
35
|
virtual ~CouponHashSet() = default;
|
|
36
36
|
virtual std::function<void(HllSketchImpl<A>*)> get_deleter() const;
|
|
37
37
|
|
|
38
38
|
protected:
|
|
39
|
-
using vector_int = std::vector<
|
|
39
|
+
using vector_int = std::vector<uint32_t, typename std::allocator_traits<A>::template rebind_alloc<uint32_t>>;
|
|
40
40
|
|
|
41
41
|
virtual CouponHashSet* copy() const;
|
|
42
42
|
virtual CouponHashSet* copyAs(target_hll_type tgtHllType) const;
|
|
43
43
|
|
|
44
|
-
virtual HllSketchImpl<A>* couponUpdate(
|
|
44
|
+
virtual HllSketchImpl<A>* couponUpdate(uint32_t coupon);
|
|
45
45
|
|
|
46
|
-
virtual
|
|
47
|
-
virtual
|
|
46
|
+
virtual uint32_t getMemDataStart() const;
|
|
47
|
+
virtual uint8_t getPreInts() const;
|
|
48
48
|
|
|
49
49
|
friend class HllSketchImplFactory<A>;
|
|
50
50
|
|
|
51
51
|
private:
|
|
52
52
|
using ChsAlloc = typename std::allocator_traits<A>::template rebind_alloc<CouponHashSet<A>>;
|
|
53
53
|
bool checkGrowOrPromote();
|
|
54
|
-
void growHashSet(
|
|
54
|
+
void growHashSet(uint8_t tgtLgCoupArrSize);
|
|
55
55
|
};
|
|
56
56
|
|
|
57
57
|
}
|
|
@@ -31,19 +31,19 @@
|
|
|
31
31
|
namespace datasketches {
|
|
32
32
|
|
|
33
33
|
template<typename A>
|
|
34
|
-
CouponList<A>::CouponList(
|
|
34
|
+
CouponList<A>::CouponList(uint8_t lgConfigK, target_hll_type tgtHllType, hll_mode mode, const A& allocator):
|
|
35
35
|
HllSketchImpl<A>(lgConfigK, tgtHllType, mode, false),
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
couponCount_(0),
|
|
37
|
+
oooFlag_(false),
|
|
38
|
+
coupons_(1ULL << (mode == hll_mode::LIST ? hll_constants::LG_INIT_LIST_SIZE : hll_constants::LG_INIT_SET_SIZE), 0, allocator)
|
|
39
39
|
{}
|
|
40
40
|
|
|
41
41
|
template<typename A>
|
|
42
42
|
CouponList<A>::CouponList(const CouponList& that, const target_hll_type tgtHllType):
|
|
43
|
-
HllSketchImpl<A>(that.
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
43
|
+
HllSketchImpl<A>(that.lgConfigK_, tgtHllType, that.mode_, false),
|
|
44
|
+
couponCount_(that.couponCount_),
|
|
45
|
+
oooFlag_(that.oooFlag_),
|
|
46
|
+
coupons_(that.coupons_)
|
|
47
47
|
{}
|
|
48
48
|
|
|
49
49
|
template<typename A>
|
|
@@ -58,48 +58,48 @@ std::function<void(HllSketchImpl<A>*)> CouponList<A>::get_deleter() const {
|
|
|
58
58
|
|
|
59
59
|
template<typename A>
|
|
60
60
|
CouponList<A>* CouponList<A>::copy() const {
|
|
61
|
-
ClAlloc cla(
|
|
61
|
+
ClAlloc cla(coupons_.get_allocator());
|
|
62
62
|
return new (cla.allocate(1)) CouponList<A>(*this);
|
|
63
63
|
}
|
|
64
64
|
|
|
65
65
|
template<typename A>
|
|
66
66
|
CouponList<A>* CouponList<A>::copyAs(target_hll_type tgtHllType) const {
|
|
67
|
-
ClAlloc cla(
|
|
67
|
+
ClAlloc cla(coupons_.get_allocator());
|
|
68
68
|
return new (cla.allocate(1)) CouponList<A>(*this, tgtHllType);
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
template<typename A>
|
|
72
72
|
CouponList<A>* CouponList<A>::newList(const void* bytes, size_t len, const A& allocator) {
|
|
73
|
-
if (len <
|
|
73
|
+
if (len < hll_constants::LIST_INT_ARR_START) {
|
|
74
74
|
throw std::out_of_range("Input data length insufficient to hold CouponHashSet");
|
|
75
75
|
}
|
|
76
76
|
|
|
77
77
|
const uint8_t* data = static_cast<const uint8_t*>(bytes);
|
|
78
|
-
if (data[
|
|
78
|
+
if (data[hll_constants::PREAMBLE_INTS_BYTE] != hll_constants::LIST_PREINTS) {
|
|
79
79
|
throw std::invalid_argument("Incorrect number of preInts in input stream");
|
|
80
80
|
}
|
|
81
|
-
if (data[
|
|
81
|
+
if (data[hll_constants::SER_VER_BYTE] != hll_constants::SER_VER) {
|
|
82
82
|
throw std::invalid_argument("Wrong ser ver in input stream");
|
|
83
83
|
}
|
|
84
|
-
if (data[
|
|
84
|
+
if (data[hll_constants::FAMILY_BYTE] != hll_constants::FAMILY_ID) {
|
|
85
85
|
throw std::invalid_argument("Input stream is not an HLL sketch");
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
-
hll_mode mode = HllSketchImpl<A>::extractCurMode(data[
|
|
88
|
+
hll_mode mode = HllSketchImpl<A>::extractCurMode(data[hll_constants::MODE_BYTE]);
|
|
89
89
|
if (mode != LIST) {
|
|
90
90
|
throw std::invalid_argument("Calling list constructor with non-list mode data");
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(data[
|
|
93
|
+
target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(data[hll_constants::MODE_BYTE]);
|
|
94
94
|
|
|
95
|
-
const
|
|
96
|
-
const bool compact = ((data[
|
|
97
|
-
const bool oooFlag = ((data[
|
|
98
|
-
const bool emptyFlag = ((data[
|
|
95
|
+
const uint8_t lgK = data[hll_constants::LG_K_BYTE];
|
|
96
|
+
const bool compact = ((data[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
|
|
97
|
+
const bool oooFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::OUT_OF_ORDER_FLAG_MASK) ? true : false);
|
|
98
|
+
const bool emptyFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::EMPTY_FLAG_MASK) ? true : false);
|
|
99
99
|
|
|
100
|
-
const
|
|
101
|
-
const
|
|
102
|
-
const size_t expectedLength =
|
|
100
|
+
const uint32_t couponCount = data[hll_constants::LIST_COUNT_BYTE];
|
|
101
|
+
const uint32_t couponsInArray = (compact ? couponCount : (1 << HllUtil<A>::computeLgArrInts(LIST, couponCount, lgK)));
|
|
102
|
+
const size_t expectedLength = hll_constants::LIST_INT_ARR_START + (couponsInArray * sizeof(uint32_t));
|
|
103
103
|
if (len < expectedLength) {
|
|
104
104
|
throw std::out_of_range("Byte array too short for sketch. Expected " + std::to_string(expectedLength)
|
|
105
105
|
+ ", found: " + std::to_string(len));
|
|
@@ -107,12 +107,12 @@ CouponList<A>* CouponList<A>::newList(const void* bytes, size_t len, const A& al
|
|
|
107
107
|
|
|
108
108
|
ClAlloc cla(allocator);
|
|
109
109
|
CouponList<A>* sketch = new (cla.allocate(1)) CouponList<A>(lgK, tgtHllType, mode, allocator);
|
|
110
|
-
sketch->
|
|
110
|
+
sketch->couponCount_ = couponCount;
|
|
111
111
|
sketch->putOutOfOrderFlag(oooFlag); // should always be false for LIST
|
|
112
112
|
|
|
113
113
|
if (!emptyFlag) {
|
|
114
114
|
// only need to read valid coupons, unlike in stream case
|
|
115
|
-
std::memcpy(sketch->
|
|
115
|
+
std::memcpy(sketch->coupons_.data(), data + hll_constants::LIST_INT_ARR_START, couponCount * sizeof(uint32_t));
|
|
116
116
|
}
|
|
117
117
|
|
|
118
118
|
return sketch;
|
|
@@ -121,44 +121,44 @@ CouponList<A>* CouponList<A>::newList(const void* bytes, size_t len, const A& al
|
|
|
121
121
|
template<typename A>
|
|
122
122
|
CouponList<A>* CouponList<A>::newList(std::istream& is, const A& allocator) {
|
|
123
123
|
uint8_t listHeader[8];
|
|
124
|
-
|
|
124
|
+
read(is, listHeader, 8 * sizeof(uint8_t));
|
|
125
125
|
|
|
126
|
-
if (listHeader[
|
|
126
|
+
if (listHeader[hll_constants::PREAMBLE_INTS_BYTE] != hll_constants::LIST_PREINTS) {
|
|
127
127
|
throw std::invalid_argument("Incorrect number of preInts in input stream");
|
|
128
128
|
}
|
|
129
|
-
if (listHeader[
|
|
129
|
+
if (listHeader[hll_constants::SER_VER_BYTE] != hll_constants::SER_VER) {
|
|
130
130
|
throw std::invalid_argument("Wrong ser ver in input stream");
|
|
131
131
|
}
|
|
132
|
-
if (listHeader[
|
|
132
|
+
if (listHeader[hll_constants::FAMILY_BYTE] != hll_constants::FAMILY_ID) {
|
|
133
133
|
throw std::invalid_argument("Input stream is not an HLL sketch");
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
-
hll_mode mode = HllSketchImpl<A>::extractCurMode(listHeader[
|
|
136
|
+
hll_mode mode = HllSketchImpl<A>::extractCurMode(listHeader[hll_constants::MODE_BYTE]);
|
|
137
137
|
if (mode != LIST) {
|
|
138
138
|
throw std::invalid_argument("Calling list constructor with non-list mode data");
|
|
139
139
|
}
|
|
140
140
|
|
|
141
|
-
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(listHeader[
|
|
141
|
+
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(listHeader[hll_constants::MODE_BYTE]);
|
|
142
142
|
|
|
143
|
-
const
|
|
144
|
-
const bool compact = ((listHeader[
|
|
145
|
-
const bool oooFlag = ((listHeader[
|
|
146
|
-
const bool emptyFlag = ((listHeader[
|
|
143
|
+
const uint8_t lgK = listHeader[hll_constants::LG_K_BYTE];
|
|
144
|
+
const bool compact = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
|
|
145
|
+
const bool oooFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::OUT_OF_ORDER_FLAG_MASK) ? true : false);
|
|
146
|
+
const bool emptyFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::EMPTY_FLAG_MASK) ? true : false);
|
|
147
147
|
|
|
148
148
|
ClAlloc cla(allocator);
|
|
149
149
|
CouponList<A>* sketch = new (cla.allocate(1)) CouponList<A>(lgK, tgtHllType, mode, allocator);
|
|
150
150
|
using coupon_list_ptr = std::unique_ptr<CouponList<A>, std::function<void(HllSketchImpl<A>*)>>;
|
|
151
151
|
coupon_list_ptr ptr(sketch, sketch->get_deleter());
|
|
152
|
-
const
|
|
153
|
-
sketch->
|
|
152
|
+
const uint32_t couponCount = listHeader[hll_constants::LIST_COUNT_BYTE];
|
|
153
|
+
sketch->couponCount_ = couponCount;
|
|
154
154
|
sketch->putOutOfOrderFlag(oooFlag); // should always be false for LIST
|
|
155
155
|
|
|
156
156
|
if (!emptyFlag) {
|
|
157
157
|
// For stream processing, need to read entire number written to stream so read
|
|
158
158
|
// pointer ends up set correctly.
|
|
159
159
|
// If not compact, still need to read empty items even though in order.
|
|
160
|
-
const
|
|
161
|
-
|
|
160
|
+
const uint32_t numToRead = (compact ? couponCount : static_cast<uint32_t>(sketch->coupons_.size()));
|
|
161
|
+
read(is, sketch->coupons_.data(), numToRead * sizeof(uint32_t));
|
|
162
162
|
}
|
|
163
163
|
|
|
164
164
|
if (!is.good())
|
|
@@ -173,17 +173,17 @@ vector_u8<A> CouponList<A>::serialize(bool compact, unsigned header_size_bytes)
|
|
|
173
173
|
vector_u8<A> byteArr(sketchSizeBytes, 0, getAllocator());
|
|
174
174
|
uint8_t* bytes = byteArr.data() + header_size_bytes;
|
|
175
175
|
|
|
176
|
-
bytes[
|
|
177
|
-
bytes[
|
|
178
|
-
bytes[
|
|
179
|
-
bytes[
|
|
180
|
-
bytes[
|
|
181
|
-
bytes[
|
|
182
|
-
bytes[
|
|
183
|
-
bytes[
|
|
184
|
-
|
|
185
|
-
if (this->
|
|
186
|
-
std::memcpy(bytes +
|
|
176
|
+
bytes[hll_constants::PREAMBLE_INTS_BYTE] = static_cast<uint8_t>(getPreInts());
|
|
177
|
+
bytes[hll_constants::SER_VER_BYTE] = static_cast<uint8_t>(hll_constants::SER_VER);
|
|
178
|
+
bytes[hll_constants::FAMILY_BYTE] = static_cast<uint8_t>(hll_constants::FAMILY_ID);
|
|
179
|
+
bytes[hll_constants::LG_K_BYTE] = static_cast<uint8_t>(this->lgConfigK_);
|
|
180
|
+
bytes[hll_constants::LG_ARR_BYTE] = count_trailing_zeros_in_u32(static_cast<uint32_t>(coupons_.size()));
|
|
181
|
+
bytes[hll_constants::FLAGS_BYTE] = this->makeFlagsByte(compact);
|
|
182
|
+
bytes[hll_constants::LIST_COUNT_BYTE] = static_cast<uint8_t>(this->mode_ == LIST ? couponCount_ : 0);
|
|
183
|
+
bytes[hll_constants::MODE_BYTE] = this->makeModeByte();
|
|
184
|
+
|
|
185
|
+
if (this->mode_ == SET) {
|
|
186
|
+
std::memcpy(bytes + hll_constants::HASH_SET_COUNT_INT, &couponCount_, sizeof(couponCount_));
|
|
187
187
|
}
|
|
188
188
|
|
|
189
189
|
// coupons
|
|
@@ -191,12 +191,12 @@ vector_u8<A> CouponList<A>::serialize(bool compact, unsigned header_size_bytes)
|
|
|
191
191
|
const int sw = (isCompact() ? 2 : 0) | (compact ? 1 : 0);
|
|
192
192
|
switch (sw) {
|
|
193
193
|
case 0: { // src updatable, dst updatable
|
|
194
|
-
std::memcpy(bytes + getMemDataStart(),
|
|
194
|
+
std::memcpy(bytes + getMemDataStart(), coupons_.data(), coupons_.size() * sizeof(uint32_t));
|
|
195
195
|
break;
|
|
196
196
|
}
|
|
197
197
|
case 1: { // src updatable, dst compact
|
|
198
198
|
bytes += getMemDataStart(); // reusing pointer for incremental writes
|
|
199
|
-
for (uint32_t coupon: *this) {
|
|
199
|
+
for (const uint32_t coupon: *this) {
|
|
200
200
|
std::memcpy(bytes, &coupon, sizeof(coupon));
|
|
201
201
|
bytes += sizeof(coupon);
|
|
202
202
|
}
|
|
@@ -213,33 +213,33 @@ vector_u8<A> CouponList<A>::serialize(bool compact, unsigned header_size_bytes)
|
|
|
213
213
|
template<typename A>
|
|
214
214
|
void CouponList<A>::serialize(std::ostream& os, const bool compact) const {
|
|
215
215
|
// header
|
|
216
|
-
const uint8_t preInts
|
|
217
|
-
|
|
218
|
-
const uint8_t serialVersion(
|
|
219
|
-
|
|
220
|
-
const uint8_t familyId(
|
|
221
|
-
|
|
222
|
-
const uint8_t lgKByte
|
|
223
|
-
|
|
224
|
-
const uint8_t lgArrIntsByte
|
|
225
|
-
|
|
226
|
-
const uint8_t flagsByte
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
if (this->
|
|
230
|
-
const uint8_t listCount(
|
|
231
|
-
|
|
216
|
+
const uint8_t preInts = getPreInts();
|
|
217
|
+
write(os, preInts);
|
|
218
|
+
const uint8_t serialVersion(hll_constants::SER_VER);
|
|
219
|
+
write(os, serialVersion);
|
|
220
|
+
const uint8_t familyId(hll_constants::FAMILY_ID);
|
|
221
|
+
write(os, familyId);
|
|
222
|
+
const uint8_t lgKByte = this->lgConfigK_;
|
|
223
|
+
write(os, lgKByte);
|
|
224
|
+
const uint8_t lgArrIntsByte = count_trailing_zeros_in_u32(static_cast<uint32_t>(coupons_.size()));
|
|
225
|
+
write(os, lgArrIntsByte);
|
|
226
|
+
const uint8_t flagsByte = this->makeFlagsByte(compact);
|
|
227
|
+
write(os, flagsByte);
|
|
228
|
+
|
|
229
|
+
if (this->mode_ == LIST) {
|
|
230
|
+
const uint8_t listCount = static_cast<uint8_t>(couponCount_);
|
|
231
|
+
write(os, listCount);
|
|
232
232
|
} else { // mode == SET
|
|
233
|
-
|
|
234
|
-
|
|
233
|
+
const uint8_t unused = 0;
|
|
234
|
+
write(os, unused);
|
|
235
235
|
}
|
|
236
236
|
|
|
237
|
-
const uint8_t modeByte
|
|
238
|
-
|
|
237
|
+
const uint8_t modeByte = this->makeModeByte();
|
|
238
|
+
write(os, modeByte);
|
|
239
239
|
|
|
240
|
-
if (this->
|
|
240
|
+
if (this->mode_ == SET) {
|
|
241
241
|
// writing as int, already stored as int
|
|
242
|
-
|
|
242
|
+
write(os, couponCount_);
|
|
243
243
|
}
|
|
244
244
|
|
|
245
245
|
// coupons
|
|
@@ -247,12 +247,12 @@ void CouponList<A>::serialize(std::ostream& os, const bool compact) const {
|
|
|
247
247
|
const int sw = (isCompact() ? 2 : 0) | (compact ? 1 : 0);
|
|
248
248
|
switch (sw) {
|
|
249
249
|
case 0: { // src updatable, dst updatable
|
|
250
|
-
|
|
250
|
+
write(os, coupons_.data(), coupons_.size() * sizeof(uint32_t));
|
|
251
251
|
break;
|
|
252
252
|
}
|
|
253
253
|
case 1: { // src updatable, dst compact
|
|
254
|
-
for (uint32_t coupon: *this) {
|
|
255
|
-
|
|
254
|
+
for (const uint32_t coupon: *this) {
|
|
255
|
+
write(os, coupon);
|
|
256
256
|
}
|
|
257
257
|
break;
|
|
258
258
|
}
|
|
@@ -265,14 +265,14 @@ void CouponList<A>::serialize(std::ostream& os, const bool compact) const {
|
|
|
265
265
|
}
|
|
266
266
|
|
|
267
267
|
template<typename A>
|
|
268
|
-
HllSketchImpl<A>* CouponList<A>::couponUpdate(
|
|
269
|
-
for (size_t i = 0; i <
|
|
270
|
-
const
|
|
271
|
-
if (couponAtIdx ==
|
|
272
|
-
|
|
273
|
-
++
|
|
274
|
-
if (
|
|
275
|
-
if (this->
|
|
268
|
+
HllSketchImpl<A>* CouponList<A>::couponUpdate(uint32_t coupon) {
|
|
269
|
+
for (size_t i = 0; i < coupons_.size(); ++i) { // search for empty slot
|
|
270
|
+
const uint32_t couponAtIdx = coupons_[i];
|
|
271
|
+
if (couponAtIdx == hll_constants::EMPTY) {
|
|
272
|
+
coupons_[i] = coupon; // the actual update
|
|
273
|
+
++couponCount_;
|
|
274
|
+
if (couponCount_ == static_cast<uint32_t>(coupons_.size())) { // array full
|
|
275
|
+
if (this->lgConfigK_ < 8) {
|
|
276
276
|
return promoteHeapListOrSetToHll(*this);
|
|
277
277
|
}
|
|
278
278
|
return promoteHeapListToSet(*this);
|
|
@@ -293,71 +293,68 @@ double CouponList<A>::getCompositeEstimate() const { return getEstimate(); }
|
|
|
293
293
|
|
|
294
294
|
template<typename A>
|
|
295
295
|
double CouponList<A>::getEstimate() const {
|
|
296
|
-
const
|
|
297
|
-
|
|
298
|
-
return fmax(est, couponCount);
|
|
296
|
+
const double est = CubicInterpolation<A>::usingXAndYTables(couponCount_);
|
|
297
|
+
return fmax(est, couponCount_);
|
|
299
298
|
}
|
|
300
299
|
|
|
301
300
|
template<typename A>
|
|
302
|
-
double CouponList<A>::getLowerBound(
|
|
301
|
+
double CouponList<A>::getLowerBound(uint8_t numStdDev) const {
|
|
303
302
|
HllUtil<A>::checkNumStdDev(numStdDev);
|
|
304
|
-
const
|
|
305
|
-
const double
|
|
306
|
-
|
|
307
|
-
return fmax(tmp, couponCount);
|
|
303
|
+
const double est = CubicInterpolation<A>::usingXAndYTables(couponCount_);
|
|
304
|
+
const double tmp = est / (1.0 + (numStdDev * hll_constants::COUPON_RSE));
|
|
305
|
+
return fmax(tmp, couponCount_);
|
|
308
306
|
}
|
|
309
307
|
|
|
310
308
|
template<typename A>
|
|
311
|
-
double CouponList<A>::getUpperBound(
|
|
309
|
+
double CouponList<A>::getUpperBound(uint8_t numStdDev) const {
|
|
312
310
|
HllUtil<A>::checkNumStdDev(numStdDev);
|
|
313
|
-
const
|
|
314
|
-
const double
|
|
315
|
-
|
|
316
|
-
return fmax(tmp, couponCount);
|
|
311
|
+
const double est = CubicInterpolation<A>::usingXAndYTables(couponCount_);
|
|
312
|
+
const double tmp = est / (1.0 - (numStdDev * hll_constants::COUPON_RSE));
|
|
313
|
+
return fmax(tmp, couponCount_);
|
|
317
314
|
}
|
|
318
315
|
|
|
319
316
|
template<typename A>
|
|
320
317
|
bool CouponList<A>::isEmpty() const { return getCouponCount() == 0; }
|
|
321
318
|
|
|
322
319
|
template<typename A>
|
|
323
|
-
|
|
324
|
-
return getMemDataStart() +
|
|
320
|
+
uint32_t CouponList<A>::getUpdatableSerializationBytes() const {
|
|
321
|
+
return getMemDataStart() + static_cast<uint32_t>(coupons_.size()) * sizeof(uint32_t);
|
|
325
322
|
}
|
|
326
323
|
|
|
327
324
|
template<typename A>
|
|
328
|
-
|
|
329
|
-
return
|
|
325
|
+
uint32_t CouponList<A>::getCouponCount() const {
|
|
326
|
+
return couponCount_;
|
|
330
327
|
}
|
|
331
328
|
|
|
332
329
|
template<typename A>
|
|
333
|
-
|
|
334
|
-
return getMemDataStart() + (
|
|
330
|
+
uint32_t CouponList<A>::getCompactSerializationBytes() const {
|
|
331
|
+
return getMemDataStart() + (couponCount_ << 2);
|
|
335
332
|
}
|
|
336
333
|
|
|
337
334
|
template<typename A>
|
|
338
|
-
|
|
339
|
-
return
|
|
335
|
+
uint32_t CouponList<A>::getMemDataStart() const {
|
|
336
|
+
return hll_constants::LIST_INT_ARR_START;
|
|
340
337
|
}
|
|
341
338
|
|
|
342
339
|
template<typename A>
|
|
343
|
-
|
|
344
|
-
return
|
|
340
|
+
uint8_t CouponList<A>::getPreInts() const {
|
|
341
|
+
return hll_constants::LIST_PREINTS;
|
|
345
342
|
}
|
|
346
343
|
|
|
347
344
|
template<typename A>
|
|
348
345
|
bool CouponList<A>::isCompact() const { return false; }
|
|
349
346
|
|
|
350
347
|
template<typename A>
|
|
351
|
-
bool CouponList<A>::isOutOfOrderFlag() const { return
|
|
348
|
+
bool CouponList<A>::isOutOfOrderFlag() const { return oooFlag_; }
|
|
352
349
|
|
|
353
350
|
template<typename A>
|
|
354
351
|
void CouponList<A>::putOutOfOrderFlag(bool oooFlag) {
|
|
355
|
-
|
|
352
|
+
oooFlag_ = oooFlag;
|
|
356
353
|
}
|
|
357
354
|
|
|
358
355
|
template<typename A>
|
|
359
356
|
A CouponList<A>::getAllocator() const {
|
|
360
|
-
return
|
|
357
|
+
return coupons_.get_allocator();
|
|
361
358
|
}
|
|
362
359
|
|
|
363
360
|
template<typename A>
|
|
@@ -372,12 +369,12 @@ HllSketchImpl<A>* CouponList<A>::promoteHeapListOrSetToHll(CouponList& src) {
|
|
|
372
369
|
|
|
373
370
|
template<typename A>
|
|
374
371
|
coupon_iterator<A> CouponList<A>::begin(bool all) const {
|
|
375
|
-
return coupon_iterator<A>(
|
|
372
|
+
return coupon_iterator<A>(coupons_.data(), coupons_.size(), 0, all);
|
|
376
373
|
}
|
|
377
374
|
|
|
378
375
|
template<typename A>
|
|
379
376
|
coupon_iterator<A> CouponList<A>::end() const {
|
|
380
|
-
return coupon_iterator<A>(
|
|
377
|
+
return coupon_iterator<A>(coupons_.data(), coupons_.size(), coupons_.size(), false);
|
|
381
378
|
}
|
|
382
379
|
|
|
383
380
|
}
|