datasketches 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/datasketches/version.rb +1 -1
- data/vendor/datasketches-cpp/CMakeLists.txt +7 -0
- data/vendor/datasketches-cpp/common/include/MurmurHash3.h +11 -7
- data/vendor/datasketches-cpp/common/include/binomial_bounds.hpp +8 -8
- data/vendor/datasketches-cpp/common/include/bounds_binomial_proportions.hpp +12 -15
- data/vendor/datasketches-cpp/common/include/common_defs.hpp +24 -0
- data/vendor/datasketches-cpp/common/include/conditional_forward.hpp +20 -8
- data/vendor/datasketches-cpp/common/include/count_zeros.hpp +2 -2
- data/vendor/datasketches-cpp/common/include/serde.hpp +7 -7
- data/vendor/datasketches-cpp/cpc/include/cpc_compressor.hpp +19 -19
- data/vendor/datasketches-cpp/cpc/include/cpc_compressor_impl.hpp +91 -89
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch.hpp +14 -1
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch_impl.hpp +121 -87
- data/vendor/datasketches-cpp/cpc/include/cpc_union_impl.hpp +14 -14
- data/vendor/datasketches-cpp/cpc/include/cpc_util.hpp +10 -10
- data/vendor/datasketches-cpp/cpc/include/icon_estimator.hpp +4 -4
- data/vendor/datasketches-cpp/cpc/include/u32_table.hpp +8 -8
- data/vendor/datasketches-cpp/cpc/include/u32_table_impl.hpp +14 -14
- data/vendor/datasketches-cpp/cpc/test/compression_test.cpp +10 -10
- data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp +25 -0
- data/vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp +1 -1
- data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp +65 -80
- data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map_impl.hpp +10 -10
- data/vendor/datasketches-cpp/fi/test/reverse_purge_hash_map_test.cpp +2 -2
- data/vendor/datasketches-cpp/hll/include/AuxHashMap-internal.hpp +60 -63
- data/vendor/datasketches-cpp/hll/include/AuxHashMap.hpp +19 -19
- data/vendor/datasketches-cpp/hll/include/CompositeInterpolationXTable-internal.hpp +15 -15
- data/vendor/datasketches-cpp/hll/include/CompositeInterpolationXTable.hpp +3 -3
- data/vendor/datasketches-cpp/hll/include/CouponHashSet-internal.hpp +74 -76
- data/vendor/datasketches-cpp/hll/include/CouponHashSet.hpp +6 -6
- data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp +110 -113
- data/vendor/datasketches-cpp/hll/include/CouponList.hpp +13 -13
- data/vendor/datasketches-cpp/hll/include/CubicInterpolation-internal.hpp +2 -4
- data/vendor/datasketches-cpp/hll/include/HarmonicNumbers-internal.hpp +1 -1
- data/vendor/datasketches-cpp/hll/include/Hll4Array-internal.hpp +80 -76
- data/vendor/datasketches-cpp/hll/include/Hll4Array.hpp +9 -9
- data/vendor/datasketches-cpp/hll/include/Hll6Array-internal.hpp +26 -26
- data/vendor/datasketches-cpp/hll/include/Hll6Array.hpp +6 -6
- data/vendor/datasketches-cpp/hll/include/Hll8Array-internal.hpp +33 -33
- data/vendor/datasketches-cpp/hll/include/Hll8Array.hpp +6 -6
- data/vendor/datasketches-cpp/hll/include/HllArray-internal.hpp +205 -209
- data/vendor/datasketches-cpp/hll/include/HllArray.hpp +36 -36
- data/vendor/datasketches-cpp/hll/include/HllSketch-internal.hpp +28 -28
- data/vendor/datasketches-cpp/hll/include/HllSketchImpl-internal.hpp +22 -22
- data/vendor/datasketches-cpp/hll/include/HllSketchImpl.hpp +13 -13
- data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp +15 -15
- data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +61 -61
- data/vendor/datasketches-cpp/hll/include/HllUtil.hpp +120 -127
- data/vendor/datasketches-cpp/hll/include/coupon_iterator-internal.hpp +9 -9
- data/vendor/datasketches-cpp/hll/include/coupon_iterator.hpp +5 -5
- data/vendor/datasketches-cpp/hll/include/hll.hpp +21 -21
- data/vendor/datasketches-cpp/hll/test/AuxHashMapTest.cpp +1 -1
- data/vendor/datasketches-cpp/hll/test/CouponHashSetTest.cpp +34 -34
- data/vendor/datasketches-cpp/hll/test/CouponListTest.cpp +25 -25
- data/vendor/datasketches-cpp/hll/test/CrossCountingTest.cpp +2 -2
- data/vendor/datasketches-cpp/hll/test/HllArrayTest.cpp +35 -35
- data/vendor/datasketches-cpp/hll/test/HllSketchTest.cpp +15 -15
- data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +10 -14
- data/vendor/datasketches-cpp/hll/test/IsomorphicTest.cpp +3 -3
- data/vendor/datasketches-cpp/hll/test/ToFromByteArrayTest.cpp +4 -4
- data/vendor/datasketches-cpp/kll/include/kll_helper.hpp +5 -4
- data/vendor/datasketches-cpp/kll/include/kll_helper_impl.hpp +6 -6
- data/vendor/datasketches-cpp/kll/include/kll_quantile_calculator.hpp +14 -6
- data/vendor/datasketches-cpp/kll/include/kll_quantile_calculator_impl.hpp +39 -24
- data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +34 -2
- data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +72 -62
- data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov.hpp +67 -0
- data/vendor/datasketches-cpp/kll/include/kolmogorov_smirnov_impl.hpp +78 -0
- data/vendor/datasketches-cpp/kll/test/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +68 -45
- data/vendor/datasketches-cpp/kll/test/kolmogorov_smirnov_test.cpp +111 -0
- data/vendor/datasketches-cpp/python/src/hll_wrapper.cpp +4 -4
- data/vendor/datasketches-cpp/python/src/theta_wrapper.cpp +6 -6
- data/vendor/datasketches-cpp/python/src/vo_wrapper.cpp +2 -2
- data/vendor/datasketches-cpp/python/tests/hll_test.py +1 -1
- data/vendor/datasketches-cpp/python/tests/vo_test.py +3 -3
- data/vendor/datasketches-cpp/req/include/req_common.hpp +2 -1
- data/vendor/datasketches-cpp/req/include/req_compactor.hpp +4 -4
- data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +26 -39
- data/vendor/datasketches-cpp/req/include/req_sketch.hpp +1 -1
- data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +9 -9
- data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +52 -52
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp +47 -56
- data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp +34 -42
- data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp +6 -6
- data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp +13 -13
- data/vendor/datasketches-cpp/setup.py +1 -1
- data/vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_sampled_sets.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser.hpp +67 -0
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser_impl.hpp +70 -0
- data/vendor/datasketches-cpp/theta/include/theta_intersection.hpp +9 -4
- data/vendor/datasketches-cpp/theta/include/theta_intersection_impl.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/theta_jaccard_similarity_base.hpp +18 -14
- data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp +42 -1
- data/vendor/datasketches-cpp/theta/include/theta_sketch_impl.hpp +107 -58
- data/vendor/datasketches-cpp/theta/include/theta_union.hpp +4 -4
- data/vendor/datasketches-cpp/theta/include/theta_union_base_impl.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/theta_union_impl.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp +2 -0
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp +33 -28
- data/vendor/datasketches-cpp/theta/test/theta_a_not_b_test.cpp +23 -1
- data/vendor/datasketches-cpp/theta/test/theta_intersection_test.cpp +21 -1
- data/vendor/datasketches-cpp/theta/test/theta_jaccard_similarity_test.cpp +58 -2
- data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp +37 -1
- data/vendor/datasketches-cpp/theta/test/theta_union_test.cpp +22 -2
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch_impl.hpp +47 -60
- data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp +51 -64
- data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +1 -1
- data/vendor/datasketches-cpp/tuple/test/tuple_a_not_b_test.cpp +17 -17
- data/vendor/datasketches-cpp/tuple/test/tuple_intersection_test.cpp +12 -12
- data/vendor/datasketches-cpp/tuple/test/tuple_jaccard_similarity_test.cpp +5 -5
- data/vendor/datasketches-cpp/tuple/test/tuple_sketch_allocation_test.cpp +1 -1
- data/vendor/datasketches-cpp/tuple/test/tuple_sketch_test.cpp +20 -20
- data/vendor/datasketches-cpp/tuple/test/tuple_union_test.cpp +12 -12
- metadata +8 -3
@@ -27,10 +27,10 @@ namespace datasketches {
|
|
27
27
|
template<typename A = std::allocator<uint8_t>>
|
28
28
|
class CompositeInterpolationXTable {
|
29
29
|
public:
|
30
|
-
static
|
30
|
+
static uint32_t get_y_stride(uint8_t logK);
|
31
31
|
|
32
|
-
static const double* get_x_arr(
|
33
|
-
static
|
32
|
+
static const double* get_x_arr(uint8_t logK);
|
33
|
+
static uint32_t get_x_arr_length();
|
34
34
|
};
|
35
35
|
|
36
36
|
}
|
@@ -28,10 +28,10 @@
|
|
28
28
|
namespace datasketches {
|
29
29
|
|
30
30
|
template<typename A>
|
31
|
-
static
|
31
|
+
static int32_t find(const uint32_t* array, uint8_t lgArrInts, uint32_t coupon);
|
32
32
|
|
33
33
|
template<typename A>
|
34
|
-
CouponHashSet<A>::CouponHashSet(
|
34
|
+
CouponHashSet<A>::CouponHashSet(uint8_t lgConfigK, target_hll_type tgtHllType, const A& allocator)
|
35
35
|
: CouponList<A>(lgConfigK, tgtHllType, hll_mode::SET, allocator)
|
36
36
|
{
|
37
37
|
if (lgConfigK <= 7) {
|
@@ -56,45 +56,45 @@ std::function<void(HllSketchImpl<A>*)> CouponHashSet<A>::get_deleter() const {
|
|
56
56
|
|
57
57
|
template<typename A>
|
58
58
|
CouponHashSet<A>* CouponHashSet<A>::newSet(const void* bytes, size_t len, const A& allocator) {
|
59
|
-
if (len <
|
59
|
+
if (len < hll_constants::HASH_SET_INT_ARR_START) { // hard-coded
|
60
60
|
throw std::out_of_range("Input data length insufficient to hold CouponHashSet");
|
61
61
|
}
|
62
62
|
|
63
63
|
const uint8_t* data = static_cast<const uint8_t*>(bytes);
|
64
|
-
if (data[
|
64
|
+
if (data[hll_constants::PREAMBLE_INTS_BYTE] != hll_constants::HASH_SET_PREINTS) {
|
65
65
|
throw std::invalid_argument("Incorrect number of preInts in input stream");
|
66
66
|
}
|
67
|
-
if (data[
|
67
|
+
if (data[hll_constants::SER_VER_BYTE] != hll_constants::SER_VER) {
|
68
68
|
throw std::invalid_argument("Wrong ser ver in input stream");
|
69
69
|
}
|
70
|
-
if (data[
|
70
|
+
if (data[hll_constants::FAMILY_BYTE] != hll_constants::FAMILY_ID) {
|
71
71
|
throw std::invalid_argument("Input stream is not an HLL sketch");
|
72
72
|
}
|
73
73
|
|
74
|
-
const hll_mode mode = HllSketchImpl<A>::extractCurMode(data[
|
74
|
+
const hll_mode mode = HllSketchImpl<A>::extractCurMode(data[hll_constants::MODE_BYTE]);
|
75
75
|
if (mode != SET) {
|
76
76
|
throw std::invalid_argument("Calling set constructor with non-set mode data");
|
77
77
|
}
|
78
78
|
|
79
|
-
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(data[
|
79
|
+
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(data[hll_constants::MODE_BYTE]);
|
80
80
|
|
81
|
-
const
|
81
|
+
const uint8_t lgK = data[hll_constants::LG_K_BYTE];
|
82
82
|
if (lgK <= 7) {
|
83
83
|
throw std::invalid_argument("Attempt to deserialize invalid CouponHashSet with lgConfigK <= 7. Found: "
|
84
84
|
+ std::to_string(lgK));
|
85
85
|
}
|
86
|
-
|
87
|
-
const bool compactFlag = ((data[
|
86
|
+
uint8_t lgArrInts = data[hll_constants::LG_ARR_BYTE];
|
87
|
+
const bool compactFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
|
88
88
|
|
89
|
-
|
90
|
-
std::memcpy(&couponCount, data +
|
91
|
-
if (lgArrInts <
|
92
|
-
lgArrInts = HllUtil
|
89
|
+
uint32_t couponCount;
|
90
|
+
std::memcpy(&couponCount, data + hll_constants::HASH_SET_COUNT_INT, sizeof(couponCount));
|
91
|
+
if (lgArrInts < hll_constants::LG_INIT_SET_SIZE) {
|
92
|
+
lgArrInts = HllUtil<>::computeLgArrInts(SET, couponCount, lgK);
|
93
93
|
}
|
94
94
|
// Don't set couponCount in sketch here;
|
95
95
|
// we'll set later if updatable, and increment with updates if compact
|
96
|
-
const
|
97
|
-
const size_t expectedLength =
|
96
|
+
const uint32_t couponsInArray = (compactFlag ? couponCount : (1 << lgArrInts));
|
97
|
+
const size_t expectedLength = hll_constants::HASH_SET_INT_ARR_START + (couponsInArray * sizeof(uint32_t));
|
98
98
|
if (len < expectedLength) {
|
99
99
|
throw std::out_of_range("Byte array too short for sketch. Expected " + std::to_string(expectedLength)
|
100
100
|
+ ", found: " + std::to_string(len));
|
@@ -104,19 +104,19 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(const void* bytes, size_t len, const
|
|
104
104
|
CouponHashSet<A>* sketch = new (chsa.allocate(1)) CouponHashSet<A>(lgK, tgtHllType, allocator);
|
105
105
|
|
106
106
|
if (compactFlag) {
|
107
|
-
const uint8_t* curPos = data +
|
108
|
-
|
109
|
-
for (
|
107
|
+
const uint8_t* curPos = data + hll_constants::HASH_SET_INT_ARR_START;
|
108
|
+
uint32_t coupon;
|
109
|
+
for (uint32_t i = 0; i < couponCount; ++i, curPos += sizeof(coupon)) {
|
110
110
|
std::memcpy(&coupon, curPos, sizeof(coupon));
|
111
111
|
sketch->couponUpdate(coupon);
|
112
112
|
}
|
113
113
|
} else {
|
114
|
-
sketch->
|
115
|
-
sketch->
|
114
|
+
sketch->coupons_.resize(1ULL << lgArrInts);
|
115
|
+
sketch->couponCount_ = couponCount;
|
116
116
|
// only need to read valid coupons, unlike in stream case
|
117
|
-
std::memcpy(sketch->
|
118
|
-
data +
|
119
|
-
couponCount * sizeof(
|
117
|
+
std::memcpy(sketch->coupons_.data(),
|
118
|
+
data + hll_constants::HASH_SET_INT_ARR_START,
|
119
|
+
couponCount * sizeof(uint32_t));
|
120
120
|
}
|
121
121
|
|
122
122
|
return sketch;
|
@@ -125,37 +125,36 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(const void* bytes, size_t len, const
|
|
125
125
|
template<typename A>
|
126
126
|
CouponHashSet<A>* CouponHashSet<A>::newSet(std::istream& is, const A& allocator) {
|
127
127
|
uint8_t listHeader[8];
|
128
|
-
|
128
|
+
read(is, listHeader, 8 * sizeof(uint8_t));
|
129
129
|
|
130
|
-
if (listHeader[
|
130
|
+
if (listHeader[hll_constants::PREAMBLE_INTS_BYTE] != hll_constants::HASH_SET_PREINTS) {
|
131
131
|
throw std::invalid_argument("Incorrect number of preInts in input stream");
|
132
132
|
}
|
133
|
-
if (listHeader[
|
133
|
+
if (listHeader[hll_constants::SER_VER_BYTE] != hll_constants::SER_VER) {
|
134
134
|
throw std::invalid_argument("Wrong ser ver in input stream");
|
135
135
|
}
|
136
|
-
if (listHeader[
|
136
|
+
if (listHeader[hll_constants::FAMILY_BYTE] != hll_constants::FAMILY_ID) {
|
137
137
|
throw std::invalid_argument("Input stream is not an HLL sketch");
|
138
138
|
}
|
139
139
|
|
140
|
-
hll_mode mode = HllSketchImpl<A>::extractCurMode(listHeader[
|
140
|
+
hll_mode mode = HllSketchImpl<A>::extractCurMode(listHeader[hll_constants::MODE_BYTE]);
|
141
141
|
if (mode != SET) {
|
142
142
|
throw std::invalid_argument("Calling set constructor with non-set mode data");
|
143
143
|
}
|
144
144
|
|
145
|
-
target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(listHeader[
|
145
|
+
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(listHeader[hll_constants::MODE_BYTE]);
|
146
146
|
|
147
|
-
const
|
147
|
+
const uint8_t lgK = listHeader[hll_constants::LG_K_BYTE];
|
148
148
|
if (lgK <= 7) {
|
149
149
|
throw std::invalid_argument("Attempt to deserialize invalid CouponHashSet with lgConfigK <= 7. Found: "
|
150
150
|
+ std::to_string(lgK));
|
151
151
|
}
|
152
|
-
|
153
|
-
const bool compactFlag = ((listHeader[
|
152
|
+
uint8_t lgArrInts = listHeader[hll_constants::LG_ARR_BYTE];
|
153
|
+
const bool compactFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
|
154
154
|
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
lgArrInts = HllUtil<A>::computeLgArrInts(SET, couponCount, lgK);
|
155
|
+
const auto couponCount = read<uint32_t>(is);
|
156
|
+
if (lgArrInts < hll_constants::LG_INIT_SET_SIZE) {
|
157
|
+
lgArrInts = HllUtil<>::computeLgArrInts(SET, couponCount, lgK);
|
159
158
|
}
|
160
159
|
|
161
160
|
ChsAlloc chsa(allocator);
|
@@ -166,16 +165,15 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(std::istream& is, const A& allocator)
|
|
166
165
|
// Don't set couponCount here;
|
167
166
|
// we'll set later if updatable, and increment with updates if compact
|
168
167
|
if (compactFlag) {
|
169
|
-
for (
|
170
|
-
|
171
|
-
is.read((char*)&coupon, sizeof(coupon));
|
168
|
+
for (uint32_t i = 0; i < couponCount; ++i) {
|
169
|
+
const auto coupon = read<uint32_t>(is);
|
172
170
|
sketch->couponUpdate(coupon);
|
173
171
|
}
|
174
172
|
} else {
|
175
|
-
sketch->
|
176
|
-
sketch->
|
173
|
+
sketch->coupons_.resize(1ULL << lgArrInts);
|
174
|
+
sketch->couponCount_ = couponCount;
|
177
175
|
// for stream processing, read entire list so read pointer ends up set correctly
|
178
|
-
|
176
|
+
read(is, sketch->coupons_.data(), sketch->coupons_.size() * sizeof(uint32_t));
|
179
177
|
}
|
180
178
|
|
181
179
|
if (!is.good())
|
@@ -186,25 +184,25 @@ CouponHashSet<A>* CouponHashSet<A>::newSet(std::istream& is, const A& allocator)
|
|
186
184
|
|
187
185
|
template<typename A>
|
188
186
|
CouponHashSet<A>* CouponHashSet<A>::copy() const {
|
189
|
-
ChsAlloc chsa(this->
|
187
|
+
ChsAlloc chsa(this->coupons_.get_allocator());
|
190
188
|
return new (chsa.allocate(1)) CouponHashSet<A>(*this);
|
191
189
|
}
|
192
190
|
|
193
191
|
template<typename A>
|
194
|
-
CouponHashSet<A>* CouponHashSet<A>::copyAs(
|
195
|
-
ChsAlloc chsa(this->
|
192
|
+
CouponHashSet<A>* CouponHashSet<A>::copyAs(target_hll_type tgtHllType) const {
|
193
|
+
ChsAlloc chsa(this->coupons_.get_allocator());
|
196
194
|
return new (chsa.allocate(1)) CouponHashSet<A>(*this, tgtHllType);
|
197
195
|
}
|
198
196
|
|
199
197
|
template<typename A>
|
200
|
-
HllSketchImpl<A>* CouponHashSet<A>::couponUpdate(
|
201
|
-
const uint8_t lgCouponArrInts = count_trailing_zeros_in_u32(this->
|
202
|
-
const
|
198
|
+
HllSketchImpl<A>* CouponHashSet<A>::couponUpdate(uint32_t coupon) {
|
199
|
+
const uint8_t lgCouponArrInts = count_trailing_zeros_in_u32(static_cast<uint32_t>(this->coupons_.size()));
|
200
|
+
const int32_t index = find<A>(this->coupons_.data(), lgCouponArrInts, coupon);
|
203
201
|
if (index >= 0) {
|
204
202
|
return this; // found duplicate, ignore
|
205
203
|
}
|
206
|
-
this->
|
207
|
-
++this->
|
204
|
+
this->coupons_[~index] = coupon; // found empty
|
205
|
+
++this->couponCount_;
|
208
206
|
if (checkGrowOrPromote()) {
|
209
207
|
return this->promoteHeapListOrSetToHll(*this);
|
210
208
|
}
|
@@ -212,20 +210,20 @@ HllSketchImpl<A>* CouponHashSet<A>::couponUpdate(int coupon) {
|
|
212
210
|
}
|
213
211
|
|
214
212
|
template<typename A>
|
215
|
-
|
216
|
-
return
|
213
|
+
uint32_t CouponHashSet<A>::getMemDataStart() const {
|
214
|
+
return hll_constants::HASH_SET_INT_ARR_START;
|
217
215
|
}
|
218
216
|
|
219
217
|
template<typename A>
|
220
|
-
|
221
|
-
return
|
218
|
+
uint8_t CouponHashSet<A>::getPreInts() const {
|
219
|
+
return hll_constants::HASH_SET_PREINTS;
|
222
220
|
}
|
223
221
|
|
224
222
|
template<typename A>
|
225
223
|
bool CouponHashSet<A>::checkGrowOrPromote() {
|
226
|
-
if (static_cast<size_t>(
|
227
|
-
const uint8_t lgCouponArrInts = count_trailing_zeros_in_u32(this->
|
228
|
-
if (lgCouponArrInts == (this->
|
224
|
+
if (static_cast<size_t>(hll_constants::RESIZE_DENOM * this->couponCount_) > (hll_constants::RESIZE_NUMER * this->coupons_.size())) {
|
225
|
+
const uint8_t lgCouponArrInts = count_trailing_zeros_in_u32(static_cast<uint32_t>(this->coupons_.size()));
|
226
|
+
if (lgCouponArrInts == (this->lgConfigK_ - 3)) { // at max size
|
229
227
|
return true; // promote to HLL
|
230
228
|
}
|
231
229
|
growHashSet(lgCouponArrInts + 1);
|
@@ -234,15 +232,15 @@ bool CouponHashSet<A>::checkGrowOrPromote() {
|
|
234
232
|
}
|
235
233
|
|
236
234
|
template<typename A>
|
237
|
-
void CouponHashSet<A>::growHashSet(
|
238
|
-
const
|
239
|
-
vector_int coupons_new(tgtLen, 0, this->
|
240
|
-
|
241
|
-
const
|
242
|
-
for (
|
243
|
-
const
|
244
|
-
if (fetched !=
|
245
|
-
const
|
235
|
+
void CouponHashSet<A>::growHashSet(uint8_t tgtLgCoupArrSize) {
|
236
|
+
const uint32_t tgtLen = 1 << tgtLgCoupArrSize;
|
237
|
+
vector_int coupons_new(tgtLen, 0, this->coupons_.get_allocator());
|
238
|
+
|
239
|
+
const uint32_t srcLen = static_cast<uint32_t>(this->coupons_.size());
|
240
|
+
for (uint32_t i = 0; i < srcLen; ++i) { // scan existing array for non-zero values
|
241
|
+
const uint32_t fetched = this->coupons_[i];
|
242
|
+
if (fetched != hll_constants::EMPTY) {
|
243
|
+
const int32_t idx = find<A>(coupons_new.data(), tgtLgCoupArrSize, fetched); // search TGT array
|
246
244
|
if (idx < 0) { // found EMPTY
|
247
245
|
coupons_new[~idx] = fetched; // insert
|
248
246
|
continue;
|
@@ -250,23 +248,23 @@ void CouponHashSet<A>::growHashSet(int tgtLgCoupArrSize) {
|
|
250
248
|
throw std::runtime_error("Error: Found duplicate coupon");
|
251
249
|
}
|
252
250
|
}
|
253
|
-
this->
|
251
|
+
this->coupons_ = std::move(coupons_new);
|
254
252
|
}
|
255
253
|
|
256
254
|
template<typename A>
|
257
|
-
static
|
258
|
-
const
|
259
|
-
|
260
|
-
const
|
255
|
+
static int32_t find(const uint32_t* array, uint8_t lgArrInts, uint32_t coupon) {
|
256
|
+
const uint32_t arrMask = (1 << lgArrInts) - 1;
|
257
|
+
uint32_t probe = coupon & arrMask;
|
258
|
+
const uint32_t loopIndex = probe;
|
261
259
|
do {
|
262
|
-
const
|
263
|
-
if (couponAtIdx ==
|
260
|
+
const uint32_t couponAtIdx = array[probe];
|
261
|
+
if (couponAtIdx == hll_constants::EMPTY) {
|
264
262
|
return ~probe; //empty
|
265
263
|
}
|
266
264
|
else if (coupon == couponAtIdx) {
|
267
265
|
return probe; //duplicate
|
268
266
|
}
|
269
|
-
const
|
267
|
+
const uint32_t stride = ((coupon & hll_constants::KEY_MASK_26) >> lgArrInts) | 1;
|
270
268
|
probe = (probe + stride) & arrMask;
|
271
269
|
} while (probe != loopIndex);
|
272
270
|
throw std::invalid_argument("Key not found and no empty slots!");
|
@@ -29,29 +29,29 @@ class CouponHashSet : public CouponList<A> {
|
|
29
29
|
public:
|
30
30
|
static CouponHashSet* newSet(const void* bytes, size_t len, const A& allocator);
|
31
31
|
static CouponHashSet* newSet(std::istream& is, const A& allocator);
|
32
|
-
CouponHashSet(
|
32
|
+
CouponHashSet(uint8_t lgConfigK, target_hll_type tgtHllType, const A& allocator);
|
33
33
|
CouponHashSet(const CouponHashSet& that, target_hll_type tgtHllType);
|
34
34
|
|
35
35
|
virtual ~CouponHashSet() = default;
|
36
36
|
virtual std::function<void(HllSketchImpl<A>*)> get_deleter() const;
|
37
37
|
|
38
38
|
protected:
|
39
|
-
using vector_int = std::vector<
|
39
|
+
using vector_int = std::vector<uint32_t, typename std::allocator_traits<A>::template rebind_alloc<uint32_t>>;
|
40
40
|
|
41
41
|
virtual CouponHashSet* copy() const;
|
42
42
|
virtual CouponHashSet* copyAs(target_hll_type tgtHllType) const;
|
43
43
|
|
44
|
-
virtual HllSketchImpl<A>* couponUpdate(
|
44
|
+
virtual HllSketchImpl<A>* couponUpdate(uint32_t coupon);
|
45
45
|
|
46
|
-
virtual
|
47
|
-
virtual
|
46
|
+
virtual uint32_t getMemDataStart() const;
|
47
|
+
virtual uint8_t getPreInts() const;
|
48
48
|
|
49
49
|
friend class HllSketchImplFactory<A>;
|
50
50
|
|
51
51
|
private:
|
52
52
|
using ChsAlloc = typename std::allocator_traits<A>::template rebind_alloc<CouponHashSet<A>>;
|
53
53
|
bool checkGrowOrPromote();
|
54
|
-
void growHashSet(
|
54
|
+
void growHashSet(uint8_t tgtLgCoupArrSize);
|
55
55
|
};
|
56
56
|
|
57
57
|
}
|
@@ -31,19 +31,19 @@
|
|
31
31
|
namespace datasketches {
|
32
32
|
|
33
33
|
template<typename A>
|
34
|
-
CouponList<A>::CouponList(
|
34
|
+
CouponList<A>::CouponList(uint8_t lgConfigK, target_hll_type tgtHllType, hll_mode mode, const A& allocator):
|
35
35
|
HllSketchImpl<A>(lgConfigK, tgtHllType, mode, false),
|
36
|
-
|
37
|
-
|
38
|
-
|
36
|
+
couponCount_(0),
|
37
|
+
oooFlag_(false),
|
38
|
+
coupons_(1ULL << (mode == hll_mode::LIST ? hll_constants::LG_INIT_LIST_SIZE : hll_constants::LG_INIT_SET_SIZE), 0, allocator)
|
39
39
|
{}
|
40
40
|
|
41
41
|
template<typename A>
|
42
42
|
CouponList<A>::CouponList(const CouponList& that, const target_hll_type tgtHllType):
|
43
|
-
HllSketchImpl<A>(that.
|
44
|
-
|
45
|
-
|
46
|
-
|
43
|
+
HllSketchImpl<A>(that.lgConfigK_, tgtHllType, that.mode_, false),
|
44
|
+
couponCount_(that.couponCount_),
|
45
|
+
oooFlag_(that.oooFlag_),
|
46
|
+
coupons_(that.coupons_)
|
47
47
|
{}
|
48
48
|
|
49
49
|
template<typename A>
|
@@ -58,48 +58,48 @@ std::function<void(HllSketchImpl<A>*)> CouponList<A>::get_deleter() const {
|
|
58
58
|
|
59
59
|
template<typename A>
|
60
60
|
CouponList<A>* CouponList<A>::copy() const {
|
61
|
-
ClAlloc cla(
|
61
|
+
ClAlloc cla(coupons_.get_allocator());
|
62
62
|
return new (cla.allocate(1)) CouponList<A>(*this);
|
63
63
|
}
|
64
64
|
|
65
65
|
template<typename A>
|
66
66
|
CouponList<A>* CouponList<A>::copyAs(target_hll_type tgtHllType) const {
|
67
|
-
ClAlloc cla(
|
67
|
+
ClAlloc cla(coupons_.get_allocator());
|
68
68
|
return new (cla.allocate(1)) CouponList<A>(*this, tgtHllType);
|
69
69
|
}
|
70
70
|
|
71
71
|
template<typename A>
|
72
72
|
CouponList<A>* CouponList<A>::newList(const void* bytes, size_t len, const A& allocator) {
|
73
|
-
if (len <
|
73
|
+
if (len < hll_constants::LIST_INT_ARR_START) {
|
74
74
|
throw std::out_of_range("Input data length insufficient to hold CouponHashSet");
|
75
75
|
}
|
76
76
|
|
77
77
|
const uint8_t* data = static_cast<const uint8_t*>(bytes);
|
78
|
-
if (data[
|
78
|
+
if (data[hll_constants::PREAMBLE_INTS_BYTE] != hll_constants::LIST_PREINTS) {
|
79
79
|
throw std::invalid_argument("Incorrect number of preInts in input stream");
|
80
80
|
}
|
81
|
-
if (data[
|
81
|
+
if (data[hll_constants::SER_VER_BYTE] != hll_constants::SER_VER) {
|
82
82
|
throw std::invalid_argument("Wrong ser ver in input stream");
|
83
83
|
}
|
84
|
-
if (data[
|
84
|
+
if (data[hll_constants::FAMILY_BYTE] != hll_constants::FAMILY_ID) {
|
85
85
|
throw std::invalid_argument("Input stream is not an HLL sketch");
|
86
86
|
}
|
87
87
|
|
88
|
-
hll_mode mode = HllSketchImpl<A>::extractCurMode(data[
|
88
|
+
hll_mode mode = HllSketchImpl<A>::extractCurMode(data[hll_constants::MODE_BYTE]);
|
89
89
|
if (mode != LIST) {
|
90
90
|
throw std::invalid_argument("Calling list constructor with non-list mode data");
|
91
91
|
}
|
92
92
|
|
93
|
-
target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(data[
|
93
|
+
target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(data[hll_constants::MODE_BYTE]);
|
94
94
|
|
95
|
-
const
|
96
|
-
const bool compact = ((data[
|
97
|
-
const bool oooFlag = ((data[
|
98
|
-
const bool emptyFlag = ((data[
|
95
|
+
const uint8_t lgK = data[hll_constants::LG_K_BYTE];
|
96
|
+
const bool compact = ((data[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
|
97
|
+
const bool oooFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::OUT_OF_ORDER_FLAG_MASK) ? true : false);
|
98
|
+
const bool emptyFlag = ((data[hll_constants::FLAGS_BYTE] & hll_constants::EMPTY_FLAG_MASK) ? true : false);
|
99
99
|
|
100
|
-
const
|
101
|
-
const
|
102
|
-
const size_t expectedLength =
|
100
|
+
const uint32_t couponCount = data[hll_constants::LIST_COUNT_BYTE];
|
101
|
+
const uint32_t couponsInArray = (compact ? couponCount : (1 << HllUtil<A>::computeLgArrInts(LIST, couponCount, lgK)));
|
102
|
+
const size_t expectedLength = hll_constants::LIST_INT_ARR_START + (couponsInArray * sizeof(uint32_t));
|
103
103
|
if (len < expectedLength) {
|
104
104
|
throw std::out_of_range("Byte array too short for sketch. Expected " + std::to_string(expectedLength)
|
105
105
|
+ ", found: " + std::to_string(len));
|
@@ -107,12 +107,12 @@ CouponList<A>* CouponList<A>::newList(const void* bytes, size_t len, const A& al
|
|
107
107
|
|
108
108
|
ClAlloc cla(allocator);
|
109
109
|
CouponList<A>* sketch = new (cla.allocate(1)) CouponList<A>(lgK, tgtHllType, mode, allocator);
|
110
|
-
sketch->
|
110
|
+
sketch->couponCount_ = couponCount;
|
111
111
|
sketch->putOutOfOrderFlag(oooFlag); // should always be false for LIST
|
112
112
|
|
113
113
|
if (!emptyFlag) {
|
114
114
|
// only need to read valid coupons, unlike in stream case
|
115
|
-
std::memcpy(sketch->
|
115
|
+
std::memcpy(sketch->coupons_.data(), data + hll_constants::LIST_INT_ARR_START, couponCount * sizeof(uint32_t));
|
116
116
|
}
|
117
117
|
|
118
118
|
return sketch;
|
@@ -121,44 +121,44 @@ CouponList<A>* CouponList<A>::newList(const void* bytes, size_t len, const A& al
|
|
121
121
|
template<typename A>
|
122
122
|
CouponList<A>* CouponList<A>::newList(std::istream& is, const A& allocator) {
|
123
123
|
uint8_t listHeader[8];
|
124
|
-
|
124
|
+
read(is, listHeader, 8 * sizeof(uint8_t));
|
125
125
|
|
126
|
-
if (listHeader[
|
126
|
+
if (listHeader[hll_constants::PREAMBLE_INTS_BYTE] != hll_constants::LIST_PREINTS) {
|
127
127
|
throw std::invalid_argument("Incorrect number of preInts in input stream");
|
128
128
|
}
|
129
|
-
if (listHeader[
|
129
|
+
if (listHeader[hll_constants::SER_VER_BYTE] != hll_constants::SER_VER) {
|
130
130
|
throw std::invalid_argument("Wrong ser ver in input stream");
|
131
131
|
}
|
132
|
-
if (listHeader[
|
132
|
+
if (listHeader[hll_constants::FAMILY_BYTE] != hll_constants::FAMILY_ID) {
|
133
133
|
throw std::invalid_argument("Input stream is not an HLL sketch");
|
134
134
|
}
|
135
135
|
|
136
|
-
hll_mode mode = HllSketchImpl<A>::extractCurMode(listHeader[
|
136
|
+
hll_mode mode = HllSketchImpl<A>::extractCurMode(listHeader[hll_constants::MODE_BYTE]);
|
137
137
|
if (mode != LIST) {
|
138
138
|
throw std::invalid_argument("Calling list constructor with non-list mode data");
|
139
139
|
}
|
140
140
|
|
141
|
-
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(listHeader[
|
141
|
+
const target_hll_type tgtHllType = HllSketchImpl<A>::extractTgtHllType(listHeader[hll_constants::MODE_BYTE]);
|
142
142
|
|
143
|
-
const
|
144
|
-
const bool compact = ((listHeader[
|
145
|
-
const bool oooFlag = ((listHeader[
|
146
|
-
const bool emptyFlag = ((listHeader[
|
143
|
+
const uint8_t lgK = listHeader[hll_constants::LG_K_BYTE];
|
144
|
+
const bool compact = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::COMPACT_FLAG_MASK) ? true : false);
|
145
|
+
const bool oooFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::OUT_OF_ORDER_FLAG_MASK) ? true : false);
|
146
|
+
const bool emptyFlag = ((listHeader[hll_constants::FLAGS_BYTE] & hll_constants::EMPTY_FLAG_MASK) ? true : false);
|
147
147
|
|
148
148
|
ClAlloc cla(allocator);
|
149
149
|
CouponList<A>* sketch = new (cla.allocate(1)) CouponList<A>(lgK, tgtHllType, mode, allocator);
|
150
150
|
using coupon_list_ptr = std::unique_ptr<CouponList<A>, std::function<void(HllSketchImpl<A>*)>>;
|
151
151
|
coupon_list_ptr ptr(sketch, sketch->get_deleter());
|
152
|
-
const
|
153
|
-
sketch->
|
152
|
+
const uint32_t couponCount = listHeader[hll_constants::LIST_COUNT_BYTE];
|
153
|
+
sketch->couponCount_ = couponCount;
|
154
154
|
sketch->putOutOfOrderFlag(oooFlag); // should always be false for LIST
|
155
155
|
|
156
156
|
if (!emptyFlag) {
|
157
157
|
// For stream processing, need to read entire number written to stream so read
|
158
158
|
// pointer ends up set correctly.
|
159
159
|
// If not compact, still need to read empty items even though in order.
|
160
|
-
const
|
161
|
-
|
160
|
+
const uint32_t numToRead = (compact ? couponCount : static_cast<uint32_t>(sketch->coupons_.size()));
|
161
|
+
read(is, sketch->coupons_.data(), numToRead * sizeof(uint32_t));
|
162
162
|
}
|
163
163
|
|
164
164
|
if (!is.good())
|
@@ -173,17 +173,17 @@ vector_u8<A> CouponList<A>::serialize(bool compact, unsigned header_size_bytes)
|
|
173
173
|
vector_u8<A> byteArr(sketchSizeBytes, 0, getAllocator());
|
174
174
|
uint8_t* bytes = byteArr.data() + header_size_bytes;
|
175
175
|
|
176
|
-
bytes[
|
177
|
-
bytes[
|
178
|
-
bytes[
|
179
|
-
bytes[
|
180
|
-
bytes[
|
181
|
-
bytes[
|
182
|
-
bytes[
|
183
|
-
bytes[
|
184
|
-
|
185
|
-
if (this->
|
186
|
-
std::memcpy(bytes +
|
176
|
+
bytes[hll_constants::PREAMBLE_INTS_BYTE] = static_cast<uint8_t>(getPreInts());
|
177
|
+
bytes[hll_constants::SER_VER_BYTE] = static_cast<uint8_t>(hll_constants::SER_VER);
|
178
|
+
bytes[hll_constants::FAMILY_BYTE] = static_cast<uint8_t>(hll_constants::FAMILY_ID);
|
179
|
+
bytes[hll_constants::LG_K_BYTE] = static_cast<uint8_t>(this->lgConfigK_);
|
180
|
+
bytes[hll_constants::LG_ARR_BYTE] = count_trailing_zeros_in_u32(static_cast<uint32_t>(coupons_.size()));
|
181
|
+
bytes[hll_constants::FLAGS_BYTE] = this->makeFlagsByte(compact);
|
182
|
+
bytes[hll_constants::LIST_COUNT_BYTE] = static_cast<uint8_t>(this->mode_ == LIST ? couponCount_ : 0);
|
183
|
+
bytes[hll_constants::MODE_BYTE] = this->makeModeByte();
|
184
|
+
|
185
|
+
if (this->mode_ == SET) {
|
186
|
+
std::memcpy(bytes + hll_constants::HASH_SET_COUNT_INT, &couponCount_, sizeof(couponCount_));
|
187
187
|
}
|
188
188
|
|
189
189
|
// coupons
|
@@ -191,12 +191,12 @@ vector_u8<A> CouponList<A>::serialize(bool compact, unsigned header_size_bytes)
|
|
191
191
|
const int sw = (isCompact() ? 2 : 0) | (compact ? 1 : 0);
|
192
192
|
switch (sw) {
|
193
193
|
case 0: { // src updatable, dst updatable
|
194
|
-
std::memcpy(bytes + getMemDataStart(),
|
194
|
+
std::memcpy(bytes + getMemDataStart(), coupons_.data(), coupons_.size() * sizeof(uint32_t));
|
195
195
|
break;
|
196
196
|
}
|
197
197
|
case 1: { // src updatable, dst compact
|
198
198
|
bytes += getMemDataStart(); // reusing pointer for incremental writes
|
199
|
-
for (uint32_t coupon: *this) {
|
199
|
+
for (const uint32_t coupon: *this) {
|
200
200
|
std::memcpy(bytes, &coupon, sizeof(coupon));
|
201
201
|
bytes += sizeof(coupon);
|
202
202
|
}
|
@@ -213,33 +213,33 @@ vector_u8<A> CouponList<A>::serialize(bool compact, unsigned header_size_bytes)
|
|
213
213
|
template<typename A>
|
214
214
|
void CouponList<A>::serialize(std::ostream& os, const bool compact) const {
|
215
215
|
// header
|
216
|
-
const uint8_t preInts
|
217
|
-
|
218
|
-
const uint8_t serialVersion(
|
219
|
-
|
220
|
-
const uint8_t familyId(
|
221
|
-
|
222
|
-
const uint8_t lgKByte
|
223
|
-
|
224
|
-
const uint8_t lgArrIntsByte
|
225
|
-
|
226
|
-
const uint8_t flagsByte
|
227
|
-
|
228
|
-
|
229
|
-
if (this->
|
230
|
-
const uint8_t listCount(
|
231
|
-
|
216
|
+
const uint8_t preInts = getPreInts();
|
217
|
+
write(os, preInts);
|
218
|
+
const uint8_t serialVersion(hll_constants::SER_VER);
|
219
|
+
write(os, serialVersion);
|
220
|
+
const uint8_t familyId(hll_constants::FAMILY_ID);
|
221
|
+
write(os, familyId);
|
222
|
+
const uint8_t lgKByte = this->lgConfigK_;
|
223
|
+
write(os, lgKByte);
|
224
|
+
const uint8_t lgArrIntsByte = count_trailing_zeros_in_u32(static_cast<uint32_t>(coupons_.size()));
|
225
|
+
write(os, lgArrIntsByte);
|
226
|
+
const uint8_t flagsByte = this->makeFlagsByte(compact);
|
227
|
+
write(os, flagsByte);
|
228
|
+
|
229
|
+
if (this->mode_ == LIST) {
|
230
|
+
const uint8_t listCount = static_cast<uint8_t>(couponCount_);
|
231
|
+
write(os, listCount);
|
232
232
|
} else { // mode == SET
|
233
|
-
|
234
|
-
|
233
|
+
const uint8_t unused = 0;
|
234
|
+
write(os, unused);
|
235
235
|
}
|
236
236
|
|
237
|
-
const uint8_t modeByte
|
238
|
-
|
237
|
+
const uint8_t modeByte = this->makeModeByte();
|
238
|
+
write(os, modeByte);
|
239
239
|
|
240
|
-
if (this->
|
240
|
+
if (this->mode_ == SET) {
|
241
241
|
// writing as int, already stored as int
|
242
|
-
|
242
|
+
write(os, couponCount_);
|
243
243
|
}
|
244
244
|
|
245
245
|
// coupons
|
@@ -247,12 +247,12 @@ void CouponList<A>::serialize(std::ostream& os, const bool compact) const {
|
|
247
247
|
const int sw = (isCompact() ? 2 : 0) | (compact ? 1 : 0);
|
248
248
|
switch (sw) {
|
249
249
|
case 0: { // src updatable, dst updatable
|
250
|
-
|
250
|
+
write(os, coupons_.data(), coupons_.size() * sizeof(uint32_t));
|
251
251
|
break;
|
252
252
|
}
|
253
253
|
case 1: { // src updatable, dst compact
|
254
|
-
for (uint32_t coupon: *this) {
|
255
|
-
|
254
|
+
for (const uint32_t coupon: *this) {
|
255
|
+
write(os, coupon);
|
256
256
|
}
|
257
257
|
break;
|
258
258
|
}
|
@@ -265,14 +265,14 @@ void CouponList<A>::serialize(std::ostream& os, const bool compact) const {
|
|
265
265
|
}
|
266
266
|
|
267
267
|
template<typename A>
|
268
|
-
HllSketchImpl<A>* CouponList<A>::couponUpdate(
|
269
|
-
for (size_t i = 0; i <
|
270
|
-
const
|
271
|
-
if (couponAtIdx ==
|
272
|
-
|
273
|
-
++
|
274
|
-
if (
|
275
|
-
if (this->
|
268
|
+
HllSketchImpl<A>* CouponList<A>::couponUpdate(uint32_t coupon) {
|
269
|
+
for (size_t i = 0; i < coupons_.size(); ++i) { // search for empty slot
|
270
|
+
const uint32_t couponAtIdx = coupons_[i];
|
271
|
+
if (couponAtIdx == hll_constants::EMPTY) {
|
272
|
+
coupons_[i] = coupon; // the actual update
|
273
|
+
++couponCount_;
|
274
|
+
if (couponCount_ == static_cast<uint32_t>(coupons_.size())) { // array full
|
275
|
+
if (this->lgConfigK_ < 8) {
|
276
276
|
return promoteHeapListOrSetToHll(*this);
|
277
277
|
}
|
278
278
|
return promoteHeapListToSet(*this);
|
@@ -293,71 +293,68 @@ double CouponList<A>::getCompositeEstimate() const { return getEstimate(); }
|
|
293
293
|
|
294
294
|
template<typename A>
|
295
295
|
double CouponList<A>::getEstimate() const {
|
296
|
-
const
|
297
|
-
|
298
|
-
return fmax(est, couponCount);
|
296
|
+
const double est = CubicInterpolation<A>::usingXAndYTables(couponCount_);
|
297
|
+
return fmax(est, couponCount_);
|
299
298
|
}
|
300
299
|
|
301
300
|
template<typename A>
|
302
|
-
double CouponList<A>::getLowerBound(
|
301
|
+
double CouponList<A>::getLowerBound(uint8_t numStdDev) const {
|
303
302
|
HllUtil<A>::checkNumStdDev(numStdDev);
|
304
|
-
const
|
305
|
-
const double
|
306
|
-
|
307
|
-
return fmax(tmp, couponCount);
|
303
|
+
const double est = CubicInterpolation<A>::usingXAndYTables(couponCount_);
|
304
|
+
const double tmp = est / (1.0 + (numStdDev * hll_constants::COUPON_RSE));
|
305
|
+
return fmax(tmp, couponCount_);
|
308
306
|
}
|
309
307
|
|
310
308
|
template<typename A>
|
311
|
-
double CouponList<A>::getUpperBound(
|
309
|
+
double CouponList<A>::getUpperBound(uint8_t numStdDev) const {
|
312
310
|
HllUtil<A>::checkNumStdDev(numStdDev);
|
313
|
-
const
|
314
|
-
const double
|
315
|
-
|
316
|
-
return fmax(tmp, couponCount);
|
311
|
+
const double est = CubicInterpolation<A>::usingXAndYTables(couponCount_);
|
312
|
+
const double tmp = est / (1.0 - (numStdDev * hll_constants::COUPON_RSE));
|
313
|
+
return fmax(tmp, couponCount_);
|
317
314
|
}
|
318
315
|
|
319
316
|
template<typename A>
|
320
317
|
bool CouponList<A>::isEmpty() const { return getCouponCount() == 0; }
|
321
318
|
|
322
319
|
template<typename A>
|
323
|
-
|
324
|
-
return getMemDataStart() +
|
320
|
+
uint32_t CouponList<A>::getUpdatableSerializationBytes() const {
|
321
|
+
return getMemDataStart() + static_cast<uint32_t>(coupons_.size()) * sizeof(uint32_t);
|
325
322
|
}
|
326
323
|
|
327
324
|
template<typename A>
|
328
|
-
|
329
|
-
return
|
325
|
+
uint32_t CouponList<A>::getCouponCount() const {
|
326
|
+
return couponCount_;
|
330
327
|
}
|
331
328
|
|
332
329
|
template<typename A>
|
333
|
-
|
334
|
-
return getMemDataStart() + (
|
330
|
+
uint32_t CouponList<A>::getCompactSerializationBytes() const {
|
331
|
+
return getMemDataStart() + (couponCount_ << 2);
|
335
332
|
}
|
336
333
|
|
337
334
|
template<typename A>
|
338
|
-
|
339
|
-
return
|
335
|
+
uint32_t CouponList<A>::getMemDataStart() const {
|
336
|
+
return hll_constants::LIST_INT_ARR_START;
|
340
337
|
}
|
341
338
|
|
342
339
|
template<typename A>
|
343
|
-
|
344
|
-
return
|
340
|
+
uint8_t CouponList<A>::getPreInts() const {
|
341
|
+
return hll_constants::LIST_PREINTS;
|
345
342
|
}
|
346
343
|
|
347
344
|
template<typename A>
|
348
345
|
bool CouponList<A>::isCompact() const { return false; }
|
349
346
|
|
350
347
|
template<typename A>
|
351
|
-
bool CouponList<A>::isOutOfOrderFlag() const { return
|
348
|
+
bool CouponList<A>::isOutOfOrderFlag() const { return oooFlag_; }
|
352
349
|
|
353
350
|
template<typename A>
|
354
351
|
void CouponList<A>::putOutOfOrderFlag(bool oooFlag) {
|
355
|
-
|
352
|
+
oooFlag_ = oooFlag;
|
356
353
|
}
|
357
354
|
|
358
355
|
template<typename A>
|
359
356
|
A CouponList<A>::getAllocator() const {
|
360
|
-
return
|
357
|
+
return coupons_.get_allocator();
|
361
358
|
}
|
362
359
|
|
363
360
|
template<typename A>
|
@@ -372,12 +369,12 @@ HllSketchImpl<A>* CouponList<A>::promoteHeapListOrSetToHll(CouponList& src) {
|
|
372
369
|
|
373
370
|
template<typename A>
|
374
371
|
coupon_iterator<A> CouponList<A>::begin(bool all) const {
|
375
|
-
return coupon_iterator<A>(
|
372
|
+
return coupon_iterator<A>(coupons_.data(), coupons_.size(), 0, all);
|
376
373
|
}
|
377
374
|
|
378
375
|
template<typename A>
|
379
376
|
coupon_iterator<A> CouponList<A>::end() const {
|
380
|
-
return coupon_iterator<A>(
|
377
|
+
return coupon_iterator<A>(coupons_.data(), coupons_.size(), coupons_.size(), false);
|
381
378
|
}
|
382
379
|
|
383
380
|
}
|