datasketches 0.2.3 → 0.2.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/README.md +8 -8
- data/ext/datasketches/kll_wrapper.cpp +7 -3
- data/ext/datasketches/theta_wrapper.cpp +20 -4
- data/lib/datasketches/version.rb +1 -1
- data/vendor/datasketches-cpp/CMakeLists.txt +25 -5
- data/vendor/datasketches-cpp/MANIFEST.in +3 -0
- data/vendor/datasketches-cpp/NOTICE +6 -5
- data/vendor/datasketches-cpp/README.md +76 -9
- data/vendor/datasketches-cpp/cmake/DataSketchesConfig.cmake.in +10 -0
- data/vendor/datasketches-cpp/common/CMakeLists.txt +18 -13
- data/vendor/datasketches-cpp/common/include/binomial_bounds.hpp +1 -0
- data/vendor/datasketches-cpp/common/include/common_defs.hpp +14 -0
- data/vendor/datasketches-cpp/{kll → common}/include/kolmogorov_smirnov.hpp +5 -3
- data/vendor/datasketches-cpp/{kll → common}/include/kolmogorov_smirnov_impl.hpp +13 -16
- data/vendor/datasketches-cpp/common/include/quantile_sketch_sorted_view.hpp +121 -0
- data/vendor/datasketches-cpp/common/include/quantile_sketch_sorted_view_impl.hpp +91 -0
- data/vendor/datasketches-cpp/common/test/test_type.hpp +2 -0
- data/vendor/datasketches-cpp/cpc/CMakeLists.txt +15 -35
- data/vendor/datasketches-cpp/cpc/include/cpc_compressor_impl.hpp +3 -1
- data/vendor/datasketches-cpp/cpc/include/cpc_confidence.hpp +1 -0
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch_impl.hpp +5 -3
- data/vendor/datasketches-cpp/cpc/include/cpc_union_impl.hpp +10 -6
- data/vendor/datasketches-cpp/cpc/test/cpc_sketch_allocation_test.cpp +17 -0
- data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp +1 -0
- data/vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp +2 -0
- data/vendor/datasketches-cpp/fi/CMakeLists.txt +5 -15
- data/vendor/datasketches-cpp/fi/include/frequent_items_sketch.hpp +37 -5
- data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp +29 -11
- data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_custom_type_test.cpp +2 -1
- data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_test.cpp +1 -0
- data/vendor/datasketches-cpp/hll/CMakeLists.txt +33 -56
- data/vendor/datasketches-cpp/hll/include/AuxHashMap-internal.hpp +2 -0
- data/vendor/datasketches-cpp/hll/include/CompositeInterpolationXTable-internal.hpp +1 -0
- data/vendor/datasketches-cpp/hll/include/CouponHashSet-internal.hpp +2 -2
- data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp +1 -0
- data/vendor/datasketches-cpp/hll/include/HllSketch-internal.hpp +6 -4
- data/vendor/datasketches-cpp/hll/include/HllSketchImpl-internal.hpp +2 -0
- data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp +2 -0
- data/vendor/datasketches-cpp/hll/test/AuxHashMapTest.cpp +1 -0
- data/vendor/datasketches-cpp/hll/test/CouponHashSetTest.cpp +1 -0
- data/vendor/datasketches-cpp/hll/test/CouponListTest.cpp +2 -0
- data/vendor/datasketches-cpp/hll/test/HllArrayTest.cpp +1 -0
- data/vendor/datasketches-cpp/hll/test/HllSketchTest.cpp +59 -0
- data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +2 -0
- data/vendor/datasketches-cpp/hll/test/TablesTest.cpp +1 -0
- data/vendor/datasketches-cpp/kll/CMakeLists.txt +5 -19
- data/vendor/datasketches-cpp/kll/include/kll_helper.hpp +0 -4
- data/vendor/datasketches-cpp/kll/include/kll_helper_impl.hpp +5 -2
- data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +108 -41
- data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +150 -132
- data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +165 -31
- data/vendor/datasketches-cpp/kll/test/kolmogorov_smirnov_test.cpp +1 -1
- data/vendor/datasketches-cpp/pyproject.toml +1 -1
- data/vendor/datasketches-cpp/python/CMakeLists.txt +8 -1
- data/vendor/datasketches-cpp/python/README.md +13 -9
- data/vendor/datasketches-cpp/python/src/datasketches.cpp +4 -0
- data/vendor/datasketches-cpp/python/src/fi_wrapper.cpp +6 -1
- data/vendor/datasketches-cpp/python/src/kll_wrapper.cpp +48 -13
- data/vendor/datasketches-cpp/python/src/ks_wrapper.cpp +68 -0
- data/vendor/datasketches-cpp/python/src/quantiles_wrapper.cpp +240 -0
- data/vendor/datasketches-cpp/python/src/req_wrapper.cpp +9 -2
- data/vendor/datasketches-cpp/python/src/vector_of_kll.cpp +1 -0
- data/vendor/datasketches-cpp/python/tests/kll_test.py +10 -4
- data/vendor/datasketches-cpp/python/tests/quantiles_test.py +126 -0
- data/vendor/datasketches-cpp/quantiles/CMakeLists.txt +42 -0
- data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch.hpp +656 -0
- data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch_impl.hpp +1373 -0
- data/vendor/datasketches-cpp/quantiles/test/CMakeLists.txt +44 -0
- data/vendor/datasketches-cpp/quantiles/test/Qk128_n1000_v0.3.0.sk +0 -0
- data/vendor/datasketches-cpp/quantiles/test/Qk128_n1000_v0.6.0.sk +0 -0
- data/vendor/datasketches-cpp/quantiles/test/Qk128_n1000_v0.8.0.sk +0 -0
- data/vendor/datasketches-cpp/quantiles/test/Qk128_n1000_v0.8.3.sk +0 -0
- data/vendor/datasketches-cpp/quantiles/test/Qk128_n50_v0.3.0.sk +0 -0
- data/vendor/datasketches-cpp/quantiles/test/Qk128_n50_v0.6.0.sk +0 -0
- data/vendor/datasketches-cpp/quantiles/test/Qk128_n50_v0.8.0.sk +0 -0
- data/vendor/datasketches-cpp/quantiles/test/Qk128_n50_v0.8.3.sk +0 -0
- data/vendor/datasketches-cpp/quantiles/test/kolmogorov_smirnov_test.cpp +110 -0
- data/vendor/datasketches-cpp/quantiles/test/quantiles_compatibility_test.cpp +129 -0
- data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_test.cpp +975 -0
- data/vendor/datasketches-cpp/req/CMakeLists.txt +6 -21
- data/vendor/datasketches-cpp/req/include/req_common.hpp +0 -5
- data/vendor/datasketches-cpp/req/include/req_compactor.hpp +6 -0
- data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +30 -2
- data/vendor/datasketches-cpp/req/include/req_sketch.hpp +73 -23
- data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +95 -63
- data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +74 -3
- data/vendor/datasketches-cpp/sampling/CMakeLists.txt +5 -9
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch.hpp +44 -7
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp +44 -33
- data/vendor/datasketches-cpp/sampling/include/var_opt_union.hpp +41 -6
- data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp +33 -15
- data/vendor/datasketches-cpp/sampling/test/var_opt_allocation_test.cpp +2 -2
- data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp +1 -0
- data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp +1 -0
- data/vendor/datasketches-cpp/setup.py +1 -1
- data/vendor/datasketches-cpp/theta/CMakeLists.txt +26 -45
- data/vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_sampled_sets.hpp +1 -0
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser_impl.hpp +92 -23
- data/vendor/datasketches-cpp/theta/include/theta_helpers.hpp +15 -0
- data/vendor/datasketches-cpp/theta/include/theta_intersection_base_impl.hpp +7 -6
- data/vendor/datasketches-cpp/theta/include/theta_set_difference_base_impl.hpp +3 -2
- data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp +32 -15
- data/vendor/datasketches-cpp/theta/include/theta_sketch_impl.hpp +150 -93
- data/vendor/datasketches-cpp/theta/include/theta_union.hpp +6 -1
- data/vendor/datasketches-cpp/theta/include/theta_union_base.hpp +3 -1
- data/vendor/datasketches-cpp/theta/include/theta_union_base_impl.hpp +9 -2
- data/vendor/datasketches-cpp/theta/include/theta_union_impl.hpp +8 -5
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp +5 -4
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp +34 -9
- data/vendor/datasketches-cpp/theta/test/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/theta/test/theta_a_not_b_test.cpp +2 -0
- data/vendor/datasketches-cpp/theta/test/theta_compact_empty_from_java_v1.sk +0 -0
- data/vendor/datasketches-cpp/theta/test/theta_compact_empty_from_java_v2.sk +0 -0
- data/vendor/datasketches-cpp/theta/test/theta_compact_estimation_from_java_v1.sk +0 -0
- data/vendor/datasketches-cpp/theta/test/theta_compact_estimation_from_java_v2.sk +0 -0
- data/vendor/datasketches-cpp/theta/test/theta_compact_exact_from_java.sk +0 -0
- data/vendor/datasketches-cpp/theta/test/theta_intersection_test.cpp +2 -0
- data/vendor/datasketches-cpp/theta/test/theta_setop_test.cpp +446 -0
- data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp +429 -1
- data/vendor/datasketches-cpp/theta/test/theta_union_test.cpp +25 -11
- data/vendor/datasketches-cpp/tuple/CMakeLists.txt +18 -33
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch.hpp +1 -1
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch_impl.hpp +3 -3
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_union.hpp +1 -1
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_union_impl.hpp +3 -3
- data/vendor/datasketches-cpp/tuple/include/tuple_sketch.hpp +29 -9
- data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp +34 -14
- data/vendor/datasketches-cpp/tuple/include/tuple_union.hpp +6 -1
- data/vendor/datasketches-cpp/tuple/include/tuple_union_impl.hpp +8 -3
- data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +16 -0
- data/vendor/datasketches-cpp/tuple/test/tuple_a_not_b_test.cpp +1 -0
- data/vendor/datasketches-cpp/tuple/test/tuple_intersection_test.cpp +1 -0
- data/vendor/datasketches-cpp/tuple/test/tuple_sketch_test.cpp +46 -8
- data/vendor/datasketches-cpp/tuple/test/tuple_union_test.cpp +8 -0
- metadata +33 -12
- data/vendor/datasketches-cpp/kll/include/kll_quantile_calculator.hpp +0 -75
- data/vendor/datasketches-cpp/kll/include/kll_quantile_calculator_impl.hpp +0 -184
- data/vendor/datasketches-cpp/req/include/req_quantile_calculator.hpp +0 -69
- data/vendor/datasketches-cpp/req/include/req_quantile_calculator_impl.hpp +0 -60
- data/vendor/datasketches-cpp/theta/test/theta_update_empty_from_java.sk +0 -0
- data/vendor/datasketches-cpp/theta/test/theta_update_estimation_from_java.sk +0 -0
@@ -0,0 +1,656 @@
|
|
1
|
+
/*
|
2
|
+
* Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
* or more contributor license agreements. See the NOTICE file
|
4
|
+
* distributed with this work for additional information
|
5
|
+
* regarding copyright ownership. The ASF licenses this file
|
6
|
+
* to you under the Apache License, Version 2.0 (the
|
7
|
+
* "License"); you may not use this file except in compliance
|
8
|
+
* with the License. You may obtain a copy of the License at
|
9
|
+
*
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
*
|
12
|
+
* Unless required by applicable law or agreed to in writing,
|
13
|
+
* software distributed under the License is distributed on an
|
14
|
+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
* KIND, either express or implied. See the License for the
|
16
|
+
* specific language governing permissions and limitations
|
17
|
+
* under the License.
|
18
|
+
*/
|
19
|
+
|
20
|
+
#ifndef _QUANTILES_SKETCH_HPP_
|
21
|
+
#define _QUANTILES_SKETCH_HPP_
|
22
|
+
|
23
|
+
#include <functional>
|
24
|
+
#include <memory>
|
25
|
+
#include <vector>
|
26
|
+
|
27
|
+
#include "quantile_sketch_sorted_view.hpp"
|
28
|
+
#include "common_defs.hpp"
|
29
|
+
#include "serde.hpp"
|
30
|
+
|
31
|
+
namespace datasketches {
|
32
|
+
|
33
|
+
/**
|
34
|
+
* This is a stochastic streaming sketch that enables near-real time analysis of the
|
35
|
+
* approximate distribution of real values from a very large stream in a single pass.
|
36
|
+
* The analysis is obtained using a getQuantiles(*) function or its inverse functions the
|
37
|
+
* Probability Mass Function from getPMF(*) and the Cumulative Distribution Function from getCDF(*).
|
38
|
+
*
|
39
|
+
* <p>Consider a large stream of one million values such as packet sizes coming into a network node.
|
40
|
+
* The absolute rank of any specific size value is simply its index in the hypothetical sorted
|
41
|
+
* array of values.
|
42
|
+
* The normalized rank (or fractional rank) is the absolute rank divided by the stream size,
|
43
|
+
* in this case one million.
|
44
|
+
* The value corresponding to the normalized rank of 0.5 represents the 50th percentile or median
|
45
|
+
* value of the distribution, or getQuantile(0.5). Similarly, the 95th percentile is obtained from
|
46
|
+
* getQuantile(0.95). Using the getQuantiles(0.0, 1.0) will return the min and max values seen by
|
47
|
+
* the sketch.</p>
|
48
|
+
*
|
49
|
+
* <p>From the min and max values, for example, 1 and 1000 bytes,
|
50
|
+
* you can obtain the PMF from getPMF(100, 500, 900) that will result in an array of
|
51
|
+
* 4 fractional values such as {.4, .3, .2, .1}, which means that
|
52
|
+
* <ul>
|
53
|
+
* <li>40% of the values were < 100,</li>
|
54
|
+
* <li>30% of the values were ≥ 100 and < 500,</li>
|
55
|
+
* <li>20% of the values were ≥ 500 and < 900, and</li>
|
56
|
+
* <li>10% of the values were ≥ 900.</li>
|
57
|
+
* </ul>
|
58
|
+
* A frequency histogram can be obtained by simply multiplying these fractions by getN(),
|
59
|
+
* which is the total count of values received.
|
60
|
+
* The getCDF(*) works similarly, but produces the cumulative distribution instead.
|
61
|
+
*
|
62
|
+
* <p>As of November 2021, this implementation produces serialized sketches which are binary-compatible
|
63
|
+
* with the equivalent Java implementation only when template parameter T = double
|
64
|
+
* (64-bit double precision values).
|
65
|
+
|
66
|
+
*
|
67
|
+
* <p>The accuracy of this sketch is a function of the configured value <i>k</i>, which also affects
|
68
|
+
* the overall size of the sketch. Accuracy of this quantile sketch is always with respect to
|
69
|
+
* the normalized rank. A <i>k</i> of 128 produces a normalized, rank error of about 1.7%.
|
70
|
+
* For example, the median value returned from getQuantile(0.5) will be between the actual values
|
71
|
+
* from the hypothetically sorted array of input values at normalized ranks of 0.483 and 0.517, with
|
72
|
+
* a confidence of about 99%.</p>
|
73
|
+
*
|
74
|
+
* <pre>
|
75
|
+
Table Guide for DoublesSketch Size in Bytes and Approximate Error:
|
76
|
+
K => | 16 32 64 128 256 512 1,024
|
77
|
+
~ Error => | 12.145% 6.359% 3.317% 1.725% 0.894% 0.463% 0.239%
|
78
|
+
N | Size in Bytes ->
|
79
|
+
------------------------------------------------------------------------
|
80
|
+
0 | 8 8 8 8 8 8 8
|
81
|
+
1 | 72 72 72 72 72 72 72
|
82
|
+
3 | 72 72 72 72 72 72 72
|
83
|
+
7 | 104 104 104 104 104 104 104
|
84
|
+
15 | 168 168 168 168 168 168 168
|
85
|
+
31 | 296 296 296 296 296 296 296
|
86
|
+
63 | 424 552 552 552 552 552 552
|
87
|
+
127 | 552 808 1,064 1,064 1,064 1,064 1,064
|
88
|
+
255 | 680 1,064 1,576 2,088 2,088 2,088 2,088
|
89
|
+
511 | 808 1,320 2,088 3,112 4,136 4,136 4,136
|
90
|
+
1,023 | 936 1,576 2,600 4,136 6,184 8,232 8,232
|
91
|
+
2,047 | 1,064 1,832 3,112 5,160 8,232 12,328 16,424
|
92
|
+
4,095 | 1,192 2,088 3,624 6,184 10,280 16,424 24,616
|
93
|
+
8,191 | 1,320 2,344 4,136 7,208 12,328 20,520 32,808
|
94
|
+
16,383 | 1,448 2,600 4,648 8,232 14,376 24,616 41,000
|
95
|
+
32,767 | 1,576 2,856 5,160 9,256 16,424 28,712 49,192
|
96
|
+
65,535 | 1,704 3,112 5,672 10,280 18,472 32,808 57,384
|
97
|
+
131,071 | 1,832 3,368 6,184 11,304 20,520 36,904 65,576
|
98
|
+
262,143 | 1,960 3,624 6,696 12,328 22,568 41,000 73,768
|
99
|
+
524,287 | 2,088 3,880 7,208 13,352 24,616 45,096 81,960
|
100
|
+
1,048,575 | 2,216 4,136 7,720 14,376 26,664 49,192 90,152
|
101
|
+
2,097,151 | 2,344 4,392 8,232 15,400 28,712 53,288 98,344
|
102
|
+
4,194,303 | 2,472 4,648 8,744 16,424 30,760 57,384 106,536
|
103
|
+
8,388,607 | 2,600 4,904 9,256 17,448 32,808 61,480 114,728
|
104
|
+
16,777,215 | 2,728 5,160 9,768 18,472 34,856 65,576 122,920
|
105
|
+
33,554,431 | 2,856 5,416 10,280 19,496 36,904 69,672 131,112
|
106
|
+
67,108,863 | 2,984 5,672 10,792 20,520 38,952 73,768 139,304
|
107
|
+
134,217,727 | 3,112 5,928 11,304 21,544 41,000 77,864 147,496
|
108
|
+
268,435,455 | 3,240 6,184 11,816 22,568 43,048 81,960 155,688
|
109
|
+
536,870,911 | 3,368 6,440 12,328 23,592 45,096 86,056 163,880
|
110
|
+
1,073,741,823 | 3,496 6,696 12,840 24,616 47,144 90,152 172,072
|
111
|
+
2,147,483,647 | 3,624 6,952 13,352 25,640 49,192 94,248 180,264
|
112
|
+
4,294,967,295 | 3,752 7,208 13,864 26,664 51,240 98,344 188,456
|
113
|
+
|
114
|
+
* </pre>
|
115
|
+
|
116
|
+
* <p>There is more documentation available on
|
117
|
+
* <a href="https://datasketches.apache.org">datasketches.apache.org</a>.</p>
|
118
|
+
*
|
119
|
+
* <p>This is an implementation of the Low Discrepancy Mergeable Quantiles Sketch
|
120
|
+
* described in section 3.2 of the journal version of the paper "Mergeable Summaries"
|
121
|
+
* by Agarwal, Cormode, Huang, Phillips, Wei, and Yi.
|
122
|
+
* <a href="http://dblp.org/rec/html/journals/tods/AgarwalCHPWY13"></a></p>
|
123
|
+
*
|
124
|
+
* <p>This algorithm is independent of the distribution of values and
|
125
|
+
* requires only that the values be comparable.</p
|
126
|
+
*
|
127
|
+
* <p>This algorithm intentionally inserts randomness into the sampling process for values that
|
128
|
+
* ultimately get retained in the sketch. The results produced by this algorithm are not
|
129
|
+
* deterministic. For example, if the same stream is inserted into two different instances of this
|
130
|
+
* sketch, the answers obtained from the two sketches may not be identical.</p>
|
131
|
+
*
|
132
|
+
* <p>Similarly, there may be directional inconsistencies. For example, the resulting array of
|
133
|
+
* values obtained from getQuantiles(fractions[]) input into the reverse directional query
|
134
|
+
* getPMF(splitPoints[]) may not result in the original fractional values.</p>
|
135
|
+
*
|
136
|
+
* @author Kevin Lang
|
137
|
+
* @author Lee Rhodes
|
138
|
+
* @author Alexander Saydakov
|
139
|
+
* @author Jon Malkin
|
140
|
+
*/
|
141
|
+
|
142
|
+
namespace quantiles_constants {
|
143
|
+
const uint16_t DEFAULT_K = 128;
|
144
|
+
const uint16_t MIN_K = 2;
|
145
|
+
const uint16_t MAX_K = 1 << 15;
|
146
|
+
}
|
147
|
+
|
148
|
+
template <typename T,
|
149
|
+
typename Comparator = std::less<T>, // strict weak ordering function (see C++ named requirements: Compare)
|
150
|
+
typename Allocator = std::allocator<T>>
|
151
|
+
class quantiles_sketch {
|
152
|
+
public:
|
153
|
+
using value_type = T;
|
154
|
+
using allocator_type = Allocator;
|
155
|
+
using comparator = Comparator;
|
156
|
+
using vector_double = std::vector<double, typename std::allocator_traits<Allocator>::template rebind_alloc<double>>;
|
157
|
+
|
158
|
+
explicit quantiles_sketch(uint16_t k = quantiles_constants::DEFAULT_K, const Allocator& allocator = Allocator());
|
159
|
+
quantiles_sketch(const quantiles_sketch& other);
|
160
|
+
quantiles_sketch(quantiles_sketch&& other) noexcept;
|
161
|
+
~quantiles_sketch();
|
162
|
+
quantiles_sketch& operator=(const quantiles_sketch& other);
|
163
|
+
quantiles_sketch& operator=(quantiles_sketch&& other) noexcept;
|
164
|
+
|
165
|
+
/**
|
166
|
+
* @brief Type converting constructor
|
167
|
+
* @param other quantiles sketch of a different type
|
168
|
+
* @param allocator instance of an Allocator
|
169
|
+
*/
|
170
|
+
template<typename From, typename FC, typename FA>
|
171
|
+
explicit quantiles_sketch(const quantiles_sketch<From, FC, FA>& other, const Allocator& allocator = Allocator());
|
172
|
+
|
173
|
+
/**
|
174
|
+
* Updates this sketch with the given data item.
|
175
|
+
* @param value an item from a stream of items
|
176
|
+
*/
|
177
|
+
template<typename FwdT>
|
178
|
+
void update(FwdT&& value);
|
179
|
+
|
180
|
+
/**
|
181
|
+
* Merges another sketch into this one.
|
182
|
+
* @param other sketch to merge into this one
|
183
|
+
*/
|
184
|
+
template<typename FwdSk>
|
185
|
+
void merge(FwdSk&& other);
|
186
|
+
|
187
|
+
/**
|
188
|
+
* Returns true if this sketch is empty.
|
189
|
+
* @return empty flag
|
190
|
+
*/
|
191
|
+
bool is_empty() const;
|
192
|
+
|
193
|
+
/**
|
194
|
+
* Returns configured parameter k
|
195
|
+
* @return parameter k
|
196
|
+
*/
|
197
|
+
uint16_t get_k() const;
|
198
|
+
|
199
|
+
/**
|
200
|
+
* Returns the length of the input stream.
|
201
|
+
* @return stream length
|
202
|
+
*/
|
203
|
+
uint64_t get_n() const;
|
204
|
+
|
205
|
+
/**
|
206
|
+
* Returns the number of retained items (samples) in the sketch.
|
207
|
+
* @return the number of retained items
|
208
|
+
*/
|
209
|
+
uint32_t get_num_retained() const;
|
210
|
+
|
211
|
+
/**
|
212
|
+
* Returns true if this sketch is in estimation mode.
|
213
|
+
* @return estimation mode flag
|
214
|
+
*/
|
215
|
+
bool is_estimation_mode() const;
|
216
|
+
|
217
|
+
/**
|
218
|
+
* Returns the min value of the stream.
|
219
|
+
* For floating point types: if the sketch is empty this returns NaN.
|
220
|
+
* For other types: if the sketch is empty this throws runtime_error.
|
221
|
+
* @return the min value of the stream
|
222
|
+
*/
|
223
|
+
const T& get_min_value() const;
|
224
|
+
|
225
|
+
/**
|
226
|
+
* Returns the max value of the stream.
|
227
|
+
* For floating point types: if the sketch is empty this returns NaN.
|
228
|
+
* For other types: if the sketch is empty this throws runtime_error.
|
229
|
+
* @return the max value of the stream
|
230
|
+
*/
|
231
|
+
const T& get_max_value() const;
|
232
|
+
|
233
|
+
/**
|
234
|
+
* Returns an instance of the comparator for this sketch.
|
235
|
+
* @return comparator
|
236
|
+
*/
|
237
|
+
Comparator get_comparator() const;
|
238
|
+
|
239
|
+
/**
|
240
|
+
* Returns the allocator for this sketch.
|
241
|
+
* @return allocator
|
242
|
+
*/
|
243
|
+
allocator_type get_allocator() const;
|
244
|
+
|
245
|
+
/**
|
246
|
+
* Returns an approximation to the value of the data item
|
247
|
+
* that would be preceded by the given fraction of a hypothetical sorted
|
248
|
+
* version of the input stream so far.
|
249
|
+
* <p>
|
250
|
+
* Note that this method has a fairly large overhead (microseconds instead of nanoseconds)
|
251
|
+
* so it should not be called multiple times to get different quantiles from the same
|
252
|
+
* sketch. Instead use get_quantiles(), which pays the overhead only once.
|
253
|
+
* <p>
|
254
|
+
* For floating point types: if the sketch is empty this returns NaN.
|
255
|
+
* For other types: if the sketch is empty this throws runtime_error.
|
256
|
+
*
|
257
|
+
* @param rank the specified fractional position in the hypothetical sorted stream.
|
258
|
+
* These are also called normalized ranks or fractional ranks.
|
259
|
+
* If rank = 0.0, the true minimum value of the stream is returned.
|
260
|
+
* If rank = 1.0, the true maximum value of the stream is returned.
|
261
|
+
*
|
262
|
+
* @return the approximation to the value at the given rank
|
263
|
+
*/
|
264
|
+
using quantile_return_type = typename quantile_sketch_sorted_view<T, Comparator, Allocator>::quantile_return_type;
|
265
|
+
template<bool inclusive = false>
|
266
|
+
quantile_return_type get_quantile(double rank) const;
|
267
|
+
|
268
|
+
/**
|
269
|
+
* This is a more efficient multiple-query version of get_quantile().
|
270
|
+
* <p>
|
271
|
+
* This returns an array that could have been generated by using get_quantile() for each
|
272
|
+
* fractional rank separately, but would be very inefficient.
|
273
|
+
* This method incurs the internal set-up overhead once and obtains multiple quantile values in
|
274
|
+
* a single query. It is strongly recommend that this method be used instead of multiple calls
|
275
|
+
* to get_quantile().
|
276
|
+
*
|
277
|
+
* <p>If the sketch is empty this returns an empty vector.
|
278
|
+
*
|
279
|
+
* @param fractions given array of fractional positions in the hypothetical sorted stream.
|
280
|
+
* These are also called normalized ranks or fractional ranks.
|
281
|
+
* These fractions must be in the interval [0.0, 1.0], inclusive.
|
282
|
+
*
|
283
|
+
* @return array of approximations to the given fractions in the same order as given fractions
|
284
|
+
* in the input array.
|
285
|
+
*/
|
286
|
+
template<bool inclusive = false>
|
287
|
+
std::vector<T, Allocator> get_quantiles(const double* fractions, uint32_t size) const;
|
288
|
+
|
289
|
+
/**
|
290
|
+
* This is a multiple-query version of get_quantile() that allows the caller to
|
291
|
+
* specify the number of evenly-spaced fractional ranks.
|
292
|
+
*
|
293
|
+
* <p>If the sketch is empty this returns an empty vector.
|
294
|
+
*
|
295
|
+
* @param num an integer that specifies the number of evenly-spaced fractional ranks.
|
296
|
+
* This must be an integer greater than 0. A value of 1 will return the min value.
|
297
|
+
* A value of 2 will return the min and the max value. A value of 3 will return the min,
|
298
|
+
* the median and the max value, etc.
|
299
|
+
*
|
300
|
+
* @return array of approximations to the given number of evenly-spaced fractional ranks.
|
301
|
+
*/
|
302
|
+
template<bool inclusive = false>
|
303
|
+
std::vector<T, Allocator> get_quantiles(uint32_t num) const;
|
304
|
+
|
305
|
+
/**
|
306
|
+
* Returns an approximation to the normalized (fractional) rank of the given value from 0 to 1,
|
307
|
+
* inclusive. When template parameter <em>inclusive=false</em> (the default), only elements strictly
|
308
|
+
* less than the provided value are included in the rank estimate. With <em>inclusive=true</em>,
|
309
|
+
* the rank estimate includes elements less than or equal to the provided value.
|
310
|
+
*
|
311
|
+
* <p>The resulting approximation has a probabilistic guarantee that can be obtained from the
|
312
|
+
* get_normalized_rank_error(false) function.
|
313
|
+
*
|
314
|
+
* <p>If the sketch is empty this returns NaN.
|
315
|
+
*
|
316
|
+
* @param value to be ranked
|
317
|
+
* @return an approximate rank of the given value
|
318
|
+
*/
|
319
|
+
template<bool inclusive = false>
|
320
|
+
double get_rank(const T& value) const;
|
321
|
+
|
322
|
+
/**
|
323
|
+
* Returns an approximation to the Probability Mass Function (PMF) of the input stream
|
324
|
+
* given a set of split points (values).
|
325
|
+
*
|
326
|
+
* <p>The resulting approximations have a probabilistic guarantee that can be obtained from the
|
327
|
+
* get_normalized_rank_error(true) function.
|
328
|
+
*
|
329
|
+
* <p>If the sketch is empty this returns an empty vector.
|
330
|
+
*
|
331
|
+
* @param split_points an array of <i>m</i> unique, monotonically increasing values
|
332
|
+
* that divide the input domain into <i>m+1</i> consecutive disjoint intervals.
|
333
|
+
* If the template parameter <em>inclusive=false</em> (the default), the definition of an "interval"
|
334
|
+
* is inclusive of the left split point and exclusive of the right
|
335
|
+
* split point, with the exception that the last interval will include the maximum value.
|
336
|
+
* If the template parameter <em>inclusive=true</em>, the definition of an "interval" is exclusive of
|
337
|
+
* the left split point and inclusive of the right split point.
|
338
|
+
* It is not necessary to include either the min or max values in these split points.
|
339
|
+
*
|
340
|
+
* @return an array of m+1 doubles each of which is an approximation
|
341
|
+
* to the fraction of the input stream values (the mass) that fall into one of those intervals.
|
342
|
+
* When <em>inclusive=false</em> (the default), the definition of an "interval" is inclusive
|
343
|
+
* of the left split point and exclusive of the right split point, with the exception that the last
|
344
|
+
* interval will include the maximum value. When <em>inclusive=true</em>,
|
345
|
+
* an "interval" is exclusive of the left split point and inclusive of the right.
|
346
|
+
*/
|
347
|
+
template<bool inclusive = false>
|
348
|
+
vector_double get_PMF(const T* split_points, uint32_t size) const;
|
349
|
+
|
350
|
+
/**
|
351
|
+
* Returns an approximation to the Cumulative Distribution Function (CDF), which is the
|
352
|
+
* cumulative analog of the PMF, of the input stream given a set of split points (values).
|
353
|
+
*
|
354
|
+
* <p>The resulting approximations have a probabilistic guarantee that can be obtained from the
|
355
|
+
* get_normalized_rank_error(false) function.
|
356
|
+
*
|
357
|
+
* <p>If the sketch is empty this returns an empty vector.
|
358
|
+
*
|
359
|
+
* @param split_points an array of <i>m</i> unique, monotonically increasing values
|
360
|
+
* that divide the input domain into <i>m+1</i> consecutive disjoint intervals.
|
361
|
+
* If the template parameter <em>inclusive=false</em> (the default), the definition of an "interval" is
|
362
|
+
* inclusive of the left split point and exclusive of the right
|
363
|
+
* split point, with the exception that the last interval will include the maximum value.
|
364
|
+
* If the template parameter <em>inclusive=true</em>, the definition of an "interval" is exclusive of
|
365
|
+
* the left split point and inclusive of the right split point.
|
366
|
+
* It is not necessary to include either the min or max values in these split points.
|
367
|
+
*
|
368
|
+
* @return an array of m+1 double values, which are a consecutive approximation to the CDF
|
369
|
+
* of the input stream given the split_points. The value at array position j of the returned
|
370
|
+
* CDF array is the sum of the returned values in positions 0 through j of the returned PMF
|
371
|
+
* array.
|
372
|
+
* When <em>inclusive=false</em> (the default), the definition of an "interval" is inclusive
|
373
|
+
* of the left split point and exclusive of the right split point, with the exception that the last
|
374
|
+
* interval will include the maximum value. When <em>inclusive=true</em>,
|
375
|
+
* an "interval" is exclusive of the left split point and inclusive of the right.
|
376
|
+
|
377
|
+
*/
|
378
|
+
template<bool inclusive = false>
|
379
|
+
vector_double get_CDF(const T* split_points, uint32_t size) const;
|
380
|
+
|
381
|
+
/**
|
382
|
+
* Computes size needed to serialize the current state of the sketch.
|
383
|
+
* This version is for fixed-size arithmetic types (integral and floating point).
|
384
|
+
* @param instance of a SerDe
|
385
|
+
* @return size in bytes needed to serialize this sketch
|
386
|
+
*/
|
387
|
+
template<typename SerDe = serde<T>, typename TT = T, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type = 0>
|
388
|
+
size_t get_serialized_size_bytes(const SerDe& serde = SerDe()) const;
|
389
|
+
|
390
|
+
/**
|
391
|
+
* Computes size needed to serialize the current state of the sketch.
|
392
|
+
* This version is for all other types and can be expensive since every item needs to be looked at.
|
393
|
+
* @param instance of a SerDe
|
394
|
+
* @return size in bytes needed to serialize this sketch
|
395
|
+
*/
|
396
|
+
template<typename SerDe = serde<T>, typename TT = T, typename std::enable_if<!std::is_arithmetic<TT>::value, int>::type = 0>
|
397
|
+
size_t get_serialized_size_bytes(const SerDe& serde = SerDe()) const;
|
398
|
+
|
399
|
+
/**
|
400
|
+
* This method serializes the sketch into a given stream in a binary form
|
401
|
+
* @param os output stream
|
402
|
+
* @param instance of a SerDe
|
403
|
+
*/
|
404
|
+
template<typename SerDe = serde<T>>
|
405
|
+
void serialize(std::ostream& os, const SerDe& serde = SerDe()) const;
|
406
|
+
|
407
|
+
// This is a convenience alias for users
|
408
|
+
// The type returned by the following serialize method
|
409
|
+
using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<Allocator>::template rebind_alloc<uint8_t>>;
|
410
|
+
|
411
|
+
/**
|
412
|
+
* This method serializes the sketch as a vector of bytes.
|
413
|
+
* An optional header can be reserved in front of the sketch.
|
414
|
+
* It is a blank space of a given size.
|
415
|
+
* This header is used in Datasketches PostgreSQL extension.
|
416
|
+
* @param header_size_bytes space to reserve in front of the sketch
|
417
|
+
* @param instance of a SerDe
|
418
|
+
* @return serialized sketch as a vector of bytes
|
419
|
+
*/
|
420
|
+
template<typename SerDe = serde<T>>
|
421
|
+
vector_bytes serialize(unsigned header_size_bytes = 0, const SerDe& serde = SerDe()) const;
|
422
|
+
|
423
|
+
/**
|
424
|
+
* This method deserializes a sketch from a given stream.
|
425
|
+
* @param is input stream
|
426
|
+
* @param instance of a SerDe
|
427
|
+
* @param instance of an Allocator
|
428
|
+
* @return an instance of a sketch
|
429
|
+
*/
|
430
|
+
template<typename SerDe = serde<T>>
|
431
|
+
static quantiles_sketch deserialize(std::istream& is, const SerDe& serde = SerDe(), const Allocator& allocator = Allocator());
|
432
|
+
|
433
|
+
/**
|
434
|
+
* This method deserializes a sketch from a given array of bytes.
|
435
|
+
* @param bytes pointer to the array of bytes
|
436
|
+
* @param size the size of the array
|
437
|
+
* @param instance of a SerDe
|
438
|
+
* @param instance of an Allocator
|
439
|
+
* @return an instance of a sketch
|
440
|
+
*/
|
441
|
+
template<typename SerDe = serde<T>>
|
442
|
+
static quantiles_sketch deserialize(const void* bytes, size_t size, const SerDe& serde = SerDe(), const Allocator& allocator = Allocator());
|
443
|
+
|
444
|
+
/**
|
445
|
+
* Gets the normalized rank error for this sketch. Constants were derived as the best fit to 99 percentile
|
446
|
+
* empirically measured max error in thousands of trials.
|
447
|
+
* @param is_pmf if true, returns the "double-sided" normalized rank error for the get_PMF() function.
|
448
|
+
* Otherwise, it is the "single-sided" normalized rank error for all the other queries.
|
449
|
+
* @return the normalized rank error for the sketch
|
450
|
+
*/
|
451
|
+
double get_normalized_rank_error(bool is_pmf) const;
|
452
|
+
|
453
|
+
/**
|
454
|
+
* Gets the normalized rank error given k and pmf. Constants were derived as the best fit to 99 percentile
|
455
|
+
* empirically measured max error in thousands of trials.
|
456
|
+
* @param k the configuration parameter
|
457
|
+
* @param is_pmf if true, returns the "double-sided" normalized rank error for the get_PMF() function.
|
458
|
+
* Otherwise, it is the "single-sided" normalized rank error for all the other queries.
|
459
|
+
* @return the normalized rank error for the given parameters
|
460
|
+
*/
|
461
|
+
static double get_normalized_rank_error(uint16_t k, bool is_pmf);
|
462
|
+
|
463
|
+
/**
|
464
|
+
* Prints a summary of the sketch.
|
465
|
+
* @param print_levels if true include information about levels
|
466
|
+
* @param print_items if true include sketch data
|
467
|
+
*/
|
468
|
+
string<Allocator> to_string(bool print_levels = false, bool print_items = false) const;
|
469
|
+
|
470
|
+
class const_iterator;
|
471
|
+
const_iterator begin() const;
|
472
|
+
const_iterator end() const;
|
473
|
+
|
474
|
+
template<bool inclusive = false>
|
475
|
+
quantile_sketch_sorted_view<T, Comparator, Allocator> get_sorted_view(bool cumulative) const;
|
476
|
+
|
477
|
+
private:
|
478
|
+
using Level = std::vector<T, Allocator>;
|
479
|
+
using VectorLevels = std::vector<Level, typename std::allocator_traits<Allocator>::template rebind_alloc<Level>>;
|
480
|
+
|
481
|
+
/* Serialized sketch layout:
|
482
|
+
* Long || Start Byte Addr:
|
483
|
+
* Addr:
|
484
|
+
* || 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
|
485
|
+
* 0 || Preamble_Longs | SerVer | FamID | Flags |----- K ---------|---- unused -----|
|
486
|
+
*
|
487
|
+
* || 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
|
488
|
+
* 1 ||---------------------------Items Seen Count (N)--------------------------------|
|
489
|
+
*
|
490
|
+
* Long 3 is the start of data, beginning with serialized min and max values, followed by
|
491
|
+
* the sketch data buffers.
|
492
|
+
*/
|
493
|
+
|
494
|
+
static const size_t EMPTY_SIZE_BYTES = 8;
|
495
|
+
static const uint8_t SERIAL_VERSION_1 = 1;
|
496
|
+
static const uint8_t SERIAL_VERSION_2 = 2;
|
497
|
+
static const uint8_t SERIAL_VERSION = 3;
|
498
|
+
static const uint8_t FAMILY = 8;
|
499
|
+
|
500
|
+
enum flags { RESERVED0, RESERVED1, IS_EMPTY, IS_COMPACT, IS_SORTED };
|
501
|
+
|
502
|
+
static const uint8_t PREAMBLE_LONGS_SHORT = 1; // for empty
|
503
|
+
static const uint8_t PREAMBLE_LONGS_FULL = 2;
|
504
|
+
static const size_t DATA_START = 16;
|
505
|
+
|
506
|
+
Allocator allocator_;
|
507
|
+
uint16_t k_;
|
508
|
+
uint64_t n_;
|
509
|
+
uint64_t bit_pattern_;
|
510
|
+
Level base_buffer_;
|
511
|
+
VectorLevels levels_;
|
512
|
+
T* min_value_;
|
513
|
+
T* max_value_;
|
514
|
+
bool is_sorted_;
|
515
|
+
|
516
|
+
// for deserialization
|
517
|
+
class item_deleter;
|
518
|
+
class items_deleter;
|
519
|
+
quantiles_sketch(uint16_t k, uint64_t n, uint64_t bit_pattern,
|
520
|
+
Level&& base_buffer, VectorLevels&& levels,
|
521
|
+
std::unique_ptr<T, item_deleter> min_value, std::unique_ptr<T, item_deleter> max_value,
|
522
|
+
bool is_sorted, const Allocator& allocator = Allocator());
|
523
|
+
|
524
|
+
void grow_base_buffer();
|
525
|
+
void process_full_base_buffer();
|
526
|
+
|
527
|
+
// returns true if size adjusted, else false
|
528
|
+
bool grow_levels_if_needed();
|
529
|
+
|
530
|
+
// buffers should be pre-sized to target capacity as appropriate
|
531
|
+
template<typename FwdV>
|
532
|
+
static void in_place_propagate_carry(uint8_t starting_level, FwdV&& buf_size_k,
|
533
|
+
Level& buf_size_2k, bool apply_as_update,
|
534
|
+
quantiles_sketch& sketch);
|
535
|
+
static void zip_buffer(Level& buf_in, Level& buf_out);
|
536
|
+
static void merge_two_size_k_buffers(Level& arr_in_1, Level& arr_in_2, Level& arr_out);
|
537
|
+
|
538
|
+
template<typename SerDe>
|
539
|
+
static Level deserialize_array(std::istream& is, uint32_t num_items, uint32_t capcacity, const SerDe& serde, const Allocator& allocator);
|
540
|
+
|
541
|
+
template<typename SerDe>
|
542
|
+
static std::pair<Level, size_t> deserialize_array(const void* bytes, size_t size, uint32_t num_items, uint32_t capcacity, const SerDe& serde, const Allocator& allocator);
|
543
|
+
|
544
|
+
static void check_k(uint16_t k);
|
545
|
+
static void check_serial_version(uint8_t serial_version);
|
546
|
+
static void check_header_validity(uint8_t preamble_longs, uint8_t flags_byte, uint8_t serial_version);
|
547
|
+
static void check_family_id(uint8_t family_id);
|
548
|
+
|
549
|
+
static uint32_t compute_retained_items(uint16_t k, uint64_t n);
|
550
|
+
static uint32_t compute_base_buffer_items(uint16_t k, uint64_t n);
|
551
|
+
static uint64_t compute_bit_pattern(uint16_t k, uint64_t n);
|
552
|
+
static uint32_t compute_valid_levels(uint64_t bit_pattern);
|
553
|
+
static uint8_t compute_levels_needed(uint16_t k, uint64_t n);
|
554
|
+
|
555
|
+
/**
|
556
|
+
* Merges the src sketch into the tgt sketch with equal values of K.
|
557
|
+
* src is modified only if elements can be moved out of it.
|
558
|
+
*/
|
559
|
+
template<typename FwdSk>
|
560
|
+
static void standard_merge(quantiles_sketch& tgt, FwdSk&& src);
|
561
|
+
|
562
|
+
/**
|
563
|
+
* Merges the src sketch into the tgt sketch with a smaller value of K.
|
564
|
+
* However, it is required that the ratio of the two K values be a power of 2.
|
565
|
+
* I.e., other.get_k() = this.get_k() * 2^(nonnegative integer).
|
566
|
+
* src is modified only if elements can be moved out of it.
|
567
|
+
*/
|
568
|
+
template<typename FwdSk>
|
569
|
+
static void downsampling_merge(quantiles_sketch& tgt, FwdSk&& src);
|
570
|
+
|
571
|
+
template<typename FwdV>
|
572
|
+
static void zip_buffer_with_stride(FwdV&& buf_in, Level& buf_out, uint16_t stride);
|
573
|
+
|
574
|
+
/**
|
575
|
+
* Returns the zero-based bit position of the lowest zero bit of <i>bits</i> starting at
|
576
|
+
* <i>startingBit</i>. If input is all ones, this returns 64.
|
577
|
+
* @param bits the input bits as a long
|
578
|
+
* @param starting_bit the zero-based starting bit position. Only the low 6 bits are used.
|
579
|
+
* @return the zero-based bit position of the lowest zero bit starting at <i>startingBit</i>.
|
580
|
+
*/
|
581
|
+
static uint8_t lowest_zero_bit_starting_at(uint64_t bits, uint8_t starting_bit);
|
582
|
+
|
583
|
+
// implementations for floating point types
|
584
|
+
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
|
585
|
+
static const TT& get_invalid_value() {
|
586
|
+
static TT value = std::numeric_limits<TT>::quiet_NaN();
|
587
|
+
return value;
|
588
|
+
}
|
589
|
+
|
590
|
+
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
|
591
|
+
static inline bool check_update_value(TT value) {
|
592
|
+
return !std::isnan(value);
|
593
|
+
}
|
594
|
+
|
595
|
+
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
|
596
|
+
static inline void check_split_points(const T* values, uint32_t size) {
|
597
|
+
for (uint32_t i = 0; i < size ; i++) {
|
598
|
+
if (std::isnan(values[i])) {
|
599
|
+
throw std::invalid_argument("Values must not be NaN");
|
600
|
+
}
|
601
|
+
if ((i < (size - 1)) && !(Comparator()(values[i], values[i + 1]))) {
|
602
|
+
throw std::invalid_argument("Values must be unique and monotonically increasing");
|
603
|
+
}
|
604
|
+
}
|
605
|
+
}
|
606
|
+
|
607
|
+
// implementations for all other types
|
608
|
+
template<typename TT = T, typename std::enable_if<!std::is_floating_point<TT>::value, int>::type = 0>
|
609
|
+
static const TT& get_invalid_value() {
|
610
|
+
throw std::runtime_error("getting quantiles from empty sketch is not supported for this type of values");
|
611
|
+
}
|
612
|
+
|
613
|
+
template<typename TT = T, typename std::enable_if<!std::is_floating_point<TT>::value, int>::type = 0>
|
614
|
+
static inline bool check_update_value(TT) {
|
615
|
+
return true;
|
616
|
+
}
|
617
|
+
|
618
|
+
template<typename TT = T, typename std::enable_if<!std::is_floating_point<TT>::value, int>::type = 0>
|
619
|
+
static inline void check_split_points(const T* values, uint32_t size) {
|
620
|
+
for (uint32_t i = 0; i < size ; i++) {
|
621
|
+
if ((i < (size - 1)) && !(Comparator()(values[i], values[i + 1]))) {
|
622
|
+
throw std::invalid_argument("Values must be unique and monotonically increasing");
|
623
|
+
}
|
624
|
+
}
|
625
|
+
}
|
626
|
+
};
|
627
|
+
|
628
|
+
|
629
|
+
template<typename T, typename C, typename A>
|
630
|
+
class quantiles_sketch<T, C, A>::const_iterator: public std::iterator<std::input_iterator_tag, T> {
|
631
|
+
public:
|
632
|
+
const_iterator& operator++();
|
633
|
+
const_iterator& operator++(int);
|
634
|
+
bool operator==(const const_iterator& other) const;
|
635
|
+
bool operator!=(const const_iterator& other) const;
|
636
|
+
std::pair<const T&, const uint64_t> operator*() const;
|
637
|
+
private:
|
638
|
+
friend class quantiles_sketch<T, C, A>;
|
639
|
+
using Level = std::vector<T, A>;
|
640
|
+
using AllocLevel = typename std::allocator_traits<A>::template rebind_alloc<Level>;
|
641
|
+
Level base_buffer_;
|
642
|
+
std::vector<Level, AllocLevel> levels_;
|
643
|
+
int level_;
|
644
|
+
uint32_t index_;
|
645
|
+
uint32_t bb_count_;
|
646
|
+
uint64_t bit_pattern_;
|
647
|
+
uint64_t weight_;
|
648
|
+
uint16_t k_;
|
649
|
+
const_iterator(const Level& base_buffer, const std::vector<Level, AllocLevel>& levels, uint16_t k, uint64_t n, bool is_end);
|
650
|
+
};
|
651
|
+
|
652
|
+
} /* namespace datasketches */
|
653
|
+
|
654
|
+
#include "quantiles_sketch_impl.hpp"
|
655
|
+
|
656
|
+
#endif // _QUANTILES_SKETCH_HPP_
|