datasketches 0.3.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/ext/datasketches/cpc_wrapper.cpp +1 -1
- data/lib/datasketches/version.rb +1 -1
- data/vendor/datasketches-cpp/CMakeLists.txt +22 -20
- data/vendor/datasketches-cpp/NOTICE +1 -1
- data/vendor/datasketches-cpp/common/include/MurmurHash3.h +25 -27
- data/vendor/datasketches-cpp/common/include/common_defs.hpp +8 -6
- data/vendor/datasketches-cpp/common/include/count_zeros.hpp +11 -0
- data/vendor/datasketches-cpp/common/include/memory_operations.hpp +5 -4
- data/vendor/datasketches-cpp/common/test/CMakeLists.txt +1 -1
- data/vendor/datasketches-cpp/common/test/integration_test.cpp +6 -0
- data/vendor/datasketches-cpp/count/CMakeLists.txt +42 -0
- data/vendor/datasketches-cpp/count/include/count_min.hpp +351 -0
- data/vendor/datasketches-cpp/count/include/count_min_impl.hpp +517 -0
- data/vendor/datasketches-cpp/count/test/CMakeLists.txt +43 -0
- data/vendor/datasketches-cpp/count/test/count_min_allocation_test.cpp +155 -0
- data/vendor/datasketches-cpp/count/test/count_min_test.cpp +306 -0
- data/vendor/datasketches-cpp/cpc/include/cpc_confidence.hpp +3 -3
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch_impl.hpp +1 -1
- data/vendor/datasketches-cpp/cpc/include/cpc_util.hpp +16 -8
- data/vendor/datasketches-cpp/density/CMakeLists.txt +42 -0
- data/vendor/datasketches-cpp/density/include/density_sketch.hpp +236 -0
- data/vendor/datasketches-cpp/density/include/density_sketch_impl.hpp +543 -0
- data/vendor/datasketches-cpp/density/test/CMakeLists.txt +35 -0
- data/vendor/datasketches-cpp/density/test/density_sketch_test.cpp +244 -0
- data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map.hpp +9 -3
- data/vendor/datasketches-cpp/hll/include/Hll4Array-internal.hpp +19 -11
- data/vendor/datasketches-cpp/hll/include/Hll4Array.hpp +2 -5
- data/vendor/datasketches-cpp/hll/include/Hll6Array-internal.hpp +19 -7
- data/vendor/datasketches-cpp/hll/include/Hll6Array.hpp +1 -1
- data/vendor/datasketches-cpp/hll/include/Hll8Array-internal.hpp +98 -42
- data/vendor/datasketches-cpp/hll/include/Hll8Array.hpp +2 -0
- data/vendor/datasketches-cpp/hll/include/HllArray-internal.hpp +92 -59
- data/vendor/datasketches-cpp/hll/include/HllArray.hpp +16 -6
- data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp +3 -21
- data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +8 -0
- data/vendor/datasketches-cpp/hll/include/HllUtil.hpp +14 -6
- data/vendor/datasketches-cpp/hll/include/coupon_iterator-internal.hpp +1 -1
- data/vendor/datasketches-cpp/hll/include/coupon_iterator.hpp +8 -2
- data/vendor/datasketches-cpp/hll/include/hll.hpp +9 -8
- data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +7 -1
- data/vendor/datasketches-cpp/kll/include/kll_helper.hpp +0 -1
- data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +8 -3
- data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +2 -2
- data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +2 -2
- data/vendor/datasketches-cpp/python/CMakeLists.txt +6 -0
- data/vendor/datasketches-cpp/python/README.md +5 -5
- data/vendor/datasketches-cpp/python/datasketches/DensityWrapper.py +87 -0
- data/vendor/datasketches-cpp/python/datasketches/KernelFunction.py +35 -0
- data/vendor/datasketches-cpp/python/datasketches/PySerDe.py +15 -9
- data/vendor/datasketches-cpp/python/datasketches/TuplePolicy.py +77 -0
- data/vendor/datasketches-cpp/python/datasketches/TupleWrapper.py +205 -0
- data/vendor/datasketches-cpp/python/datasketches/__init__.py +17 -1
- data/vendor/datasketches-cpp/python/include/kernel_function.hpp +98 -0
- data/vendor/datasketches-cpp/python/include/py_object_lt.hpp +37 -0
- data/vendor/datasketches-cpp/python/include/py_object_ostream.hpp +48 -0
- data/vendor/datasketches-cpp/python/include/quantile_conditional.hpp +104 -0
- data/vendor/datasketches-cpp/python/include/tuple_policy.hpp +136 -0
- data/vendor/datasketches-cpp/python/src/count_wrapper.cpp +101 -0
- data/vendor/datasketches-cpp/python/src/cpc_wrapper.cpp +16 -30
- data/vendor/datasketches-cpp/python/src/datasketches.cpp +6 -0
- data/vendor/datasketches-cpp/python/src/density_wrapper.cpp +95 -0
- data/vendor/datasketches-cpp/python/src/fi_wrapper.cpp +127 -73
- data/vendor/datasketches-cpp/python/src/hll_wrapper.cpp +28 -36
- data/vendor/datasketches-cpp/python/src/kll_wrapper.cpp +108 -160
- data/vendor/datasketches-cpp/python/src/py_serde.cpp +5 -4
- data/vendor/datasketches-cpp/python/src/quantiles_wrapper.cpp +99 -148
- data/vendor/datasketches-cpp/python/src/req_wrapper.cpp +117 -178
- data/vendor/datasketches-cpp/python/src/theta_wrapper.cpp +67 -73
- data/vendor/datasketches-cpp/python/src/tuple_wrapper.cpp +215 -0
- data/vendor/datasketches-cpp/python/src/vo_wrapper.cpp +1 -1
- data/vendor/datasketches-cpp/python/tests/count_min_test.py +86 -0
- data/vendor/datasketches-cpp/python/tests/cpc_test.py +10 -10
- data/vendor/datasketches-cpp/python/tests/density_test.py +93 -0
- data/vendor/datasketches-cpp/python/tests/fi_test.py +41 -2
- data/vendor/datasketches-cpp/python/tests/hll_test.py +19 -20
- data/vendor/datasketches-cpp/python/tests/kll_test.py +40 -6
- data/vendor/datasketches-cpp/python/tests/quantiles_test.py +39 -5
- data/vendor/datasketches-cpp/python/tests/req_test.py +38 -5
- data/vendor/datasketches-cpp/python/tests/theta_test.py +16 -14
- data/vendor/datasketches-cpp/python/tests/tuple_test.py +206 -0
- data/vendor/datasketches-cpp/python/tests/vo_test.py +7 -0
- data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch.hpp +8 -3
- data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch_impl.hpp +4 -4
- data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_test.cpp +1 -1
- data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +0 -2
- data/vendor/datasketches-cpp/req/include/req_sketch.hpp +8 -3
- data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +2 -2
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch.hpp +20 -6
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp +30 -16
- data/vendor/datasketches-cpp/sampling/include/var_opt_union.hpp +5 -1
- data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp +19 -15
- data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp +33 -14
- data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp +0 -2
- data/vendor/datasketches-cpp/setup.py +1 -1
- data/vendor/datasketches-cpp/theta/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/theta/include/bit_packing.hpp +6279 -0
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser.hpp +14 -8
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser_impl.hpp +60 -46
- data/vendor/datasketches-cpp/theta/include/theta_helpers.hpp +4 -2
- data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp +58 -10
- data/vendor/datasketches-cpp/theta/include/theta_sketch_impl.hpp +430 -130
- data/vendor/datasketches-cpp/theta/include/theta_union_base_impl.hpp +9 -9
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp +16 -4
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp +2 -2
- data/vendor/datasketches-cpp/theta/test/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/theta/test/bit_packing_test.cpp +80 -0
- data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp +42 -3
- data/vendor/datasketches-cpp/theta/test/theta_union_test.cpp +25 -0
- data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp +2 -1
- data/vendor/datasketches-cpp/version.cfg.in +1 -1
- metadata +31 -3
@@ -17,199 +17,138 @@
|
|
17
17
|
* under the License.
|
18
18
|
*/
|
19
19
|
|
20
|
+
#include "py_object_lt.hpp"
|
21
|
+
#include "py_object_ostream.hpp"
|
22
|
+
#include "quantile_conditional.hpp"
|
20
23
|
#include "req_sketch.hpp"
|
21
24
|
|
22
25
|
#include <pybind11/pybind11.h>
|
23
26
|
#include <pybind11/stl.h>
|
24
27
|
#include <pybind11/numpy.h>
|
25
|
-
#include <sstream>
|
26
28
|
#include <vector>
|
27
29
|
#include <stdexcept>
|
28
30
|
|
29
31
|
namespace py = pybind11;
|
30
32
|
|
31
|
-
|
32
|
-
|
33
|
-
namespace python {
|
34
|
-
|
35
|
-
template<typename T>
|
36
|
-
req_sketch<T> req_sketch_deserialize(py::bytes sk_bytes) {
|
37
|
-
std::string sk_str = sk_bytes; // implicit cast
|
38
|
-
return req_sketch<T>::deserialize(sk_str.c_str(), sk_str.length());
|
39
|
-
}
|
40
|
-
|
41
|
-
template<typename T>
|
42
|
-
py::object req_sketch_serialize(const req_sketch<T>& sk) {
|
43
|
-
auto ser_result = sk.serialize();
|
44
|
-
return py::bytes((char*)ser_result.data(), ser_result.size());
|
45
|
-
}
|
46
|
-
|
47
|
-
// maybe possible to disambiguate the static vs method rank error calls, but
|
48
|
-
// this is easier for now
|
49
|
-
template<typename T>
|
50
|
-
double req_sketch_generic_normalized_rank_error(uint16_t k, bool pmf) {
|
51
|
-
return req_sketch<T>::get_normalized_rank_error(k, pmf);
|
52
|
-
}
|
53
|
-
|
54
|
-
template<typename T>
|
55
|
-
py::list req_sketch_get_quantiles(const req_sketch<T>& sk,
|
56
|
-
std::vector<double>& ranks,
|
57
|
-
bool inclusive) {
|
58
|
-
size_t n_quantiles = ranks.size();
|
59
|
-
auto result = sk.get_quantiles(ranks.data(), n_quantiles, inclusive);
|
60
|
-
// returning as std::vector<> would copy values to a list anyway
|
61
|
-
py::list list(n_quantiles);
|
62
|
-
for (size_t i = 0; i < n_quantiles; ++i) {
|
63
|
-
list[i] = result[i];
|
64
|
-
}
|
65
|
-
return list;
|
66
|
-
}
|
67
|
-
|
68
|
-
template<typename T>
|
69
|
-
py::list req_sketch_get_pmf(const req_sketch<T>& sk,
|
70
|
-
std::vector<T>& split_points,
|
71
|
-
bool inclusive) {
|
72
|
-
size_t n_points = split_points.size();
|
73
|
-
auto result = sk.get_PMF(split_points.data(), n_points, inclusive);
|
74
|
-
py::list list(n_points + 1);
|
75
|
-
for (size_t i = 0; i <= n_points; ++i) {
|
76
|
-
list[i] = result[i];
|
77
|
-
}
|
78
|
-
return list;
|
79
|
-
}
|
80
|
-
|
81
|
-
template<typename T>
|
82
|
-
py::list req_sketch_get_cdf(const req_sketch<T>& sk,
|
83
|
-
std::vector<T>& split_points,
|
84
|
-
bool inclusive) {
|
85
|
-
size_t n_points = split_points.size();
|
86
|
-
auto result = sk.get_CDF(split_points.data(), n_points, inclusive);
|
87
|
-
py::list list(n_points + 1);
|
88
|
-
for (size_t i = 0; i <= n_points; ++i) {
|
89
|
-
list[i] = result[i];
|
90
|
-
}
|
91
|
-
return list;
|
92
|
-
}
|
93
|
-
|
94
|
-
template<typename T>
|
95
|
-
void req_sketch_update(req_sketch<T>& sk, py::array_t<T, py::array::c_style | py::array::forcecast> items) {
|
96
|
-
if (items.ndim() != 1) {
|
97
|
-
throw std::invalid_argument("input data must have only one dimension. Found: "
|
98
|
-
+ std::to_string(items.ndim()));
|
99
|
-
}
|
100
|
-
|
101
|
-
auto data = items.template unchecked<1>();
|
102
|
-
for (uint32_t i = 0; i < data.size(); ++i) {
|
103
|
-
sk.update(data(i));
|
104
|
-
}
|
105
|
-
}
|
106
|
-
|
107
|
-
}
|
108
|
-
}
|
109
|
-
|
110
|
-
namespace dspy = datasketches::python;
|
111
|
-
|
112
|
-
template<typename T>
|
33
|
+
template<typename T, typename C>
|
113
34
|
void bind_req_sketch(py::module &m, const char* name) {
|
114
35
|
using namespace datasketches;
|
115
36
|
|
116
|
-
py::class_<req_sketch<T>>(m, name)
|
37
|
+
auto req_class = py::class_<req_sketch<T, C>>(m, name)
|
117
38
|
.def(py::init<uint16_t, bool>(), py::arg("k")=12, py::arg("is_hra")=true)
|
118
|
-
.def(py::init<const req_sketch<T>&>())
|
119
|
-
.def("update", (void (req_sketch<T>::*)(const T&)) &req_sketch<T>::update, py::arg("item"),
|
120
|
-
|
121
|
-
.def("
|
122
|
-
|
123
|
-
.def("
|
124
|
-
|
125
|
-
.def("
|
126
|
-
|
127
|
-
.def("
|
128
|
-
|
129
|
-
.def("
|
130
|
-
|
131
|
-
.def("
|
132
|
-
|
133
|
-
.def("
|
134
|
-
|
135
|
-
.def("
|
136
|
-
|
137
|
-
.def("
|
138
|
-
|
139
|
-
.def("
|
140
|
-
|
141
|
-
.def("
|
142
|
-
|
143
|
-
.def("
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
.def("
|
208
|
-
|
209
|
-
|
39
|
+
.def(py::init<const req_sketch<T, C>&>())
|
40
|
+
.def("update", (void (req_sketch<T, C>::*)(const T&)) &req_sketch<T, C>::update, py::arg("item"),
|
41
|
+
"Updates the sketch with the given value")
|
42
|
+
.def("merge", (void (req_sketch<T, C>::*)(const req_sketch<T, C>&)) &req_sketch<T, C>::merge, py::arg("sketch"),
|
43
|
+
"Merges the provided sketch into this one")
|
44
|
+
.def("__str__", &req_sketch<T, C>::to_string, py::arg("print_levels")=false, py::arg("print_items")=false,
|
45
|
+
"Produces a string summary of the sketch")
|
46
|
+
.def("to_string", &req_sketch<T, C>::to_string, py::arg("print_levels")=false, py::arg("print_items")=false,
|
47
|
+
"Produces a string summary of the sketch")
|
48
|
+
.def("is_hra", &req_sketch<T, C>::is_HRA,
|
49
|
+
"Returns True if the sketch is in High Rank Accuracy mode, otherwise False")
|
50
|
+
.def("is_empty", &req_sketch<T, C>::is_empty,
|
51
|
+
"Returns True if the sketch is empty, otherwise False")
|
52
|
+
.def("get_k", &req_sketch<T, C>::get_k,
|
53
|
+
"Returns the configured parameter k")
|
54
|
+
.def("get_n", &req_sketch<T, C>::get_n,
|
55
|
+
"Returns the length of the input stream")
|
56
|
+
.def("get_num_retained", &req_sketch<T, C>::get_num_retained,
|
57
|
+
"Returns the number of retained items (samples) in the sketch")
|
58
|
+
.def("is_estimation_mode", &req_sketch<T, C>::is_estimation_mode,
|
59
|
+
"Returns True if the sketch is in estimation mode, otherwise False")
|
60
|
+
.def("get_min_value", &req_sketch<T, C>::get_min_item,
|
61
|
+
"Returns the minimum value from the stream. If empty, req_floats_sketch returns nan; req_ints_sketch throws a RuntimeError")
|
62
|
+
.def("get_max_value", &req_sketch<T, C>::get_max_item,
|
63
|
+
"Returns the maximum value from the stream. If empty, req_floats_sketch returns nan; req_ints_sketch throws a RuntimeError")
|
64
|
+
.def("get_quantile", &req_sketch<T, C>::get_quantile, py::arg("rank"), py::arg("inclusive")=false,
|
65
|
+
"Returns an approximation to the data value "
|
66
|
+
"associated with the given normalized rank in a hypothetical sorted "
|
67
|
+
"version of the input stream so far.\n"
|
68
|
+
"For req_floats_sketch: if the sketch is empty this returns nan. "
|
69
|
+
"For req_ints_sketch: if the sketch is empty this throws a RuntimeError.")
|
70
|
+
.def(
|
71
|
+
"get_quantiles",
|
72
|
+
[](const req_sketch<T, C>& sk, const std::vector<double>& ranks, bool inclusive) {
|
73
|
+
return sk.get_quantiles(ranks.data(), ranks.size(), inclusive);
|
74
|
+
},
|
75
|
+
py::arg("ranks"), py::arg("inclusive")=false,
|
76
|
+
"This returns an array that could have been generated by using get_quantile() for each "
|
77
|
+
"normalized rank separately.\n"
|
78
|
+
"If the sketch is empty this returns an empty vector.\n"
|
79
|
+
"Deprecated. Will be removed in the next major version. Use get_quantile() instead."
|
80
|
+
)
|
81
|
+
.def("get_rank", &req_sketch<T, C>::get_rank, py::arg("value"), py::arg("inclusive")=false,
|
82
|
+
"Returns an approximation to the normalized rank of the given value from 0 to 1, inclusive.\n"
|
83
|
+
"The resulting approximation has a probabilistic guarantee that can be obtained from the "
|
84
|
+
"get_normalized_rank_error(False) function.\n"
|
85
|
+
"With the parameter inclusive=true the weight of the given value is included into the rank."
|
86
|
+
"Otherwise the rank equals the sum of the weights of values less than the given value.\n"
|
87
|
+
"If the sketch is empty this returns nan.")
|
88
|
+
.def(
|
89
|
+
"get_pmf",
|
90
|
+
[](const req_sketch<T, C>& sk, const std::vector<T>& split_points, bool inclusive) {
|
91
|
+
return sk.get_PMF(split_points.data(), split_points.size(), inclusive);
|
92
|
+
},
|
93
|
+
py::arg("split_points"), py::arg("inclusive")=false,
|
94
|
+
"Returns an approximation to the Probability Mass Function (PMF) of the input stream "
|
95
|
+
"given a set of split points (values).\n"
|
96
|
+
"The resulting approximations have a probabilistic guarantee that can be obtained from the "
|
97
|
+
"get_normalized_rank_error(True) function.\n"
|
98
|
+
"If the sketch is empty this returns an empty vector.\n"
|
99
|
+
"split_points is an array of m unique, monotonically increasing float values "
|
100
|
+
"that divide the real number line into m+1 consecutive disjoint intervals.\n"
|
101
|
+
"If the parameter inclusive=false, the definition of an 'interval' is inclusive of the left split point (or minimum value) and "
|
102
|
+
"exclusive of the right split point, with the exception that the last interval will include "
|
103
|
+
"the maximum value.\n"
|
104
|
+
"If the parameter inclusive=true, the definition of an 'interval' is exclusive of the left split point (or minimum value) and "
|
105
|
+
"inclusive of the right split point.\n"
|
106
|
+
"It is not necessary to include either the min or max values in these split points."
|
107
|
+
)
|
108
|
+
.def(
|
109
|
+
"get_cdf",
|
110
|
+
[](const req_sketch<T, C>& sk, const std::vector<T>& split_points, bool inclusive) {
|
111
|
+
return sk.get_CDF(split_points.data(), split_points.size(), inclusive);
|
112
|
+
},
|
113
|
+
py::arg("split_points"), py::arg("inclusive")=false,
|
114
|
+
"Returns an approximation to the Cumulative Distribution Function (CDF), which is the "
|
115
|
+
"cumulative analog of the PMF, of the input stream given a set of split points (values).\n"
|
116
|
+
"The resulting approximations have a probabilistic guarantee that can be obtained from the "
|
117
|
+
"get_normalized_rank_error(True) function.\n"
|
118
|
+
"If the sketch is empty this returns an empty vector.\n"
|
119
|
+
"split_points is an array of m unique, monotonically increasing float values "
|
120
|
+
"that divide the real number line into m+1 consecutive disjoint intervals.\n"
|
121
|
+
"If the parameter inclusive=false, the definition of an 'interval' is inclusive of the left split point (or minimum value) and "
|
122
|
+
"exclusive of the right split point, with the exception that the last interval will include "
|
123
|
+
"the maximum value.\n"
|
124
|
+
"If the parameter inclusive=true, the definition of an 'interval' is exclusive of the left split point (or minimum value) and "
|
125
|
+
"inclusive of the right split point.\n"
|
126
|
+
"It is not necessary to include either the min or max values in these split points."
|
127
|
+
)
|
128
|
+
.def("get_rank_lower_bound", &req_sketch<T, C>::get_rank_lower_bound, py::arg("rank"), py::arg("num_std_dev"),
|
129
|
+
"Returns an approximate lower bound on the given normalized rank.\n"
|
130
|
+
"Normalized rank must be a value between 0.0 and 1.0 (inclusive); "
|
131
|
+
"the number of standard deviations must be 1, 2, or 3.")
|
132
|
+
.def("get_rank_upper_bound", &req_sketch<T, C>::get_rank_upper_bound, py::arg("rank"), py::arg("num_std_dev"),
|
133
|
+
"Returns an approximate upper bound on the given normalized rank.\n"
|
134
|
+
"Normalized rank must be a value between 0.0 and 1.0 (inclusive); "
|
135
|
+
"the number of standard deviations must be 1, 2, or 3.")
|
136
|
+
.def_static("get_RSE", &req_sketch<T, C>::get_RSE,
|
137
|
+
py::arg("k"), py::arg("rank"), py::arg("is_hra"), py::arg("n"),
|
138
|
+
"Returns an a priori estimate of relative standard error (RSE, expressed as a number in [0,1]). "
|
139
|
+
"Derived from Lemma 12 in http://arxiv.org/abs/2004.01668v2, but the constant factors have been "
|
140
|
+
"modified based on empirical measurements, for a given value of parameter k.\n"
|
141
|
+
"Normalized rank must be a value between 0.0 and 1.0 (inclusive). If is_hra is True, uses high "
|
142
|
+
"rank accuracy mode, else low rank accuracy. N is an estimate of the total number of points "
|
143
|
+
"provided to the sketch.")
|
144
|
+
.def("__iter__", [](const req_sketch<T, C>& s) { return py::make_iterator(s.begin(), s.end()); });
|
145
|
+
|
146
|
+
add_serialization<T>(req_class);
|
147
|
+
add_vector_update<T>(req_class);
|
210
148
|
}
|
211
149
|
|
212
150
|
void init_req(py::module &m) {
|
213
|
-
bind_req_sketch<int
|
214
|
-
bind_req_sketch<float
|
151
|
+
bind_req_sketch<int, std::less<int>>(m, "req_ints_sketch");
|
152
|
+
bind_req_sketch<float, std::less<float>>(m, "req_floats_sketch");
|
153
|
+
bind_req_sketch<py::object, py_object_lt>(m, "req_items_sketch");
|
215
154
|
}
|
@@ -17,7 +17,6 @@
|
|
17
17
|
* under the License.
|
18
18
|
*/
|
19
19
|
|
20
|
-
#include <sstream>
|
21
20
|
#include <pybind11/pybind11.h>
|
22
21
|
#include <pybind11/stl.h>
|
23
22
|
|
@@ -28,51 +27,8 @@
|
|
28
27
|
#include "theta_jaccard_similarity.hpp"
|
29
28
|
#include "common_defs.hpp"
|
30
29
|
|
31
|
-
|
32
30
|
namespace py = pybind11;
|
33
31
|
|
34
|
-
namespace datasketches {
|
35
|
-
namespace python {
|
36
|
-
|
37
|
-
update_theta_sketch update_theta_sketch_factory(uint8_t lg_k, double p, uint64_t seed) {
|
38
|
-
update_theta_sketch::builder builder;
|
39
|
-
builder.set_lg_k(lg_k);
|
40
|
-
builder.set_p(p);
|
41
|
-
builder.set_seed(seed);
|
42
|
-
return builder.build();
|
43
|
-
}
|
44
|
-
|
45
|
-
theta_union theta_union_factory(uint8_t lg_k, double p, uint64_t seed) {
|
46
|
-
theta_union::builder builder;
|
47
|
-
builder.set_lg_k(lg_k);
|
48
|
-
builder.set_p(p);
|
49
|
-
builder.set_seed(seed);
|
50
|
-
return builder.build();
|
51
|
-
}
|
52
|
-
|
53
|
-
uint16_t theta_sketch_get_seed_hash(const theta_sketch& sk) {
|
54
|
-
return sk.get_seed_hash();
|
55
|
-
}
|
56
|
-
|
57
|
-
py::object compact_theta_sketch_serialize(const compact_theta_sketch& sk) {
|
58
|
-
auto serResult = sk.serialize();
|
59
|
-
return py::bytes((char*)serResult.data(), serResult.size());
|
60
|
-
}
|
61
|
-
|
62
|
-
compact_theta_sketch compact_theta_sketch_deserialize(py::bytes skBytes, uint64_t seed) {
|
63
|
-
std::string skStr = skBytes; // implicit cast
|
64
|
-
return compact_theta_sketch::deserialize(skStr.c_str(), skStr.length(), seed);
|
65
|
-
}
|
66
|
-
|
67
|
-
py::list theta_jaccard_sim_computation(const theta_sketch& sketch_a, const theta_sketch& sketch_b, uint64_t seed) {
|
68
|
-
return py::cast(theta_jaccard_similarity::jaccard(sketch_a, sketch_b, seed));
|
69
|
-
}
|
70
|
-
|
71
|
-
}
|
72
|
-
}
|
73
|
-
|
74
|
-
namespace dspy = datasketches::python;
|
75
|
-
|
76
32
|
void init_theta(py::module &m) {
|
77
33
|
using namespace datasketches;
|
78
34
|
|
@@ -93,17 +49,24 @@ void init_theta(py::module &m) {
|
|
93
49
|
"Returns True if sketch is in estimation mode, otherwise False")
|
94
50
|
.def("get_theta", &theta_sketch::get_theta,
|
95
51
|
"Returns theta (effective sampling rate) as a fraction from 0 to 1")
|
52
|
+
.def("get_theta64", &theta_sketch::get_theta64,
|
53
|
+
"Returns theta as 64-bit value")
|
96
54
|
.def("get_num_retained", &theta_sketch::get_num_retained,
|
97
|
-
"
|
98
|
-
.def("get_seed_hash", &
|
55
|
+
"Returns the number of items currently in the sketch")
|
56
|
+
.def("get_seed_hash", &theta_sketch::get_seed_hash,
|
99
57
|
"Returns a hash of the seed used in the sketch")
|
100
58
|
.def("is_ordered", &theta_sketch::is_ordered,
|
101
59
|
"Returns True if the sketch entries are sorted, otherwise False")
|
60
|
+
.def("__iter__", [](const theta_sketch& s) { return py::make_iterator(s.begin(), s.end()); })
|
102
61
|
;
|
103
62
|
|
104
63
|
py::class_<update_theta_sketch, theta_sketch>(m, "update_theta_sketch")
|
105
|
-
.def(
|
106
|
-
|
64
|
+
.def(
|
65
|
+
py::init([](uint8_t lg_k, double p, uint64_t seed) {
|
66
|
+
return update_theta_sketch::builder().set_lg_k(lg_k).set_p(p).set_seed(seed).build();
|
67
|
+
}),
|
68
|
+
py::arg("lg_k")=theta_constants::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED
|
69
|
+
)
|
107
70
|
.def(py::init<const update_theta_sketch&>())
|
108
71
|
.def("update", (void (update_theta_sketch::*)(int64_t)) &update_theta_sketch::update, py::arg("datum"),
|
109
72
|
"Updates the sketch with the given integral value")
|
@@ -118,16 +81,30 @@ void init_theta(py::module &m) {
|
|
118
81
|
py::class_<compact_theta_sketch, theta_sketch>(m, "compact_theta_sketch")
|
119
82
|
.def(py::init<const compact_theta_sketch&>())
|
120
83
|
.def(py::init<const theta_sketch&, bool>())
|
121
|
-
.def(
|
122
|
-
"
|
123
|
-
|
84
|
+
.def(
|
85
|
+
"serialize",
|
86
|
+
[](const compact_theta_sketch& sk) {
|
87
|
+
auto bytes = sk.serialize();
|
88
|
+
return py::bytes(reinterpret_cast<const char*>(bytes.data()), bytes.size());
|
89
|
+
},
|
90
|
+
"Serializes the sketch into a bytes object"
|
91
|
+
)
|
92
|
+
.def_static(
|
93
|
+
"deserialize",
|
94
|
+
[](const std::string& bytes, uint64_t seed) {
|
95
|
+
return compact_theta_sketch::deserialize(bytes.data(), bytes.size(), seed);
|
96
|
+
},
|
124
97
|
py::arg("bytes"), py::arg("seed")=DEFAULT_SEED,
|
125
|
-
"Reads a bytes object and returns the corresponding compact_theta_sketch"
|
126
|
-
|
98
|
+
"Reads a bytes object and returns the corresponding compact_theta_sketch"
|
99
|
+
);
|
127
100
|
|
128
101
|
py::class_<theta_union>(m, "theta_union")
|
129
|
-
.def(
|
130
|
-
|
102
|
+
.def(
|
103
|
+
py::init([](uint8_t lg_k, double p, uint64_t seed) {
|
104
|
+
return theta_union::builder().set_lg_k(lg_k).set_p(p).set_seed(seed).build();
|
105
|
+
}),
|
106
|
+
py::arg("lg_k")=theta_constants::DEFAULT_LG_K, py::arg("p")=1.0, py::arg("seed")=DEFAULT_SEED
|
107
|
+
)
|
131
108
|
.def("update", &theta_union::update<const theta_sketch&>, py::arg("sketch"),
|
132
109
|
"Updates the union with the given sketch")
|
133
110
|
.def("get_result", &theta_union::get_result, py::arg("ordered")=true,
|
@@ -147,26 +124,43 @@ void init_theta(py::module &m) {
|
|
147
124
|
|
148
125
|
py::class_<theta_a_not_b>(m, "theta_a_not_b")
|
149
126
|
.def(py::init<uint64_t>(), py::arg("seed")=DEFAULT_SEED)
|
150
|
-
.def(
|
151
|
-
|
127
|
+
.def(
|
128
|
+
"compute",
|
129
|
+
&theta_a_not_b::compute<const theta_sketch&, const theta_sketch&>,
|
130
|
+
py::arg("a"), py::arg("b"), py::arg("ordered")=true,
|
131
|
+
"Returns a sketch with the result of applying the A-not-B operation on the given inputs"
|
132
|
+
)
|
152
133
|
;
|
153
134
|
|
154
135
|
py::class_<theta_jaccard_similarity>(m, "theta_jaccard_similarity")
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
136
|
+
.def_static(
|
137
|
+
"jaccard",
|
138
|
+
[](const theta_sketch& sketch_a, const theta_sketch& sketch_b, uint64_t seed) {
|
139
|
+
return theta_jaccard_similarity::jaccard(sketch_a, sketch_b, seed);
|
140
|
+
},
|
141
|
+
py::arg("sketch_a"), py::arg("sketch_b"), py::arg("seed")=DEFAULT_SEED,
|
142
|
+
"Returns a list with {lower_bound, estimate, upper_bound} of the Jaccard similarity between sketches"
|
143
|
+
)
|
144
|
+
.def_static(
|
145
|
+
"exactly_equal",
|
146
|
+
&theta_jaccard_similarity::exactly_equal<const theta_sketch&, const theta_sketch&>,
|
147
|
+
py::arg("sketch_a"), py::arg("sketch_b"), py::arg("seed")=DEFAULT_SEED,
|
148
|
+
"Returns True if sketch_a and sketch_b are equivalent, otherwise False"
|
149
|
+
)
|
150
|
+
.def_static(
|
151
|
+
"similarity_test",
|
152
|
+
&theta_jaccard_similarity::similarity_test<const theta_sketch&, const theta_sketch&>,
|
153
|
+
py::arg("actual"), py::arg("expected"), py::arg("threshold"), py::arg("seed")=DEFAULT_SEED,
|
154
|
+
"Tests similarity of an actual sketch against an expected sketch. Computers the lower bound of the Jaccard "
|
155
|
+
"index J_{LB} of the actual and expected sketches. If J_{LB} >= threshold, then the sketches are considered "
|
156
|
+
"to be similar with a confidence of 97.7% and returns True, otherwise False.")
|
157
|
+
.def_static(
|
158
|
+
"dissimilarity_test",
|
159
|
+
&theta_jaccard_similarity::dissimilarity_test<const theta_sketch&, const theta_sketch&>,
|
160
|
+
py::arg("actual"), py::arg("expected"), py::arg("threshold"), py::arg("seed")=DEFAULT_SEED,
|
161
|
+
"Tests dissimilarity of an actual sketch against an expected sketch. Computers the lower bound of the Jaccard "
|
162
|
+
"index J_{UB} of the actual and expected sketches. If J_{UB} <= threshold, then the sketches are considered "
|
163
|
+
"to be dissimilar with a confidence of 97.7% and returns True, otherwise False."
|
164
|
+
)
|
171
165
|
;
|
172
166
|
}
|