datasketches 0.3.1 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/NOTICE +1 -1
- data/README.md +0 -2
- data/ext/datasketches/cpc_wrapper.cpp +3 -3
- data/ext/datasketches/kll_wrapper.cpp +0 -10
- data/lib/datasketches/version.rb +1 -1
- data/lib/datasketches.rb +1 -1
- data/vendor/datasketches-cpp/CMakeLists.txt +23 -20
- data/vendor/datasketches-cpp/CODE_OF_CONDUCT.md +3 -0
- data/vendor/datasketches-cpp/CONTRIBUTING.md +50 -0
- data/vendor/datasketches-cpp/Doxyfile +2827 -0
- data/vendor/datasketches-cpp/LICENSE +0 -76
- data/vendor/datasketches-cpp/README.md +1 -3
- data/vendor/datasketches-cpp/common/CMakeLists.txt +12 -11
- data/vendor/datasketches-cpp/common/include/MurmurHash3.h +25 -27
- data/vendor/datasketches-cpp/common/include/common_defs.hpp +15 -10
- data/vendor/datasketches-cpp/common/include/count_zeros.hpp +11 -2
- data/vendor/datasketches-cpp/common/include/kolmogorov_smirnov.hpp +9 -6
- data/vendor/datasketches-cpp/common/include/memory_operations.hpp +5 -4
- data/vendor/datasketches-cpp/common/include/optional.hpp +148 -0
- data/vendor/datasketches-cpp/common/include/quantiles_sorted_view.hpp +95 -2
- data/vendor/datasketches-cpp/common/include/quantiles_sorted_view_impl.hpp +1 -1
- data/vendor/datasketches-cpp/common/include/serde.hpp +69 -20
- data/vendor/datasketches-cpp/common/test/CMakeLists.txt +2 -2
- data/vendor/datasketches-cpp/common/test/integration_test.cpp +6 -0
- data/vendor/datasketches-cpp/common/test/optional_test.cpp +85 -0
- data/vendor/datasketches-cpp/common/test/test_allocator.hpp +14 -14
- data/vendor/datasketches-cpp/{python/src/__init__.py → count/CMakeLists.txt} +25 -1
- data/vendor/datasketches-cpp/count/include/count_min.hpp +405 -0
- data/vendor/datasketches-cpp/count/include/count_min_impl.hpp +497 -0
- data/vendor/datasketches-cpp/{MANIFEST.in → count/test/CMakeLists.txt} +23 -20
- data/vendor/datasketches-cpp/count/test/count_min_allocation_test.cpp +155 -0
- data/vendor/datasketches-cpp/count/test/count_min_test.cpp +303 -0
- data/vendor/datasketches-cpp/cpc/include/cpc_common.hpp +14 -20
- data/vendor/datasketches-cpp/cpc/include/cpc_compressor.hpp +7 -4
- data/vendor/datasketches-cpp/cpc/include/cpc_compressor_impl.hpp +17 -17
- data/vendor/datasketches-cpp/cpc/include/cpc_confidence.hpp +3 -3
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch.hpp +40 -40
- data/vendor/datasketches-cpp/cpc/include/cpc_sketch_impl.hpp +14 -11
- data/vendor/datasketches-cpp/cpc/include/cpc_union.hpp +35 -11
- data/vendor/datasketches-cpp/cpc/include/cpc_union_impl.hpp +8 -8
- data/vendor/datasketches-cpp/cpc/include/cpc_util.hpp +16 -8
- data/vendor/datasketches-cpp/cpc/include/u32_table.hpp +3 -2
- data/vendor/datasketches-cpp/cpc/include/u32_table_impl.hpp +5 -5
- data/vendor/datasketches-cpp/cpc/test/CMakeLists.txt +20 -7
- data/vendor/datasketches-cpp/cpc/test/cpc_sketch_deserialize_from_java_test.cpp +60 -0
- data/vendor/datasketches-cpp/cpc/test/cpc_sketch_serialize_for_java.cpp +38 -0
- data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp +4 -29
- data/vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp +4 -4
- data/vendor/datasketches-cpp/{tox.ini → density/CMakeLists.txt} +24 -8
- data/vendor/datasketches-cpp/density/include/density_sketch.hpp +256 -0
- data/vendor/datasketches-cpp/density/include/density_sketch_impl.hpp +543 -0
- data/vendor/datasketches-cpp/{python/datasketches/__init__.py → density/test/CMakeLists.txt} +15 -3
- data/vendor/datasketches-cpp/density/test/density_sketch_test.cpp +244 -0
- data/vendor/datasketches-cpp/fi/include/frequent_items_sketch.hpp +21 -9
- data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp +6 -4
- data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map.hpp +9 -3
- data/vendor/datasketches-cpp/fi/test/CMakeLists.txt +14 -1
- data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_deserialize_from_java_test.cpp +95 -0
- data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_serialize_for_java.cpp +83 -0
- data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_test.cpp +3 -42
- data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp +2 -2
- data/vendor/datasketches-cpp/hll/include/CouponList.hpp +3 -1
- data/vendor/datasketches-cpp/hll/include/Hll4Array-internal.hpp +19 -11
- data/vendor/datasketches-cpp/hll/include/Hll4Array.hpp +2 -5
- data/vendor/datasketches-cpp/hll/include/Hll6Array-internal.hpp +19 -7
- data/vendor/datasketches-cpp/hll/include/Hll6Array.hpp +1 -1
- data/vendor/datasketches-cpp/hll/include/Hll8Array-internal.hpp +98 -42
- data/vendor/datasketches-cpp/hll/include/Hll8Array.hpp +2 -0
- data/vendor/datasketches-cpp/hll/include/HllArray-internal.hpp +94 -61
- data/vendor/datasketches-cpp/hll/include/HllArray.hpp +20 -8
- data/vendor/datasketches-cpp/hll/include/HllSketch-internal.hpp +4 -4
- data/vendor/datasketches-cpp/hll/include/HllSketchImpl.hpp +3 -1
- data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp +3 -21
- data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +8 -0
- data/vendor/datasketches-cpp/hll/include/HllUtil.hpp +14 -18
- data/vendor/datasketches-cpp/hll/include/coupon_iterator-internal.hpp +1 -1
- data/vendor/datasketches-cpp/hll/include/coupon_iterator.hpp +8 -2
- data/vendor/datasketches-cpp/hll/include/hll.hpp +79 -65
- data/vendor/datasketches-cpp/hll/test/CMakeLists.txt +14 -1
- data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +7 -1
- data/vendor/datasketches-cpp/hll/test/ToFromByteArrayTest.cpp +0 -68
- data/vendor/datasketches-cpp/hll/test/hll_sketch_deserialize_from_java_test.cpp +69 -0
- data/vendor/datasketches-cpp/hll/test/hll_sketch_serialize_for_java.cpp +52 -0
- data/vendor/datasketches-cpp/kll/include/kll_helper.hpp +0 -1
- data/vendor/datasketches-cpp/kll/include/kll_helper_impl.hpp +2 -2
- data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +79 -53
- data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +61 -132
- data/vendor/datasketches-cpp/kll/test/CMakeLists.txt +14 -1
- data/vendor/datasketches-cpp/kll/test/kll_sketch_deserialize_from_java_test.cpp +103 -0
- data/vendor/datasketches-cpp/kll/test/kll_sketch_serialize_for_java.cpp +62 -0
- data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +5 -40
- data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch.hpp +76 -54
- data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch_impl.hpp +66 -136
- data/vendor/datasketches-cpp/quantiles/test/CMakeLists.txt +14 -1
- data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_deserialize_from_java_test.cpp +84 -0
- data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_serialize_for_java.cpp +52 -0
- data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_test.cpp +15 -39
- data/vendor/datasketches-cpp/req/include/req_common.hpp +7 -3
- data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +2 -4
- data/vendor/datasketches-cpp/req/include/req_sketch.hpp +105 -26
- data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +50 -111
- data/vendor/datasketches-cpp/req/test/CMakeLists.txt +14 -1
- data/vendor/datasketches-cpp/req/test/req_sketch_deserialize_from_java_test.cpp +55 -0
- data/vendor/datasketches-cpp/{tuple/include/array_of_doubles_intersection_impl.hpp → req/test/req_sketch_serialize_for_java.cpp} +12 -7
- data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +3 -89
- data/vendor/datasketches-cpp/sampling/CMakeLists.txt +4 -0
- data/vendor/datasketches-cpp/sampling/include/ebpps_sample.hpp +210 -0
- data/vendor/datasketches-cpp/sampling/include/ebpps_sample_impl.hpp +535 -0
- data/vendor/datasketches-cpp/sampling/include/ebpps_sketch.hpp +281 -0
- data/vendor/datasketches-cpp/sampling/include/ebpps_sketch_impl.hpp +531 -0
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch.hpp +89 -32
- data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp +33 -19
- data/vendor/datasketches-cpp/sampling/include/var_opt_union.hpp +13 -10
- data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp +23 -19
- data/vendor/datasketches-cpp/sampling/test/CMakeLists.txt +55 -8
- data/vendor/datasketches-cpp/sampling/test/ebpps_allocation_test.cpp +96 -0
- data/vendor/datasketches-cpp/sampling/test/ebpps_sample_test.cpp +137 -0
- data/vendor/datasketches-cpp/sampling/test/ebpps_sketch_test.cpp +266 -0
- data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_deserialize_from_java_test.cpp +81 -0
- data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_serialize_for_java.cpp +54 -0
- data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp +33 -51
- data/vendor/datasketches-cpp/sampling/test/var_opt_union_deserialize_from_java_test.cpp +50 -0
- data/vendor/datasketches-cpp/sampling/test/var_opt_union_serialize_for_java.cpp +56 -0
- data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp +0 -20
- data/vendor/datasketches-cpp/theta/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/theta/include/bit_packing.hpp +6279 -0
- data/vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_sampled_sets.hpp +1 -0
- data/vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_theta_sketched_sets.hpp +7 -6
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser.hpp +14 -8
- data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser_impl.hpp +60 -46
- data/vendor/datasketches-cpp/theta/include/theta_a_not_b.hpp +20 -5
- data/vendor/datasketches-cpp/theta/include/theta_constants.hpp +10 -4
- data/vendor/datasketches-cpp/theta/include/theta_helpers.hpp +3 -1
- data/vendor/datasketches-cpp/theta/include/theta_intersection.hpp +13 -5
- data/vendor/datasketches-cpp/theta/include/theta_intersection_base_impl.hpp +5 -5
- data/vendor/datasketches-cpp/theta/include/theta_intersection_impl.hpp +3 -3
- data/vendor/datasketches-cpp/theta/include/theta_jaccard_similarity.hpp +2 -1
- data/vendor/datasketches-cpp/theta/include/theta_jaccard_similarity_base.hpp +1 -0
- data/vendor/datasketches-cpp/theta/include/theta_set_difference_base_impl.hpp +1 -1
- data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp +180 -33
- data/vendor/datasketches-cpp/theta/include/theta_sketch_impl.hpp +430 -130
- data/vendor/datasketches-cpp/theta/include/theta_union.hpp +17 -10
- data/vendor/datasketches-cpp/theta/include/theta_union_base_impl.hpp +10 -10
- data/vendor/datasketches-cpp/theta/include/theta_union_impl.hpp +3 -3
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp +21 -6
- data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp +13 -3
- data/vendor/datasketches-cpp/theta/test/CMakeLists.txt +15 -1
- data/vendor/datasketches-cpp/theta/test/bit_packing_test.cpp +80 -0
- data/vendor/datasketches-cpp/theta/test/theta_sketch_deserialize_from_java_test.cpp +57 -0
- data/vendor/datasketches-cpp/theta/test/theta_sketch_serialize_for_java.cpp +61 -0
- data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp +39 -188
- data/vendor/datasketches-cpp/theta/test/theta_union_test.cpp +25 -0
- data/vendor/datasketches-cpp/tuple/CMakeLists.txt +8 -7
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch.hpp +19 -144
- data/vendor/datasketches-cpp/tuple/include/{array_of_doubles_a_not_b.hpp → array_tuple_a_not_b.hpp} +24 -16
- data/vendor/datasketches-cpp/tuple/include/{array_of_doubles_a_not_b_impl.hpp → array_tuple_a_not_b_impl.hpp} +4 -4
- data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection.hpp +65 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection_impl.hpp +31 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_sketch.hpp +237 -0
- data/vendor/datasketches-cpp/tuple/include/{array_of_doubles_sketch_impl.hpp → array_tuple_sketch_impl.hpp} +40 -41
- data/vendor/datasketches-cpp/tuple/include/array_tuple_union.hpp +81 -0
- data/vendor/datasketches-cpp/tuple/include/array_tuple_union_impl.hpp +43 -0
- data/vendor/datasketches-cpp/tuple/include/tuple_a_not_b.hpp +11 -2
- data/vendor/datasketches-cpp/tuple/include/tuple_intersection.hpp +17 -10
- data/vendor/datasketches-cpp/tuple/include/tuple_jaccard_similarity.hpp +2 -1
- data/vendor/datasketches-cpp/tuple/include/tuple_sketch.hpp +95 -32
- data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp +2 -1
- data/vendor/datasketches-cpp/tuple/include/tuple_union.hpp +19 -11
- data/vendor/datasketches-cpp/tuple/test/CMakeLists.txt +16 -1
- data/vendor/datasketches-cpp/tuple/test/aod_sketch_deserialize_from_java_test.cpp +76 -0
- data/vendor/datasketches-cpp/tuple/test/aod_sketch_serialize_for_java.cpp +62 -0
- data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +5 -129
- data/vendor/datasketches-cpp/tuple/test/engagement_test.cpp +85 -89
- data/vendor/datasketches-cpp/tuple/test/tuple_jaccard_similarity_test.cpp +3 -1
- data/vendor/datasketches-cpp/tuple/test/tuple_sketch_deserialize_from_java_test.cpp +47 -0
- data/vendor/datasketches-cpp/{python/src/datasketches.cpp → tuple/test/tuple_sketch_serialize_for_java.cpp} +16 -30
- data/vendor/datasketches-cpp/tuple/test/tuple_sketch_test.cpp +1 -1
- data/vendor/datasketches-cpp/version.cfg.in +1 -1
- metadata +61 -79
- data/vendor/datasketches-cpp/fi/test/items_sketch_string_from_java.sk +0 -0
- data/vendor/datasketches-cpp/fi/test/items_sketch_string_utf8_from_java.sk +0 -0
- data/vendor/datasketches-cpp/fi/test/longs_sketch_from_java.sk +0 -0
- data/vendor/datasketches-cpp/hll/test/array6_from_java.sk +0 -0
- data/vendor/datasketches-cpp/hll/test/compact_array4_from_java.sk +0 -0
- data/vendor/datasketches-cpp/hll/test/compact_set_from_java.sk +0 -0
- data/vendor/datasketches-cpp/hll/test/list_from_java.sk +0 -0
- data/vendor/datasketches-cpp/hll/test/updatable_array4_from_java.sk +0 -0
- data/vendor/datasketches-cpp/hll/test/updatable_set_from_java.sk +0 -0
- data/vendor/datasketches-cpp/kll/test/kll_sketch_from_java.sk +0 -0
- data/vendor/datasketches-cpp/pyproject.toml +0 -23
- data/vendor/datasketches-cpp/python/CMakeLists.txt +0 -81
- data/vendor/datasketches-cpp/python/README.md +0 -85
- data/vendor/datasketches-cpp/python/datasketches/PySerDe.py +0 -104
- data/vendor/datasketches-cpp/python/include/py_serde.hpp +0 -113
- data/vendor/datasketches-cpp/python/jupyter/CPCSketch.ipynb +0 -345
- data/vendor/datasketches-cpp/python/jupyter/FrequentItemsSketch.ipynb +0 -354
- data/vendor/datasketches-cpp/python/jupyter/HLLSketch.ipynb +0 -346
- data/vendor/datasketches-cpp/python/jupyter/KLLSketch.ipynb +0 -463
- data/vendor/datasketches-cpp/python/jupyter/ThetaSketchNotebook.ipynb +0 -403
- data/vendor/datasketches-cpp/python/pybind11Path.cmd +0 -21
- data/vendor/datasketches-cpp/python/src/cpc_wrapper.cpp +0 -90
- data/vendor/datasketches-cpp/python/src/fi_wrapper.cpp +0 -128
- data/vendor/datasketches-cpp/python/src/hll_wrapper.cpp +0 -134
- data/vendor/datasketches-cpp/python/src/kll_wrapper.cpp +0 -210
- data/vendor/datasketches-cpp/python/src/ks_wrapper.cpp +0 -68
- data/vendor/datasketches-cpp/python/src/py_serde.cpp +0 -111
- data/vendor/datasketches-cpp/python/src/quantiles_wrapper.cpp +0 -204
- data/vendor/datasketches-cpp/python/src/req_wrapper.cpp +0 -215
- data/vendor/datasketches-cpp/python/src/theta_wrapper.cpp +0 -172
- data/vendor/datasketches-cpp/python/src/vector_of_kll.cpp +0 -490
- data/vendor/datasketches-cpp/python/src/vo_wrapper.cpp +0 -173
- data/vendor/datasketches-cpp/python/tests/__init__.py +0 -16
- data/vendor/datasketches-cpp/python/tests/cpc_test.py +0 -64
- data/vendor/datasketches-cpp/python/tests/fi_test.py +0 -110
- data/vendor/datasketches-cpp/python/tests/hll_test.py +0 -130
- data/vendor/datasketches-cpp/python/tests/kll_test.py +0 -125
- data/vendor/datasketches-cpp/python/tests/quantiles_test.py +0 -126
- data/vendor/datasketches-cpp/python/tests/req_test.py +0 -126
- data/vendor/datasketches-cpp/python/tests/theta_test.py +0 -146
- data/vendor/datasketches-cpp/python/tests/vector_of_kll_test.py +0 -148
- data/vendor/datasketches-cpp/python/tests/vo_test.py +0 -125
- data/vendor/datasketches-cpp/req/test/req_float_empty_from_java.sk +0 -0
- data/vendor/datasketches-cpp/req/test/req_float_estimation_from_java.sk +0 -0
- data/vendor/datasketches-cpp/req/test/req_float_exact_from_java.sk +0 -0
- data/vendor/datasketches-cpp/req/test/req_float_raw_items_from_java.sk +0 -0
- data/vendor/datasketches-cpp/req/test/req_float_single_item_from_java.sk +0 -0
- data/vendor/datasketches-cpp/sampling/test/binaries_from_java.txt +0 -67
- data/vendor/datasketches-cpp/sampling/test/varopt_sketch_long_sampling.sk +0 -0
- data/vendor/datasketches-cpp/sampling/test/varopt_sketch_string_exact.sk +0 -0
- data/vendor/datasketches-cpp/sampling/test/varopt_union_double_sampling.sk +0 -0
- data/vendor/datasketches-cpp/setup.py +0 -110
- data/vendor/datasketches-cpp/theta/test/theta_compact_empty_from_java.sk +0 -0
- data/vendor/datasketches-cpp/theta/test/theta_compact_estimation_from_java.sk +0 -0
- data/vendor/datasketches-cpp/theta/test/theta_compact_exact_from_java.sk +0 -0
- data/vendor/datasketches-cpp/theta/test/theta_compact_single_item_from_java.sk +0 -0
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_intersection.hpp +0 -52
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_union.hpp +0 -81
- data/vendor/datasketches-cpp/tuple/include/array_of_doubles_union_impl.hpp +0 -43
- data/vendor/datasketches-cpp/tuple/test/aod_1_compact_empty_from_java.sk +0 -1
- data/vendor/datasketches-cpp/tuple/test/aod_1_compact_estimation_from_java.sk +0 -0
- data/vendor/datasketches-cpp/tuple/test/aod_1_compact_non_empty_no_entries_from_java.sk +0 -0
- data/vendor/datasketches-cpp/tuple/test/aod_2_compact_exact_from_java.sk +0 -0
- data/vendor/datasketches-cpp/tuple/test/aod_3_compact_empty_from_java.sk +0 -1
@@ -1,64 +0,0 @@
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
3
|
-
# distributed with this work for additional information
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
6
|
-
# "License"); you may not use this file except in compliance
|
7
|
-
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
12
|
-
# software distributed under the License is distributed on an
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
-
# KIND, either express or implied. See the License for the
|
15
|
-
# specific language governing permissions and limitations
|
16
|
-
# under the License.
|
17
|
-
|
18
|
-
import unittest
|
19
|
-
from datasketches import cpc_sketch, cpc_union
|
20
|
-
|
21
|
-
class CpcTest(unittest.TestCase):
|
22
|
-
def test_cpc_example(self):
|
23
|
-
k = 12 # 2^k = 4096 rows in the table
|
24
|
-
n = 1 << 18 # ~256k unique values
|
25
|
-
|
26
|
-
# create a couple sketches and inject some values
|
27
|
-
# we'll have 1/4 of the values overlap
|
28
|
-
cpc = cpc_sketch(k)
|
29
|
-
cpc2 = cpc_sketch(k)
|
30
|
-
offset = int(3 * n / 4) # it's a float w/o cast
|
31
|
-
# because we hash on the bits, not an abstract numeric value,
|
32
|
-
# cpc.update(1) and cpc.update(1.0) give different results.
|
33
|
-
for i in range(0, n):
|
34
|
-
cpc.update(i)
|
35
|
-
cpc2.update(i + offset)
|
36
|
-
|
37
|
-
# although we provide get_composite_estimate() and get_estimate(),
|
38
|
-
# the latter will always give the best available estimate. we
|
39
|
-
# recommend using get_estimate().
|
40
|
-
# we can check that the upper and lower bounds bracket the
|
41
|
-
# estimate, without needing to know the exact value.
|
42
|
-
self.assertLessEqual(cpc.get_lower_bound(1), cpc.get_estimate())
|
43
|
-
self.assertGreaterEqual(cpc.get_upper_bound(1), cpc.get_estimate())
|
44
|
-
|
45
|
-
# unioning uses a separate class, but we need to get_result()
|
46
|
-
# tp query the unioned sketches
|
47
|
-
union = cpc_union(k)
|
48
|
-
union.update(cpc)
|
49
|
-
union.update(cpc2)
|
50
|
-
result = union.get_result()
|
51
|
-
|
52
|
-
# since our process here (including post-union CPC) is
|
53
|
-
# deterministic, we have checked and know the exact
|
54
|
-
# answer is within one standard deviation of the estimate
|
55
|
-
self.assertLessEqual(result.get_lower_bound(1), 7 * n / 4)
|
56
|
-
self.assertGreaterEqual(result.get_upper_bound(1), 7 * n / 4)
|
57
|
-
|
58
|
-
# serialize for storage and reconstruct
|
59
|
-
sk_bytes = result.serialize()
|
60
|
-
new_cpc = cpc_sketch.deserialize(sk_bytes)
|
61
|
-
self.assertFalse(new_cpc.is_empty())
|
62
|
-
|
63
|
-
if __name__ == '__main__':
|
64
|
-
unittest.main()
|
@@ -1,110 +0,0 @@
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
3
|
-
# distributed with this work for additional information
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
6
|
-
# "License"); you may not use this file except in compliance
|
7
|
-
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
12
|
-
# software distributed under the License is distributed on an
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
-
# KIND, either express or implied. See the License for the
|
15
|
-
# specific language governing permissions and limitations
|
16
|
-
# under the License.
|
17
|
-
|
18
|
-
import unittest
|
19
|
-
from datasketches import frequent_strings_sketch, frequent_items_error_type
|
20
|
-
|
21
|
-
class FiTest(unittest.TestCase):
|
22
|
-
def test_fi_example(self):
|
23
|
-
k = 3 # a small value so we can easily fill the sketch
|
24
|
-
fi = frequent_strings_sketch(k)
|
25
|
-
|
26
|
-
# we'll use a small number of distinct items so we
|
27
|
-
# can use exponentially increasing weights and have
|
28
|
-
# some frequent items, decreasing so we have some
|
29
|
-
# small items inserted after a purge
|
30
|
-
n = 8
|
31
|
-
for i in range(0, n):
|
32
|
-
fi.update(str(i), 2 ** (n - i))
|
33
|
-
|
34
|
-
# there are two ways to extract items :
|
35
|
-
# * NO_FALSE_POSITIVES includes all items with a lower bound
|
36
|
-
# above the a posteriori error
|
37
|
-
# * NO_FALSE_NEGATIVES includes all items with an uper bound
|
38
|
-
# above the a posteriori error
|
39
|
-
# a more complete discussion may be found at
|
40
|
-
# https://datasketches.github.io/docs/Frequency/FrequentItemsOverview.html
|
41
|
-
items_no_fp = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_POSITIVES)
|
42
|
-
items_no_fn = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_NEGATIVES)
|
43
|
-
self.assertLessEqual(len(items_no_fp), len(items_no_fn))
|
44
|
-
|
45
|
-
# the items list returns a decreasing weight-sorted list, and
|
46
|
-
# for each item we have (item, estimate, lower_bound, upper_bound)
|
47
|
-
item = items_no_fp[1]
|
48
|
-
self.assertLessEqual(item[2], item[1]) # lower bound vs estimate
|
49
|
-
self.assertLessEqual(item[1], item[3]) # estimate vs upper bound
|
50
|
-
|
51
|
-
# we can also query directly for a specific item
|
52
|
-
id = items_no_fn[0][0]
|
53
|
-
est = fi.get_estimate(id)
|
54
|
-
lb = fi.get_lower_bound(id)
|
55
|
-
ub = fi.get_upper_bound(id)
|
56
|
-
self.assertLessEqual(lb, est)
|
57
|
-
self.assertLessEqual(est, ub)
|
58
|
-
|
59
|
-
# the values are zero if the item isn't in our list
|
60
|
-
self.assertEqual(fi.get_estimate("NaN"), 0)
|
61
|
-
|
62
|
-
# now create a second sketch with a lot of unique
|
63
|
-
# values but all with equal weight (of 1) such that
|
64
|
-
# the total weight is much larger than the first sketch
|
65
|
-
fi2 = frequent_strings_sketch(k)
|
66
|
-
wt = fi.get_total_weight()
|
67
|
-
for i in range(0, 4*wt):
|
68
|
-
fi2.update(str(i))
|
69
|
-
|
70
|
-
# merge the second sketch into the first
|
71
|
-
fi.merge(fi2)
|
72
|
-
|
73
|
-
# we can see that the weight is much larger
|
74
|
-
self.assertEqual(5 * wt, fi.get_total_weight())
|
75
|
-
|
76
|
-
# querying with NO_FALSE_POSITIVES means we don't find anything
|
77
|
-
# heavy enough to return
|
78
|
-
items_no_fp = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_POSITIVES)
|
79
|
-
self.assertEqual(len(items_no_fp), 0)
|
80
|
-
|
81
|
-
# we do, however, find a few potential heavy items
|
82
|
-
# if querying with NO_FALSE_NEGATIVES
|
83
|
-
items_no_fn = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_NEGATIVES)
|
84
|
-
self.assertGreater(len(items_no_fn), 0)
|
85
|
-
|
86
|
-
# finally, serialize and reconstruct
|
87
|
-
fi_bytes = fi.serialize()
|
88
|
-
self.assertEqual(len(fi_bytes), fi.get_serialized_size_bytes())
|
89
|
-
new_fi = frequent_strings_sketch.deserialize(fi_bytes)
|
90
|
-
|
91
|
-
# and now interrogate the sketch
|
92
|
-
self.assertFalse(new_fi.is_empty())
|
93
|
-
self.assertGreater(new_fi.get_num_active_items(), 0)
|
94
|
-
self.assertEqual(5 * wt, new_fi.get_total_weight())
|
95
|
-
|
96
|
-
|
97
|
-
def test_fi_sketch(self):
|
98
|
-
# only testing a few things not used in the above example
|
99
|
-
k = 12
|
100
|
-
wt = 10000
|
101
|
-
fi = frequent_strings_sketch(k)
|
102
|
-
|
103
|
-
self.assertAlmostEqual(fi.get_sketch_epsilon(), 0.0008545, delta=1e-6)
|
104
|
-
|
105
|
-
sk_apriori_error = fi.get_sketch_epsilon() * wt
|
106
|
-
reference_apriori_error = frequent_strings_sketch.get_apriori_error(k, wt)
|
107
|
-
self.assertAlmostEqual(sk_apriori_error, reference_apriori_error, delta=1e-6)
|
108
|
-
|
109
|
-
if __name__ == '__main__':
|
110
|
-
unittest.main()
|
@@ -1,130 +0,0 @@
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
3
|
-
# distributed with this work for additional information
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
6
|
-
# "License"); you may not use this file except in compliance
|
7
|
-
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
12
|
-
# software distributed under the License is distributed on an
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
-
# KIND, either express or implied. See the License for the
|
15
|
-
# specific language governing permissions and limitations
|
16
|
-
# under the License.
|
17
|
-
|
18
|
-
import unittest
|
19
|
-
from datasketches import hll_sketch, hll_union, tgt_hll_type
|
20
|
-
|
21
|
-
class HllTest(unittest.TestCase):
|
22
|
-
def test_hll_example(self):
|
23
|
-
k = 12 # 2^k = 4096 rows in the table
|
24
|
-
n = 1 << 18 # ~256k unique values
|
25
|
-
|
26
|
-
# create a couple sketches and inject some values
|
27
|
-
# we'll have 1/4 of the values overlap
|
28
|
-
hll = hll_sketch(k, tgt_hll_type.HLL_8)
|
29
|
-
hll2 = hll_sketch(k, tgt_hll_type.HLL_6)
|
30
|
-
offset = int(3 * n / 4) # it's a float w/o cast
|
31
|
-
# because we hash on the bits, not an abstract numeric value,
|
32
|
-
# hll.update(1) and hll.update(1.0) give different results.
|
33
|
-
for i in range(0, n):
|
34
|
-
hll.update(i)
|
35
|
-
hll2.update(i + offset)
|
36
|
-
|
37
|
-
# we can check that the upper and lower bounds bracket the
|
38
|
-
# estimate, without needing to know the exact value.
|
39
|
-
self.assertLessEqual(hll.get_lower_bound(1), hll.get_estimate())
|
40
|
-
self.assertGreaterEqual(hll.get_upper_bound(1), hll.get_estimate())
|
41
|
-
|
42
|
-
# unioning uses a separate class, and we can either get a result
|
43
|
-
# sketch or query the union object directly
|
44
|
-
union = hll_union(k)
|
45
|
-
union.update(hll)
|
46
|
-
union.update(hll2)
|
47
|
-
result = union.get_result()
|
48
|
-
self.assertEqual(result.get_estimate(), union.get_estimate())
|
49
|
-
|
50
|
-
# since our process here (including post-union HLL) is
|
51
|
-
# deterministic, we have checked and know the exact
|
52
|
-
# answer is within one standard deviation of the estimate
|
53
|
-
self.assertLessEqual(union.get_lower_bound(1), 7 * n / 4)
|
54
|
-
self.assertGreaterEqual(union.get_upper_bound(1), 7 * n / 4)
|
55
|
-
|
56
|
-
# serialize for storage and reconstruct
|
57
|
-
sk_bytes = result.serialize_compact()
|
58
|
-
self.assertEqual(len(sk_bytes), result.get_compact_serialization_bytes())
|
59
|
-
new_hll = hll_sketch.deserialize(sk_bytes)
|
60
|
-
|
61
|
-
# the sketch can self-report its configuration and status
|
62
|
-
self.assertEqual(new_hll.lg_config_k, k)
|
63
|
-
self.assertEqual(new_hll.tgt_type, tgt_hll_type.HLL_4)
|
64
|
-
self.assertFalse(new_hll.is_empty())
|
65
|
-
|
66
|
-
# if we want to reduce some object overhead, we can also reset
|
67
|
-
new_hll.reset()
|
68
|
-
self.assertTrue(new_hll.is_empty())
|
69
|
-
|
70
|
-
def test_hll_sketch(self):
|
71
|
-
k = 8
|
72
|
-
n = 117
|
73
|
-
hll = self.generate_sketch(n, k, tgt_hll_type.HLL_6)
|
74
|
-
hll.update('string data')
|
75
|
-
hll.update(3.14159) # double data
|
76
|
-
|
77
|
-
self.assertLessEqual(hll.get_lower_bound(1), hll.get_estimate())
|
78
|
-
self.assertGreaterEqual(hll.get_upper_bound(1), hll.get_estimate())
|
79
|
-
|
80
|
-
self.assertEqual(hll.lg_config_k, k)
|
81
|
-
self.assertEqual(hll.tgt_type, tgt_hll_type.HLL_6)
|
82
|
-
|
83
|
-
bytes_compact = hll.serialize_compact()
|
84
|
-
bytes_update = hll.serialize_updatable()
|
85
|
-
self.assertEqual(len(bytes_compact), hll.get_compact_serialization_bytes())
|
86
|
-
self.assertEqual(len(bytes_update), hll.get_updatable_serialization_bytes())
|
87
|
-
|
88
|
-
self.assertFalse(hll.is_compact())
|
89
|
-
self.assertFalse(hll.is_empty())
|
90
|
-
|
91
|
-
self.assertTrue(isinstance(hll_sketch.deserialize(bytes_compact), hll_sketch))
|
92
|
-
self.assertTrue(isinstance(hll_sketch.deserialize(bytes_update), hll_sketch))
|
93
|
-
|
94
|
-
self.assertIsNotNone(hll_sketch.get_rel_err(True, False, 12, 1))
|
95
|
-
self.assertIsNotNone(hll_sketch.get_max_updatable_serialization_bytes(20, tgt_hll_type.HLL_6))
|
96
|
-
|
97
|
-
hll.reset()
|
98
|
-
self.assertTrue(hll.is_empty())
|
99
|
-
|
100
|
-
def test_hll_union(self):
|
101
|
-
k = 7
|
102
|
-
n = 53
|
103
|
-
union = hll_union(k)
|
104
|
-
|
105
|
-
sk = self.generate_sketch(n, k, tgt_hll_type.HLL_4, 0)
|
106
|
-
union.update(sk)
|
107
|
-
sk = self.generate_sketch(3 * n, k, tgt_hll_type.HLL_4, n)
|
108
|
-
union.update(sk)
|
109
|
-
union.update('string data')
|
110
|
-
union.update(1.4142136)
|
111
|
-
|
112
|
-
self.assertLessEqual(union.get_lower_bound(1), union.get_estimate())
|
113
|
-
self.assertGreaterEqual(union.get_upper_bound(1), union.get_estimate())
|
114
|
-
|
115
|
-
self.assertEqual(union.lg_config_k, k)
|
116
|
-
self.assertFalse(union.is_empty())
|
117
|
-
|
118
|
-
sk = union.get_result()
|
119
|
-
self.assertTrue(isinstance(sk, hll_sketch))
|
120
|
-
self.assertEqual(sk.tgt_type, tgt_hll_type.HLL_4)
|
121
|
-
|
122
|
-
def generate_sketch(self, n, k, sk_type=tgt_hll_type.HLL_4, st_idx=0):
|
123
|
-
sk = hll_sketch(k, sk_type)
|
124
|
-
for i in range(st_idx, st_idx + n):
|
125
|
-
sk.update(i)
|
126
|
-
return sk
|
127
|
-
|
128
|
-
|
129
|
-
if __name__ == '__main__':
|
130
|
-
unittest.main()
|
@@ -1,125 +0,0 @@
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
3
|
-
# distributed with this work for additional information
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
6
|
-
# "License"); you may not use this file except in compliance
|
7
|
-
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
12
|
-
# software distributed under the License is distributed on an
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
-
# KIND, either express or implied. See the License for the
|
15
|
-
# specific language governing permissions and limitations
|
16
|
-
# under the License.
|
17
|
-
|
18
|
-
import unittest
|
19
|
-
from datasketches import kll_ints_sketch, kll_floats_sketch, kll_doubles_sketch, ks_test
|
20
|
-
import numpy as np
|
21
|
-
|
22
|
-
class KllTest(unittest.TestCase):
|
23
|
-
def test_kll_example(self):
|
24
|
-
k = 160
|
25
|
-
n = 2 ** 20
|
26
|
-
|
27
|
-
# create a sketch and inject ~1 million N(0,1) points as an array and as a single item
|
28
|
-
kll = kll_floats_sketch(k)
|
29
|
-
kll.update(np.random.normal(size=n-1))
|
30
|
-
kll.update(0.0)
|
31
|
-
|
32
|
-
# 0 should be near the median
|
33
|
-
self.assertAlmostEqual(0.5, kll.get_rank(0.0), delta=0.035)
|
34
|
-
|
35
|
-
# the median should be near 0
|
36
|
-
self.assertAlmostEqual(0.0, kll.get_quantile(0.5), delta=0.035)
|
37
|
-
|
38
|
-
# we also track the min/max independently from the rest of the data
|
39
|
-
# which lets us know the full observed data range
|
40
|
-
self.assertLessEqual(kll.get_min_value(), kll.get_quantile(0.01))
|
41
|
-
self.assertLessEqual(0.0, kll.get_rank(kll.get_min_value()))
|
42
|
-
self.assertGreaterEqual(kll.get_max_value(), kll.get_quantile(0.99))
|
43
|
-
self.assertGreaterEqual(1.0, kll.get_rank(kll.get_max_value()))
|
44
|
-
|
45
|
-
# we can also extract a list of values at a time,
|
46
|
-
# here the values should give us something close to [-2, -1, 0, 1, 2].
|
47
|
-
# then get the CDF, which will return something close to
|
48
|
-
# the original values used in get_quantiles()
|
49
|
-
# finally, can check the normalized rank error bound
|
50
|
-
pts = kll.get_quantiles([0.0228, 0.1587, 0.5, 0.8413, 0.9772])
|
51
|
-
cdf = kll.get_cdf(pts) # include 1.0 at end to account for all probability mass
|
52
|
-
self.assertEqual(len(cdf), len(pts)+1)
|
53
|
-
err = kll.normalized_rank_error(False)
|
54
|
-
self.assertEqual(err, kll_floats_sketch.get_normalized_rank_error(k, False))
|
55
|
-
|
56
|
-
# and a few basic queries about the sketch
|
57
|
-
self.assertFalse(kll.is_empty())
|
58
|
-
self.assertTrue(kll.is_estimation_mode())
|
59
|
-
self.assertEqual(kll.get_n(), n)
|
60
|
-
self.assertEqual(kll.get_k(), k)
|
61
|
-
self.assertLess(kll.get_num_retained(), n)
|
62
|
-
|
63
|
-
# merging itself will double the number of items the sketch has seen
|
64
|
-
kll.merge(kll)
|
65
|
-
self.assertEqual(kll.get_n(), 2*n)
|
66
|
-
|
67
|
-
# we can then serialize and reconstruct the sketch
|
68
|
-
kll_bytes = kll.serialize()
|
69
|
-
new_kll = kll.deserialize(kll_bytes)
|
70
|
-
self.assertEqual(kll.get_num_retained(), new_kll.get_num_retained())
|
71
|
-
self.assertEqual(kll.get_min_value(), new_kll.get_min_value())
|
72
|
-
self.assertEqual(kll.get_max_value(), new_kll.get_max_value())
|
73
|
-
self.assertEqual(kll.get_quantile(0.7), new_kll.get_quantile(0.7))
|
74
|
-
self.assertEqual(kll.get_rank(0.0), new_kll.get_rank(0.0))
|
75
|
-
|
76
|
-
# A Kolmogorov-Smirnov Test of kll and new_kll should match, even for
|
77
|
-
# a fairly small p-value -- cannot reject the null hypothesis that
|
78
|
-
# they come from the same distribution (since they do)
|
79
|
-
self.assertFalse(ks_test(kll, new_kll, 0.001))
|
80
|
-
|
81
|
-
|
82
|
-
def test_kll_ints_sketch(self):
|
83
|
-
k = 100
|
84
|
-
n = 10
|
85
|
-
kll = kll_ints_sketch(k)
|
86
|
-
for i in range(0, n):
|
87
|
-
kll.update(i)
|
88
|
-
|
89
|
-
self.assertEqual(kll.get_min_value(), 0)
|
90
|
-
self.assertEqual(kll.get_max_value(), n-1)
|
91
|
-
self.assertEqual(kll.get_n(), n)
|
92
|
-
self.assertFalse(kll.is_empty())
|
93
|
-
self.assertFalse(kll.is_estimation_mode()) # n < k
|
94
|
-
self.assertEqual(kll.get_k(), k)
|
95
|
-
|
96
|
-
pmf = kll.get_pmf([round(n/2)])
|
97
|
-
self.assertIsNotNone(pmf)
|
98
|
-
self.assertEqual(len(pmf), 2)
|
99
|
-
|
100
|
-
cdf = kll.get_cdf([round(n/2)])
|
101
|
-
self.assertIsNotNone(cdf)
|
102
|
-
self.assertEqual(len(cdf), 2)
|
103
|
-
|
104
|
-
self.assertEqual(kll.get_quantile(0.5), round(n/2))
|
105
|
-
quants = kll.get_quantiles([0.25, 0.5, 0.75])
|
106
|
-
self.assertIsNotNone(quants)
|
107
|
-
self.assertEqual(len(quants), 3)
|
108
|
-
|
109
|
-
self.assertEqual(kll.get_rank(round(n/2)), 0.5)
|
110
|
-
|
111
|
-
# merge self
|
112
|
-
kll.merge(kll)
|
113
|
-
self.assertEqual(kll.get_n(), 2 * n)
|
114
|
-
|
115
|
-
sk_bytes = kll.serialize()
|
116
|
-
self.assertTrue(isinstance(kll_ints_sketch.deserialize(sk_bytes), kll_ints_sketch))
|
117
|
-
|
118
|
-
def test_kll_doubles_sketch(self):
|
119
|
-
# already tested float and ints and it's templatized, so just make sure it instantiates properly
|
120
|
-
k = 75
|
121
|
-
kll = kll_doubles_sketch(k)
|
122
|
-
self.assertTrue(kll.is_empty())
|
123
|
-
|
124
|
-
if __name__ == '__main__':
|
125
|
-
unittest.main()
|
@@ -1,126 +0,0 @@
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
3
|
-
# distributed with this work for additional information
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
6
|
-
# "License"); you may not use this file except in compliance
|
7
|
-
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
12
|
-
# software distributed under the License is distributed on an
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
-
# KIND, either express or implied. See the License for the
|
15
|
-
# specific language governing permissions and limitations
|
16
|
-
# under the License.
|
17
|
-
|
18
|
-
import unittest
|
19
|
-
from datasketches import quantiles_ints_sketch, quantiles_floats_sketch, quantiles_doubles_sketch, ks_test
|
20
|
-
import numpy as np
|
21
|
-
|
22
|
-
class QuantilesTest(unittest.TestCase):
|
23
|
-
def test_quantiles_example(self):
|
24
|
-
k = 128
|
25
|
-
n = 2 ** 20
|
26
|
-
|
27
|
-
# create a sketch and inject ~1 million N(0,1) points as an array and as a single item
|
28
|
-
quantiles = quantiles_floats_sketch(k)
|
29
|
-
quantiles.update(np.random.normal(size=n-1))
|
30
|
-
quantiles.update(0.0)
|
31
|
-
|
32
|
-
# 0 should be near the median
|
33
|
-
self.assertAlmostEqual(0.5, quantiles.get_rank(0.0), delta=0.035)
|
34
|
-
|
35
|
-
# the median should be near 0
|
36
|
-
self.assertAlmostEqual(0.0, quantiles.get_quantile(0.5), delta=0.035)
|
37
|
-
|
38
|
-
# we also track the min/max independently from the rest of the data
|
39
|
-
# which lets us know the full observed data range
|
40
|
-
self.assertLessEqual(quantiles.get_min_value(), quantiles.get_quantile(0.01))
|
41
|
-
self.assertLessEqual(0.0, quantiles.get_rank(quantiles.get_min_value()))
|
42
|
-
self.assertGreaterEqual(quantiles.get_max_value(), quantiles.get_quantile(0.99))
|
43
|
-
self.assertGreaterEqual(1.0, quantiles.get_rank(quantiles.get_max_value()))
|
44
|
-
|
45
|
-
# we can also extract a list of values at a time,
|
46
|
-
# here the values should give us something close to [-2, -1, 0, 1, 2].
|
47
|
-
# then get the CDF, which will return something close to
|
48
|
-
# the original values used in get_quantiles()
|
49
|
-
# finally, can check the normalized rank error bound
|
50
|
-
pts = quantiles.get_quantiles([0.0228, 0.1587, 0.5, 0.8413, 0.9772])
|
51
|
-
cdf = quantiles.get_cdf(pts) # include 1.0 at end to account for all probability mass
|
52
|
-
self.assertEqual(len(cdf), len(pts)+1)
|
53
|
-
err = quantiles.normalized_rank_error(False)
|
54
|
-
self.assertEqual(err, quantiles_floats_sketch.get_normalized_rank_error(k, False))
|
55
|
-
|
56
|
-
# and a few basic queries about the sketch
|
57
|
-
self.assertFalse(quantiles.is_empty())
|
58
|
-
self.assertTrue(quantiles.is_estimation_mode())
|
59
|
-
self.assertEqual(quantiles.get_n(), n)
|
60
|
-
self.assertEqual(quantiles.get_k(), k)
|
61
|
-
self.assertLess(quantiles.get_num_retained(), n)
|
62
|
-
|
63
|
-
# merging itself will double the number of items the sketch has seen
|
64
|
-
quantiles.merge(quantiles)
|
65
|
-
self.assertEqual(quantiles.get_n(), 2*n)
|
66
|
-
|
67
|
-
# we can then serialize and reconstruct the sketch
|
68
|
-
quantiles_bytes = quantiles.serialize()
|
69
|
-
new_quantiles = quantiles.deserialize(quantiles_bytes)
|
70
|
-
self.assertEqual(quantiles.get_num_retained(), new_quantiles.get_num_retained())
|
71
|
-
self.assertEqual(quantiles.get_min_value(), new_quantiles.get_min_value())
|
72
|
-
self.assertEqual(quantiles.get_max_value(), new_quantiles.get_max_value())
|
73
|
-
self.assertEqual(quantiles.get_quantile(0.7), new_quantiles.get_quantile(0.7))
|
74
|
-
self.assertEqual(quantiles.get_rank(0.0), new_quantiles.get_rank(0.0))
|
75
|
-
|
76
|
-
# If we create a new sketch with a very different distribution, a Kolmogorov-Smirnov Test
|
77
|
-
# of the two should return True: we can reject the null hypothesis that the sketches
|
78
|
-
# come from the same distributions.
|
79
|
-
unif_quantiles = quantiles_floats_sketch(k)
|
80
|
-
unif_quantiles.update(np.random.uniform(10, 20, size=n-1))
|
81
|
-
self.assertTrue(ks_test(quantiles, unif_quantiles, 0.001))
|
82
|
-
|
83
|
-
def test_quantiles_ints_sketch(self):
|
84
|
-
k = 128
|
85
|
-
n = 10
|
86
|
-
quantiles = quantiles_ints_sketch(k)
|
87
|
-
for i in range(0, n):
|
88
|
-
quantiles.update(i)
|
89
|
-
|
90
|
-
self.assertEqual(quantiles.get_min_value(), 0)
|
91
|
-
self.assertEqual(quantiles.get_max_value(), n-1)
|
92
|
-
self.assertEqual(quantiles.get_n(), n)
|
93
|
-
self.assertFalse(quantiles.is_empty())
|
94
|
-
self.assertFalse(quantiles.is_estimation_mode()) # n < k
|
95
|
-
self.assertEqual(quantiles.get_k(), k)
|
96
|
-
|
97
|
-
pmf = quantiles.get_pmf([round(n/2)])
|
98
|
-
self.assertIsNotNone(pmf)
|
99
|
-
self.assertEqual(len(pmf), 2)
|
100
|
-
|
101
|
-
cdf = quantiles.get_cdf([round(n/2)])
|
102
|
-
self.assertIsNotNone(cdf)
|
103
|
-
self.assertEqual(len(cdf), 2)
|
104
|
-
|
105
|
-
self.assertEqual(quantiles.get_quantile(0.5), round(n/2))
|
106
|
-
quants = quantiles.get_quantiles([0.25, 0.5, 0.75])
|
107
|
-
self.assertIsNotNone(quants)
|
108
|
-
self.assertEqual(len(quants), 3)
|
109
|
-
|
110
|
-
self.assertEqual(quantiles.get_rank(round(n/2)), 0.5)
|
111
|
-
|
112
|
-
# merge self
|
113
|
-
quantiles.merge(quantiles)
|
114
|
-
self.assertEqual(quantiles.get_n(), 2 * n)
|
115
|
-
|
116
|
-
sk_bytes = quantiles.serialize()
|
117
|
-
self.assertTrue(isinstance(quantiles_ints_sketch.deserialize(sk_bytes), quantiles_ints_sketch))
|
118
|
-
|
119
|
-
def test_quantiles_doubles_sketch(self):
|
120
|
-
# already tested floats and ints and it's templatized, so just make sure it instantiates properly
|
121
|
-
k = 128
|
122
|
-
quantiles = quantiles_doubles_sketch(k)
|
123
|
-
self.assertTrue(quantiles.is_empty())
|
124
|
-
|
125
|
-
if __name__ == '__main__':
|
126
|
-
unittest.main()
|
@@ -1,126 +0,0 @@
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
3
|
-
# distributed with this work for additional information
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
6
|
-
# "License"); you may not use this file except in compliance
|
7
|
-
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
12
|
-
# software distributed under the License is distributed on an
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
-
# KIND, either express or implied. See the License for the
|
15
|
-
# specific language governing permissions and limitations
|
16
|
-
# under the License.
|
17
|
-
|
18
|
-
import unittest
|
19
|
-
from datasketches import req_ints_sketch, req_floats_sketch
|
20
|
-
import numpy as np
|
21
|
-
|
22
|
-
class reqTest(unittest.TestCase):
|
23
|
-
def test_req_example(self):
|
24
|
-
k = 12
|
25
|
-
n = 2 ** 20
|
26
|
-
|
27
|
-
# create a sketch and inject ~1 million N(0,1) points as an array and as a single item
|
28
|
-
req = req_floats_sketch(k, True) # high rank accuracy
|
29
|
-
req.update(np.random.normal(size=n-1))
|
30
|
-
req.update(0.0)
|
31
|
-
|
32
|
-
# 0 should be near the median
|
33
|
-
self.assertAlmostEqual(0.5, req.get_rank(0.0), delta=0.045)
|
34
|
-
|
35
|
-
# the median should be near 0
|
36
|
-
self.assertAlmostEqual(0.0, req.get_quantile(0.5), delta=0.045)
|
37
|
-
|
38
|
-
# we also track the min/max independently from the rest of the data
|
39
|
-
# which lets us know the full observed data range
|
40
|
-
self.assertLessEqual(req.get_min_value(), req.get_quantile(0.01))
|
41
|
-
self.assertLessEqual(0.0, req.get_rank(req.get_min_value()))
|
42
|
-
self.assertGreaterEqual(req.get_max_value(), req.get_quantile(0.99))
|
43
|
-
self.assertGreaterEqual(1.0, req.get_rank(req.get_max_value()))
|
44
|
-
|
45
|
-
# we can also extract a list of values at a time,
|
46
|
-
# here the values should give us something close to [-2, -1, 0, 1, 2].
|
47
|
-
# then get the CDF, which will return something close to
|
48
|
-
# the original values used in get_quantiles()
|
49
|
-
# finally, can check the normalized rank error bound
|
50
|
-
pts = req.get_quantiles([0.0228, 0.1587, 0.5, 0.8413, 0.9772])
|
51
|
-
cdf = req.get_cdf(pts) # include 1.0 at end to account for all probability mass
|
52
|
-
self.assertEqual(len(cdf), len(pts)+1)
|
53
|
-
|
54
|
-
# For relative error quantiles, the error depends on the actual rank
|
55
|
-
# so we need to use that to detemrine the bounds
|
56
|
-
est = req.get_rank(0.999, True)
|
57
|
-
lb = req.get_rank_lower_bound(est, 1)
|
58
|
-
ub = req.get_rank_upper_bound(est, 1)
|
59
|
-
self.assertLessEqual(lb, est)
|
60
|
-
self.assertLessEqual(est, ub)
|
61
|
-
|
62
|
-
# and a few basic queries about the sketch
|
63
|
-
self.assertFalse(req.is_empty())
|
64
|
-
self.assertTrue(req.is_estimation_mode())
|
65
|
-
self.assertEqual(req.get_n(), n)
|
66
|
-
self.assertLess(req.get_num_retained(), n)
|
67
|
-
self.assertEqual(req.get_k(), k)
|
68
|
-
|
69
|
-
# merging itself will double the number of items the sketch has seen
|
70
|
-
req.merge(req)
|
71
|
-
self.assertEqual(req.get_n(), 2*n)
|
72
|
-
|
73
|
-
# we can then serialize and reconstruct the sketch
|
74
|
-
req_bytes = req.serialize()
|
75
|
-
new_req = req.deserialize(req_bytes)
|
76
|
-
self.assertEqual(req.get_num_retained(), new_req.get_num_retained())
|
77
|
-
self.assertEqual(req.get_min_value(), new_req.get_min_value())
|
78
|
-
self.assertEqual(req.get_max_value(), new_req.get_max_value())
|
79
|
-
self.assertEqual(req.get_quantile(0.7), new_req.get_quantile(0.7))
|
80
|
-
self.assertEqual(req.get_rank(0.0), new_req.get_rank(0.0))
|
81
|
-
|
82
|
-
def test_req_ints_sketch(self):
|
83
|
-
k = 100
|
84
|
-
n = 10
|
85
|
-
req = req_ints_sketch(k)
|
86
|
-
for i in range(0, n):
|
87
|
-
req.update(i)
|
88
|
-
|
89
|
-
self.assertEqual(req.get_min_value(), 0)
|
90
|
-
self.assertEqual(req.get_max_value(), n-1)
|
91
|
-
self.assertEqual(req.get_n(), n)
|
92
|
-
self.assertFalse(req.is_empty())
|
93
|
-
self.assertFalse(req.is_estimation_mode()) # n < k
|
94
|
-
self.assertEqual(req.get_k(), k)
|
95
|
-
|
96
|
-
pmf = req.get_pmf([round(n/2)])
|
97
|
-
self.assertIsNotNone(pmf)
|
98
|
-
self.assertEqual(len(pmf), 2)
|
99
|
-
|
100
|
-
cdf = req.get_cdf([round(n/2)])
|
101
|
-
self.assertIsNotNone(cdf)
|
102
|
-
self.assertEqual(len(cdf), 2)
|
103
|
-
|
104
|
-
self.assertEqual(req.get_quantile(0.5), round(n/2))
|
105
|
-
quants = req.get_quantiles([0.25, 0.5, 0.75])
|
106
|
-
self.assertIsNotNone(quants)
|
107
|
-
self.assertEqual(len(quants), 3)
|
108
|
-
|
109
|
-
self.assertEqual(req.get_rank(round(n/2)), 0.5)
|
110
|
-
|
111
|
-
# merge self
|
112
|
-
req.merge(req)
|
113
|
-
self.assertEqual(req.get_n(), 2 * n)
|
114
|
-
|
115
|
-
sk_bytes = req.serialize()
|
116
|
-
self.assertTrue(isinstance(req_ints_sketch.deserialize(sk_bytes), req_ints_sketch))
|
117
|
-
|
118
|
-
def test_req_floats_sketch(self):
|
119
|
-
# already tested ints and it's templatized, so just make sure it instantiates properly
|
120
|
-
k = 75
|
121
|
-
req = req_floats_sketch(k, False) # low rank accuracy
|
122
|
-
self.assertTrue(req.is_empty())
|
123
|
-
self.assertFalse(req.is_hra())
|
124
|
-
|
125
|
-
if __name__ == '__main__':
|
126
|
-
unittest.main()
|