datasketches 0.3.1 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (245) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -0
  3. data/NOTICE +1 -1
  4. data/README.md +0 -2
  5. data/ext/datasketches/cpc_wrapper.cpp +3 -3
  6. data/ext/datasketches/kll_wrapper.cpp +0 -10
  7. data/lib/datasketches/version.rb +1 -1
  8. data/lib/datasketches.rb +1 -1
  9. data/vendor/datasketches-cpp/CMakeLists.txt +23 -20
  10. data/vendor/datasketches-cpp/CODE_OF_CONDUCT.md +3 -0
  11. data/vendor/datasketches-cpp/CONTRIBUTING.md +50 -0
  12. data/vendor/datasketches-cpp/Doxyfile +2827 -0
  13. data/vendor/datasketches-cpp/LICENSE +0 -76
  14. data/vendor/datasketches-cpp/README.md +1 -3
  15. data/vendor/datasketches-cpp/common/CMakeLists.txt +12 -11
  16. data/vendor/datasketches-cpp/common/include/MurmurHash3.h +25 -27
  17. data/vendor/datasketches-cpp/common/include/common_defs.hpp +15 -10
  18. data/vendor/datasketches-cpp/common/include/count_zeros.hpp +11 -2
  19. data/vendor/datasketches-cpp/common/include/kolmogorov_smirnov.hpp +9 -6
  20. data/vendor/datasketches-cpp/common/include/memory_operations.hpp +5 -4
  21. data/vendor/datasketches-cpp/common/include/optional.hpp +148 -0
  22. data/vendor/datasketches-cpp/common/include/quantiles_sorted_view.hpp +95 -2
  23. data/vendor/datasketches-cpp/common/include/quantiles_sorted_view_impl.hpp +1 -1
  24. data/vendor/datasketches-cpp/common/include/serde.hpp +69 -20
  25. data/vendor/datasketches-cpp/common/test/CMakeLists.txt +2 -2
  26. data/vendor/datasketches-cpp/common/test/integration_test.cpp +6 -0
  27. data/vendor/datasketches-cpp/common/test/optional_test.cpp +85 -0
  28. data/vendor/datasketches-cpp/common/test/test_allocator.hpp +14 -14
  29. data/vendor/datasketches-cpp/{python/src/__init__.py → count/CMakeLists.txt} +25 -1
  30. data/vendor/datasketches-cpp/count/include/count_min.hpp +405 -0
  31. data/vendor/datasketches-cpp/count/include/count_min_impl.hpp +497 -0
  32. data/vendor/datasketches-cpp/{MANIFEST.in → count/test/CMakeLists.txt} +23 -20
  33. data/vendor/datasketches-cpp/count/test/count_min_allocation_test.cpp +155 -0
  34. data/vendor/datasketches-cpp/count/test/count_min_test.cpp +303 -0
  35. data/vendor/datasketches-cpp/cpc/include/cpc_common.hpp +14 -20
  36. data/vendor/datasketches-cpp/cpc/include/cpc_compressor.hpp +7 -4
  37. data/vendor/datasketches-cpp/cpc/include/cpc_compressor_impl.hpp +17 -17
  38. data/vendor/datasketches-cpp/cpc/include/cpc_confidence.hpp +3 -3
  39. data/vendor/datasketches-cpp/cpc/include/cpc_sketch.hpp +40 -40
  40. data/vendor/datasketches-cpp/cpc/include/cpc_sketch_impl.hpp +14 -11
  41. data/vendor/datasketches-cpp/cpc/include/cpc_union.hpp +35 -11
  42. data/vendor/datasketches-cpp/cpc/include/cpc_union_impl.hpp +8 -8
  43. data/vendor/datasketches-cpp/cpc/include/cpc_util.hpp +16 -8
  44. data/vendor/datasketches-cpp/cpc/include/u32_table.hpp +3 -2
  45. data/vendor/datasketches-cpp/cpc/include/u32_table_impl.hpp +5 -5
  46. data/vendor/datasketches-cpp/cpc/test/CMakeLists.txt +20 -7
  47. data/vendor/datasketches-cpp/cpc/test/cpc_sketch_deserialize_from_java_test.cpp +60 -0
  48. data/vendor/datasketches-cpp/cpc/test/cpc_sketch_serialize_for_java.cpp +38 -0
  49. data/vendor/datasketches-cpp/cpc/test/cpc_sketch_test.cpp +4 -29
  50. data/vendor/datasketches-cpp/cpc/test/cpc_union_test.cpp +4 -4
  51. data/vendor/datasketches-cpp/{tox.ini → density/CMakeLists.txt} +24 -8
  52. data/vendor/datasketches-cpp/density/include/density_sketch.hpp +256 -0
  53. data/vendor/datasketches-cpp/density/include/density_sketch_impl.hpp +543 -0
  54. data/vendor/datasketches-cpp/{python/datasketches/__init__.py → density/test/CMakeLists.txt} +15 -3
  55. data/vendor/datasketches-cpp/density/test/density_sketch_test.cpp +244 -0
  56. data/vendor/datasketches-cpp/fi/include/frequent_items_sketch.hpp +21 -9
  57. data/vendor/datasketches-cpp/fi/include/frequent_items_sketch_impl.hpp +6 -4
  58. data/vendor/datasketches-cpp/fi/include/reverse_purge_hash_map.hpp +9 -3
  59. data/vendor/datasketches-cpp/fi/test/CMakeLists.txt +14 -1
  60. data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_deserialize_from_java_test.cpp +95 -0
  61. data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_serialize_for_java.cpp +83 -0
  62. data/vendor/datasketches-cpp/fi/test/frequent_items_sketch_test.cpp +3 -42
  63. data/vendor/datasketches-cpp/hll/include/CouponList-internal.hpp +2 -2
  64. data/vendor/datasketches-cpp/hll/include/CouponList.hpp +3 -1
  65. data/vendor/datasketches-cpp/hll/include/Hll4Array-internal.hpp +19 -11
  66. data/vendor/datasketches-cpp/hll/include/Hll4Array.hpp +2 -5
  67. data/vendor/datasketches-cpp/hll/include/Hll6Array-internal.hpp +19 -7
  68. data/vendor/datasketches-cpp/hll/include/Hll6Array.hpp +1 -1
  69. data/vendor/datasketches-cpp/hll/include/Hll8Array-internal.hpp +98 -42
  70. data/vendor/datasketches-cpp/hll/include/Hll8Array.hpp +2 -0
  71. data/vendor/datasketches-cpp/hll/include/HllArray-internal.hpp +94 -61
  72. data/vendor/datasketches-cpp/hll/include/HllArray.hpp +20 -8
  73. data/vendor/datasketches-cpp/hll/include/HllSketch-internal.hpp +4 -4
  74. data/vendor/datasketches-cpp/hll/include/HllSketchImpl.hpp +3 -1
  75. data/vendor/datasketches-cpp/hll/include/HllSketchImplFactory.hpp +3 -21
  76. data/vendor/datasketches-cpp/hll/include/HllUnion-internal.hpp +8 -0
  77. data/vendor/datasketches-cpp/hll/include/HllUtil.hpp +14 -18
  78. data/vendor/datasketches-cpp/hll/include/coupon_iterator-internal.hpp +1 -1
  79. data/vendor/datasketches-cpp/hll/include/coupon_iterator.hpp +8 -2
  80. data/vendor/datasketches-cpp/hll/include/hll.hpp +79 -65
  81. data/vendor/datasketches-cpp/hll/test/CMakeLists.txt +14 -1
  82. data/vendor/datasketches-cpp/hll/test/HllUnionTest.cpp +7 -1
  83. data/vendor/datasketches-cpp/hll/test/ToFromByteArrayTest.cpp +0 -68
  84. data/vendor/datasketches-cpp/hll/test/hll_sketch_deserialize_from_java_test.cpp +69 -0
  85. data/vendor/datasketches-cpp/hll/test/hll_sketch_serialize_for_java.cpp +52 -0
  86. data/vendor/datasketches-cpp/kll/include/kll_helper.hpp +0 -1
  87. data/vendor/datasketches-cpp/kll/include/kll_helper_impl.hpp +2 -2
  88. data/vendor/datasketches-cpp/kll/include/kll_sketch.hpp +79 -53
  89. data/vendor/datasketches-cpp/kll/include/kll_sketch_impl.hpp +61 -132
  90. data/vendor/datasketches-cpp/kll/test/CMakeLists.txt +14 -1
  91. data/vendor/datasketches-cpp/kll/test/kll_sketch_deserialize_from_java_test.cpp +103 -0
  92. data/vendor/datasketches-cpp/kll/test/kll_sketch_serialize_for_java.cpp +62 -0
  93. data/vendor/datasketches-cpp/kll/test/kll_sketch_test.cpp +5 -40
  94. data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch.hpp +76 -54
  95. data/vendor/datasketches-cpp/quantiles/include/quantiles_sketch_impl.hpp +66 -136
  96. data/vendor/datasketches-cpp/quantiles/test/CMakeLists.txt +14 -1
  97. data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_deserialize_from_java_test.cpp +84 -0
  98. data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_serialize_for_java.cpp +52 -0
  99. data/vendor/datasketches-cpp/quantiles/test/quantiles_sketch_test.cpp +15 -39
  100. data/vendor/datasketches-cpp/req/include/req_common.hpp +7 -3
  101. data/vendor/datasketches-cpp/req/include/req_compactor_impl.hpp +2 -4
  102. data/vendor/datasketches-cpp/req/include/req_sketch.hpp +105 -26
  103. data/vendor/datasketches-cpp/req/include/req_sketch_impl.hpp +50 -111
  104. data/vendor/datasketches-cpp/req/test/CMakeLists.txt +14 -1
  105. data/vendor/datasketches-cpp/req/test/req_sketch_deserialize_from_java_test.cpp +55 -0
  106. data/vendor/datasketches-cpp/{tuple/include/array_of_doubles_intersection_impl.hpp → req/test/req_sketch_serialize_for_java.cpp} +12 -7
  107. data/vendor/datasketches-cpp/req/test/req_sketch_test.cpp +3 -89
  108. data/vendor/datasketches-cpp/sampling/CMakeLists.txt +4 -0
  109. data/vendor/datasketches-cpp/sampling/include/ebpps_sample.hpp +210 -0
  110. data/vendor/datasketches-cpp/sampling/include/ebpps_sample_impl.hpp +535 -0
  111. data/vendor/datasketches-cpp/sampling/include/ebpps_sketch.hpp +281 -0
  112. data/vendor/datasketches-cpp/sampling/include/ebpps_sketch_impl.hpp +531 -0
  113. data/vendor/datasketches-cpp/sampling/include/var_opt_sketch.hpp +89 -32
  114. data/vendor/datasketches-cpp/sampling/include/var_opt_sketch_impl.hpp +33 -19
  115. data/vendor/datasketches-cpp/sampling/include/var_opt_union.hpp +13 -10
  116. data/vendor/datasketches-cpp/sampling/include/var_opt_union_impl.hpp +23 -19
  117. data/vendor/datasketches-cpp/sampling/test/CMakeLists.txt +55 -8
  118. data/vendor/datasketches-cpp/sampling/test/ebpps_allocation_test.cpp +96 -0
  119. data/vendor/datasketches-cpp/sampling/test/ebpps_sample_test.cpp +137 -0
  120. data/vendor/datasketches-cpp/sampling/test/ebpps_sketch_test.cpp +266 -0
  121. data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_deserialize_from_java_test.cpp +81 -0
  122. data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_serialize_for_java.cpp +54 -0
  123. data/vendor/datasketches-cpp/sampling/test/var_opt_sketch_test.cpp +33 -51
  124. data/vendor/datasketches-cpp/sampling/test/var_opt_union_deserialize_from_java_test.cpp +50 -0
  125. data/vendor/datasketches-cpp/sampling/test/var_opt_union_serialize_for_java.cpp +56 -0
  126. data/vendor/datasketches-cpp/sampling/test/var_opt_union_test.cpp +0 -20
  127. data/vendor/datasketches-cpp/theta/CMakeLists.txt +1 -0
  128. data/vendor/datasketches-cpp/theta/include/bit_packing.hpp +6279 -0
  129. data/vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_sampled_sets.hpp +1 -0
  130. data/vendor/datasketches-cpp/theta/include/bounds_on_ratios_in_theta_sketched_sets.hpp +7 -6
  131. data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser.hpp +14 -8
  132. data/vendor/datasketches-cpp/theta/include/compact_theta_sketch_parser_impl.hpp +60 -46
  133. data/vendor/datasketches-cpp/theta/include/theta_a_not_b.hpp +20 -5
  134. data/vendor/datasketches-cpp/theta/include/theta_constants.hpp +10 -4
  135. data/vendor/datasketches-cpp/theta/include/theta_helpers.hpp +3 -1
  136. data/vendor/datasketches-cpp/theta/include/theta_intersection.hpp +13 -5
  137. data/vendor/datasketches-cpp/theta/include/theta_intersection_base_impl.hpp +5 -5
  138. data/vendor/datasketches-cpp/theta/include/theta_intersection_impl.hpp +3 -3
  139. data/vendor/datasketches-cpp/theta/include/theta_jaccard_similarity.hpp +2 -1
  140. data/vendor/datasketches-cpp/theta/include/theta_jaccard_similarity_base.hpp +1 -0
  141. data/vendor/datasketches-cpp/theta/include/theta_set_difference_base_impl.hpp +1 -1
  142. data/vendor/datasketches-cpp/theta/include/theta_sketch.hpp +180 -33
  143. data/vendor/datasketches-cpp/theta/include/theta_sketch_impl.hpp +430 -130
  144. data/vendor/datasketches-cpp/theta/include/theta_union.hpp +17 -10
  145. data/vendor/datasketches-cpp/theta/include/theta_union_base_impl.hpp +10 -10
  146. data/vendor/datasketches-cpp/theta/include/theta_union_impl.hpp +3 -3
  147. data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base.hpp +21 -6
  148. data/vendor/datasketches-cpp/theta/include/theta_update_sketch_base_impl.hpp +13 -3
  149. data/vendor/datasketches-cpp/theta/test/CMakeLists.txt +15 -1
  150. data/vendor/datasketches-cpp/theta/test/bit_packing_test.cpp +80 -0
  151. data/vendor/datasketches-cpp/theta/test/theta_sketch_deserialize_from_java_test.cpp +57 -0
  152. data/vendor/datasketches-cpp/theta/test/theta_sketch_serialize_for_java.cpp +61 -0
  153. data/vendor/datasketches-cpp/theta/test/theta_sketch_test.cpp +39 -188
  154. data/vendor/datasketches-cpp/theta/test/theta_union_test.cpp +25 -0
  155. data/vendor/datasketches-cpp/tuple/CMakeLists.txt +8 -7
  156. data/vendor/datasketches-cpp/tuple/include/array_of_doubles_sketch.hpp +19 -144
  157. data/vendor/datasketches-cpp/tuple/include/{array_of_doubles_a_not_b.hpp → array_tuple_a_not_b.hpp} +24 -16
  158. data/vendor/datasketches-cpp/tuple/include/{array_of_doubles_a_not_b_impl.hpp → array_tuple_a_not_b_impl.hpp} +4 -4
  159. data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection.hpp +65 -0
  160. data/vendor/datasketches-cpp/tuple/include/array_tuple_intersection_impl.hpp +31 -0
  161. data/vendor/datasketches-cpp/tuple/include/array_tuple_sketch.hpp +237 -0
  162. data/vendor/datasketches-cpp/tuple/include/{array_of_doubles_sketch_impl.hpp → array_tuple_sketch_impl.hpp} +40 -41
  163. data/vendor/datasketches-cpp/tuple/include/array_tuple_union.hpp +81 -0
  164. data/vendor/datasketches-cpp/tuple/include/array_tuple_union_impl.hpp +43 -0
  165. data/vendor/datasketches-cpp/tuple/include/tuple_a_not_b.hpp +11 -2
  166. data/vendor/datasketches-cpp/tuple/include/tuple_intersection.hpp +17 -10
  167. data/vendor/datasketches-cpp/tuple/include/tuple_jaccard_similarity.hpp +2 -1
  168. data/vendor/datasketches-cpp/tuple/include/tuple_sketch.hpp +95 -32
  169. data/vendor/datasketches-cpp/tuple/include/tuple_sketch_impl.hpp +2 -1
  170. data/vendor/datasketches-cpp/tuple/include/tuple_union.hpp +19 -11
  171. data/vendor/datasketches-cpp/tuple/test/CMakeLists.txt +16 -1
  172. data/vendor/datasketches-cpp/tuple/test/aod_sketch_deserialize_from_java_test.cpp +76 -0
  173. data/vendor/datasketches-cpp/tuple/test/aod_sketch_serialize_for_java.cpp +62 -0
  174. data/vendor/datasketches-cpp/tuple/test/array_of_doubles_sketch_test.cpp +5 -129
  175. data/vendor/datasketches-cpp/tuple/test/engagement_test.cpp +85 -89
  176. data/vendor/datasketches-cpp/tuple/test/tuple_jaccard_similarity_test.cpp +3 -1
  177. data/vendor/datasketches-cpp/tuple/test/tuple_sketch_deserialize_from_java_test.cpp +47 -0
  178. data/vendor/datasketches-cpp/{python/src/datasketches.cpp → tuple/test/tuple_sketch_serialize_for_java.cpp} +16 -30
  179. data/vendor/datasketches-cpp/tuple/test/tuple_sketch_test.cpp +1 -1
  180. data/vendor/datasketches-cpp/version.cfg.in +1 -1
  181. metadata +61 -79
  182. data/vendor/datasketches-cpp/fi/test/items_sketch_string_from_java.sk +0 -0
  183. data/vendor/datasketches-cpp/fi/test/items_sketch_string_utf8_from_java.sk +0 -0
  184. data/vendor/datasketches-cpp/fi/test/longs_sketch_from_java.sk +0 -0
  185. data/vendor/datasketches-cpp/hll/test/array6_from_java.sk +0 -0
  186. data/vendor/datasketches-cpp/hll/test/compact_array4_from_java.sk +0 -0
  187. data/vendor/datasketches-cpp/hll/test/compact_set_from_java.sk +0 -0
  188. data/vendor/datasketches-cpp/hll/test/list_from_java.sk +0 -0
  189. data/vendor/datasketches-cpp/hll/test/updatable_array4_from_java.sk +0 -0
  190. data/vendor/datasketches-cpp/hll/test/updatable_set_from_java.sk +0 -0
  191. data/vendor/datasketches-cpp/kll/test/kll_sketch_from_java.sk +0 -0
  192. data/vendor/datasketches-cpp/pyproject.toml +0 -23
  193. data/vendor/datasketches-cpp/python/CMakeLists.txt +0 -81
  194. data/vendor/datasketches-cpp/python/README.md +0 -85
  195. data/vendor/datasketches-cpp/python/datasketches/PySerDe.py +0 -104
  196. data/vendor/datasketches-cpp/python/include/py_serde.hpp +0 -113
  197. data/vendor/datasketches-cpp/python/jupyter/CPCSketch.ipynb +0 -345
  198. data/vendor/datasketches-cpp/python/jupyter/FrequentItemsSketch.ipynb +0 -354
  199. data/vendor/datasketches-cpp/python/jupyter/HLLSketch.ipynb +0 -346
  200. data/vendor/datasketches-cpp/python/jupyter/KLLSketch.ipynb +0 -463
  201. data/vendor/datasketches-cpp/python/jupyter/ThetaSketchNotebook.ipynb +0 -403
  202. data/vendor/datasketches-cpp/python/pybind11Path.cmd +0 -21
  203. data/vendor/datasketches-cpp/python/src/cpc_wrapper.cpp +0 -90
  204. data/vendor/datasketches-cpp/python/src/fi_wrapper.cpp +0 -128
  205. data/vendor/datasketches-cpp/python/src/hll_wrapper.cpp +0 -134
  206. data/vendor/datasketches-cpp/python/src/kll_wrapper.cpp +0 -210
  207. data/vendor/datasketches-cpp/python/src/ks_wrapper.cpp +0 -68
  208. data/vendor/datasketches-cpp/python/src/py_serde.cpp +0 -111
  209. data/vendor/datasketches-cpp/python/src/quantiles_wrapper.cpp +0 -204
  210. data/vendor/datasketches-cpp/python/src/req_wrapper.cpp +0 -215
  211. data/vendor/datasketches-cpp/python/src/theta_wrapper.cpp +0 -172
  212. data/vendor/datasketches-cpp/python/src/vector_of_kll.cpp +0 -490
  213. data/vendor/datasketches-cpp/python/src/vo_wrapper.cpp +0 -173
  214. data/vendor/datasketches-cpp/python/tests/__init__.py +0 -16
  215. data/vendor/datasketches-cpp/python/tests/cpc_test.py +0 -64
  216. data/vendor/datasketches-cpp/python/tests/fi_test.py +0 -110
  217. data/vendor/datasketches-cpp/python/tests/hll_test.py +0 -130
  218. data/vendor/datasketches-cpp/python/tests/kll_test.py +0 -125
  219. data/vendor/datasketches-cpp/python/tests/quantiles_test.py +0 -126
  220. data/vendor/datasketches-cpp/python/tests/req_test.py +0 -126
  221. data/vendor/datasketches-cpp/python/tests/theta_test.py +0 -146
  222. data/vendor/datasketches-cpp/python/tests/vector_of_kll_test.py +0 -148
  223. data/vendor/datasketches-cpp/python/tests/vo_test.py +0 -125
  224. data/vendor/datasketches-cpp/req/test/req_float_empty_from_java.sk +0 -0
  225. data/vendor/datasketches-cpp/req/test/req_float_estimation_from_java.sk +0 -0
  226. data/vendor/datasketches-cpp/req/test/req_float_exact_from_java.sk +0 -0
  227. data/vendor/datasketches-cpp/req/test/req_float_raw_items_from_java.sk +0 -0
  228. data/vendor/datasketches-cpp/req/test/req_float_single_item_from_java.sk +0 -0
  229. data/vendor/datasketches-cpp/sampling/test/binaries_from_java.txt +0 -67
  230. data/vendor/datasketches-cpp/sampling/test/varopt_sketch_long_sampling.sk +0 -0
  231. data/vendor/datasketches-cpp/sampling/test/varopt_sketch_string_exact.sk +0 -0
  232. data/vendor/datasketches-cpp/sampling/test/varopt_union_double_sampling.sk +0 -0
  233. data/vendor/datasketches-cpp/setup.py +0 -110
  234. data/vendor/datasketches-cpp/theta/test/theta_compact_empty_from_java.sk +0 -0
  235. data/vendor/datasketches-cpp/theta/test/theta_compact_estimation_from_java.sk +0 -0
  236. data/vendor/datasketches-cpp/theta/test/theta_compact_exact_from_java.sk +0 -0
  237. data/vendor/datasketches-cpp/theta/test/theta_compact_single_item_from_java.sk +0 -0
  238. data/vendor/datasketches-cpp/tuple/include/array_of_doubles_intersection.hpp +0 -52
  239. data/vendor/datasketches-cpp/tuple/include/array_of_doubles_union.hpp +0 -81
  240. data/vendor/datasketches-cpp/tuple/include/array_of_doubles_union_impl.hpp +0 -43
  241. data/vendor/datasketches-cpp/tuple/test/aod_1_compact_empty_from_java.sk +0 -1
  242. data/vendor/datasketches-cpp/tuple/test/aod_1_compact_estimation_from_java.sk +0 -0
  243. data/vendor/datasketches-cpp/tuple/test/aod_1_compact_non_empty_no_entries_from_java.sk +0 -0
  244. data/vendor/datasketches-cpp/tuple/test/aod_2_compact_exact_from_java.sk +0 -0
  245. data/vendor/datasketches-cpp/tuple/test/aod_3_compact_empty_from_java.sk +0 -1
@@ -1,146 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- import unittest
19
-
20
- from datasketches import theta_sketch, update_theta_sketch
21
- from datasketches import compact_theta_sketch, theta_union
22
- from datasketches import theta_intersection, theta_a_not_b
23
- from datasketches import theta_jaccard_similarity
24
-
25
- class ThetaTest(unittest.TestCase):
26
- def test_theta_basic_example(self):
27
- k = 12 # 2^k = 4096 rows in the table
28
- n = 1 << 18 # ~256k unique values
29
-
30
- # create a sketch and inject some values
31
- sk = self.generate_theta_sketch(n, k)
32
-
33
- # we can check that the upper and lower bounds bracket the
34
- # estimate, without needing to know the exact value.
35
- self.assertLessEqual(sk.get_lower_bound(1), sk.get_estimate())
36
- self.assertGreaterEqual(sk.get_upper_bound(1), sk.get_estimate())
37
-
38
- # because this sketch is deterministically generated, we can
39
- # also compare against the exact value
40
- self.assertLessEqual(sk.get_lower_bound(1), n)
41
- self.assertGreaterEqual(sk.get_upper_bound(1), n)
42
-
43
- # compact and serialize for storage, then reconstruct
44
- sk_bytes = sk.compact().serialize()
45
- new_sk = compact_theta_sketch.deserialize(sk_bytes)
46
-
47
- # estimate remains unchanged
48
- self.assertFalse(sk.is_empty())
49
- self.assertEqual(sk.get_estimate(), new_sk.get_estimate())
50
-
51
- def test_theta_set_operations(self):
52
- k = 12 # 2^k = 4096 rows in the table
53
- n = 1 << 18 # ~256k unique values
54
-
55
- # we'll have 1/4 of the values overlap
56
- offset = int(3 * n / 4) # it's a float w/o cast
57
-
58
- # create a couple sketches and inject some values
59
- sk1 = self.generate_theta_sketch(n, k)
60
- sk2 = self.generate_theta_sketch(n, k, offset)
61
-
62
- # UNIONS
63
- # create a union object
64
- union = theta_union(k)
65
- union.update(sk1)
66
- union.update(sk2)
67
-
68
- # getting result from union returns a compact_theta_sketch
69
- # compact theta sketches can be used in additional unions
70
- # or set operations but cannot accept further item updates
71
- result = union.get_result()
72
- self.assertTrue(isinstance(result, compact_theta_sketch))
73
-
74
- # since our process here is deterministic, we have
75
- # checked and know the exact answer is within one
76
- # standard deviation of the estimate
77
- self.assertLessEqual(result.get_lower_bound(1), 7 * n / 4)
78
- self.assertGreaterEqual(result.get_upper_bound(1), 7 * n / 4)
79
-
80
-
81
- # INTERSECTIONS
82
- # create an intersection object
83
- intersect = theta_intersection() # no lg_k
84
- intersect.update(sk1)
85
- intersect.update(sk2)
86
-
87
- # has_result() indicates the intersection has been used,
88
- # although the result may be the empty set
89
- self.assertTrue(intersect.has_result())
90
-
91
- # as with unions, the result is a compact sketch
92
- result = intersect.get_result()
93
- self.assertTrue(isinstance(result, compact_theta_sketch))
94
-
95
- # we know the sets overlap by 1/4
96
- self.assertLessEqual(result.get_lower_bound(1), n / 4)
97
- self.assertGreaterEqual(result.get_upper_bound(1), n / 4)
98
-
99
-
100
- # A NOT B
101
- # create an a_not_b object
102
- anb = theta_a_not_b() # no lg_k
103
- result = anb.compute(sk1, sk2)
104
-
105
- # as with unions, the result is a compact sketch
106
- self.assertTrue(isinstance(result, compact_theta_sketch))
107
-
108
- # we know the sets overlap by 1/4, so the remainder is 3/4
109
- self.assertLessEqual(result.get_lower_bound(1), 3 * n / 4)
110
- self.assertGreaterEqual(result.get_upper_bound(1), 3 * n / 4)
111
-
112
-
113
- # JACCARD SIMILARITY
114
- # Jaccard Similarity measure returns (lower_bound, estimate, upper_bound)
115
- jac = theta_jaccard_similarity.jaccard(sk1, sk2)
116
-
117
- # we can check that results are in the expected order
118
- self.assertLess(jac[0], jac[1])
119
- self.assertLess(jac[1], jac[2])
120
-
121
- # checks for sketch equivalency
122
- self.assertTrue(theta_jaccard_similarity.exactly_equal(sk1, sk1))
123
- self.assertFalse(theta_jaccard_similarity.exactly_equal(sk1, sk2))
124
-
125
- # we can apply a check for similarity or dissimilarity at a
126
- # given threshhold, at 97.7% confidence.
127
-
128
- # check that the Jaccard Index is at most (upper bound) 0.2.
129
- # exact result would be 1/7
130
- self.assertTrue(theta_jaccard_similarity.dissimilarity_test(sk1, sk2, 0.2))
131
-
132
- # check that the Jaccard Index is at least (lower bound) 0.7
133
- # exact result would be 3/4, using result from A NOT B test
134
- self.assertTrue(theta_jaccard_similarity.similarity_test(sk1, result, 0.7))
135
-
136
-
137
- def generate_theta_sketch(self, n, k, offset=0):
138
- sk = update_theta_sketch(k)
139
- for i in range(0, n):
140
- sk.update(i + offset)
141
- return sk
142
-
143
- if __name__ == '__main__':
144
- unittest.main()
145
-
146
-
@@ -1,148 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- import unittest
19
- from datasketches import (vector_of_kll_ints_sketches,
20
- vector_of_kll_floats_sketches)
21
- import numpy as np
22
-
23
- class VectorOfKllSketchesTest(unittest.TestCase):
24
- def test_vector_of_kll_floats_sketches_example(self):
25
- k = 200
26
- d = 3
27
- n = 2 ** 20
28
-
29
- # create a sketch and inject ~1 million N(0,1) points
30
- kll = vector_of_kll_floats_sketches(k, d)
31
- # Track the min/max for each sketch to test later
32
- smin = np.zeros(d) + np.inf
33
- smax = np.zeros(d) - np.inf
34
-
35
- for i in range(0, n):
36
- dat = np.random.randn(d)
37
- smin = np.amin([smin, dat], axis=0)
38
- smax = np.amax([smax, dat], axis=0)
39
- kll.update(dat)
40
-
41
- # 0 should be near the median
42
- np.testing.assert_allclose(0.5, kll.get_ranks(0.0), atol=0.035)
43
- # the median should be near 0
44
- np.testing.assert_allclose(0.0, kll.get_quantiles(0.5), atol=0.035)
45
- # we also track the min/max independently from the rest of the data
46
- # which lets us know the full observed data range
47
- np.testing.assert_allclose(kll.get_min_values(), smin)
48
- np.testing.assert_allclose(kll.get_max_values(), smax)
49
- np.testing.assert_array_less(kll.get_min_values(), kll.get_quantiles(0.01)[:,0])
50
- np.testing.assert_array_less(kll.get_quantiles(0.99)[:,0], kll.get_max_values())
51
-
52
- # we can also extract a list of values at a time,
53
- # here the values should give us something close to [-2, -1, 0, 1, 2].
54
- # then get the CDF, which will return something close to
55
- # the original values used in get_quantiles()
56
- # finally, can check the normalized rank error bound
57
- pts = kll.get_quantiles([0.0228, 0.1587, 0.5, 0.8413, 0.9772])
58
- # use the mean pts for the CDF, include 1.0 at end to account for all probability mass
59
- meanpts = np.mean(pts, axis=0)
60
- cdf = kll.get_cdf(meanpts)
61
- self.assertEqual(cdf.shape[0], pts.shape[0])
62
- self.assertEqual(cdf.shape[1], pts.shape[1]+1)
63
-
64
- # and a few basic queries about the sketch
65
- self.assertFalse(np.all(kll.is_empty()))
66
- self.assertTrue(np.all(kll.is_estimation_mode()))
67
- self.assertTrue(np.all(kll.get_n() == n))
68
- self.assertTrue(np.all(kll.get_num_retained() < n))
69
-
70
- # we can combine sketches across all dimensions and get the reuslt
71
- result = kll.collapse()
72
- self.assertEqual(result.get_n(), d * n)
73
-
74
- # merging a copy of itself will double the number of items the sketch has seen
75
- kll_copy = vector_of_kll_floats_sketches(kll)
76
- kll.merge(kll_copy)
77
- np.testing.assert_equal(kll.get_n(), 2*n)
78
-
79
- # we can then serialize and reconstruct the sketch
80
- kll_bytes = kll.serialize() # serializes each sketch as a list
81
- new_kll = vector_of_kll_floats_sketches(k, d)
82
- for s in range(len(kll_bytes)):
83
- new_kll.deserialize(kll_bytes[s], s)
84
-
85
- # everything should be exactly equal
86
- np.testing.assert_equal(kll.get_num_retained(), new_kll.get_num_retained())
87
- np.testing.assert_equal;(kll.get_min_values(), new_kll.get_min_values())
88
- np.testing.assert_equal(kll.get_max_values(), new_kll.get_max_values())
89
- np.testing.assert_equal(kll.get_quantiles(0.7), new_kll.get_quantiles(0.7))
90
- np.testing.assert_equal(kll.get_ranks(0.0), new_kll.get_ranks(0.0))
91
-
92
- def test_kll_ints_sketches(self):
93
- # already tested floats and it's templatized, so just make sure it instantiates properly
94
- k = 100
95
- d = 5
96
- kll = vector_of_kll_ints_sketches(k, d)
97
- self.assertTrue(np.all(kll.is_empty()))
98
-
99
- def test_kll_2Dupdates(self):
100
- # 1D case tested in the first example
101
- # 2D case will follow same idea, but focusing on update()
102
- k = 200
103
- d = 3
104
- # we'll do ~250k updates of 4 values each (total ~1mil updates, as above)
105
- n = 2 ** 18
106
- nbatch = 4
107
-
108
- # create a sketch and inject ~1 million N(0,1) points
109
- kll = vector_of_kll_floats_sketches(k, d)
110
- # Track the min/max for each sketch to test later
111
- smin = np.zeros(d) + np.inf
112
- smax = np.zeros(d) - np.inf
113
-
114
- for i in range(0, n):
115
- dat = np.random.randn(nbatch, d)
116
- smin = np.amin(np.row_stack((smin, dat)), axis=0)
117
- smax = np.amax(np.row_stack((smax, dat)), axis=0)
118
- kll.update(dat)
119
-
120
- # 0 should be near the median
121
- np.testing.assert_allclose(0.5, kll.get_ranks(0.0), atol=0.035)
122
- # the median should be near 0
123
- np.testing.assert_allclose(0.0, kll.get_quantiles(0.5), atol=0.035)
124
- # we also track the min/max independently from the rest of the data
125
- # which lets us know the full observed data range
126
- np.testing.assert_allclose(kll.get_min_values(), smin)
127
- np.testing.assert_allclose(kll.get_max_values(), smax)
128
-
129
- def test_kll_3Dupdates(self):
130
- # now test 3D update, which should fail
131
- k = 200
132
- d = 3
133
-
134
- # create a sketch
135
- kll = vector_of_kll_floats_sketches(k, d)
136
-
137
- # we'll try 1 3D update
138
- dat = np.random.randn(10, 7, d)
139
- try:
140
- kll.update(dat)
141
- except:
142
- # this is what we expect
143
- pass
144
- # the sketches should still be empty
145
- self.assertTrue(np.all(kll.is_empty()))
146
-
147
- if __name__ == '__main__':
148
- unittest.main()
@@ -1,125 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- import unittest
19
- from datasketches import var_opt_sketch, var_opt_union, PyIntsSerDe, PyStringsSerDe
20
-
21
- class VoTest(unittest.TestCase):
22
- def test_vo_example(self):
23
- k = 50 # a small value so we can easily fill the sketch
24
- vo = var_opt_sketch(k)
25
-
26
- # varopt sampling reduces to standard reservoir sampling
27
- # if the items are all equally weighted, although the
28
- # algorithm will be significantly slower than an optimized
29
- # reservoir sampler
30
- n = 5 * k
31
- for i in range(0, n):
32
- vo.update(i)
33
-
34
- # we can also add a heavy item, using a negative weight for
35
- # easy filtering later. keep in mind that "heavy" is a
36
- # relative concept, so using a fixed multiple of n may not
37
- # be considered a heavy item for larger values of n
38
- vo.update(-1, 1000 * n)
39
- self.assertEqual(k, vo.k)
40
- self.assertEqual(k, vo.num_samples)
41
- self.assertEqual(n + 1, vo.n)
42
- self.assertFalse(vo.is_empty())
43
-
44
- # we can easily get the list of items in the sample
45
- items = vo.get_samples()
46
- self.assertEqual(len(items), k)
47
-
48
- # we can also apply a predicate to the sketch to get an estimate
49
- # (with optimally minimal variance) of the subset sum of items
50
- # matching that predicate among the entire population
51
-
52
- # we'll use a lambda here, but any function operating on a single
53
- # item which returns a boolean value should work
54
- summary = vo.estimate_subset_sum(lambda x: x < 0)
55
- self.assertEqual(summary['estimate'], 1000 * n)
56
- self.assertEqual(summary['total_sketch_weight'], 1001 * n)
57
-
58
- # a regular function is similarly handled
59
- def geq_zero(x):
60
- return x >= 0
61
- summary = vo.estimate_subset_sum(geq_zero)
62
- self.assertEqual(summary['estimate'], n)
63
- self.assertEqual(summary['total_sketch_weight'], 1001 * n)
64
-
65
- # next we'll create a second, smaller sketch with
66
- # only heavier items relative to the previous sketch,
67
- # but with the sketch in sampling mode
68
- k2 = 5
69
- vo2 = var_opt_sketch(k2)
70
- # for weight, use the estimate of all items >=0 from before
71
- wt = summary['estimate']
72
- for i in range(0, k2 + 1):
73
- vo2.update((2 * n) + i, wt)
74
-
75
- # now union the sketches, demonstrating how the
76
- # union's k may not be equal to that of either
77
- # input value
78
- union = var_opt_union(k)
79
- union.update(vo)
80
- union.update(vo2)
81
-
82
- result = union.get_result()
83
- self.assertEqual(n + k2 + 2, result.n)
84
- self.assertFalse(result.is_empty())
85
- self.assertGreater(result.k, k2)
86
- self.assertLess(result.k, k)
87
-
88
- # we can compare what information is available from both
89
- # the union and a sketch.
90
- print(union)
91
-
92
- # if we want to print the list of items, there must be a
93
- # __str__() method for each item (which need not be the same
94
- # type; they're all generic python objects when used from
95
- # python), otherwise you may trigger an exception.
96
- # to_string() is provided as a convenience to avoid direct
97
- # calls to __str__() with parameters.
98
- print(result.to_string(True))
99
-
100
- # finally, we can serialize the sketch by providing an
101
- # appropriate serde class.
102
- expected_size = result.get_serialized_size_bytes(PyIntsSerDe())
103
- b = result.serialize(PyIntsSerDe())
104
- self.assertEqual(expected_size, len(b))
105
-
106
- # if we try to deserialize with the wrong serde, things break
107
- try:
108
- var_opt_sketch.deserialize(b, PyStringsSerDe())
109
- self.fail()
110
- except:
111
- # expected; do nothing
112
- self.assertTrue(True)
113
-
114
- # using the correct serde gives us back a copy of the original
115
- rebuilt = var_opt_sketch.deserialize(b, PyIntsSerDe())
116
- self.assertEqual(result.k, rebuilt.k)
117
- self.assertEqual(result.num_samples, rebuilt.num_samples)
118
- self.assertEqual(result.n, rebuilt.n)
119
- summary1 = result.estimate_subset_sum(geq_zero)
120
- summary2 = rebuilt.estimate_subset_sum(geq_zero)
121
- self.assertEqual(summary1['estimate'], summary2['estimate'])
122
- self.assertEqual(summary1['total_sketch_weight'], summary2['total_sketch_weight'])
123
-
124
- if __name__ == '__main__':
125
- unittest.main()
@@ -1,67 +0,0 @@
1
- /*
2
- * Licensed to the Apache Software Foundation (ASF) under one
3
- * or more contributor license agreements. See the NOTICE file
4
- * distributed with this work for additional information
5
- * regarding copyright ownership. The ASF licenses this file
6
- * to you under the Apache License, Version 2.0 (the
7
- * "License"); you may not use this file except in compliance
8
- * with the License. You may obtain a copy of the License at
9
- *
10
- * http://www.apache.org/licenses/LICENSE-2.0
11
- *
12
- * Unless required by applicable law or agreed to in writing,
13
- * software distributed under the License is distributed on an
14
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- * KIND, either express or implied. See the License for the
16
- * specific language governing permissions and limitations
17
- * under the License.
18
- */
19
-
20
- Code snippets used to generate to generate the binary images from Java.
21
- Heavy items have negative weights to allow a simple predicate to filter
22
- heavy vs light sketch entires.
23
-
24
-
25
- varopt_sketch_long_sampling.bin:
26
- final VarOptItemsSketch<String> sk = VarOptItemsSketch.newInstance(1024);
27
- for (int i = 1; i <= 200; ++i) {
28
- sk.update(Integer.toString(i), 1000.0 / i);
29
- }
30
- byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
31
-
32
-
33
- varopt_sketch_string_exact.bin:
34
- final VarOptItemsSketch<Long> sk = VarOptItemsSketch.newInstance(1024);
35
- for (long i = 0; i < 2000; ++i) {
36
- sk.update(i, 1.0);
37
- }
38
- sk.update(-1L, 100000.0);
39
- sk.update(-2L, 110000.0);
40
- sk.update(-3L, 120000.0);
41
- byte[] bytes = sk.toByteArray(new ArrayOfLongsSerDe());
42
-
43
-
44
- varopt_union_double_sampling.bin:
45
- // parallels small samplign sketch test
46
- final int kSmall = 16;
47
- final int n1 = 32;
48
- final int n2 = 64;
49
- final int kMax = 128;
50
-
51
- // small k sketch, but sampling
52
- VarOptItemsSketch<Double> sketch = VarOptItemsSketch.newInstance(kSmall);
53
- for (int i = 0; i < n1; ++i) {
54
- sketch.update(1.0 * i, 1.0);
55
- }
56
- sketch.update(-1.0, n1 * n1); // add a heavy item
57
-
58
- final VarOptItemsUnion<Double> union = VarOptItemsUnion.newInstance(kMax);
59
- union.update(sketch);
60
-
61
- // another one, but different n to get a different per-item weight
62
- sketch = VarOptItemsSketch.newInstance(kSmall);
63
- for (int i = 0; i < n2; ++i) {
64
- sketch.update(1.0 * i, 1.0);
65
- }
66
- union.update(sketch);
67
- byte[] bytes = union.toByteArray(new ArrayOfDoublesSerDe());
@@ -1,110 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- # Modified from:
19
- # http://www.benjack.io/2018/02/02/python-cpp-revisited.html
20
-
21
- import os
22
- import sys
23
- import platform
24
- import subprocess
25
- import re
26
- from datetime import datetime, timezone
27
-
28
- from setuptools import setup, find_packages, Extension
29
- from setuptools.command.build_ext import build_ext
30
-
31
- class CMakeExtension(Extension):
32
- def __init__(self, name, sourcedir=''):
33
- Extension.__init__(self, name, sources=[])
34
- self.sourcedir = os.path.abspath(sourcedir)
35
-
36
- class CMakeBuild(build_ext):
37
- def run(self):
38
- try:
39
- subprocess.check_output(['cmake', '--version'])
40
- except OSError:
41
- raise RuntimeError(
42
- "CMake >= 3.12 must be installed to build the following extensions: " +
43
- ", ".join(e.name for e in self.extensions))
44
-
45
- for ext in self.extensions:
46
- self.build_extension(ext)
47
-
48
- def build_extension(self, ext):
49
- extdir = os.path.abspath(
50
- os.path.dirname(self.get_ext_fullpath(ext.name)))
51
- cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir]
52
- cmake_args += ['-DWITH_PYTHON=True']
53
- cmake_args += ['-DCMAKE_CXX_STANDARD=11']
54
- # ensure we use a consistent python version
55
- cmake_args += ['-DPython3_EXECUTABLE=' + sys.executable]
56
- cfg = 'Debug' if self.debug else 'Release'
57
- build_args = ['--config', cfg]
58
-
59
- if platform.system() == "Windows":
60
- cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(
61
- cfg.upper(),
62
- extdir)]
63
- if sys.maxsize > 2**32:
64
- cmake_args += ['-T', 'host=x64']
65
- cmake_args += ['-DCMAKE_GENERATOR_PLATFORM=x64']
66
- build_args += ['--', '/m']
67
- else:
68
- cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
69
- build_args += ['--', '-j2']
70
-
71
- env = os.environ.copy()
72
- env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(
73
- env.get('CXXFLAGS', ''),
74
- self.distribution.get_version())
75
- if not os.path.exists(self.build_temp):
76
- os.makedirs(self.build_temp)
77
- subprocess.check_call(['cmake', ext.sourcedir] + cmake_args,
78
- cwd=self.build_temp, env=env)
79
- subprocess.check_call(['cmake', '--build', '.', '--target', 'python'] + build_args,
80
- cwd=self.build_temp, env=env)
81
- print() # add an empty line to pretty print
82
-
83
- # Read and parse the version format
84
- # @DT@ -> datestamp
85
- # @HHMM@ -> .devHHMM to indicate development version
86
- # Releases should have a fixed version with no @ variables
87
- with open('version.cfg.in', 'r') as file:
88
- ds_version = file.read().rstrip()
89
- dt = datetime.now(timezone.utc)
90
- ds_version = re.sub('@DT@', dt.strftime('%Y%m%d'), ds_version)
91
- ds_version = re.sub('@HHMM@', 'dev' + dt.strftime('%H%M'), ds_version)
92
-
93
- setup(
94
- name='datasketches',
95
- version=ds_version,
96
- author='Apache Software Foundation',
97
- author_email='dev@datasketches.apache.org',
98
- description='The Apache DataSketches Library for Python',
99
- license='Apache License 2.0',
100
- url='http://datasketches.apache.org',
101
- long_description=open('python/README.md').read(),
102
- long_description_content_type='text/markdown',
103
- packages=find_packages(where='python',exclude=['src','*tests*']), # src not needed if only the .so
104
- package_dir={'':'python'},
105
- # may need to add all source paths for sdist packages w/o MANIFEST.in
106
- ext_modules=[CMakeExtension('datasketches')],
107
- cmdclass={'build_ext': CMakeBuild},
108
- install_requires=['numpy'],
109
- zip_safe=False
110
- )
@@ -1,52 +0,0 @@
1
- /*
2
- * Licensed to the Apache Software Foundation (ASF) under one
3
- * or more contributor license agreements. See the NOTICE file
4
- * distributed with this work for additional information
5
- * regarding copyright ownership. The ASF licenses this file
6
- * to you under the Apache License, Version 2.0 (the
7
- * "License"); you may not use this file except in compliance
8
- * with the License. You may obtain a copy of the License at
9
- *
10
- * http://www.apache.org/licenses/LICENSE-2.0
11
- *
12
- * Unless required by applicable law or agreed to in writing,
13
- * software distributed under the License is distributed on an
14
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- * KIND, either express or implied. See the License for the
16
- * specific language governing permissions and limitations
17
- * under the License.
18
- */
19
-
20
- #ifndef ARRAY_OF_DOUBLES_INTERSECTION_HPP_
21
- #define ARRAY_OF_DOUBLES_INTERSECTION_HPP_
22
-
23
- #include <vector>
24
- #include <memory>
25
-
26
- #include "array_of_doubles_sketch.hpp"
27
- #include "tuple_intersection.hpp"
28
-
29
- namespace datasketches {
30
-
31
- template<
32
- typename Policy,
33
- typename Allocator = std::allocator<double>
34
- >
35
- class array_of_doubles_intersection: public tuple_intersection<aod<Allocator>, Policy, AllocAOD<Allocator>> {
36
- public:
37
- using Summary = aod<Allocator>;
38
- using AllocSummary = AllocAOD<Allocator>;
39
- using Base = tuple_intersection<Summary, Policy, AllocSummary>;
40
- using CompactSketch = compact_array_of_doubles_sketch_alloc<Allocator>;
41
- using resize_factor = theta_constants::resize_factor;
42
-
43
- explicit array_of_doubles_intersection(uint64_t seed = DEFAULT_SEED, const Policy& policy = Policy(), const Allocator& allocator = Allocator());
44
-
45
- CompactSketch get_result(bool ordered = true) const;
46
- };
47
-
48
- } /* namespace datasketches */
49
-
50
- #include "array_of_doubles_intersection_impl.hpp"
51
-
52
- #endif