datasketches 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/ext/datasketches/vo_wrapper.cpp +1 -1
- data/lib/datasketches/version.rb +1 -1
- data/vendor/datasketches-cpp/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/LICENSE +35 -7
- data/vendor/datasketches-cpp/NOTICE +2 -2
- data/vendor/datasketches-cpp/common/CMakeLists.txt +2 -1
- data/vendor/datasketches-cpp/common/include/common_defs.hpp +1 -0
- data/vendor/datasketches-cpp/common/include/quantiles_sorted_view_impl.hpp +5 -7
- data/vendor/datasketches-cpp/common/include/xxhash64.h +202 -0
- data/vendor/datasketches-cpp/filters/CMakeLists.txt +43 -0
- data/vendor/datasketches-cpp/filters/include/bit_array_ops.hpp +180 -0
- data/vendor/datasketches-cpp/filters/include/bloom_filter.hpp +753 -0
- data/vendor/datasketches-cpp/filters/include/bloom_filter_builder_impl.hpp +132 -0
- data/vendor/datasketches-cpp/filters/include/bloom_filter_impl.hpp +908 -0
- data/vendor/datasketches-cpp/filters/test/CMakeLists.txt +60 -0
- data/vendor/datasketches-cpp/filters/test/bit_array_ops_test.cpp +107 -0
- data/vendor/datasketches-cpp/filters/test/bloom_filter_allocation_test.cpp +75 -0
- data/vendor/datasketches-cpp/filters/test/bloom_filter_deserialize_from_java_test.cpp +51 -0
- data/vendor/datasketches-cpp/filters/test/bloom_filter_serialize_for_java.cpp +45 -0
- data/vendor/datasketches-cpp/filters/test/bloom_filter_test.cpp +406 -0
- data/vendor/datasketches-cpp/tdigest/include/tdigest.hpp +51 -1
- data/vendor/datasketches-cpp/tdigest/include/tdigest_impl.hpp +38 -1
- data/vendor/datasketches-cpp/tdigest/test/tdigest_test.cpp +12 -3
- data/vendor/datasketches-cpp/theta/include/bit_packing.hpp +5 -5
- data/vendor/datasketches-cpp/theta/test/bit_packing_test.cpp +41 -39
- data/vendor/datasketches-cpp/version.cfg.in +1 -1
- metadata +18 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6a0f499815550712d69187bde082bfbbf0a2b076f8d4ae747cf9c85b59cdbd55
|
4
|
+
data.tar.gz: 46cc4162f0a894a5dfc1c000ab74a4baf95404f515a69a19bede6aed28630279
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 63ac3ca31eeb6bd10bc4ededf78f15143211e38a34fcc627d9ecdc500097b862c6e4a73e0032b5338851041facf0359c9c142e4f80e83fd76cc0ed1c264201e8
|
7
|
+
data.tar.gz: 9345d158a0e83d4b4a70703ddf47a686ab2a82c6dd31c5e55a1823be3a009c53ac807bc81e759812a682f23e56c2af43aab59baa1d2b2341eeac8dbdf619f26d
|
data/CHANGELOG.md
CHANGED
@@ -8,7 +8,7 @@ using datasketches::var_opt_sketch;
|
|
8
8
|
|
9
9
|
template<typename T>
|
10
10
|
void bind_vo_sketch(Rice::Module &m, const char* name) {
|
11
|
-
Rice::define_class_under<var_opt_sketch<T>>(m,
|
11
|
+
Rice::define_class_under<var_opt_sketch<T>>(m, name)
|
12
12
|
.define_constructor(Rice::Constructor<var_opt_sketch<T>, uint32_t>())
|
13
13
|
.define_method("k", &var_opt_sketch<T>::get_k)
|
14
14
|
.define_method("n", &var_opt_sketch<T>::get_n)
|
data/lib/datasketches/version.rb
CHANGED
@@ -207,9 +207,9 @@ APPENDIX A: How to apply the Apache License to your work.
|
|
207
207
|
|
208
208
|
APPENDIX B: Additional licenses relevant to this product.
|
209
209
|
|
210
|
-
This product includes a number of source files with code that has been
|
211
|
-
adapted from 3rd party sources including sources that may be subject
|
212
|
-
to different copyright notices and license terms. Your use of
|
210
|
+
This product includes a number of source files with code that has been
|
211
|
+
adapted from 3rd party sources including sources that may be subject
|
212
|
+
to different copyright notices and license terms. Your use of
|
213
213
|
the source code for these subcomponents is subject to the terms and
|
214
214
|
conditions of the following licenses.
|
215
215
|
|
@@ -221,7 +221,7 @@ APPENDIX B: Additional licenses relevant to this product.
|
|
221
221
|
https://github.com/catchorg/Catch2/blob/v2.x/LICENSE.txt
|
222
222
|
|
223
223
|
Boost Software License - Version 1.0 - August 17th, 2003
|
224
|
-
|
224
|
+
|
225
225
|
Permission is hereby granted, free of charge, to any person or organization
|
226
226
|
obtaining a copy of the software and accompanying documentation covered by
|
227
227
|
this license (the "Software") to use, reproduce, display, distribute,
|
@@ -248,6 +248,35 @@ APPENDIX B: Additional licenses relevant to this product.
|
|
248
248
|
Found in the Catch2 unit test code that is downloaded from github.com as part
|
249
249
|
of CMake configuration if configured to build tests.
|
250
250
|
|
251
|
+
=============================================================
|
252
|
+
MIT License
|
253
|
+
=============================================================
|
254
|
+
|
255
|
+
Original source:
|
256
|
+
https://github.com/stbrumme/xxhash/blob/master/LICENSE
|
257
|
+
|
258
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
259
|
+
of this software and associated documentation files (the "Software"),
|
260
|
+
to deal in the Software without restriction, including without limitation
|
261
|
+
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
262
|
+
and/or sell copies of the Software, and to permit persons to whom the Software
|
263
|
+
is furnished to do so, subject to the following conditions:
|
264
|
+
|
265
|
+
The above copyright notice and this permission notice shall be included
|
266
|
+
in all copies or substantial portions of the Software.
|
267
|
+
|
268
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
269
|
+
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
270
|
+
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
271
|
+
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
272
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
273
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
274
|
+
|
275
|
+
Code Location:
|
276
|
+
common/include/xxhash64.h
|
277
|
+
Original source code:
|
278
|
+
Copyright (c) 2018 Stephan Brumme
|
279
|
+
https://github.com/stbrumme/xxhash/blob/master/xxhash64.h
|
251
280
|
|
252
281
|
=============================================================
|
253
282
|
Public Domain
|
@@ -255,7 +284,7 @@ APPENDIX B: Additional licenses relevant to this product.
|
|
255
284
|
Original source code:
|
256
285
|
https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
|
257
286
|
Placed in the Public Domain by Austin Appleby
|
258
|
-
|
287
|
+
|
259
288
|
Code Locations:
|
260
289
|
common/include/MurmurHash3.h
|
261
290
|
that is adapted from the above.
|
@@ -263,8 +292,7 @@ APPENDIX B: Additional licenses relevant to this product.
|
|
263
292
|
Original source code:
|
264
293
|
* https://graphics.stanford.edu/~seander/bithacks.html
|
265
294
|
* Placed in the Public Domain by Sean Eron Anderson
|
266
|
-
|
295
|
+
|
267
296
|
Code Locations:
|
268
297
|
* common/include/ceiling_power_of_2.hpp
|
269
298
|
that is adapted from the above.
|
270
|
-
|
@@ -1,9 +1,9 @@
|
|
1
1
|
Apache DataSketches C++
|
2
|
-
Copyright
|
2
|
+
Copyright 2025 The Apache Software Foundation
|
3
3
|
|
4
4
|
Copyright 2015-2018 Yahoo Inc.
|
5
5
|
Copyright 2019-2020 Verizon Media
|
6
|
-
Copyright 2021 Yahoo Inc.
|
6
|
+
Copyright 2021- Yahoo Inc.
|
7
7
|
|
8
8
|
This product includes software developed at
|
9
9
|
The Apache Software Foundation (http://www.apache.org/).
|
@@ -31,7 +31,7 @@ target_include_directories(common
|
|
31
31
|
|
32
32
|
install(TARGETS common EXPORT ${PROJECT_NAME})
|
33
33
|
|
34
|
-
install(FILES
|
34
|
+
install(FILES
|
35
35
|
${CMAKE_CURRENT_BINARY_DIR}/include/version.hpp
|
36
36
|
include/binomial_bounds.hpp
|
37
37
|
include/bounds_binomial_proportions.hpp
|
@@ -49,4 +49,5 @@ install(FILES
|
|
49
49
|
include/quantiles_sorted_view_impl.hpp
|
50
50
|
include/quantiles_sorted_view.hpp
|
51
51
|
include/serde.hpp
|
52
|
+
include/xxhash64.h
|
52
53
|
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
|
@@ -42,6 +42,7 @@ namespace random_utils {
|
|
42
42
|
static std::random_device rd; // possibly unsafe in MinGW with GCC < 9.2
|
43
43
|
static thread_local std::mt19937_64 rand(rd());
|
44
44
|
static thread_local std::uniform_real_distribution<> next_double(0.0, 1.0);
|
45
|
+
static thread_local std::uniform_int_distribution<uint64_t> next_uint64(0, UINT64_MAX);
|
45
46
|
|
46
47
|
// thread-safe random bit
|
47
48
|
static thread_local std::independent_bits_engine<std::mt19937, 1, uint32_t>
|
@@ -86,19 +86,17 @@ auto quantiles_sorted_view<T, C, A>::get_quantile(double rank, bool inclusive) c
|
|
86
86
|
template<typename T, typename C, typename A>
|
87
87
|
auto quantiles_sorted_view<T, C, A>::get_CDF(const T* split_points, uint32_t size, bool inclusive) const -> vector_double {
|
88
88
|
if (entries_.empty()) throw std::runtime_error("operation is undefined for an empty sketch");
|
89
|
-
vector_double buckets(entries_.get_allocator());
|
90
|
-
if (entries_.size() == 0) return buckets;
|
91
89
|
check_split_points(split_points, size);
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
90
|
+
vector_double ranks(entries_.get_allocator());
|
91
|
+
ranks.reserve(size + 1);
|
92
|
+
for (uint32_t i = 0; i < size; ++i) ranks.push_back(get_rank(split_points[i], inclusive));
|
93
|
+
ranks.push_back(1);
|
94
|
+
return ranks;
|
96
95
|
}
|
97
96
|
|
98
97
|
template<typename T, typename C, typename A>
|
99
98
|
auto quantiles_sorted_view<T, C, A>::get_PMF(const T* split_points, uint32_t size, bool inclusive) const -> vector_double {
|
100
99
|
auto buckets = get_CDF(split_points, size, inclusive);
|
101
|
-
if (buckets.size() == 0) return buckets;
|
102
100
|
for (uint32_t i = size; i > 0; --i) {
|
103
101
|
buckets[i] -= buckets[i - 1];
|
104
102
|
}
|
@@ -0,0 +1,202 @@
|
|
1
|
+
// //////////////////////////////////////////////////////////
|
2
|
+
// xxhash64.h
|
3
|
+
// Copyright (c) 2016 Stephan Brumme. All rights reserved.
|
4
|
+
// see http://create.stephan-brumme.com/disclaimer.html
|
5
|
+
//
|
6
|
+
|
7
|
+
#pragma once
|
8
|
+
#include <stdint.h> // for uint32_t and uint64_t
|
9
|
+
|
10
|
+
/// XXHash (64 bit), based on Yann Collet's descriptions, see http://cyan4973.github.io/xxHash/
|
11
|
+
/** How to use:
|
12
|
+
uint64_t myseed = 0;
|
13
|
+
XXHash64 myhash(myseed);
|
14
|
+
myhash.add(pointerToSomeBytes, numberOfBytes);
|
15
|
+
myhash.add(pointerToSomeMoreBytes, numberOfMoreBytes); // call add() as often as you like to ...
|
16
|
+
// and compute hash:
|
17
|
+
uint64_t result = myhash.hash();
|
18
|
+
|
19
|
+
// or all of the above in one single line:
|
20
|
+
uint64_t result2 = XXHash64::hash(mypointer, numBytes, myseed);
|
21
|
+
|
22
|
+
Note: my code is NOT endian-aware !
|
23
|
+
**/
|
24
|
+
class XXHash64
|
25
|
+
{
|
26
|
+
public:
|
27
|
+
/// create new XXHash (64 bit)
|
28
|
+
/** @param seed your seed value, even zero is a valid seed **/
|
29
|
+
explicit XXHash64(uint64_t seed)
|
30
|
+
{
|
31
|
+
state[0] = seed + Prime1 + Prime2;
|
32
|
+
state[1] = seed + Prime2;
|
33
|
+
state[2] = seed;
|
34
|
+
state[3] = seed - Prime1;
|
35
|
+
bufferSize = 0;
|
36
|
+
totalLength = 0;
|
37
|
+
}
|
38
|
+
|
39
|
+
/// add a chunk of bytes
|
40
|
+
/** @param input pointer to a continuous block of data
|
41
|
+
@param length number of bytes
|
42
|
+
@return false if parameters are invalid / zero **/
|
43
|
+
bool add(const void* input, uint64_t length)
|
44
|
+
{
|
45
|
+
// no data ?
|
46
|
+
if (!input || length == 0)
|
47
|
+
return false;
|
48
|
+
|
49
|
+
totalLength += length;
|
50
|
+
// byte-wise access
|
51
|
+
const unsigned char* data = (const unsigned char*)input;
|
52
|
+
|
53
|
+
// unprocessed old data plus new data still fit in temporary buffer ?
|
54
|
+
if (bufferSize + length < MaxBufferSize)
|
55
|
+
{
|
56
|
+
// just add new data
|
57
|
+
while (length-- > 0)
|
58
|
+
buffer[bufferSize++] = *data++;
|
59
|
+
return true;
|
60
|
+
}
|
61
|
+
|
62
|
+
// point beyond last byte
|
63
|
+
const unsigned char* stop = data + length;
|
64
|
+
const unsigned char* stopBlock = stop - MaxBufferSize;
|
65
|
+
|
66
|
+
// some data left from previous update ?
|
67
|
+
if (bufferSize > 0)
|
68
|
+
{
|
69
|
+
// make sure temporary buffer is full (16 bytes)
|
70
|
+
while (bufferSize < MaxBufferSize)
|
71
|
+
buffer[bufferSize++] = *data++;
|
72
|
+
|
73
|
+
// process these 32 bytes (4x8)
|
74
|
+
process(buffer, state[0], state[1], state[2], state[3]);
|
75
|
+
}
|
76
|
+
|
77
|
+
// copying state to local variables helps optimizer A LOT
|
78
|
+
uint64_t s0 = state[0], s1 = state[1], s2 = state[2], s3 = state[3];
|
79
|
+
// 32 bytes at once
|
80
|
+
while (data <= stopBlock)
|
81
|
+
{
|
82
|
+
// local variables s0..s3 instead of state[0]..state[3] are much faster
|
83
|
+
process(data, s0, s1, s2, s3);
|
84
|
+
data += 32;
|
85
|
+
}
|
86
|
+
// copy back
|
87
|
+
state[0] = s0; state[1] = s1; state[2] = s2; state[3] = s3;
|
88
|
+
|
89
|
+
// copy remainder to temporary buffer
|
90
|
+
bufferSize = stop - data;
|
91
|
+
for (uint64_t i = 0; i < bufferSize; i++)
|
92
|
+
buffer[i] = data[i];
|
93
|
+
|
94
|
+
// done
|
95
|
+
return true;
|
96
|
+
}
|
97
|
+
|
98
|
+
/// get current hash
|
99
|
+
/** @return 64 bit XXHash **/
|
100
|
+
uint64_t hash() const
|
101
|
+
{
|
102
|
+
// fold 256 bit state into one single 64 bit value
|
103
|
+
uint64_t result;
|
104
|
+
if (totalLength >= MaxBufferSize)
|
105
|
+
{
|
106
|
+
result = rotateLeft(state[0], 1) +
|
107
|
+
rotateLeft(state[1], 7) +
|
108
|
+
rotateLeft(state[2], 12) +
|
109
|
+
rotateLeft(state[3], 18);
|
110
|
+
result = (result ^ processSingle(0, state[0])) * Prime1 + Prime4;
|
111
|
+
result = (result ^ processSingle(0, state[1])) * Prime1 + Prime4;
|
112
|
+
result = (result ^ processSingle(0, state[2])) * Prime1 + Prime4;
|
113
|
+
result = (result ^ processSingle(0, state[3])) * Prime1 + Prime4;
|
114
|
+
}
|
115
|
+
else
|
116
|
+
{
|
117
|
+
// internal state wasn't set in add(), therefore original seed is still stored in state2
|
118
|
+
result = state[2] + Prime5;
|
119
|
+
}
|
120
|
+
|
121
|
+
result += totalLength;
|
122
|
+
|
123
|
+
// process remaining bytes in temporary buffer
|
124
|
+
const unsigned char* data = buffer;
|
125
|
+
// point beyond last byte
|
126
|
+
const unsigned char* stop = data + bufferSize;
|
127
|
+
|
128
|
+
// at least 8 bytes left ? => eat 8 bytes per step
|
129
|
+
for (; data + 8 <= stop; data += 8)
|
130
|
+
result = rotateLeft(result ^ processSingle(0, *(uint64_t*)data), 27) * Prime1 + Prime4;
|
131
|
+
|
132
|
+
// 4 bytes left ? => eat those
|
133
|
+
if (data + 4 <= stop)
|
134
|
+
{
|
135
|
+
result = rotateLeft(result ^ (*(uint32_t*)data) * Prime1, 23) * Prime2 + Prime3;
|
136
|
+
data += 4;
|
137
|
+
}
|
138
|
+
|
139
|
+
// take care of remaining 0..3 bytes, eat 1 byte per step
|
140
|
+
while (data != stop)
|
141
|
+
result = rotateLeft(result ^ (*data++) * Prime5, 11) * Prime1;
|
142
|
+
|
143
|
+
// mix bits
|
144
|
+
result ^= result >> 33;
|
145
|
+
result *= Prime2;
|
146
|
+
result ^= result >> 29;
|
147
|
+
result *= Prime3;
|
148
|
+
result ^= result >> 32;
|
149
|
+
return result;
|
150
|
+
}
|
151
|
+
|
152
|
+
|
153
|
+
/// combine constructor, add() and hash() in one static function (C style)
|
154
|
+
/** @param input pointer to a continuous block of data
|
155
|
+
@param length number of bytes
|
156
|
+
@param seed your seed value, e.g. zero is a valid seed
|
157
|
+
@return 64 bit XXHash **/
|
158
|
+
static uint64_t hash(const void* input, uint64_t length, uint64_t seed)
|
159
|
+
{
|
160
|
+
XXHash64 hasher(seed);
|
161
|
+
hasher.add(input, length);
|
162
|
+
return hasher.hash();
|
163
|
+
}
|
164
|
+
|
165
|
+
private:
|
166
|
+
/// magic constants :-)
|
167
|
+
static const uint64_t Prime1 = 11400714785074694791ULL;
|
168
|
+
static const uint64_t Prime2 = 14029467366897019727ULL;
|
169
|
+
static const uint64_t Prime3 = 1609587929392839161ULL;
|
170
|
+
static const uint64_t Prime4 = 9650029242287828579ULL;
|
171
|
+
static const uint64_t Prime5 = 2870177450012600261ULL;
|
172
|
+
|
173
|
+
/// temporarily store up to 31 bytes between multiple add() calls
|
174
|
+
static const uint64_t MaxBufferSize = 31+1;
|
175
|
+
|
176
|
+
uint64_t state[4];
|
177
|
+
unsigned char buffer[MaxBufferSize];
|
178
|
+
uint64_t bufferSize;
|
179
|
+
uint64_t totalLength;
|
180
|
+
|
181
|
+
/// rotate bits, should compile to a single CPU instruction (ROL)
|
182
|
+
static inline uint64_t rotateLeft(uint64_t x, unsigned char bits)
|
183
|
+
{
|
184
|
+
return (x << bits) | (x >> (64 - bits));
|
185
|
+
}
|
186
|
+
|
187
|
+
/// process a single 64 bit value
|
188
|
+
static inline uint64_t processSingle(uint64_t previous, uint64_t input)
|
189
|
+
{
|
190
|
+
return rotateLeft(previous + input * Prime2, 31) * Prime1;
|
191
|
+
}
|
192
|
+
|
193
|
+
/// process a block of 4x4 bytes, this is the main part of the XXHash32 algorithm
|
194
|
+
static inline void process(const void* data, uint64_t& state0, uint64_t& state1, uint64_t& state2, uint64_t& state3)
|
195
|
+
{
|
196
|
+
const uint64_t* block = (const uint64_t*) data;
|
197
|
+
state0 = processSingle(state0, block[0]);
|
198
|
+
state1 = processSingle(state1, block[1]);
|
199
|
+
state2 = processSingle(state2, block[2]);
|
200
|
+
state3 = processSingle(state3, block[3]);
|
201
|
+
}
|
202
|
+
};
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
add_library(filters INTERFACE)
|
19
|
+
|
20
|
+
add_library(${PROJECT_NAME}::FILTERS ALIAS filters)
|
21
|
+
|
22
|
+
if (BUILD_TESTS)
|
23
|
+
add_subdirectory(test)
|
24
|
+
endif()
|
25
|
+
|
26
|
+
target_include_directories(filters
|
27
|
+
INTERFACE
|
28
|
+
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
|
29
|
+
$<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>
|
30
|
+
)
|
31
|
+
|
32
|
+
target_link_libraries(filters INTERFACE common)
|
33
|
+
|
34
|
+
install(TARGETS filters
|
35
|
+
EXPORT ${PROJECT_NAME}
|
36
|
+
)
|
37
|
+
|
38
|
+
install(FILES
|
39
|
+
include/bloom_filter.hpp
|
40
|
+
include/bloom_filter_impl.hpp
|
41
|
+
include/bloom_filter_builder_impl.hpp
|
42
|
+
include/bit_array_ops.hpp
|
43
|
+
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
|
@@ -0,0 +1,180 @@
|
|
1
|
+
/*
|
2
|
+
* Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
* or more contributor license agreements. See the NOTICE file
|
4
|
+
* distributed with this work for additional information
|
5
|
+
* regarding copyright ownership. The ASF licenses this file
|
6
|
+
* to you under the Apache License, Version 2.0 (the
|
7
|
+
* "License"); you may not use this file except in compliance
|
8
|
+
* with the License. You may obtain a copy of the License at
|
9
|
+
*
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
*
|
12
|
+
* Unless required by applicable law or agreed to in writing,
|
13
|
+
* software distributed under the License is distributed on an
|
14
|
+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
* KIND, either express or implied. See the License for the
|
16
|
+
* specific language governing permissions and limitations
|
17
|
+
* under the License.
|
18
|
+
*/
|
19
|
+
|
20
|
+
#ifndef _BIT_ARRAY_OPS_HPP_
|
21
|
+
#define _BIT_ARRAY_OPS_HPP_
|
22
|
+
|
23
|
+
#include <bitset>
|
24
|
+
|
25
|
+
namespace datasketches {
|
26
|
+
|
27
|
+
/**
|
28
|
+
* This class comprises methods that operate one or more arrays of bits (uint8_t*) to
|
29
|
+
* provide bit array operations. The class does not take ownership of memory and operates On
|
30
|
+
* arrays in-place. Sizes of the arrays, in bytes, are passed in as arguments.
|
31
|
+
*
|
32
|
+
* None of the methods in this class perform bounds checks. The caller is responsible for ensuring
|
33
|
+
* that indices are within the array bounds.
|
34
|
+
*
|
35
|
+
* Implementation assumes the actual arrays are multiples of 64 bits in length.
|
36
|
+
*/
|
37
|
+
namespace bit_array_ops {
|
38
|
+
|
39
|
+
/**
|
40
|
+
* Get the value of a bit at the given index.
|
41
|
+
* @param array the array of bits
|
42
|
+
* @param index the index of the bit to get
|
43
|
+
* @return the value of the bit at the given index.
|
44
|
+
*/
|
45
|
+
static inline bool get_bit(uint8_t* array, uint64_t index) {
|
46
|
+
return (array[index >> 3] & (1 << (index & 7))) != 0;
|
47
|
+
}
|
48
|
+
|
49
|
+
/**
|
50
|
+
* Set the bit at the given index to 1.
|
51
|
+
* @param array the array of bits
|
52
|
+
* @param index the index of the bit to set.
|
53
|
+
*/
|
54
|
+
static inline void set_bit(uint8_t* array, uint64_t index) {
|
55
|
+
array[index >> 3] |= (1 << (index & 7));
|
56
|
+
}
|
57
|
+
|
58
|
+
/**
|
59
|
+
* Set the bit at the given index to 0.
|
60
|
+
* @param array the array of bits
|
61
|
+
* @param index the index of the bit to clear.
|
62
|
+
*/
|
63
|
+
static inline void clear_bit(uint8_t* array, uint64_t index) {
|
64
|
+
array[index >> 3] &= ~(1 << (index & 7));
|
65
|
+
}
|
66
|
+
|
67
|
+
/**
|
68
|
+
* Assign the value of the bit at the given index.
|
69
|
+
* @param array the array of bits
|
70
|
+
* @param index the index of the bit to set.
|
71
|
+
*/
|
72
|
+
static inline void assign_bit(uint8_t* array, uint64_t index, bool value) {
|
73
|
+
// read-only checks handled by set_bit() and clear_bit()
|
74
|
+
if (value) {
|
75
|
+
set_bit(array, index);
|
76
|
+
} else {
|
77
|
+
clear_bit(array, index);
|
78
|
+
}
|
79
|
+
}
|
80
|
+
|
81
|
+
/**
|
82
|
+
* Gets the value of a bit at the specified index and sets it to true
|
83
|
+
* @param array the array of bits
|
84
|
+
* @param index the index of the bit to get and set
|
85
|
+
* @return the value of the bit at the specified index
|
86
|
+
*/
|
87
|
+
static inline bool get_and_set_bit(uint8_t* array, uint64_t index) {
|
88
|
+
const uint64_t offset = index >> 3;
|
89
|
+
const uint8_t mask = 1 << (index & 7);
|
90
|
+
if ((array[offset] & mask) != 0) {
|
91
|
+
return true;
|
92
|
+
} else {
|
93
|
+
array[offset] |= mask;
|
94
|
+
return false;
|
95
|
+
}
|
96
|
+
}
|
97
|
+
|
98
|
+
/**
|
99
|
+
* @brief Gets the number of bits set in the bit array.
|
100
|
+
* @param array the array of bits
|
101
|
+
* @param length_bytes the length of the array, in bytes
|
102
|
+
* @return the number of bits set in the bit array.
|
103
|
+
*/
|
104
|
+
static inline uint64_t count_num_bits_set(uint8_t* array, uint64_t length_bytes) {
|
105
|
+
uint64_t num_bits_set = 0;
|
106
|
+
|
107
|
+
// we rounded up to a multiple of 64 so we know we can use 64-bit operations
|
108
|
+
const uint64_t* array64 = reinterpret_cast<const uint64_t*>(array);
|
109
|
+
// Calculate the number of 64-bit chunks
|
110
|
+
uint64_t num_longs = length_bytes / 8; // 8 bytes per 64 bits
|
111
|
+
for (uint64_t i = 0; i < num_longs; ++i) {
|
112
|
+
// Wrap the 64-bit chunk with std::bitset for easy bit counting
|
113
|
+
std::bitset<64> bits(array64[i]);
|
114
|
+
num_bits_set += bits.count();
|
115
|
+
}
|
116
|
+
return num_bits_set;
|
117
|
+
}
|
118
|
+
|
119
|
+
/**
|
120
|
+
* Performs a union operation on one bit array with another bit array.
|
121
|
+
* This operation modifies the tgt bit array to be the union of its original bits and the bits of the src array.
|
122
|
+
* The union operation is equivalent to a bitwise OR operation between the two arrays.
|
123
|
+
*
|
124
|
+
* @param tgt the array of bits into which the results are written
|
125
|
+
* @param src the array of bits to union into tgt
|
126
|
+
* @param length_bytes the length of the two arrays, in bytes
|
127
|
+
* @return the number of bits set in the resulting array
|
128
|
+
*/
|
129
|
+
static inline uint64_t union_with(uint8_t* tgt, const uint8_t* src, uint64_t length_bytes) {
|
130
|
+
uint64_t num_bits_set = 0;
|
131
|
+
for (uint64_t i = 0; i < length_bytes; ++i) {
|
132
|
+
tgt[i] |= src[i];
|
133
|
+
std::bitset<8> bits(tgt[i]);
|
134
|
+
num_bits_set += bits.count();
|
135
|
+
}
|
136
|
+
return num_bits_set;
|
137
|
+
}
|
138
|
+
|
139
|
+
/**
|
140
|
+
* Performs an intersection operation on one bit array with another bit array.
|
141
|
+
* This operation modifies the tgt bit array to contain only the bits that are set in both that array and the src array.
|
142
|
+
* The intersection operation is equivalent to a bitwise AND operation between the two arrays.
|
143
|
+
*
|
144
|
+
* @param tgt the array of bits into which the results are written
|
145
|
+
* @param src the array of bits to intersect with tgt
|
146
|
+
* @param length_bytes the length of the two arrays, in bytes
|
147
|
+
* @return the number of bits set in the resulting array
|
148
|
+
*/
|
149
|
+
static inline uint64_t intersect(uint8_t* tgt, const uint8_t* src, uint64_t length_bytes) {
|
150
|
+
uint64_t num_bits_set = 0;
|
151
|
+
for (uint64_t i = 0; i < length_bytes; ++i) {
|
152
|
+
tgt[i] &= src[i];
|
153
|
+
std::bitset<8> bits(tgt[i]);
|
154
|
+
num_bits_set += bits.count();
|
155
|
+
}
|
156
|
+
return num_bits_set;
|
157
|
+
}
|
158
|
+
|
159
|
+
/**
|
160
|
+
* Inverts the bits of this bit array.
|
161
|
+
* This operation modifies the bit array by flipping all its bits; 0s become 1s and 1s become 0s.
|
162
|
+
* @param array the array of bits
|
163
|
+
* @param length_bytes the length of the array, in bytes
|
164
|
+
* @return the number of bits set in the resulting array
|
165
|
+
*/
|
166
|
+
static inline uint64_t invert(uint8_t* array, uint64_t length_bytes) {
|
167
|
+
uint64_t num_bits_set = 0;
|
168
|
+
for (uint64_t i = 0; i < length_bytes; ++i) {
|
169
|
+
array[i] = ~array[i];
|
170
|
+
std::bitset<8> bits(array[i]);
|
171
|
+
num_bits_set += bits.count();
|
172
|
+
}
|
173
|
+
return num_bits_set;
|
174
|
+
}
|
175
|
+
|
176
|
+
} // namespace bit_array_ops
|
177
|
+
|
178
|
+
} // namespace datasketches
|
179
|
+
|
180
|
+
#endif // _BIT_ARRAY_OPS_HPP_
|