datasketches 0.4.3 → 0.4.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/ext/datasketches/vo_wrapper.cpp +1 -1
- data/lib/datasketches/version.rb +1 -1
- data/vendor/datasketches-cpp/CMakeLists.txt +1 -0
- data/vendor/datasketches-cpp/LICENSE +35 -7
- data/vendor/datasketches-cpp/NOTICE +2 -2
- data/vendor/datasketches-cpp/common/CMakeLists.txt +2 -1
- data/vendor/datasketches-cpp/common/include/common_defs.hpp +1 -0
- data/vendor/datasketches-cpp/common/include/quantiles_sorted_view_impl.hpp +5 -7
- data/vendor/datasketches-cpp/common/include/xxhash64.h +202 -0
- data/vendor/datasketches-cpp/filters/CMakeLists.txt +43 -0
- data/vendor/datasketches-cpp/filters/include/bit_array_ops.hpp +180 -0
- data/vendor/datasketches-cpp/filters/include/bloom_filter.hpp +753 -0
- data/vendor/datasketches-cpp/filters/include/bloom_filter_builder_impl.hpp +132 -0
- data/vendor/datasketches-cpp/filters/include/bloom_filter_impl.hpp +908 -0
- data/vendor/datasketches-cpp/filters/test/CMakeLists.txt +60 -0
- data/vendor/datasketches-cpp/filters/test/bit_array_ops_test.cpp +107 -0
- data/vendor/datasketches-cpp/filters/test/bloom_filter_allocation_test.cpp +75 -0
- data/vendor/datasketches-cpp/filters/test/bloom_filter_deserialize_from_java_test.cpp +51 -0
- data/vendor/datasketches-cpp/filters/test/bloom_filter_serialize_for_java.cpp +45 -0
- data/vendor/datasketches-cpp/filters/test/bloom_filter_test.cpp +406 -0
- data/vendor/datasketches-cpp/tdigest/include/tdigest.hpp +51 -1
- data/vendor/datasketches-cpp/tdigest/include/tdigest_impl.hpp +38 -1
- data/vendor/datasketches-cpp/tdigest/test/tdigest_test.cpp +12 -3
- data/vendor/datasketches-cpp/theta/include/bit_packing.hpp +5 -5
- data/vendor/datasketches-cpp/theta/test/bit_packing_test.cpp +41 -39
- data/vendor/datasketches-cpp/version.cfg.in +1 -1
- metadata +17 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 427f1c95c4dc0782a7003f24b155b39d0b01560e57ff2046785d3254e79250c0
|
4
|
+
data.tar.gz: 820d72e4c907cb7d59bd808d2b837692032f038dac06cf96d0767531cc68370e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 03abee1fa7222628c3e3b00c9fed91218abf17a9dd8116aa6c55e37112a33506c11333533580a3804ce764bbce17ac4db08450e3cf82a52a36e924abd2391762
|
7
|
+
data.tar.gz: e838d14d5f43706dd85a51a45074e224131bc0ecd9157dfc350b2f990c6d54b9338dfb48fc0ad14d86f2d74ef2b855a385e4a67aa3be6e173bf5610887e18e04
|
data/CHANGELOG.md
CHANGED
@@ -8,7 +8,7 @@ using datasketches::var_opt_sketch;
|
|
8
8
|
|
9
9
|
template<typename T>
|
10
10
|
void bind_vo_sketch(Rice::Module &m, const char* name) {
|
11
|
-
Rice::define_class_under<var_opt_sketch<T>>(m,
|
11
|
+
Rice::define_class_under<var_opt_sketch<T>>(m, name)
|
12
12
|
.define_constructor(Rice::Constructor<var_opt_sketch<T>, uint32_t>())
|
13
13
|
.define_method("k", &var_opt_sketch<T>::get_k)
|
14
14
|
.define_method("n", &var_opt_sketch<T>::get_n)
|
data/lib/datasketches/version.rb
CHANGED
@@ -207,9 +207,9 @@ APPENDIX A: How to apply the Apache License to your work.
|
|
207
207
|
|
208
208
|
APPENDIX B: Additional licenses relevant to this product.
|
209
209
|
|
210
|
-
This product includes a number of source files with code that has been
|
211
|
-
adapted from 3rd party sources including sources that may be subject
|
212
|
-
to different copyright notices and license terms. Your use of
|
210
|
+
This product includes a number of source files with code that has been
|
211
|
+
adapted from 3rd party sources including sources that may be subject
|
212
|
+
to different copyright notices and license terms. Your use of
|
213
213
|
the source code for these subcomponents is subject to the terms and
|
214
214
|
conditions of the following licenses.
|
215
215
|
|
@@ -221,7 +221,7 @@ APPENDIX B: Additional licenses relevant to this product.
|
|
221
221
|
https://github.com/catchorg/Catch2/blob/v2.x/LICENSE.txt
|
222
222
|
|
223
223
|
Boost Software License - Version 1.0 - August 17th, 2003
|
224
|
-
|
224
|
+
|
225
225
|
Permission is hereby granted, free of charge, to any person or organization
|
226
226
|
obtaining a copy of the software and accompanying documentation covered by
|
227
227
|
this license (the "Software") to use, reproduce, display, distribute,
|
@@ -248,6 +248,35 @@ APPENDIX B: Additional licenses relevant to this product.
|
|
248
248
|
Found in the Catch2 unit test code that is downloaded from github.com as part
|
249
249
|
of CMake configuration if configured to build tests.
|
250
250
|
|
251
|
+
=============================================================
|
252
|
+
MIT License
|
253
|
+
=============================================================
|
254
|
+
|
255
|
+
Original source:
|
256
|
+
https://github.com/stbrumme/xxhash/blob/master/LICENSE
|
257
|
+
|
258
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
259
|
+
of this software and associated documentation files (the "Software"),
|
260
|
+
to deal in the Software without restriction, including without limitation
|
261
|
+
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
262
|
+
and/or sell copies of the Software, and to permit persons to whom the Software
|
263
|
+
is furnished to do so, subject to the following conditions:
|
264
|
+
|
265
|
+
The above copyright notice and this permission notice shall be included
|
266
|
+
in all copies or substantial portions of the Software.
|
267
|
+
|
268
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
269
|
+
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
270
|
+
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
271
|
+
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
272
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
273
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
274
|
+
|
275
|
+
Code Location:
|
276
|
+
common/include/xxhash64.h
|
277
|
+
Original source code:
|
278
|
+
Copyright (c) 2018 Stephan Brumme
|
279
|
+
https://github.com/stbrumme/xxhash/blob/master/xxhash64.h
|
251
280
|
|
252
281
|
=============================================================
|
253
282
|
Public Domain
|
@@ -255,7 +284,7 @@ APPENDIX B: Additional licenses relevant to this product.
|
|
255
284
|
Original source code:
|
256
285
|
https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
|
257
286
|
Placed in the Public Domain by Austin Appleby
|
258
|
-
|
287
|
+
|
259
288
|
Code Locations:
|
260
289
|
common/include/MurmurHash3.h
|
261
290
|
that is adapted from the above.
|
@@ -263,8 +292,7 @@ APPENDIX B: Additional licenses relevant to this product.
|
|
263
292
|
Original source code:
|
264
293
|
* https://graphics.stanford.edu/~seander/bithacks.html
|
265
294
|
* Placed in the Public Domain by Sean Eron Anderson
|
266
|
-
|
295
|
+
|
267
296
|
Code Locations:
|
268
297
|
* common/include/ceiling_power_of_2.hpp
|
269
298
|
that is adapted from the above.
|
270
|
-
|
@@ -1,9 +1,9 @@
|
|
1
1
|
Apache DataSketches C++
|
2
|
-
Copyright
|
2
|
+
Copyright 2025 The Apache Software Foundation
|
3
3
|
|
4
4
|
Copyright 2015-2018 Yahoo Inc.
|
5
5
|
Copyright 2019-2020 Verizon Media
|
6
|
-
Copyright 2021 Yahoo Inc.
|
6
|
+
Copyright 2021- Yahoo Inc.
|
7
7
|
|
8
8
|
This product includes software developed at
|
9
9
|
The Apache Software Foundation (http://www.apache.org/).
|
@@ -31,7 +31,7 @@ target_include_directories(common
|
|
31
31
|
|
32
32
|
install(TARGETS common EXPORT ${PROJECT_NAME})
|
33
33
|
|
34
|
-
install(FILES
|
34
|
+
install(FILES
|
35
35
|
${CMAKE_CURRENT_BINARY_DIR}/include/version.hpp
|
36
36
|
include/binomial_bounds.hpp
|
37
37
|
include/bounds_binomial_proportions.hpp
|
@@ -49,4 +49,5 @@ install(FILES
|
|
49
49
|
include/quantiles_sorted_view_impl.hpp
|
50
50
|
include/quantiles_sorted_view.hpp
|
51
51
|
include/serde.hpp
|
52
|
+
include/xxhash64.h
|
52
53
|
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
|
@@ -42,6 +42,7 @@ namespace random_utils {
|
|
42
42
|
static std::random_device rd; // possibly unsafe in MinGW with GCC < 9.2
|
43
43
|
static thread_local std::mt19937_64 rand(rd());
|
44
44
|
static thread_local std::uniform_real_distribution<> next_double(0.0, 1.0);
|
45
|
+
static thread_local std::uniform_int_distribution<uint64_t> next_uint64(0, UINT64_MAX);
|
45
46
|
|
46
47
|
// thread-safe random bit
|
47
48
|
static thread_local std::independent_bits_engine<std::mt19937, 1, uint32_t>
|
@@ -86,19 +86,17 @@ auto quantiles_sorted_view<T, C, A>::get_quantile(double rank, bool inclusive) c
|
|
86
86
|
template<typename T, typename C, typename A>
|
87
87
|
auto quantiles_sorted_view<T, C, A>::get_CDF(const T* split_points, uint32_t size, bool inclusive) const -> vector_double {
|
88
88
|
if (entries_.empty()) throw std::runtime_error("operation is undefined for an empty sketch");
|
89
|
-
vector_double buckets(entries_.get_allocator());
|
90
|
-
if (entries_.size() == 0) return buckets;
|
91
89
|
check_split_points(split_points, size);
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
90
|
+
vector_double ranks(entries_.get_allocator());
|
91
|
+
ranks.reserve(size + 1);
|
92
|
+
for (uint32_t i = 0; i < size; ++i) ranks.push_back(get_rank(split_points[i], inclusive));
|
93
|
+
ranks.push_back(1);
|
94
|
+
return ranks;
|
96
95
|
}
|
97
96
|
|
98
97
|
template<typename T, typename C, typename A>
|
99
98
|
auto quantiles_sorted_view<T, C, A>::get_PMF(const T* split_points, uint32_t size, bool inclusive) const -> vector_double {
|
100
99
|
auto buckets = get_CDF(split_points, size, inclusive);
|
101
|
-
if (buckets.size() == 0) return buckets;
|
102
100
|
for (uint32_t i = size; i > 0; --i) {
|
103
101
|
buckets[i] -= buckets[i - 1];
|
104
102
|
}
|
@@ -0,0 +1,202 @@
|
|
1
|
+
// //////////////////////////////////////////////////////////
|
2
|
+
// xxhash64.h
|
3
|
+
// Copyright (c) 2016 Stephan Brumme. All rights reserved.
|
4
|
+
// see http://create.stephan-brumme.com/disclaimer.html
|
5
|
+
//
|
6
|
+
|
7
|
+
#pragma once
|
8
|
+
#include <stdint.h> // for uint32_t and uint64_t
|
9
|
+
|
10
|
+
/// XXHash (64 bit), based on Yann Collet's descriptions, see http://cyan4973.github.io/xxHash/
|
11
|
+
/** How to use:
|
12
|
+
uint64_t myseed = 0;
|
13
|
+
XXHash64 myhash(myseed);
|
14
|
+
myhash.add(pointerToSomeBytes, numberOfBytes);
|
15
|
+
myhash.add(pointerToSomeMoreBytes, numberOfMoreBytes); // call add() as often as you like to ...
|
16
|
+
// and compute hash:
|
17
|
+
uint64_t result = myhash.hash();
|
18
|
+
|
19
|
+
// or all of the above in one single line:
|
20
|
+
uint64_t result2 = XXHash64::hash(mypointer, numBytes, myseed);
|
21
|
+
|
22
|
+
Note: my code is NOT endian-aware !
|
23
|
+
**/
|
24
|
+
class XXHash64
|
25
|
+
{
|
26
|
+
public:
|
27
|
+
/// create new XXHash (64 bit)
|
28
|
+
/** @param seed your seed value, even zero is a valid seed **/
|
29
|
+
explicit XXHash64(uint64_t seed)
|
30
|
+
{
|
31
|
+
state[0] = seed + Prime1 + Prime2;
|
32
|
+
state[1] = seed + Prime2;
|
33
|
+
state[2] = seed;
|
34
|
+
state[3] = seed - Prime1;
|
35
|
+
bufferSize = 0;
|
36
|
+
totalLength = 0;
|
37
|
+
}
|
38
|
+
|
39
|
+
/// add a chunk of bytes
|
40
|
+
/** @param input pointer to a continuous block of data
|
41
|
+
@param length number of bytes
|
42
|
+
@return false if parameters are invalid / zero **/
|
43
|
+
bool add(const void* input, uint64_t length)
|
44
|
+
{
|
45
|
+
// no data ?
|
46
|
+
if (!input || length == 0)
|
47
|
+
return false;
|
48
|
+
|
49
|
+
totalLength += length;
|
50
|
+
// byte-wise access
|
51
|
+
const unsigned char* data = (const unsigned char*)input;
|
52
|
+
|
53
|
+
// unprocessed old data plus new data still fit in temporary buffer ?
|
54
|
+
if (bufferSize + length < MaxBufferSize)
|
55
|
+
{
|
56
|
+
// just add new data
|
57
|
+
while (length-- > 0)
|
58
|
+
buffer[bufferSize++] = *data++;
|
59
|
+
return true;
|
60
|
+
}
|
61
|
+
|
62
|
+
// point beyond last byte
|
63
|
+
const unsigned char* stop = data + length;
|
64
|
+
const unsigned char* stopBlock = stop - MaxBufferSize;
|
65
|
+
|
66
|
+
// some data left from previous update ?
|
67
|
+
if (bufferSize > 0)
|
68
|
+
{
|
69
|
+
// make sure temporary buffer is full (16 bytes)
|
70
|
+
while (bufferSize < MaxBufferSize)
|
71
|
+
buffer[bufferSize++] = *data++;
|
72
|
+
|
73
|
+
// process these 32 bytes (4x8)
|
74
|
+
process(buffer, state[0], state[1], state[2], state[3]);
|
75
|
+
}
|
76
|
+
|
77
|
+
// copying state to local variables helps optimizer A LOT
|
78
|
+
uint64_t s0 = state[0], s1 = state[1], s2 = state[2], s3 = state[3];
|
79
|
+
// 32 bytes at once
|
80
|
+
while (data <= stopBlock)
|
81
|
+
{
|
82
|
+
// local variables s0..s3 instead of state[0]..state[3] are much faster
|
83
|
+
process(data, s0, s1, s2, s3);
|
84
|
+
data += 32;
|
85
|
+
}
|
86
|
+
// copy back
|
87
|
+
state[0] = s0; state[1] = s1; state[2] = s2; state[3] = s3;
|
88
|
+
|
89
|
+
// copy remainder to temporary buffer
|
90
|
+
bufferSize = stop - data;
|
91
|
+
for (uint64_t i = 0; i < bufferSize; i++)
|
92
|
+
buffer[i] = data[i];
|
93
|
+
|
94
|
+
// done
|
95
|
+
return true;
|
96
|
+
}
|
97
|
+
|
98
|
+
/// get current hash
|
99
|
+
/** @return 64 bit XXHash **/
|
100
|
+
uint64_t hash() const
|
101
|
+
{
|
102
|
+
// fold 256 bit state into one single 64 bit value
|
103
|
+
uint64_t result;
|
104
|
+
if (totalLength >= MaxBufferSize)
|
105
|
+
{
|
106
|
+
result = rotateLeft(state[0], 1) +
|
107
|
+
rotateLeft(state[1], 7) +
|
108
|
+
rotateLeft(state[2], 12) +
|
109
|
+
rotateLeft(state[3], 18);
|
110
|
+
result = (result ^ processSingle(0, state[0])) * Prime1 + Prime4;
|
111
|
+
result = (result ^ processSingle(0, state[1])) * Prime1 + Prime4;
|
112
|
+
result = (result ^ processSingle(0, state[2])) * Prime1 + Prime4;
|
113
|
+
result = (result ^ processSingle(0, state[3])) * Prime1 + Prime4;
|
114
|
+
}
|
115
|
+
else
|
116
|
+
{
|
117
|
+
// internal state wasn't set in add(), therefore original seed is still stored in state2
|
118
|
+
result = state[2] + Prime5;
|
119
|
+
}
|
120
|
+
|
121
|
+
result += totalLength;
|
122
|
+
|
123
|
+
// process remaining bytes in temporary buffer
|
124
|
+
const unsigned char* data = buffer;
|
125
|
+
// point beyond last byte
|
126
|
+
const unsigned char* stop = data + bufferSize;
|
127
|
+
|
128
|
+
// at least 8 bytes left ? => eat 8 bytes per step
|
129
|
+
for (; data + 8 <= stop; data += 8)
|
130
|
+
result = rotateLeft(result ^ processSingle(0, *(uint64_t*)data), 27) * Prime1 + Prime4;
|
131
|
+
|
132
|
+
// 4 bytes left ? => eat those
|
133
|
+
if (data + 4 <= stop)
|
134
|
+
{
|
135
|
+
result = rotateLeft(result ^ (*(uint32_t*)data) * Prime1, 23) * Prime2 + Prime3;
|
136
|
+
data += 4;
|
137
|
+
}
|
138
|
+
|
139
|
+
// take care of remaining 0..3 bytes, eat 1 byte per step
|
140
|
+
while (data != stop)
|
141
|
+
result = rotateLeft(result ^ (*data++) * Prime5, 11) * Prime1;
|
142
|
+
|
143
|
+
// mix bits
|
144
|
+
result ^= result >> 33;
|
145
|
+
result *= Prime2;
|
146
|
+
result ^= result >> 29;
|
147
|
+
result *= Prime3;
|
148
|
+
result ^= result >> 32;
|
149
|
+
return result;
|
150
|
+
}
|
151
|
+
|
152
|
+
|
153
|
+
/// combine constructor, add() and hash() in one static function (C style)
|
154
|
+
/** @param input pointer to a continuous block of data
|
155
|
+
@param length number of bytes
|
156
|
+
@param seed your seed value, e.g. zero is a valid seed
|
157
|
+
@return 64 bit XXHash **/
|
158
|
+
static uint64_t hash(const void* input, uint64_t length, uint64_t seed)
|
159
|
+
{
|
160
|
+
XXHash64 hasher(seed);
|
161
|
+
hasher.add(input, length);
|
162
|
+
return hasher.hash();
|
163
|
+
}
|
164
|
+
|
165
|
+
private:
|
166
|
+
/// magic constants :-)
|
167
|
+
static const uint64_t Prime1 = 11400714785074694791ULL;
|
168
|
+
static const uint64_t Prime2 = 14029467366897019727ULL;
|
169
|
+
static const uint64_t Prime3 = 1609587929392839161ULL;
|
170
|
+
static const uint64_t Prime4 = 9650029242287828579ULL;
|
171
|
+
static const uint64_t Prime5 = 2870177450012600261ULL;
|
172
|
+
|
173
|
+
/// temporarily store up to 31 bytes between multiple add() calls
|
174
|
+
static const uint64_t MaxBufferSize = 31+1;
|
175
|
+
|
176
|
+
uint64_t state[4];
|
177
|
+
unsigned char buffer[MaxBufferSize];
|
178
|
+
uint64_t bufferSize;
|
179
|
+
uint64_t totalLength;
|
180
|
+
|
181
|
+
/// rotate bits, should compile to a single CPU instruction (ROL)
|
182
|
+
static inline uint64_t rotateLeft(uint64_t x, unsigned char bits)
|
183
|
+
{
|
184
|
+
return (x << bits) | (x >> (64 - bits));
|
185
|
+
}
|
186
|
+
|
187
|
+
/// process a single 64 bit value
|
188
|
+
static inline uint64_t processSingle(uint64_t previous, uint64_t input)
|
189
|
+
{
|
190
|
+
return rotateLeft(previous + input * Prime2, 31) * Prime1;
|
191
|
+
}
|
192
|
+
|
193
|
+
/// process a block of 4x4 bytes, this is the main part of the XXHash32 algorithm
|
194
|
+
static inline void process(const void* data, uint64_t& state0, uint64_t& state1, uint64_t& state2, uint64_t& state3)
|
195
|
+
{
|
196
|
+
const uint64_t* block = (const uint64_t*) data;
|
197
|
+
state0 = processSingle(state0, block[0]);
|
198
|
+
state1 = processSingle(state1, block[1]);
|
199
|
+
state2 = processSingle(state2, block[2]);
|
200
|
+
state3 = processSingle(state3, block[3]);
|
201
|
+
}
|
202
|
+
};
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
add_library(filters INTERFACE)
|
19
|
+
|
20
|
+
add_library(${PROJECT_NAME}::FILTERS ALIAS filters)
|
21
|
+
|
22
|
+
if (BUILD_TESTS)
|
23
|
+
add_subdirectory(test)
|
24
|
+
endif()
|
25
|
+
|
26
|
+
target_include_directories(filters
|
27
|
+
INTERFACE
|
28
|
+
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
|
29
|
+
$<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>
|
30
|
+
)
|
31
|
+
|
32
|
+
target_link_libraries(filters INTERFACE common)
|
33
|
+
|
34
|
+
install(TARGETS filters
|
35
|
+
EXPORT ${PROJECT_NAME}
|
36
|
+
)
|
37
|
+
|
38
|
+
install(FILES
|
39
|
+
include/bloom_filter.hpp
|
40
|
+
include/bloom_filter_impl.hpp
|
41
|
+
include/bloom_filter_builder_impl.hpp
|
42
|
+
include/bit_array_ops.hpp
|
43
|
+
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
|
@@ -0,0 +1,180 @@
|
|
1
|
+
/*
|
2
|
+
* Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
* or more contributor license agreements. See the NOTICE file
|
4
|
+
* distributed with this work for additional information
|
5
|
+
* regarding copyright ownership. The ASF licenses this file
|
6
|
+
* to you under the Apache License, Version 2.0 (the
|
7
|
+
* "License"); you may not use this file except in compliance
|
8
|
+
* with the License. You may obtain a copy of the License at
|
9
|
+
*
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
*
|
12
|
+
* Unless required by applicable law or agreed to in writing,
|
13
|
+
* software distributed under the License is distributed on an
|
14
|
+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
* KIND, either express or implied. See the License for the
|
16
|
+
* specific language governing permissions and limitations
|
17
|
+
* under the License.
|
18
|
+
*/
|
19
|
+
|
20
|
+
#ifndef _BIT_ARRAY_OPS_HPP_
|
21
|
+
#define _BIT_ARRAY_OPS_HPP_
|
22
|
+
|
23
|
+
#include <bitset>
|
24
|
+
|
25
|
+
namespace datasketches {
|
26
|
+
|
27
|
+
/**
|
28
|
+
* This class comprises methods that operate one or more arrays of bits (uint8_t*) to
|
29
|
+
* provide bit array operations. The class does not take ownership of memory and operates On
|
30
|
+
* arrays in-place. Sizes of the arrays, in bytes, are passed in as arguments.
|
31
|
+
*
|
32
|
+
* None of the methods in this class perform bounds checks. The caller is responsible for ensuring
|
33
|
+
* that indices are within the array bounds.
|
34
|
+
*
|
35
|
+
* Implementation assumes the actual arrays are multiples of 64 bits in length.
|
36
|
+
*/
|
37
|
+
namespace bit_array_ops {
|
38
|
+
|
39
|
+
/**
|
40
|
+
* Get the value of a bit at the given index.
|
41
|
+
* @param array the array of bits
|
42
|
+
* @param index the index of the bit to get
|
43
|
+
* @return the value of the bit at the given index.
|
44
|
+
*/
|
45
|
+
static inline bool get_bit(uint8_t* array, uint64_t index) {
|
46
|
+
return (array[index >> 3] & (1 << (index & 7))) != 0;
|
47
|
+
}
|
48
|
+
|
49
|
+
/**
|
50
|
+
* Set the bit at the given index to 1.
|
51
|
+
* @param array the array of bits
|
52
|
+
* @param index the index of the bit to set.
|
53
|
+
*/
|
54
|
+
static inline void set_bit(uint8_t* array, uint64_t index) {
|
55
|
+
array[index >> 3] |= (1 << (index & 7));
|
56
|
+
}
|
57
|
+
|
58
|
+
/**
|
59
|
+
* Set the bit at the given index to 0.
|
60
|
+
* @param array the array of bits
|
61
|
+
* @param index the index of the bit to clear.
|
62
|
+
*/
|
63
|
+
static inline void clear_bit(uint8_t* array, uint64_t index) {
|
64
|
+
array[index >> 3] &= ~(1 << (index & 7));
|
65
|
+
}
|
66
|
+
|
67
|
+
/**
|
68
|
+
* Assign the value of the bit at the given index.
|
69
|
+
* @param array the array of bits
|
70
|
+
* @param index the index of the bit to set.
|
71
|
+
*/
|
72
|
+
static inline void assign_bit(uint8_t* array, uint64_t index, bool value) {
|
73
|
+
// read-only checks handled by set_bit() and clear_bit()
|
74
|
+
if (value) {
|
75
|
+
set_bit(array, index);
|
76
|
+
} else {
|
77
|
+
clear_bit(array, index);
|
78
|
+
}
|
79
|
+
}
|
80
|
+
|
81
|
+
/**
|
82
|
+
* Gets the value of a bit at the specified index and sets it to true
|
83
|
+
* @param array the array of bits
|
84
|
+
* @param index the index of the bit to get and set
|
85
|
+
* @return the value of the bit at the specified index
|
86
|
+
*/
|
87
|
+
static inline bool get_and_set_bit(uint8_t* array, uint64_t index) {
|
88
|
+
const uint64_t offset = index >> 3;
|
89
|
+
const uint8_t mask = 1 << (index & 7);
|
90
|
+
if ((array[offset] & mask) != 0) {
|
91
|
+
return true;
|
92
|
+
} else {
|
93
|
+
array[offset] |= mask;
|
94
|
+
return false;
|
95
|
+
}
|
96
|
+
}
|
97
|
+
|
98
|
+
/**
|
99
|
+
* @brief Gets the number of bits set in the bit array.
|
100
|
+
* @param array the array of bits
|
101
|
+
* @param length_bytes the length of the array, in bytes
|
102
|
+
* @return the number of bits set in the bit array.
|
103
|
+
*/
|
104
|
+
static inline uint64_t count_num_bits_set(uint8_t* array, uint64_t length_bytes) {
|
105
|
+
uint64_t num_bits_set = 0;
|
106
|
+
|
107
|
+
// we rounded up to a multiple of 64 so we know we can use 64-bit operations
|
108
|
+
const uint64_t* array64 = reinterpret_cast<const uint64_t*>(array);
|
109
|
+
// Calculate the number of 64-bit chunks
|
110
|
+
uint64_t num_longs = length_bytes / 8; // 8 bytes per 64 bits
|
111
|
+
for (uint64_t i = 0; i < num_longs; ++i) {
|
112
|
+
// Wrap the 64-bit chunk with std::bitset for easy bit counting
|
113
|
+
std::bitset<64> bits(array64[i]);
|
114
|
+
num_bits_set += bits.count();
|
115
|
+
}
|
116
|
+
return num_bits_set;
|
117
|
+
}
|
118
|
+
|
119
|
+
/**
|
120
|
+
* Performs a union operation on one bit array with another bit array.
|
121
|
+
* This operation modifies the tgt bit array to be the union of its original bits and the bits of the src array.
|
122
|
+
* The union operation is equivalent to a bitwise OR operation between the two arrays.
|
123
|
+
*
|
124
|
+
* @param tgt the array of bits into which the results are written
|
125
|
+
* @param src the array of bits to union into tgt
|
126
|
+
* @param length_bytes the length of the two arrays, in bytes
|
127
|
+
* @return the number of bits set in the resulting array
|
128
|
+
*/
|
129
|
+
static inline uint64_t union_with(uint8_t* tgt, const uint8_t* src, uint64_t length_bytes) {
|
130
|
+
uint64_t num_bits_set = 0;
|
131
|
+
for (uint64_t i = 0; i < length_bytes; ++i) {
|
132
|
+
tgt[i] |= src[i];
|
133
|
+
std::bitset<8> bits(tgt[i]);
|
134
|
+
num_bits_set += bits.count();
|
135
|
+
}
|
136
|
+
return num_bits_set;
|
137
|
+
}
|
138
|
+
|
139
|
+
/**
|
140
|
+
* Performs an intersection operation on one bit array with another bit array.
|
141
|
+
* This operation modifies the tgt bit array to contain only the bits that are set in both that array and the src array.
|
142
|
+
* The intersection operation is equivalent to a bitwise AND operation between the two arrays.
|
143
|
+
*
|
144
|
+
* @param tgt the array of bits into which the results are written
|
145
|
+
* @param src the array of bits to intersect with tgt
|
146
|
+
* @param length_bytes the length of the two arrays, in bytes
|
147
|
+
* @return the number of bits set in the resulting array
|
148
|
+
*/
|
149
|
+
static inline uint64_t intersect(uint8_t* tgt, const uint8_t* src, uint64_t length_bytes) {
|
150
|
+
uint64_t num_bits_set = 0;
|
151
|
+
for (uint64_t i = 0; i < length_bytes; ++i) {
|
152
|
+
tgt[i] &= src[i];
|
153
|
+
std::bitset<8> bits(tgt[i]);
|
154
|
+
num_bits_set += bits.count();
|
155
|
+
}
|
156
|
+
return num_bits_set;
|
157
|
+
}
|
158
|
+
|
159
|
+
/**
|
160
|
+
* Inverts the bits of this bit array.
|
161
|
+
* This operation modifies the bit array by flipping all its bits; 0s become 1s and 1s become 0s.
|
162
|
+
* @param array the array of bits
|
163
|
+
* @param length_bytes the length of the array, in bytes
|
164
|
+
* @return the number of bits set in the resulting array
|
165
|
+
*/
|
166
|
+
static inline uint64_t invert(uint8_t* array, uint64_t length_bytes) {
|
167
|
+
uint64_t num_bits_set = 0;
|
168
|
+
for (uint64_t i = 0; i < length_bytes; ++i) {
|
169
|
+
array[i] = ~array[i];
|
170
|
+
std::bitset<8> bits(array[i]);
|
171
|
+
num_bits_set += bits.count();
|
172
|
+
}
|
173
|
+
return num_bits_set;
|
174
|
+
}
|
175
|
+
|
176
|
+
} // namespace bit_array_ops
|
177
|
+
|
178
|
+
} // namespace datasketches
|
179
|
+
|
180
|
+
#endif // _BIT_ARRAY_OPS_HPP_
|