StrIdx 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/unordered_dense.h ADDED
@@ -0,0 +1,2032 @@
1
+ ///////////////////////// ankerl::unordered_dense::{map, set} /////////////////////////
2
+
3
+ // A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion.
4
+ // Version 4.4.0
5
+ // https://github.com/martinus/unordered_dense
6
+ //
7
+ // Licensed under the MIT License <http://opensource.org/licenses/MIT>.
8
+ // SPDX-License-Identifier: MIT
9
+ // Copyright (c) 2022-2023 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
10
+ //
11
+ // Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ // of this software and associated documentation files (the "Software"), to deal
13
+ // in the Software without restriction, including without limitation the rights
14
+ // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ // copies of the Software, and to permit persons to whom the Software is
16
+ // furnished to do so, subject to the following conditions:
17
+ //
18
+ // The above copyright notice and this permission notice shall be included in all
19
+ // copies or substantial portions of the Software.
20
+ //
21
+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
+ // SOFTWARE.
28
+
29
+ #ifndef ANKERL_UNORDERED_DENSE_H
30
+ #define ANKERL_UNORDERED_DENSE_H
31
+
32
+ // see https://semver.org/spec/v2.0.0.html
33
+ #define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes
34
+ #define ANKERL_UNORDERED_DENSE_VERSION_MINOR 4 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality
35
+ #define ANKERL_UNORDERED_DENSE_VERSION_PATCH 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes
36
+
37
+ // API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/
38
+
39
+ // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
40
+ #define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch
41
+ // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
42
+ #define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch)
43
+ #define ANKERL_UNORDERED_DENSE_NAMESPACE \
44
+ ANKERL_UNORDERED_DENSE_VERSION_CONCAT( \
45
+ ANKERL_UNORDERED_DENSE_VERSION_MAJOR, ANKERL_UNORDERED_DENSE_VERSION_MINOR, ANKERL_UNORDERED_DENSE_VERSION_PATCH)
46
+
47
+ #if defined(_MSVC_LANG)
48
+ # define ANKERL_UNORDERED_DENSE_CPP_VERSION _MSVC_LANG
49
+ #else
50
+ # define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus
51
+ #endif
52
+
53
+ #if defined(__GNUC__)
54
+ // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
55
+ # define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__))
56
+ #elif defined(_MSC_VER)
57
+ // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
58
+ # define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop))
59
+ #endif
60
+
61
+ // exceptions
62
+ #if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
63
+ # define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 1 // NOLINT(cppcoreguidelines-macro-usage)
64
+ #else
65
+ # define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 0 // NOLINT(cppcoreguidelines-macro-usage)
66
+ #endif
67
+ #ifdef _MSC_VER
68
+ # define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline)
69
+ #else
70
+ # define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline))
71
+ #endif
72
+
73
+ // defined in unordered_dense.cpp
74
+ #if !defined(ANKERL_UNORDERED_DENSE_EXPORT)
75
+ # define ANKERL_UNORDERED_DENSE_EXPORT
76
+ #endif
77
+
78
+ #if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L
79
+ # error ankerl::unordered_dense requires C++17 or higher
80
+ #else
81
+ # include <array> // for array
82
+ # include <cstdint> // for uint64_t, uint32_t, uint8_t, UINT64_C
83
+ # include <cstring> // for size_t, memcpy, memset
84
+ # include <functional> // for equal_to, hash
85
+ # include <initializer_list> // for initializer_list
86
+ # include <iterator> // for pair, distance
87
+ # include <limits> // for numeric_limits
88
+ # include <memory> // for allocator, allocator_traits, shared_ptr
89
+ # include <optional> // for optional
90
+ # include <stdexcept> // for out_of_range
91
+ # include <string> // for basic_string
92
+ # include <string_view> // for basic_string_view, hash
93
+ # include <tuple> // for forward_as_tuple
94
+ # include <type_traits> // for enable_if_t, declval, conditional_t, ena...
95
+ # include <utility> // for forward, exchange, pair, as_const, piece...
96
+ # include <vector> // for vector
97
+ # if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() == 0
98
+ # include <cstdlib> // for abort
99
+ # endif
100
+
101
+ # if defined(__has_include)
102
+ # if __has_include(<memory_resource>)
103
+ # define ANKERL_UNORDERED_DENSE_PMR std::pmr // NOLINT(cppcoreguidelines-macro-usage)
104
+ # include <memory_resource> // for polymorphic_allocator
105
+ # elif __has_include(<experimental/memory_resource>)
106
+ # define ANKERL_UNORDERED_DENSE_PMR std::experimental::pmr // NOLINT(cppcoreguidelines-macro-usage)
107
+ # include <experimental/memory_resource> // for polymorphic_allocator
108
+ # endif
109
+ # endif
110
+
111
+ # if defined(_MSC_VER) && defined(_M_X64)
112
+ # include <intrin.h>
113
+ # pragma intrinsic(_umul128)
114
+ # endif
115
+
116
+ # if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
117
+ # define ANKERL_UNORDERED_DENSE_LIKELY(x) __builtin_expect(x, 1) // NOLINT(cppcoreguidelines-macro-usage)
118
+ # define ANKERL_UNORDERED_DENSE_UNLIKELY(x) __builtin_expect(x, 0) // NOLINT(cppcoreguidelines-macro-usage)
119
+ # else
120
+ # define ANKERL_UNORDERED_DENSE_LIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
121
+ # define ANKERL_UNORDERED_DENSE_UNLIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
122
+ # endif
123
+
124
+ namespace ankerl::unordered_dense {
125
+ inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE {
126
+
127
+ namespace detail {
128
+
129
+ # if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS()
130
+
131
+ // make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
132
+ // inlinings more difficult. Throws are also generally the slow path.
133
+ [[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_not_found() {
134
+ throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found");
135
+ }
136
+ [[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_bucket_overflow() {
137
+ throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size");
138
+ }
139
+ [[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() {
140
+ throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements");
141
+ }
142
+
143
+ # else
144
+
145
+ [[noreturn]] inline void on_error_key_not_found() {
146
+ abort();
147
+ }
148
+ [[noreturn]] inline void on_error_bucket_overflow() {
149
+ abort();
150
+ }
151
+ [[noreturn]] inline void on_error_too_many_elements() {
152
+ abort();
153
+ }
154
+
155
+ # endif
156
+
157
+ } // namespace detail
158
+
159
+ // hash ///////////////////////////////////////////////////////////////////////
160
+
161
+ // This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash
162
+ // No big-endian support (because different values on different machines don't matter),
163
+ // hardcodes seed and the secret, reformats the code, and clang-tidy fixes.
164
+ namespace detail::wyhash {
165
+
166
+ inline void mum(uint64_t* a, uint64_t* b) {
167
+ # if defined(__SIZEOF_INT128__)
168
+ __uint128_t r = *a;
169
+ r *= *b;
170
+ *a = static_cast<uint64_t>(r);
171
+ *b = static_cast<uint64_t>(r >> 64U);
172
+ # elif defined(_MSC_VER) && defined(_M_X64)
173
+ *a = _umul128(*a, *b, b);
174
+ # else
175
+ uint64_t ha = *a >> 32U;
176
+ uint64_t hb = *b >> 32U;
177
+ uint64_t la = static_cast<uint32_t>(*a);
178
+ uint64_t lb = static_cast<uint32_t>(*b);
179
+ uint64_t hi{};
180
+ uint64_t lo{};
181
+ uint64_t rh = ha * hb;
182
+ uint64_t rm0 = ha * lb;
183
+ uint64_t rm1 = hb * la;
184
+ uint64_t rl = la * lb;
185
+ uint64_t t = rl + (rm0 << 32U);
186
+ auto c = static_cast<uint64_t>(t < rl);
187
+ lo = t + (rm1 << 32U);
188
+ c += static_cast<uint64_t>(lo < t);
189
+ hi = rh + (rm0 >> 32U) + (rm1 >> 32U) + c;
190
+ *a = lo;
191
+ *b = hi;
192
+ # endif
193
+ }
194
+
195
+ // multiply and xor mix function, aka MUM
196
+ [[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t {
197
+ mum(&a, &b);
198
+ return a ^ b;
199
+ }
200
+
201
+ // read functions. WARNING: we don't care about endianness, so results are different on big endian!
202
+ [[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t {
203
+ uint64_t v{};
204
+ std::memcpy(&v, p, 8U);
205
+ return v;
206
+ }
207
+
208
+ [[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t {
209
+ uint32_t v{};
210
+ std::memcpy(&v, p, 4);
211
+ return v;
212
+ }
213
+
214
+ // reads 1, 2, or 3 bytes
215
+ [[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t {
216
+ return (static_cast<uint64_t>(p[0]) << 16U) | (static_cast<uint64_t>(p[k >> 1U]) << 8U) | p[k - 1];
217
+ }
218
+
219
+ [[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t {
220
+ static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f),
221
+ UINT64_C(0xe7037ed1a0b428db),
222
+ UINT64_C(0x8ebc6af09c88c6e3),
223
+ UINT64_C(0x589965cc75374cc3)};
224
+
225
+ auto const* p = static_cast<uint8_t const*>(key);
226
+ uint64_t seed = secret[0];
227
+ uint64_t a{};
228
+ uint64_t b{};
229
+ if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) {
230
+ if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) {
231
+ a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U));
232
+ b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U));
233
+ } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) {
234
+ a = r3(p, len);
235
+ b = 0;
236
+ } else {
237
+ a = 0;
238
+ b = 0;
239
+ }
240
+ } else {
241
+ size_t i = len;
242
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) {
243
+ uint64_t see1 = seed;
244
+ uint64_t see2 = seed;
245
+ do {
246
+ seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
247
+ see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1);
248
+ see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2);
249
+ p += 48;
250
+ i -= 48;
251
+ } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48));
252
+ seed ^= see1 ^ see2;
253
+ }
254
+ while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) {
255
+ seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
256
+ i -= 16;
257
+ p += 16;
258
+ }
259
+ a = r8(p + i - 16);
260
+ b = r8(p + i - 8);
261
+ }
262
+
263
+ return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed));
264
+ }
265
+
266
+ [[nodiscard]] inline auto hash(uint64_t x) -> uint64_t {
267
+ return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15));
268
+ }
269
+
270
+ } // namespace detail::wyhash
271
+
272
+ ANKERL_UNORDERED_DENSE_EXPORT template <typename T, typename Enable = void>
273
+ struct hash {
274
+ auto operator()(T const& obj) const noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>())))
275
+ -> uint64_t {
276
+ return std::hash<T>{}(obj);
277
+ }
278
+ };
279
+
280
+ template <typename CharT>
281
+ struct hash<std::basic_string<CharT>> {
282
+ using is_avalanching = void;
283
+ auto operator()(std::basic_string<CharT> const& str) const noexcept -> uint64_t {
284
+ return detail::wyhash::hash(str.data(), sizeof(CharT) * str.size());
285
+ }
286
+ };
287
+
288
+ template <typename CharT>
289
+ struct hash<std::basic_string_view<CharT>> {
290
+ using is_avalanching = void;
291
+ auto operator()(std::basic_string_view<CharT> const& sv) const noexcept -> uint64_t {
292
+ return detail::wyhash::hash(sv.data(), sizeof(CharT) * sv.size());
293
+ }
294
+ };
295
+
296
+ template <class T>
297
+ struct hash<T*> {
298
+ using is_avalanching = void;
299
+ auto operator()(T* ptr) const noexcept -> uint64_t {
300
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
301
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr));
302
+ }
303
+ };
304
+
305
+ template <class T>
306
+ struct hash<std::unique_ptr<T>> {
307
+ using is_avalanching = void;
308
+ auto operator()(std::unique_ptr<T> const& ptr) const noexcept -> uint64_t {
309
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
310
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
311
+ }
312
+ };
313
+
314
+ template <class T>
315
+ struct hash<std::shared_ptr<T>> {
316
+ using is_avalanching = void;
317
+ auto operator()(std::shared_ptr<T> const& ptr) const noexcept -> uint64_t {
318
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
319
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
320
+ }
321
+ };
322
+
323
+ template <typename Enum>
324
+ struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
325
+ using is_avalanching = void;
326
+ auto operator()(Enum e) const noexcept -> uint64_t {
327
+ using underlying = typename std::underlying_type_t<Enum>;
328
+ return detail::wyhash::hash(static_cast<underlying>(e));
329
+ }
330
+ };
331
+
332
+ template <typename... Args>
333
+ struct tuple_hash_helper {
334
+ // Converts the value into 64bit. If it is an integral type, just cast it. Mixing is doing the rest.
335
+ // If it isn't an integral we need to hash it.
336
+ template <typename Arg>
337
+ [[nodiscard]] constexpr static auto to64(Arg const& arg) -> uint64_t {
338
+ if constexpr (std::is_integral_v<Arg> || std::is_enum_v<Arg>) {
339
+ return static_cast<uint64_t>(arg);
340
+ } else {
341
+ return hash<Arg>{}(arg);
342
+ }
343
+ }
344
+
345
+ [[nodiscard]] static auto mix64(uint64_t state, uint64_t v) -> uint64_t {
346
+ return detail::wyhash::mix(state + v, uint64_t{0x9ddfea08eb382d69});
347
+ }
348
+
349
+ // Creates a buffer that holds all the data from each element of the tuple. If possible we memcpy the data directly. If
350
+ // not, we hash the object and use this for the array. Size of the array is known at compile time, and memcpy is optimized
351
+ // away, so filling the buffer is highly efficient. Finally, call wyhash with this buffer.
352
+ template <typename T, std::size_t... Idx>
353
+ [[nodiscard]] static auto calc_hash(T const& t, std::index_sequence<Idx...>) noexcept -> uint64_t {
354
+ auto h = uint64_t{};
355
+ ((h = mix64(h, to64(std::get<Idx>(t)))), ...);
356
+ return h;
357
+ }
358
+ };
359
+
360
+ template <typename... Args>
361
+ struct hash<std::tuple<Args...>> : tuple_hash_helper<Args...> {
362
+ using is_avalanching = void;
363
+ auto operator()(std::tuple<Args...> const& t) const noexcept -> uint64_t {
364
+ return tuple_hash_helper<Args...>::calc_hash(t, std::index_sequence_for<Args...>{});
365
+ }
366
+ };
367
+
368
+ template <typename A, typename B>
369
+ struct hash<std::pair<A, B>> : tuple_hash_helper<A, B> {
370
+ using is_avalanching = void;
371
+ auto operator()(std::pair<A, B> const& t) const noexcept -> uint64_t {
372
+ return tuple_hash_helper<A, B>::calc_hash(t, std::index_sequence_for<A, B>{});
373
+ }
374
+ };
375
+
376
+ // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
377
+ # define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T) \
378
+ template <> \
379
+ struct hash<T> { \
380
+ using is_avalanching = void; \
381
+ auto operator()(T const& obj) const noexcept -> uint64_t { \
382
+ return detail::wyhash::hash(static_cast<uint64_t>(obj)); \
383
+ } \
384
+ }
385
+
386
+ # if defined(__GNUC__) && !defined(__clang__)
387
+ # pragma GCC diagnostic push
388
+ # pragma GCC diagnostic ignored "-Wuseless-cast"
389
+ # endif
390
+ // see https://en.cppreference.com/w/cpp/utility/hash
391
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool);
392
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char);
393
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char);
394
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char);
395
+ # if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L && defined(__cpp_char8_t)
396
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t);
397
+ # endif
398
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t);
399
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char32_t);
400
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(wchar_t);
401
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(short);
402
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned short);
403
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(int);
404
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned int);
405
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long);
406
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long long);
407
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long);
408
+ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long);
409
+
410
+ # if defined(__GNUC__) && !defined(__clang__)
411
+ # pragma GCC diagnostic pop
412
+ # endif
413
+
414
+ // bucket_type //////////////////////////////////////////////////////////
415
+
416
+ namespace bucket_type {
417
+
418
+ struct standard {
419
+ static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
420
+ static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
421
+
422
+ uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
423
+ uint32_t m_value_idx; // index into the m_values vector.
424
+ };
425
+
426
+ ANKERL_UNORDERED_DENSE_PACK(struct big {
427
+ static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
428
+ static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
429
+
430
+ uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
431
+ size_t m_value_idx; // index into the m_values vector.
432
+ });
433
+
434
+ } // namespace bucket_type
435
+
436
+ namespace detail {
437
+
438
+ struct nonesuch {};
439
+
440
+ template <class Default, class AlwaysVoid, template <class...> class Op, class... Args>
441
+ struct detector {
442
+ using value_t = std::false_type;
443
+ using type = Default;
444
+ };
445
+
446
+ template <class Default, template <class...> class Op, class... Args>
447
+ struct detector<Default, std::void_t<Op<Args...>>, Op, Args...> {
448
+ using value_t = std::true_type;
449
+ using type = Op<Args...>;
450
+ };
451
+
452
+ template <template <class...> class Op, class... Args>
453
+ using is_detected = typename detail::detector<detail::nonesuch, void, Op, Args...>::value_t;
454
+
455
+ template <template <class...> class Op, class... Args>
456
+ constexpr bool is_detected_v = is_detected<Op, Args...>::value;
457
+
458
+ template <typename T>
459
+ using detect_avalanching = typename T::is_avalanching;
460
+
461
+ template <typename T>
462
+ using detect_is_transparent = typename T::is_transparent;
463
+
464
+ template <typename T>
465
+ using detect_iterator = typename T::iterator;
466
+
467
+ template <typename T>
468
+ using detect_reserve = decltype(std::declval<T&>().reserve(size_t{}));
469
+
470
+ // enable_if helpers
471
+
472
+ template <typename Mapped>
473
+ constexpr bool is_map_v = !std::is_void_v<Mapped>;
474
+
475
+ // clang-format off
476
+ template <typename Hash, typename KeyEqual>
477
+ constexpr bool is_transparent_v = is_detected_v<detect_is_transparent, Hash> && is_detected_v<detect_is_transparent, KeyEqual>;
478
+ // clang-format on
479
+
480
+ template <typename From, typename To1, typename To2>
481
+ constexpr bool is_neither_convertible_v = !std::is_convertible_v<From, To1> && !std::is_convertible_v<From, To2>;
482
+
483
+ template <typename T>
484
+ constexpr bool has_reserve = is_detected_v<detect_reserve, T>;
485
+
486
+ // base type for map has mapped_type
487
+ template <class T>
488
+ struct base_table_type_map {
489
+ using mapped_type = T;
490
+ };
491
+
492
+ // base type for set doesn't have mapped_type
493
+ struct base_table_type_set {};
494
+
495
+ } // namespace detail
496
+
497
+ // Very much like std::deque, but faster for indexing (in most cases). As of now this doesn't implement the full std::vector
498
+ // API, but merely what's necessary to work as an underlying container for ankerl::unordered_dense::{map, set}.
499
+ // It allocates blocks of equal size and puts them into the m_blocks vector. That means it can grow simply by adding a new
500
+ // block to the back of m_blocks, and doesn't double its size like an std::vector. The disadvantage is that memory is not
501
+ // linear and thus there is one more indirection necessary for indexing.
502
+ template <typename T, typename Allocator = std::allocator<T>, size_t MaxSegmentSizeBytes = 4096>
503
+ class segmented_vector {
504
+ template <bool IsConst>
505
+ class iter_t;
506
+
507
+ public:
508
+ using allocator_type = Allocator;
509
+ using pointer = typename std::allocator_traits<allocator_type>::pointer;
510
+ using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;
511
+ using difference_type = typename std::allocator_traits<allocator_type>::difference_type;
512
+ using value_type = T;
513
+ using size_type = std::size_t;
514
+ using reference = T&;
515
+ using const_reference = T const&;
516
+ using iterator = iter_t<false>;
517
+ using const_iterator = iter_t<true>;
518
+
519
+ private:
520
+ using vec_alloc = typename std::allocator_traits<Allocator>::template rebind_alloc<pointer>;
521
+ std::vector<pointer, vec_alloc> m_blocks{};
522
+ size_t m_size{};
523
+
524
+ // Calculates the maximum number for x in (s << x) <= max_val
525
+ static constexpr auto num_bits_closest(size_t max_val, size_t s) -> size_t {
526
+ auto f = size_t{0};
527
+ while (s << (f + 1) <= max_val) {
528
+ ++f;
529
+ }
530
+ return f;
531
+ }
532
+
533
+ using self_t = segmented_vector<T, Allocator, MaxSegmentSizeBytes>;
534
+ static constexpr auto num_bits = num_bits_closest(MaxSegmentSizeBytes, sizeof(T));
535
+ static constexpr auto num_elements_in_block = 1U << num_bits;
536
+ static constexpr auto mask = num_elements_in_block - 1U;
537
+
538
+ /**
539
+ * Iterator class doubles as const_iterator and iterator
540
+ */
541
+ template <bool IsConst>
542
+ class iter_t {
543
+ using ptr_t = typename std::conditional_t<IsConst, segmented_vector::const_pointer const*, segmented_vector::pointer*>;
544
+ ptr_t m_data{};
545
+ size_t m_idx{};
546
+
547
+ template <bool B>
548
+ friend class iter_t;
549
+
550
+ public:
551
+ using difference_type = segmented_vector::difference_type;
552
+ using value_type = T;
553
+ using reference = typename std::conditional_t<IsConst, value_type const&, value_type&>;
554
+ using pointer = typename std::conditional_t<IsConst, segmented_vector::const_pointer, segmented_vector::pointer>;
555
+ using iterator_category = std::forward_iterator_tag;
556
+
557
+ iter_t() noexcept = default;
558
+
559
+ template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
560
+ // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
561
+ constexpr iter_t(iter_t<OtherIsConst> const& other) noexcept
562
+ : m_data(other.m_data)
563
+ , m_idx(other.m_idx) {}
564
+
565
+ constexpr iter_t(ptr_t data, size_t idx) noexcept
566
+ : m_data(data)
567
+ , m_idx(idx) {}
568
+
569
+ template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
570
+ constexpr auto operator=(iter_t<OtherIsConst> const& other) noexcept -> iter_t& {
571
+ m_data = other.m_data;
572
+ m_idx = other.m_idx;
573
+ return *this;
574
+ }
575
+
576
+ constexpr auto operator++() noexcept -> iter_t& {
577
+ ++m_idx;
578
+ return *this;
579
+ }
580
+
581
+ constexpr auto operator+(difference_type diff) noexcept -> iter_t {
582
+ return {m_data, static_cast<size_t>(static_cast<difference_type>(m_idx) + diff)};
583
+ }
584
+
585
+ template <bool OtherIsConst>
586
+ constexpr auto operator-(iter_t<OtherIsConst> const& other) noexcept -> difference_type {
587
+ return static_cast<difference_type>(m_idx) - static_cast<difference_type>(other.m_idx);
588
+ }
589
+
590
+ constexpr auto operator*() const noexcept -> reference {
591
+ return m_data[m_idx >> num_bits][m_idx & mask];
592
+ }
593
+
594
+ constexpr auto operator->() const noexcept -> pointer {
595
+ return &m_data[m_idx >> num_bits][m_idx & mask];
596
+ }
597
+
598
+ template <bool O>
599
+ constexpr auto operator==(iter_t<O> const& o) const noexcept -> bool {
600
+ return m_idx == o.m_idx;
601
+ }
602
+
603
+ template <bool O>
604
+ constexpr auto operator!=(iter_t<O> const& o) const noexcept -> bool {
605
+ return !(*this == o);
606
+ }
607
+ };
608
+
609
+ // slow path: need to allocate a new segment every once in a while
610
+ void increase_capacity() {
611
+ auto ba = Allocator(m_blocks.get_allocator());
612
+ pointer block = std::allocator_traits<Allocator>::allocate(ba, num_elements_in_block);
613
+ m_blocks.push_back(block);
614
+ }
615
+
616
+ // Moves everything from other
617
+ void append_everything_from(segmented_vector&& other) {
618
+ reserve(size() + other.size());
619
+ for (auto&& o : other) {
620
+ emplace_back(std::move(o));
621
+ }
622
+ }
623
+
624
+ // Copies everything from other
625
+ void append_everything_from(segmented_vector const& other) {
626
+ reserve(size() + other.size());
627
+ for (auto const& o : other) {
628
+ emplace_back(o);
629
+ }
630
+ }
631
+
632
+ void dealloc() {
633
+ auto ba = Allocator(m_blocks.get_allocator());
634
+ for (auto ptr : m_blocks) {
635
+ std::allocator_traits<Allocator>::deallocate(ba, ptr, num_elements_in_block);
636
+ }
637
+ }
638
+
639
+ [[nodiscard]] static constexpr auto calc_num_blocks_for_capacity(size_t capacity) {
640
+ return (capacity + num_elements_in_block - 1U) / num_elements_in_block;
641
+ }
642
+
643
+ public:
644
+ segmented_vector() = default;
645
+
646
+ // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
647
+ segmented_vector(Allocator alloc)
648
+ : m_blocks(vec_alloc(alloc)) {}
649
+
650
+ segmented_vector(segmented_vector&& other, Allocator alloc)
651
+ : segmented_vector(alloc) {
652
+ *this = std::move(other);
653
+ }
654
+
655
+ segmented_vector(segmented_vector const& other, Allocator alloc)
656
+ : m_blocks(vec_alloc(alloc)) {
657
+ append_everything_from(other);
658
+ }
659
+
660
+ segmented_vector(segmented_vector&& other) noexcept
661
+ : segmented_vector(std::move(other), get_allocator()) {}
662
+
663
+ segmented_vector(segmented_vector const& other) {
664
+ append_everything_from(other);
665
+ }
666
+
667
+ auto operator=(segmented_vector const& other) -> segmented_vector& {
668
+ if (this == &other) {
669
+ return *this;
670
+ }
671
+ clear();
672
+ append_everything_from(other);
673
+ return *this;
674
+ }
675
+
676
+ auto operator=(segmented_vector&& other) noexcept -> segmented_vector& {
677
+ clear();
678
+ dealloc();
679
+ if (other.get_allocator() == get_allocator()) {
680
+ m_blocks = std::move(other.m_blocks);
681
+ m_size = std::exchange(other.m_size, {});
682
+ } else {
683
+ // make sure to construct with other's allocator!
684
+ m_blocks = std::vector<pointer, vec_alloc>(vec_alloc(other.get_allocator()));
685
+ append_everything_from(std::move(other));
686
+ }
687
+ return *this;
688
+ }
689
+
690
+ ~segmented_vector() {
691
+ clear();
692
+ dealloc();
693
+ }
694
+
695
+ [[nodiscard]] constexpr auto size() const -> size_t {
696
+ return m_size;
697
+ }
698
+
699
+ [[nodiscard]] constexpr auto capacity() const -> size_t {
700
+ return m_blocks.size() * num_elements_in_block;
701
+ }
702
+
703
+ // Indexing is highly performance critical
704
+ [[nodiscard]] constexpr auto operator[](size_t i) const noexcept -> T const& {
705
+ return m_blocks[i >> num_bits][i & mask];
706
+ }
707
+
708
+ [[nodiscard]] constexpr auto operator[](size_t i) noexcept -> T& {
709
+ return m_blocks[i >> num_bits][i & mask];
710
+ }
711
+
712
+ [[nodiscard]] constexpr auto begin() -> iterator {
713
+ return {m_blocks.data(), 0U};
714
+ }
715
+ [[nodiscard]] constexpr auto begin() const -> const_iterator {
716
+ return {m_blocks.data(), 0U};
717
+ }
718
+ [[nodiscard]] constexpr auto cbegin() const -> const_iterator {
719
+ return {m_blocks.data(), 0U};
720
+ }
721
+
722
+ [[nodiscard]] constexpr auto end() -> iterator {
723
+ return {m_blocks.data(), m_size};
724
+ }
725
+ [[nodiscard]] constexpr auto end() const -> const_iterator {
726
+ return {m_blocks.data(), m_size};
727
+ }
728
+ [[nodiscard]] constexpr auto cend() const -> const_iterator {
729
+ return {m_blocks.data(), m_size};
730
+ }
731
+
732
+ [[nodiscard]] constexpr auto back() -> reference {
733
+ return operator[](m_size - 1);
734
+ }
735
+ [[nodiscard]] constexpr auto back() const -> const_reference {
736
+ return operator[](m_size - 1);
737
+ }
738
+
739
+ void pop_back() {
740
+ back().~T();
741
+ --m_size;
742
+ }
743
+
744
+ [[nodiscard]] auto empty() const {
745
+ return 0 == m_size;
746
+ }
747
+
748
+ void reserve(size_t new_capacity) {
749
+ m_blocks.reserve(calc_num_blocks_for_capacity(new_capacity));
750
+ while (new_capacity > capacity()) {
751
+ increase_capacity();
752
+ }
753
+ }
754
+
755
+ [[nodiscard]] auto get_allocator() const -> allocator_type {
756
+ return allocator_type{m_blocks.get_allocator()};
757
+ }
758
+
759
+ template <class... Args>
760
+ auto emplace_back(Args&&... args) -> reference {
761
+ if (m_size == capacity()) {
762
+ increase_capacity();
763
+ }
764
+ auto* ptr = static_cast<void*>(&operator[](m_size));
765
+ auto& ref = *new (ptr) T(std::forward<Args>(args)...);
766
+ ++m_size;
767
+ return ref;
768
+ }
769
+
770
+ void clear() {
771
+ if constexpr (!std::is_trivially_destructible_v<T>) {
772
+ for (size_t i = 0, s = size(); i < s; ++i) {
773
+ operator[](i).~T();
774
+ }
775
+ }
776
+ m_size = 0;
777
+ }
778
+
779
+ void shrink_to_fit() {
780
+ auto ba = Allocator(m_blocks.get_allocator());
781
+ auto num_blocks_required = calc_num_blocks_for_capacity(m_size);
782
+ while (m_blocks.size() > num_blocks_required) {
783
+ std::allocator_traits<Allocator>::deallocate(ba, m_blocks.back(), num_elements_in_block);
784
+ m_blocks.pop_back();
785
+ }
786
+ m_blocks.shrink_to_fit();
787
+ }
788
+ };
789
+
790
+ namespace detail {
791
+
792
+ // This is it, the table. Doubles as map and set, and uses `void` for T when its used as a set.
793
+ template <class Key,
794
+ class T, // when void, treat it as a set.
795
+ class Hash,
796
+ class KeyEqual,
797
+ class AllocatorOrContainer,
798
+ class Bucket,
799
+ bool IsSegmented>
800
+ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, base_table_type_set> {
801
+ using underlying_value_type = typename std::conditional_t<is_map_v<T>, std::pair<Key, T>, Key>;
802
+ using underlying_container_type = std::conditional_t<IsSegmented,
803
+ segmented_vector<underlying_value_type, AllocatorOrContainer>,
804
+ std::vector<underlying_value_type, AllocatorOrContainer>>;
805
+
806
+ public:
807
+ using value_container_type = std::
808
+ conditional_t<is_detected_v<detect_iterator, AllocatorOrContainer>, AllocatorOrContainer, underlying_container_type>;
809
+
810
+ private:
811
+ using bucket_alloc =
812
+ typename std::allocator_traits<typename value_container_type::allocator_type>::template rebind_alloc<Bucket>;
813
+ using bucket_alloc_traits = std::allocator_traits<bucket_alloc>;
814
+
815
+ static constexpr uint8_t initial_shifts = 64 - 2; // 2^(64-m_shift) number of buckets
816
+ static constexpr float default_max_load_factor = 0.8F;
817
+
818
+ public:
819
+ using key_type = Key;
820
+ using value_type = typename value_container_type::value_type;
821
+ using size_type = typename value_container_type::size_type;
822
+ using difference_type = typename value_container_type::difference_type;
823
+ using hasher = Hash;
824
+ using key_equal = KeyEqual;
825
+ using allocator_type = typename value_container_type::allocator_type;
826
+ using reference = typename value_container_type::reference;
827
+ using const_reference = typename value_container_type::const_reference;
828
+ using pointer = typename value_container_type::pointer;
829
+ using const_pointer = typename value_container_type::const_pointer;
830
+ using const_iterator = typename value_container_type::const_iterator;
831
+ using iterator = std::conditional_t<is_map_v<T>, typename value_container_type::iterator, const_iterator>;
832
+ using bucket_type = Bucket;
833
+
834
+ private:
835
+ using value_idx_type = decltype(Bucket::m_value_idx);
836
+ using dist_and_fingerprint_type = decltype(Bucket::m_dist_and_fingerprint);
837
+
838
+ static_assert(std::is_trivially_destructible_v<Bucket>, "assert there's no need to call destructor / std::destroy");
839
+ static_assert(std::is_trivially_copyable_v<Bucket>, "assert we can just memset / memcpy");
840
+
841
+ value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes.
842
+ using bucket_pointer = typename std::allocator_traits<bucket_alloc>::pointer;
843
+ bucket_pointer m_buckets{};
844
+ size_t m_num_buckets = 0;
845
+ size_t m_max_bucket_capacity = 0;
846
+ float m_max_load_factor = default_max_load_factor;
847
+ Hash m_hash{};
848
+ KeyEqual m_equal{};
849
+ uint8_t m_shifts = initial_shifts;
850
+
851
+ [[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type {
852
+ return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == m_num_buckets)
853
+ ? 0
854
+ : static_cast<value_idx_type>(bucket_idx + 1U);
855
+ }
856
+
857
+ // Helper to access bucket through pointer types
858
+ [[nodiscard]] static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& {
859
+ return *(bucket_ptr + static_cast<typename std::allocator_traits<bucket_alloc>::difference_type>(offset));
860
+ }
861
+
862
+ // use the dist_inc and dist_dec functions so that uint16_t types work without warning
863
+ [[nodiscard]] static constexpr auto dist_inc(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
864
+ return static_cast<dist_and_fingerprint_type>(x + Bucket::dist_inc);
865
+ }
866
+
867
+ [[nodiscard]] static constexpr auto dist_dec(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
868
+ return static_cast<dist_and_fingerprint_type>(x - Bucket::dist_inc);
869
+ }
870
+
871
+ // The goal of mixed_hash is to always produce a high quality 64bit hash.
872
+ template <typename K>
873
+ [[nodiscard]] constexpr auto mixed_hash(K const& key) const -> uint64_t {
874
+ if constexpr (is_detected_v<detect_avalanching, Hash>) {
875
+ // we know that the hash is good because is_avalanching.
876
+ if constexpr (sizeof(decltype(m_hash(key))) < sizeof(uint64_t)) {
877
+ // 32bit hash and is_avalanching => multiply with a constant to avalanche bits upwards
878
+ return m_hash(key) * UINT64_C(0x9ddfea08eb382d69);
879
+ } else {
880
+ // 64bit and is_avalanching => only use the hash itself.
881
+ return m_hash(key);
882
+ }
883
+ } else {
884
+ // not is_avalanching => apply wyhash
885
+ return wyhash::hash(m_hash(key));
886
+ }
887
+ }
888
+
889
+ [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type {
890
+ return Bucket::dist_inc | (static_cast<dist_and_fingerprint_type>(hash) & Bucket::fingerprint_mask);
891
+ }
892
+
893
+ [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> value_idx_type {
894
+ return static_cast<value_idx_type>(hash >> m_shifts);
895
+ }
896
+
897
+ [[nodiscard]] static constexpr auto get_key(value_type const& vt) -> key_type const& {
898
+ if constexpr (is_map_v<T>) {
899
+ return vt.first;
900
+ } else {
901
+ return vt;
902
+ }
903
+ }
904
+
905
+ template <typename K>
906
+ [[nodiscard]] auto next_while_less(K const& key) const -> Bucket {
907
+ auto hash = mixed_hash(key);
908
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
909
+ auto bucket_idx = bucket_idx_from_hash(hash);
910
+
911
+ while (dist_and_fingerprint < at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
912
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
913
+ bucket_idx = next(bucket_idx);
914
+ }
915
+ return {dist_and_fingerprint, bucket_idx};
916
+ }
917
+
918
+ void place_and_shift_up(Bucket bucket, value_idx_type place) {
919
+ while (0 != at(m_buckets, place).m_dist_and_fingerprint) {
920
+ bucket = std::exchange(at(m_buckets, place), bucket);
921
+ bucket.m_dist_and_fingerprint = dist_inc(bucket.m_dist_and_fingerprint);
922
+ place = next(place);
923
+ }
924
+ at(m_buckets, place) = bucket;
925
+ }
926
+
927
+ [[nodiscard]] static constexpr auto calc_num_buckets(uint8_t shifts) -> size_t {
928
+ return (std::min)(max_bucket_count(), size_t{1} << (64U - shifts));
929
+ }
930
+
931
+ [[nodiscard]] constexpr auto calc_shifts_for_size(size_t s) const -> uint8_t {
932
+ auto shifts = initial_shifts;
933
+ while (shifts > 0 && static_cast<size_t>(static_cast<float>(calc_num_buckets(shifts)) * max_load_factor()) < s) {
934
+ --shifts;
935
+ }
936
+ return shifts;
937
+ }
938
+
939
+ // assumes m_values has data, m_buckets=m_buckets_end=nullptr, m_shifts is INITIAL_SHIFTS
940
+ void copy_buckets(table const& other) {
941
+ // assumes m_values has already the correct data copied over.
942
+ if (empty()) {
943
+ // when empty, at least allocate an initial buckets and clear them.
944
+ allocate_buckets_from_shift();
945
+ clear_buckets();
946
+ } else {
947
+ m_shifts = other.m_shifts;
948
+ allocate_buckets_from_shift();
949
+ std::memcpy(m_buckets, other.m_buckets, sizeof(Bucket) * bucket_count());
950
+ }
951
+ }
952
+
953
+ /**
954
+ * True when no element can be added any more without increasing the size
955
+ */
956
+ [[nodiscard]] auto is_full() const -> bool {
957
+ return size() > m_max_bucket_capacity;
958
+ }
959
+
960
+ void deallocate_buckets() {
961
+ auto ba = bucket_alloc(m_values.get_allocator());
962
+ if (nullptr != m_buckets) {
963
+ bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
964
+ m_buckets = nullptr;
965
+ }
966
+ m_num_buckets = 0;
967
+ m_max_bucket_capacity = 0;
968
+ }
969
+
970
+ void allocate_buckets_from_shift() {
971
+ auto ba = bucket_alloc(m_values.get_allocator());
972
+ m_num_buckets = calc_num_buckets(m_shifts);
973
+ m_buckets = bucket_alloc_traits::allocate(ba, m_num_buckets);
974
+ if (m_num_buckets == max_bucket_count()) {
975
+ // reached the maximum, make sure we can use each bucket
976
+ m_max_bucket_capacity = max_bucket_count();
977
+ } else {
978
+ m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(m_num_buckets) * max_load_factor());
979
+ }
980
+ }
981
+
982
+ void clear_buckets() {
983
+ if (m_buckets != nullptr) {
984
+ std::memset(&*m_buckets, 0, sizeof(Bucket) * bucket_count());
985
+ }
986
+ }
987
+
988
+ void clear_and_fill_buckets_from_values() {
989
+ clear_buckets();
990
+ for (value_idx_type value_idx = 0, end_idx = static_cast<value_idx_type>(m_values.size()); value_idx < end_idx;
991
+ ++value_idx) {
992
+ auto const& key = get_key(m_values[value_idx]);
993
+ auto [dist_and_fingerprint, bucket] = next_while_less(key);
994
+
995
+ // we know for certain that key has not yet been inserted, so no need to check it.
996
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket);
997
+ }
998
+ }
999
+
1000
+ void increase_size() {
1001
+ if (m_max_bucket_capacity == max_bucket_count()) {
1002
+ // remove the value again, we can't add it!
1003
+ m_values.pop_back();
1004
+ on_error_bucket_overflow();
1005
+ }
1006
+ --m_shifts;
1007
+ deallocate_buckets();
1008
+ allocate_buckets_from_shift();
1009
+ clear_and_fill_buckets_from_values();
1010
+ }
1011
+
1012
+ template <typename Op>
1013
+ void do_erase(value_idx_type bucket_idx, Op handle_erased_value) {
1014
+ auto const value_idx_to_remove = at(m_buckets, bucket_idx).m_value_idx;
1015
+
1016
+ // shift down until either empty or an element with correct spot is found
1017
+ auto next_bucket_idx = next(bucket_idx);
1018
+ while (at(m_buckets, next_bucket_idx).m_dist_and_fingerprint >= Bucket::dist_inc * 2) {
1019
+ at(m_buckets, bucket_idx) = {dist_dec(at(m_buckets, next_bucket_idx).m_dist_and_fingerprint),
1020
+ at(m_buckets, next_bucket_idx).m_value_idx};
1021
+ bucket_idx = std::exchange(next_bucket_idx, next(next_bucket_idx));
1022
+ }
1023
+ at(m_buckets, bucket_idx) = {};
1024
+ handle_erased_value(std::move(m_values[value_idx_to_remove]));
1025
+
1026
+ // update m_values
1027
+ if (value_idx_to_remove != m_values.size() - 1) {
1028
+ // no luck, we'll have to replace the value with the last one and update the index accordingly
1029
+ auto& val = m_values[value_idx_to_remove];
1030
+ val = std::move(m_values.back());
1031
+
1032
+ // update the values_idx of the moved entry. No need to play the info game, just look until we find the values_idx
1033
+ auto mh = mixed_hash(get_key(val));
1034
+ bucket_idx = bucket_idx_from_hash(mh);
1035
+
1036
+ auto const values_idx_back = static_cast<value_idx_type>(m_values.size() - 1);
1037
+ while (values_idx_back != at(m_buckets, bucket_idx).m_value_idx) {
1038
+ bucket_idx = next(bucket_idx);
1039
+ }
1040
+ at(m_buckets, bucket_idx).m_value_idx = value_idx_to_remove;
1041
+ }
1042
+ m_values.pop_back();
1043
+ }
1044
+
1045
+ template <typename K, typename Op>
1046
+ auto do_erase_key(K&& key, Op handle_erased_value) -> size_t {
1047
+ if (empty()) {
1048
+ return 0;
1049
+ }
1050
+
1051
+ auto [dist_and_fingerprint, bucket_idx] = next_while_less(key);
1052
+
1053
+ while (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
1054
+ !m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
1055
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1056
+ bucket_idx = next(bucket_idx);
1057
+ }
1058
+
1059
+ if (dist_and_fingerprint != at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
1060
+ return 0;
1061
+ }
1062
+ do_erase(bucket_idx, handle_erased_value);
1063
+ return 1;
1064
+ }
1065
+
1066
+ template <class K, class M>
1067
+ auto do_insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
1068
+ auto it_isinserted = try_emplace(std::forward<K>(key), std::forward<M>(mapped));
1069
+ if (!it_isinserted.second) {
1070
+ it_isinserted.first->second = std::forward<M>(mapped);
1071
+ }
1072
+ return it_isinserted;
1073
+ }
1074
+
1075
+ template <typename... Args>
1076
+ auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, value_idx_type bucket_idx, Args&&... args)
1077
+ -> std::pair<iterator, bool> {
1078
+
1079
+ // emplace the new value. If that throws an exception, no harm done; index is still in a valid state
1080
+ m_values.emplace_back(std::forward<Args>(args)...);
1081
+
1082
+ auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
1083
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
1084
+ increase_size();
1085
+ } else {
1086
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
1087
+ }
1088
+
1089
+ // place element and shift up until we find an empty spot
1090
+ return {begin() + static_cast<difference_type>(value_idx), true};
1091
+ }
1092
+
1093
+ template <typename K, typename... Args>
1094
+ auto do_try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
1095
+ auto hash = mixed_hash(key);
1096
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
1097
+ auto bucket_idx = bucket_idx_from_hash(hash);
1098
+
1099
+ while (true) {
1100
+ auto* bucket = &at(m_buckets, bucket_idx);
1101
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
1102
+ if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
1103
+ return {begin() + static_cast<difference_type>(bucket->m_value_idx), false};
1104
+ }
1105
+ } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
1106
+ return do_place_element(dist_and_fingerprint,
1107
+ bucket_idx,
1108
+ std::piecewise_construct,
1109
+ std::forward_as_tuple(std::forward<K>(key)),
1110
+ std::forward_as_tuple(std::forward<Args>(args)...));
1111
+ }
1112
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1113
+ bucket_idx = next(bucket_idx);
1114
+ }
1115
+ }
1116
+
1117
+ template <typename K>
1118
+ auto do_find(K const& key) -> iterator {
1119
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(empty())) {
1120
+ return end();
1121
+ }
1122
+
1123
+ auto mh = mixed_hash(key);
1124
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(mh);
1125
+ auto bucket_idx = bucket_idx_from_hash(mh);
1126
+ auto* bucket = &at(m_buckets, bucket_idx);
1127
+
1128
+ // unrolled loop. *Always* check a few directly, then enter the loop. This is faster.
1129
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
1130
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
1131
+ }
1132
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1133
+ bucket_idx = next(bucket_idx);
1134
+ bucket = &at(m_buckets, bucket_idx);
1135
+
1136
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
1137
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
1138
+ }
1139
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1140
+ bucket_idx = next(bucket_idx);
1141
+ bucket = &at(m_buckets, bucket_idx);
1142
+
1143
+ while (true) {
1144
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
1145
+ if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
1146
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
1147
+ }
1148
+ } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
1149
+ return end();
1150
+ }
1151
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1152
+ bucket_idx = next(bucket_idx);
1153
+ bucket = &at(m_buckets, bucket_idx);
1154
+ }
1155
+ }
1156
+
1157
+ template <typename K>
1158
+ auto do_find(K const& key) const -> const_iterator {
1159
+ return const_cast<table*>(this)->do_find(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
1160
+ }
1161
+
1162
+ template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1163
+ auto do_at(K const& key) -> Q& {
1164
+ if (auto it = find(key); ANKERL_UNORDERED_DENSE_LIKELY(end() != it)) {
1165
+ return it->second;
1166
+ }
1167
+ on_error_key_not_found();
1168
+ }
1169
+
1170
+ template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1171
+ auto do_at(K const& key) const -> Q const& {
1172
+ return const_cast<table*>(this)->at(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
1173
+ }
1174
+
1175
+ public:
1176
+ explicit table(size_t bucket_count,
1177
+ Hash const& hash = Hash(),
1178
+ KeyEqual const& equal = KeyEqual(),
1179
+ allocator_type const& alloc_or_container = allocator_type())
1180
+ : m_values(alloc_or_container)
1181
+ , m_hash(hash)
1182
+ , m_equal(equal) {
1183
+ if (0 != bucket_count) {
1184
+ reserve(bucket_count);
1185
+ } else {
1186
+ allocate_buckets_from_shift();
1187
+ clear_buckets();
1188
+ }
1189
+ }
1190
+
1191
+ table()
1192
+ : table(0) {}
1193
+
1194
+ table(size_t bucket_count, allocator_type const& alloc)
1195
+ : table(bucket_count, Hash(), KeyEqual(), alloc) {}
1196
+
1197
+ table(size_t bucket_count, Hash const& hash, allocator_type const& alloc)
1198
+ : table(bucket_count, hash, KeyEqual(), alloc) {}
1199
+
1200
+ explicit table(allocator_type const& alloc)
1201
+ : table(0, Hash(), KeyEqual(), alloc) {}
1202
+
1203
+ template <class InputIt>
1204
+ table(InputIt first,
1205
+ InputIt last,
1206
+ size_type bucket_count = 0,
1207
+ Hash const& hash = Hash(),
1208
+ KeyEqual const& equal = KeyEqual(),
1209
+ allocator_type const& alloc = allocator_type())
1210
+ : table(bucket_count, hash, equal, alloc) {
1211
+ insert(first, last);
1212
+ }
1213
+
1214
+ template <class InputIt>
1215
+ table(InputIt first, InputIt last, size_type bucket_count, allocator_type const& alloc)
1216
+ : table(first, last, bucket_count, Hash(), KeyEqual(), alloc) {}
1217
+
1218
+ template <class InputIt>
1219
+ table(InputIt first, InputIt last, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
1220
+ : table(first, last, bucket_count, hash, KeyEqual(), alloc) {}
1221
+
1222
+ table(table const& other)
1223
+ : table(other, other.m_values.get_allocator()) {}
1224
+
1225
+ table(table const& other, allocator_type const& alloc)
1226
+ : m_values(other.m_values, alloc)
1227
+ , m_max_load_factor(other.m_max_load_factor)
1228
+ , m_hash(other.m_hash)
1229
+ , m_equal(other.m_equal) {
1230
+ copy_buckets(other);
1231
+ }
1232
+
1233
+ table(table&& other) noexcept
1234
+ : table(std::move(other), other.m_values.get_allocator()) {}
1235
+
1236
+ table(table&& other, allocator_type const& alloc) noexcept
1237
+ : m_values(alloc) {
1238
+ *this = std::move(other);
1239
+ }
1240
+
1241
+ table(std::initializer_list<value_type> ilist,
1242
+ size_t bucket_count = 0,
1243
+ Hash const& hash = Hash(),
1244
+ KeyEqual const& equal = KeyEqual(),
1245
+ allocator_type const& alloc = allocator_type())
1246
+ : table(bucket_count, hash, equal, alloc) {
1247
+ insert(ilist);
1248
+ }
1249
+
1250
+ table(std::initializer_list<value_type> ilist, size_type bucket_count, allocator_type const& alloc)
1251
+ : table(ilist, bucket_count, Hash(), KeyEqual(), alloc) {}
1252
+
1253
+ table(std::initializer_list<value_type> init, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
1254
+ : table(init, bucket_count, hash, KeyEqual(), alloc) {}
1255
+
1256
+ ~table() {
1257
+ if (nullptr != m_buckets) {
1258
+ auto ba = bucket_alloc(m_values.get_allocator());
1259
+ bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
1260
+ }
1261
+ }
1262
+
1263
+ auto operator=(table const& other) -> table& {
1264
+ if (&other != this) {
1265
+ deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
1266
+ m_values = other.m_values;
1267
+ m_max_load_factor = other.m_max_load_factor;
1268
+ m_hash = other.m_hash;
1269
+ m_equal = other.m_equal;
1270
+ m_shifts = initial_shifts;
1271
+ copy_buckets(other);
1272
+ }
1273
+ return *this;
1274
+ }
1275
+
1276
+ auto operator=(table&& other) noexcept(noexcept(std::is_nothrow_move_assignable_v<value_container_type> &&
1277
+ std::is_nothrow_move_assignable_v<Hash> &&
1278
+ std::is_nothrow_move_assignable_v<KeyEqual>)) -> table& {
1279
+ if (&other != this) {
1280
+ deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
1281
+ m_values = std::move(other.m_values);
1282
+ other.m_values.clear();
1283
+
1284
+ // we can only reuse m_buckets when both maps have the same allocator!
1285
+ if (get_allocator() == other.get_allocator()) {
1286
+ m_buckets = std::exchange(other.m_buckets, nullptr);
1287
+ m_num_buckets = std::exchange(other.m_num_buckets, 0);
1288
+ m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0);
1289
+ m_shifts = std::exchange(other.m_shifts, initial_shifts);
1290
+ m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor);
1291
+ m_hash = std::exchange(other.m_hash, {});
1292
+ m_equal = std::exchange(other.m_equal, {});
1293
+ other.allocate_buckets_from_shift();
1294
+ other.clear_buckets();
1295
+ } else {
1296
+ // set max_load_factor *before* copying the other's buckets, so we have the same
1297
+ // behavior
1298
+ m_max_load_factor = other.m_max_load_factor;
1299
+
1300
+ // copy_buckets sets m_buckets, m_num_buckets, m_max_bucket_capacity, m_shifts
1301
+ copy_buckets(other);
1302
+ // clear's the other's buckets so other is now already usable.
1303
+ other.clear_buckets();
1304
+ m_hash = other.m_hash;
1305
+ m_equal = other.m_equal;
1306
+ }
1307
+ // map "other" is now already usable, it's empty.
1308
+ }
1309
+ return *this;
1310
+ }
1311
+
1312
+ auto operator=(std::initializer_list<value_type> ilist) -> table& {
1313
+ clear();
1314
+ insert(ilist);
1315
+ return *this;
1316
+ }
1317
+
1318
+ auto get_allocator() const noexcept -> allocator_type {
1319
+ return m_values.get_allocator();
1320
+ }
1321
+
1322
+ // iterators //////////////////////////////////////////////////////////////
1323
+
1324
+ auto begin() noexcept -> iterator {
1325
+ return m_values.begin();
1326
+ }
1327
+
1328
+ auto begin() const noexcept -> const_iterator {
1329
+ return m_values.begin();
1330
+ }
1331
+
1332
+ auto cbegin() const noexcept -> const_iterator {
1333
+ return m_values.cbegin();
1334
+ }
1335
+
1336
+ auto end() noexcept -> iterator {
1337
+ return m_values.end();
1338
+ }
1339
+
1340
+ auto cend() const noexcept -> const_iterator {
1341
+ return m_values.cend();
1342
+ }
1343
+
1344
+ auto end() const noexcept -> const_iterator {
1345
+ return m_values.end();
1346
+ }
1347
+
1348
+ // capacity ///////////////////////////////////////////////////////////////
1349
+
1350
+ [[nodiscard]] auto empty() const noexcept -> bool {
1351
+ return m_values.empty();
1352
+ }
1353
+
1354
+ [[nodiscard]] auto size() const noexcept -> size_t {
1355
+ return m_values.size();
1356
+ }
1357
+
1358
+ [[nodiscard]] static constexpr auto max_size() noexcept -> size_t {
1359
+ if constexpr ((std::numeric_limits<value_idx_type>::max)() == (std::numeric_limits<size_t>::max)()) {
1360
+ return size_t{1} << (sizeof(value_idx_type) * 8 - 1);
1361
+ } else {
1362
+ return size_t{1} << (sizeof(value_idx_type) * 8);
1363
+ }
1364
+ }
1365
+
1366
+ // modifiers //////////////////////////////////////////////////////////////
1367
+
1368
+ void clear() {
1369
+ m_values.clear();
1370
+ clear_buckets();
1371
+ }
1372
+
1373
+ auto insert(value_type const& value) -> std::pair<iterator, bool> {
1374
+ return emplace(value);
1375
+ }
1376
+
1377
+ auto insert(value_type&& value) -> std::pair<iterator, bool> {
1378
+ return emplace(std::move(value));
1379
+ }
1380
+
1381
+ template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
1382
+ auto insert(P&& value) -> std::pair<iterator, bool> {
1383
+ return emplace(std::forward<P>(value));
1384
+ }
1385
+
1386
+ auto insert(const_iterator /*hint*/, value_type const& value) -> iterator {
1387
+ return insert(value).first;
1388
+ }
1389
+
1390
+ auto insert(const_iterator /*hint*/, value_type&& value) -> iterator {
1391
+ return insert(std::move(value)).first;
1392
+ }
1393
+
1394
+ template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
1395
+ auto insert(const_iterator /*hint*/, P&& value) -> iterator {
1396
+ return insert(std::forward<P>(value)).first;
1397
+ }
1398
+
1399
+ template <class InputIt>
1400
+ void insert(InputIt first, InputIt last) {
1401
+ while (first != last) {
1402
+ insert(*first);
1403
+ ++first;
1404
+ }
1405
+ }
1406
+
1407
+ void insert(std::initializer_list<value_type> ilist) {
1408
+ insert(ilist.begin(), ilist.end());
1409
+ }
1410
+
1411
+ // nonstandard API: *this is emptied.
1412
+ // Also see "A Standard flat_map" https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p0429r9.pdf
1413
+ auto extract() && -> value_container_type {
1414
+ return std::move(m_values);
1415
+ }
1416
+
1417
+ // nonstandard API:
1418
+ // Discards the internally held container and replaces it with the one passed. Erases non-unique elements.
1419
+ auto replace(value_container_type&& container) {
1420
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(container.size() > max_size())) {
1421
+ on_error_too_many_elements();
1422
+ }
1423
+ auto shifts = calc_shifts_for_size(container.size());
1424
+ if (0 == m_num_buckets || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
1425
+ m_shifts = shifts;
1426
+ deallocate_buckets();
1427
+ allocate_buckets_from_shift();
1428
+ }
1429
+ clear_buckets();
1430
+
1431
+ m_values = std::move(container);
1432
+
1433
+ // can't use clear_and_fill_buckets_from_values() because container elements might not be unique
1434
+ auto value_idx = value_idx_type{};
1435
+
1436
+ // loop until we reach the end of the container. duplicated entries will be replaced with back().
1437
+ while (value_idx != static_cast<value_idx_type>(m_values.size())) {
1438
+ auto const& key = get_key(m_values[value_idx]);
1439
+
1440
+ auto hash = mixed_hash(key);
1441
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
1442
+ auto bucket_idx = bucket_idx_from_hash(hash);
1443
+
1444
+ bool key_found = false;
1445
+ while (true) {
1446
+ auto const& bucket = at(m_buckets, bucket_idx);
1447
+ if (dist_and_fingerprint > bucket.m_dist_and_fingerprint) {
1448
+ break;
1449
+ }
1450
+ if (dist_and_fingerprint == bucket.m_dist_and_fingerprint &&
1451
+ m_equal(key, get_key(m_values[bucket.m_value_idx]))) {
1452
+ key_found = true;
1453
+ break;
1454
+ }
1455
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1456
+ bucket_idx = next(bucket_idx);
1457
+ }
1458
+
1459
+ if (key_found) {
1460
+ if (value_idx != static_cast<value_idx_type>(m_values.size() - 1)) {
1461
+ m_values[value_idx] = std::move(m_values.back());
1462
+ }
1463
+ m_values.pop_back();
1464
+ } else {
1465
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
1466
+ ++value_idx;
1467
+ }
1468
+ }
1469
+ }
1470
+
1471
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1472
+ auto insert_or_assign(Key const& key, M&& mapped) -> std::pair<iterator, bool> {
1473
+ return do_insert_or_assign(key, std::forward<M>(mapped));
1474
+ }
1475
+
1476
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1477
+ auto insert_or_assign(Key&& key, M&& mapped) -> std::pair<iterator, bool> {
1478
+ return do_insert_or_assign(std::move(key), std::forward<M>(mapped));
1479
+ }
1480
+
1481
+ template <typename K,
1482
+ typename M,
1483
+ typename Q = T,
1484
+ typename H = Hash,
1485
+ typename KE = KeyEqual,
1486
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1487
+ auto insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
1488
+ return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped));
1489
+ }
1490
+
1491
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1492
+ auto insert_or_assign(const_iterator /*hint*/, Key const& key, M&& mapped) -> iterator {
1493
+ return do_insert_or_assign(key, std::forward<M>(mapped)).first;
1494
+ }
1495
+
1496
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1497
+ auto insert_or_assign(const_iterator /*hint*/, Key&& key, M&& mapped) -> iterator {
1498
+ return do_insert_or_assign(std::move(key), std::forward<M>(mapped)).first;
1499
+ }
1500
+
1501
+ template <typename K,
1502
+ typename M,
1503
+ typename Q = T,
1504
+ typename H = Hash,
1505
+ typename KE = KeyEqual,
1506
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1507
+ auto insert_or_assign(const_iterator /*hint*/, K&& key, M&& mapped) -> iterator {
1508
+ return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped)).first;
1509
+ }
1510
+
1511
+ // Single arguments for unordered_set can be used without having to construct the value_type
1512
+ template <class K,
1513
+ typename Q = T,
1514
+ typename H = Hash,
1515
+ typename KE = KeyEqual,
1516
+ std::enable_if_t<!is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1517
+ auto emplace(K&& key) -> std::pair<iterator, bool> {
1518
+ auto hash = mixed_hash(key);
1519
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
1520
+ auto bucket_idx = bucket_idx_from_hash(hash);
1521
+
1522
+ while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
1523
+ if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
1524
+ m_equal(key, m_values[at(m_buckets, bucket_idx).m_value_idx])) {
1525
+ // found it, return without ever actually creating anything
1526
+ return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
1527
+ }
1528
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1529
+ bucket_idx = next(bucket_idx);
1530
+ }
1531
+
1532
+ // value is new, insert element first, so when exception happens we are in a valid state
1533
+ return do_place_element(dist_and_fingerprint, bucket_idx, std::forward<K>(key));
1534
+ }
1535
+
1536
+ template <class... Args>
1537
+ auto emplace(Args&&... args) -> std::pair<iterator, bool> {
1538
+ // we have to instantiate the value_type to be able to access the key.
1539
+ // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop.
1540
+ auto& key = get_key(m_values.emplace_back(std::forward<Args>(args)...));
1541
+ auto hash = mixed_hash(key);
1542
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
1543
+ auto bucket_idx = bucket_idx_from_hash(hash);
1544
+
1545
+ while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
1546
+ if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
1547
+ m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
1548
+ m_values.pop_back(); // value was already there, so get rid of it
1549
+ return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
1550
+ }
1551
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
1552
+ bucket_idx = next(bucket_idx);
1553
+ }
1554
+
1555
+ // value is new, place the bucket and shift up until we find an empty spot
1556
+ auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
1557
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
1558
+ // increase_size just rehashes all the data we have in m_values
1559
+ increase_size();
1560
+ } else {
1561
+ // place element and shift up until we find an empty spot
1562
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
1563
+ }
1564
+ return {begin() + static_cast<difference_type>(value_idx), true};
1565
+ }
1566
+
1567
+ template <class... Args>
1568
+ auto emplace_hint(const_iterator /*hint*/, Args&&... args) -> iterator {
1569
+ return emplace(std::forward<Args>(args)...).first;
1570
+ }
1571
+
1572
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1573
+ auto try_emplace(Key const& key, Args&&... args) -> std::pair<iterator, bool> {
1574
+ return do_try_emplace(key, std::forward<Args>(args)...);
1575
+ }
1576
+
1577
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1578
+ auto try_emplace(Key&& key, Args&&... args) -> std::pair<iterator, bool> {
1579
+ return do_try_emplace(std::move(key), std::forward<Args>(args)...);
1580
+ }
1581
+
1582
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1583
+ auto try_emplace(const_iterator /*hint*/, Key const& key, Args&&... args) -> iterator {
1584
+ return do_try_emplace(key, std::forward<Args>(args)...).first;
1585
+ }
1586
+
1587
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1588
+ auto try_emplace(const_iterator /*hint*/, Key&& key, Args&&... args) -> iterator {
1589
+ return do_try_emplace(std::move(key), std::forward<Args>(args)...).first;
1590
+ }
1591
+
1592
+ template <
1593
+ typename K,
1594
+ typename... Args,
1595
+ typename Q = T,
1596
+ typename H = Hash,
1597
+ typename KE = KeyEqual,
1598
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
1599
+ bool> = true>
1600
+ auto try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
1601
+ return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...);
1602
+ }
1603
+
1604
+ template <
1605
+ typename K,
1606
+ typename... Args,
1607
+ typename Q = T,
1608
+ typename H = Hash,
1609
+ typename KE = KeyEqual,
1610
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
1611
+ bool> = true>
1612
+ auto try_emplace(const_iterator /*hint*/, K&& key, Args&&... args) -> iterator {
1613
+ return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...).first;
1614
+ }
1615
+
1616
+ auto erase(iterator it) -> iterator {
1617
+ auto hash = mixed_hash(get_key(*it));
1618
+ auto bucket_idx = bucket_idx_from_hash(hash);
1619
+
1620
+ auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
1621
+ while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
1622
+ bucket_idx = next(bucket_idx);
1623
+ }
1624
+
1625
+ do_erase(bucket_idx, [](value_type&& /*unused*/) {
1626
+ });
1627
+ return begin() + static_cast<difference_type>(value_idx_to_remove);
1628
+ }
1629
+
1630
+ auto extract(iterator it) -> value_type {
1631
+ auto hash = mixed_hash(get_key(*it));
1632
+ auto bucket_idx = bucket_idx_from_hash(hash);
1633
+
1634
+ auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
1635
+ while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
1636
+ bucket_idx = next(bucket_idx);
1637
+ }
1638
+
1639
+ auto tmp = std::optional<value_type>{};
1640
+ do_erase(bucket_idx, [&tmp](value_type&& val) {
1641
+ tmp = std::move(val);
1642
+ });
1643
+ return std::move(tmp).value();
1644
+ }
1645
+
1646
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1647
+ auto erase(const_iterator it) -> iterator {
1648
+ return erase(begin() + (it - cbegin()));
1649
+ }
1650
+
1651
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1652
+ auto extract(const_iterator it) -> value_type {
1653
+ return extract(begin() + (it - cbegin()));
1654
+ }
1655
+
1656
+ auto erase(const_iterator first, const_iterator last) -> iterator {
1657
+ auto const idx_first = first - cbegin();
1658
+ auto const idx_last = last - cbegin();
1659
+ auto const first_to_last = std::distance(first, last);
1660
+ auto const last_to_end = std::distance(last, cend());
1661
+
1662
+ // remove elements from left to right which moves elements from the end back
1663
+ auto const mid = idx_first + (std::min)(first_to_last, last_to_end);
1664
+ auto idx = idx_first;
1665
+ while (idx != mid) {
1666
+ erase(begin() + idx);
1667
+ ++idx;
1668
+ }
1669
+
1670
+ // all elements from the right are moved, now remove the last element until all done
1671
+ idx = idx_last;
1672
+ while (idx != mid) {
1673
+ --idx;
1674
+ erase(begin() + idx);
1675
+ }
1676
+
1677
+ return begin() + idx_first;
1678
+ }
1679
+
1680
+ auto erase(Key const& key) -> size_t {
1681
+ return do_erase_key(key, [](value_type&& /*unused*/) {
1682
+ });
1683
+ }
1684
+
1685
+ auto extract(Key const& key) -> std::optional<value_type> {
1686
+ auto tmp = std::optional<value_type>{};
1687
+ do_erase_key(key, [&tmp](value_type&& val) {
1688
+ tmp = std::move(val);
1689
+ });
1690
+ return tmp;
1691
+ }
1692
+
1693
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1694
+ auto erase(K&& key) -> size_t {
1695
+ return do_erase_key(std::forward<K>(key), [](value_type&& /*unused*/) {
1696
+ });
1697
+ }
1698
+
1699
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1700
+ auto extract(K&& key) -> std::optional<value_type> {
1701
+ auto tmp = std::optional<value_type>{};
1702
+ do_erase_key(std::forward<K>(key), [&tmp](value_type&& val) {
1703
+ tmp = std::move(val);
1704
+ });
1705
+ return tmp;
1706
+ }
1707
+
1708
+ void swap(table& other) noexcept(noexcept(std::is_nothrow_swappable_v<value_container_type> &&
1709
+ std::is_nothrow_swappable_v<Hash> && std::is_nothrow_swappable_v<KeyEqual>)) {
1710
+ using std::swap;
1711
+ swap(other, *this);
1712
+ }
1713
+
1714
+ // lookup /////////////////////////////////////////////////////////////////
1715
+
1716
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1717
+ auto at(key_type const& key) -> Q& {
1718
+ return do_at(key);
1719
+ }
1720
+
1721
+ template <typename K,
1722
+ typename Q = T,
1723
+ typename H = Hash,
1724
+ typename KE = KeyEqual,
1725
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1726
+ auto at(K const& key) -> Q& {
1727
+ return do_at(key);
1728
+ }
1729
+
1730
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1731
+ auto at(key_type const& key) const -> Q const& {
1732
+ return do_at(key);
1733
+ }
1734
+
1735
+ template <typename K,
1736
+ typename Q = T,
1737
+ typename H = Hash,
1738
+ typename KE = KeyEqual,
1739
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1740
+ auto at(K const& key) const -> Q const& {
1741
+ return do_at(key);
1742
+ }
1743
+
1744
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1745
+ auto operator[](Key const& key) -> Q& {
1746
+ return try_emplace(key).first->second;
1747
+ }
1748
+
1749
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
1750
+ auto operator[](Key&& key) -> Q& {
1751
+ return try_emplace(std::move(key)).first->second;
1752
+ }
1753
+
1754
+ template <typename K,
1755
+ typename Q = T,
1756
+ typename H = Hash,
1757
+ typename KE = KeyEqual,
1758
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
1759
+ auto operator[](K&& key) -> Q& {
1760
+ return try_emplace(std::forward<K>(key)).first->second;
1761
+ }
1762
+
1763
+ auto count(Key const& key) const -> size_t {
1764
+ return find(key) == end() ? 0 : 1;
1765
+ }
1766
+
1767
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1768
+ auto count(K const& key) const -> size_t {
1769
+ return find(key) == end() ? 0 : 1;
1770
+ }
1771
+
1772
+ auto find(Key const& key) -> iterator {
1773
+ return do_find(key);
1774
+ }
1775
+
1776
+ auto find(Key const& key) const -> const_iterator {
1777
+ return do_find(key);
1778
+ }
1779
+
1780
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1781
+ auto find(K const& key) -> iterator {
1782
+ return do_find(key);
1783
+ }
1784
+
1785
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1786
+ auto find(K const& key) const -> const_iterator {
1787
+ return do_find(key);
1788
+ }
1789
+
1790
+ auto contains(Key const& key) const -> bool {
1791
+ return find(key) != end();
1792
+ }
1793
+
1794
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1795
+ auto contains(K const& key) const -> bool {
1796
+ return find(key) != end();
1797
+ }
1798
+
1799
+ auto equal_range(Key const& key) -> std::pair<iterator, iterator> {
1800
+ auto it = do_find(key);
1801
+ return {it, it == end() ? end() : it + 1};
1802
+ }
1803
+
1804
+ auto equal_range(const Key& key) const -> std::pair<const_iterator, const_iterator> {
1805
+ auto it = do_find(key);
1806
+ return {it, it == end() ? end() : it + 1};
1807
+ }
1808
+
1809
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1810
+ auto equal_range(K const& key) -> std::pair<iterator, iterator> {
1811
+ auto it = do_find(key);
1812
+ return {it, it == end() ? end() : it + 1};
1813
+ }
1814
+
1815
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
1816
+ auto equal_range(K const& key) const -> std::pair<const_iterator, const_iterator> {
1817
+ auto it = do_find(key);
1818
+ return {it, it == end() ? end() : it + 1};
1819
+ }
1820
+
1821
+ // bucket interface ///////////////////////////////////////////////////////
1822
+
1823
+ auto bucket_count() const noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
1824
+ return m_num_buckets;
1825
+ }
1826
+
1827
+ static constexpr auto max_bucket_count() noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
1828
+ return max_size();
1829
+ }
1830
+
1831
+ // hash policy ////////////////////////////////////////////////////////////
1832
+
1833
+ [[nodiscard]] auto load_factor() const -> float {
1834
+ return bucket_count() ? static_cast<float>(size()) / static_cast<float>(bucket_count()) : 0.0F;
1835
+ }
1836
+
1837
+ [[nodiscard]] auto max_load_factor() const -> float {
1838
+ return m_max_load_factor;
1839
+ }
1840
+
1841
+ void max_load_factor(float ml) {
1842
+ m_max_load_factor = ml;
1843
+ if (m_num_buckets != max_bucket_count()) {
1844
+ m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(bucket_count()) * max_load_factor());
1845
+ }
1846
+ }
1847
+
1848
+ void rehash(size_t count) {
1849
+ count = (std::min)(count, max_size());
1850
+ auto shifts = calc_shifts_for_size((std::max)(count, size()));
1851
+ if (shifts != m_shifts) {
1852
+ m_shifts = shifts;
1853
+ deallocate_buckets();
1854
+ m_values.shrink_to_fit();
1855
+ allocate_buckets_from_shift();
1856
+ clear_and_fill_buckets_from_values();
1857
+ }
1858
+ }
1859
+
1860
+ void reserve(size_t capa) {
1861
+ capa = (std::min)(capa, max_size());
1862
+ if constexpr (has_reserve<value_container_type>) {
1863
+ // std::deque doesn't have reserve(). Make sure we only call when available
1864
+ m_values.reserve(capa);
1865
+ }
1866
+ auto shifts = calc_shifts_for_size((std::max)(capa, size()));
1867
+ if (0 == m_num_buckets || shifts < m_shifts) {
1868
+ m_shifts = shifts;
1869
+ deallocate_buckets();
1870
+ allocate_buckets_from_shift();
1871
+ clear_and_fill_buckets_from_values();
1872
+ }
1873
+ }
1874
+
1875
+ // observers //////////////////////////////////////////////////////////////
1876
+
1877
+ auto hash_function() const -> hasher {
1878
+ return m_hash;
1879
+ }
1880
+
1881
+ auto key_eq() const -> key_equal {
1882
+ return m_equal;
1883
+ }
1884
+
1885
+ // nonstandard API: expose the underlying values container
1886
+ [[nodiscard]] auto values() const noexcept -> value_container_type const& {
1887
+ return m_values;
1888
+ }
1889
+
1890
+ // non-member functions ///////////////////////////////////////////////////
1891
+
1892
+ friend auto operator==(table const& a, table const& b) -> bool {
1893
+ if (&a == &b) {
1894
+ return true;
1895
+ }
1896
+ if (a.size() != b.size()) {
1897
+ return false;
1898
+ }
1899
+ for (auto const& b_entry : b) {
1900
+ auto it = a.find(get_key(b_entry));
1901
+ if constexpr (is_map_v<T>) {
1902
+ // map: check that key is here, then also check that value is the same
1903
+ if (a.end() == it || !(b_entry.second == it->second)) {
1904
+ return false;
1905
+ }
1906
+ } else {
1907
+ // set: only check that the key is here
1908
+ if (a.end() == it) {
1909
+ return false;
1910
+ }
1911
+ }
1912
+ }
1913
+ return true;
1914
+ }
1915
+
1916
+ friend auto operator!=(table const& a, table const& b) -> bool {
1917
+ return !(a == b);
1918
+ }
1919
+ };
1920
+
1921
+ } // namespace detail
1922
+
1923
+ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1924
+ class T,
1925
+ class Hash = hash<Key>,
1926
+ class KeyEqual = std::equal_to<Key>,
1927
+ class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
1928
+ class Bucket = bucket_type::standard>
1929
+ using map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
1930
+
1931
+ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1932
+ class T,
1933
+ class Hash = hash<Key>,
1934
+ class KeyEqual = std::equal_to<Key>,
1935
+ class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
1936
+ class Bucket = bucket_type::standard>
1937
+ using segmented_map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
1938
+
1939
+ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1940
+ class Hash = hash<Key>,
1941
+ class KeyEqual = std::equal_to<Key>,
1942
+ class AllocatorOrContainer = std::allocator<Key>,
1943
+ class Bucket = bucket_type::standard>
1944
+ using set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
1945
+
1946
+ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1947
+ class Hash = hash<Key>,
1948
+ class KeyEqual = std::equal_to<Key>,
1949
+ class AllocatorOrContainer = std::allocator<Key>,
1950
+ class Bucket = bucket_type::standard>
1951
+ using segmented_set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
1952
+
1953
+ # if defined(ANKERL_UNORDERED_DENSE_PMR)
1954
+
1955
+ namespace pmr {
1956
+
1957
+ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1958
+ class T,
1959
+ class Hash = hash<Key>,
1960
+ class KeyEqual = std::equal_to<Key>,
1961
+ class Bucket = bucket_type::standard>
1962
+ using map =
1963
+ detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, false>;
1964
+
1965
+ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1966
+ class T,
1967
+ class Hash = hash<Key>,
1968
+ class KeyEqual = std::equal_to<Key>,
1969
+ class Bucket = bucket_type::standard>
1970
+ using segmented_map =
1971
+ detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, true>;
1972
+
1973
+ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1974
+ class Hash = hash<Key>,
1975
+ class KeyEqual = std::equal_to<Key>,
1976
+ class Bucket = bucket_type::standard>
1977
+ using set = detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, false>;
1978
+
1979
+ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
1980
+ class Hash = hash<Key>,
1981
+ class KeyEqual = std::equal_to<Key>,
1982
+ class Bucket = bucket_type::standard>
1983
+ using segmented_set =
1984
+ detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, true>;
1985
+
1986
+ } // namespace pmr
1987
+
1988
+ # endif
1989
+
1990
+ // deduction guides ///////////////////////////////////////////////////////////
1991
+
1992
+ // deduction guides for alias templates are only possible since C++20
1993
+ // see https://en.cppreference.com/w/cpp/language/class_template_argument_deduction
1994
+
1995
+ } // namespace ANKERL_UNORDERED_DENSE_NAMESPACE
1996
+ } // namespace ankerl::unordered_dense
1997
+
1998
+ // std extensions /////////////////////////////////////////////////////////////
1999
+
2000
+ namespace std { // NOLINT(cert-dcl58-cpp)
2001
+
2002
+ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
2003
+ class T,
2004
+ class Hash,
2005
+ class KeyEqual,
2006
+ class AllocatorOrContainer,
2007
+ class Bucket,
2008
+ class Pred,
2009
+ bool IsSegmented>
2010
+ // NOLINTNEXTLINE(cert-dcl58-cpp)
2011
+ auto erase_if(ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>& map,
2012
+ Pred pred) -> size_t {
2013
+ using map_t = ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>;
2014
+
2015
+ // going back to front because erase() invalidates the end iterator
2016
+ auto const old_size = map.size();
2017
+ auto idx = old_size;
2018
+ while (idx) {
2019
+ --idx;
2020
+ auto it = map.begin() + static_cast<typename map_t::difference_type>(idx);
2021
+ if (pred(*it)) {
2022
+ map.erase(it);
2023
+ }
2024
+ }
2025
+
2026
+ return old_size - map.size();
2027
+ }
2028
+
2029
+ } // namespace std
2030
+
2031
+ #endif
2032
+ #endif