rrudb 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/.yardopts +1 -0
  3. data/LICENSE.txt +22 -0
  4. data/README.md +26 -0
  5. data/examples/example.rb +39 -0
  6. data/ext/rudb/NuDB/include/nudb/CMakeLists.txt +104 -0
  7. data/ext/rudb/NuDB/include/nudb/_experimental/basic_seconds_clock.hpp +200 -0
  8. data/ext/rudb/NuDB/include/nudb/_experimental/chrono_util.hpp +58 -0
  9. data/ext/rudb/NuDB/include/nudb/_experimental/test/fail_file.hpp +343 -0
  10. data/ext/rudb/NuDB/include/nudb/_experimental/test/temp_dir.hpp +73 -0
  11. data/ext/rudb/NuDB/include/nudb/_experimental/test/test_store.hpp +451 -0
  12. data/ext/rudb/NuDB/include/nudb/_experimental/test/xor_shift_engine.hpp +105 -0
  13. data/ext/rudb/NuDB/include/nudb/_experimental/util.hpp +288 -0
  14. data/ext/rudb/NuDB/include/nudb/basic_store.hpp +461 -0
  15. data/ext/rudb/NuDB/include/nudb/concepts.hpp +205 -0
  16. data/ext/rudb/NuDB/include/nudb/context.hpp +144 -0
  17. data/ext/rudb/NuDB/include/nudb/create.hpp +117 -0
  18. data/ext/rudb/NuDB/include/nudb/detail/arena.hpp +296 -0
  19. data/ext/rudb/NuDB/include/nudb/detail/bucket.hpp +473 -0
  20. data/ext/rudb/NuDB/include/nudb/detail/buffer.hpp +86 -0
  21. data/ext/rudb/NuDB/include/nudb/detail/bulkio.hpp +196 -0
  22. data/ext/rudb/NuDB/include/nudb/detail/cache.hpp +236 -0
  23. data/ext/rudb/NuDB/include/nudb/detail/endian.hpp +93 -0
  24. data/ext/rudb/NuDB/include/nudb/detail/field.hpp +265 -0
  25. data/ext/rudb/NuDB/include/nudb/detail/format.hpp +630 -0
  26. data/ext/rudb/NuDB/include/nudb/detail/gentex.hpp +259 -0
  27. data/ext/rudb/NuDB/include/nudb/detail/mutex.hpp +26 -0
  28. data/ext/rudb/NuDB/include/nudb/detail/pool.hpp +243 -0
  29. data/ext/rudb/NuDB/include/nudb/detail/store_base.hpp +45 -0
  30. data/ext/rudb/NuDB/include/nudb/detail/stream.hpp +149 -0
  31. data/ext/rudb/NuDB/include/nudb/detail/xxhash.hpp +328 -0
  32. data/ext/rudb/NuDB/include/nudb/error.hpp +257 -0
  33. data/ext/rudb/NuDB/include/nudb/file.hpp +55 -0
  34. data/ext/rudb/NuDB/include/nudb/impl/basic_store.ipp +785 -0
  35. data/ext/rudb/NuDB/include/nudb/impl/context.ipp +241 -0
  36. data/ext/rudb/NuDB/include/nudb/impl/create.ipp +163 -0
  37. data/ext/rudb/NuDB/include/nudb/impl/error.ipp +175 -0
  38. data/ext/rudb/NuDB/include/nudb/impl/posix_file.ipp +248 -0
  39. data/ext/rudb/NuDB/include/nudb/impl/recover.ipp +209 -0
  40. data/ext/rudb/NuDB/include/nudb/impl/rekey.ipp +248 -0
  41. data/ext/rudb/NuDB/include/nudb/impl/verify.ipp +634 -0
  42. data/ext/rudb/NuDB/include/nudb/impl/visit.ipp +96 -0
  43. data/ext/rudb/NuDB/include/nudb/impl/win32_file.ipp +264 -0
  44. data/ext/rudb/NuDB/include/nudb/native_file.hpp +76 -0
  45. data/ext/rudb/NuDB/include/nudb/nudb.hpp +27 -0
  46. data/ext/rudb/NuDB/include/nudb/posix_file.hpp +228 -0
  47. data/ext/rudb/NuDB/include/nudb/progress.hpp +32 -0
  48. data/ext/rudb/NuDB/include/nudb/recover.hpp +73 -0
  49. data/ext/rudb/NuDB/include/nudb/rekey.hpp +110 -0
  50. data/ext/rudb/NuDB/include/nudb/store.hpp +27 -0
  51. data/ext/rudb/NuDB/include/nudb/type_traits.hpp +63 -0
  52. data/ext/rudb/NuDB/include/nudb/verify.hpp +200 -0
  53. data/ext/rudb/NuDB/include/nudb/version.hpp +21 -0
  54. data/ext/rudb/NuDB/include/nudb/visit.hpp +63 -0
  55. data/ext/rudb/NuDB/include/nudb/win32_file.hpp +246 -0
  56. data/ext/rudb/NuDB/include/nudb/xxhasher.hpp +45 -0
  57. data/ext/rudb/extconf.rb +12 -0
  58. data/ext/rudb/rudb.cpp +234 -0
  59. data/lib/rudb/version.rb +3 -0
  60. data/lib/rudb.rb +1 -0
  61. metadata +104 -0
@@ -0,0 +1,248 @@
1
+ //
2
+ // Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
3
+ //
4
+ // Distributed under the Boost Software License, Version 1.0. (See accompanying
5
+ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6
+ //
7
+
8
+ #ifndef NUDB_IMPL_POSIX_FILE_IPP
9
+ #define NUDB_IMPL_POSIX_FILE_IPP
10
+
11
+ #include <boost/assert.hpp>
12
+ #include <limits.h>
13
+
14
+ namespace nudb {
15
+
16
+ inline
17
+ posix_file::
18
+ ~posix_file()
19
+ {
20
+ close();
21
+ }
22
+
23
+ inline
24
+ posix_file::
25
+ posix_file(posix_file &&other)
26
+ : fd_(other.fd_)
27
+ {
28
+ other.fd_ = -1;
29
+ }
30
+
31
+ inline
32
+ posix_file&
33
+ posix_file::
34
+ operator=(posix_file&& other)
35
+ {
36
+ if(&other == this)
37
+ return *this;
38
+ close();
39
+ fd_ = other.fd_;
40
+ other.fd_ = -1;
41
+ return *this;
42
+ }
43
+
44
+ inline
45
+ void
46
+ posix_file::
47
+ close()
48
+ {
49
+ if(fd_ != -1)
50
+ {
51
+ ::close(fd_);
52
+ fd_ = -1;
53
+ }
54
+ }
55
+
56
+ inline
57
+ void
58
+ posix_file::
59
+ create(file_mode mode, path_type const& path, error_code& ec)
60
+ {
61
+ auto const result = flags(mode);
62
+ BOOST_ASSERT(! is_open());
63
+ fd_ = ::open(path.c_str(), result.first | O_CREAT | O_EXCL, 0644);
64
+ if(fd_ == -1)
65
+ return last_err(ec);
66
+ #ifndef __APPLE__
67
+ if(::posix_fadvise(fd_, 0, 0, result.second) != 0)
68
+ return last_err(ec);
69
+ #endif
70
+ }
71
+
72
+ inline
73
+ void
74
+ posix_file::
75
+ open(file_mode mode, path_type const& path, error_code& ec)
76
+ {
77
+ BOOST_ASSERT(! is_open());
78
+ auto const result = flags(mode);
79
+ fd_ = ::open(path.c_str(), result.first);
80
+ if(fd_ == -1)
81
+ return last_err(ec);
82
+ #ifndef __APPLE__
83
+ if(::posix_fadvise(fd_, 0, 0, result.second) != 0)
84
+ return last_err(ec);
85
+ #endif
86
+ }
87
+
88
+ inline
89
+ void
90
+ posix_file::
91
+ erase(path_type const& path, error_code& ec)
92
+ {
93
+ if(::unlink(path.c_str()) != 0)
94
+ {
95
+ int const ev = errno;
96
+ return err(ev, ec);
97
+ }
98
+ }
99
+
100
+ inline
101
+ std::uint64_t
102
+ posix_file::
103
+ size(error_code& ec) const
104
+ {
105
+ static_assert(sizeof(stat::st_size) == sizeof(std::uint64_t), "");
106
+ struct stat st;
107
+ if(::fstat(fd_, &st) != 0)
108
+ {
109
+ last_err(ec);
110
+ return 0;
111
+ }
112
+ return st.st_size;
113
+ }
114
+ inline
115
+ void
116
+ posix_file::
117
+ read(std::uint64_t offset,
118
+ void* buffer, std::size_t bytes, error_code& ec)
119
+ {
120
+ static_assert(sizeof(off_t) >= sizeof(offset), "");
121
+ while(bytes > 0)
122
+ {
123
+ auto const amount = static_cast<ssize_t>(
124
+ std::min(bytes, static_cast<std::size_t>(SSIZE_MAX)));
125
+ auto const n = ::pread(fd_, buffer, amount, offset);
126
+ if(n == -1)
127
+ {
128
+ auto const ev = errno;
129
+ if(ev == EINTR)
130
+ continue;
131
+ return err(ev, ec);
132
+ }
133
+ if(n == 0)
134
+ {
135
+ ec = error::short_read;
136
+ return;
137
+ }
138
+ offset += n;
139
+ bytes -= n;
140
+ buffer = reinterpret_cast<char*>(buffer) + n;
141
+ }
142
+ }
143
+
144
+ inline
145
+ void
146
+ posix_file::
147
+ write(std::uint64_t offset,
148
+ void const* buffer, std::size_t bytes, error_code& ec)
149
+ {
150
+ static_assert(sizeof(off_t) >= sizeof(offset), "");
151
+ while(bytes > 0)
152
+ {
153
+ auto const amount = static_cast<ssize_t>(
154
+ std::min(bytes, static_cast<std::size_t>(SSIZE_MAX)));
155
+ auto const n = ::pwrite(fd_, buffer, amount, offset);
156
+ if(n == -1)
157
+ {
158
+ auto const ev = errno;
159
+ if(ev == EINTR)
160
+ continue;
161
+ return err(ev, ec);
162
+ }
163
+ offset += n;
164
+ bytes -= n;
165
+ buffer = reinterpret_cast<char const*>(buffer) + n;
166
+ }
167
+ }
168
+
169
+ inline
170
+ void
171
+ posix_file::
172
+ sync(error_code& ec)
173
+ {
174
+ for(;;)
175
+ {
176
+ if(::fsync(fd_) == 0)
177
+ break;
178
+ auto const ev = errno;
179
+ if(ev == EINTR)
180
+ continue;
181
+ return err(ev, ec);
182
+ }
183
+ }
184
+
185
+ inline
186
+ void
187
+ posix_file::
188
+ trunc(std::uint64_t length, error_code& ec)
189
+ {
190
+ for(;;)
191
+ {
192
+ if(::ftruncate(fd_, length) == 0)
193
+ break;
194
+ auto const ev = errno;
195
+ if(ev == EINTR)
196
+ continue;
197
+ return err(ev, ec);
198
+ }
199
+ }
200
+
201
+ inline
202
+ std::pair<int, int>
203
+ posix_file::
204
+ flags(file_mode mode)
205
+ {
206
+ std::pair<int, int> result;
207
+ switch(mode)
208
+ {
209
+ case file_mode::scan:
210
+ result.first =
211
+ O_RDONLY;
212
+ #ifndef __APPLE__
213
+ result.second =
214
+ POSIX_FADV_SEQUENTIAL;
215
+ #endif
216
+ break;
217
+ case file_mode::read:
218
+ result.first =
219
+ O_RDONLY;
220
+ #ifndef __APPLE__
221
+ result.second =
222
+ POSIX_FADV_RANDOM;
223
+ #endif
224
+ break;
225
+ case file_mode::append:
226
+ result.first =
227
+ O_RDWR |
228
+ O_APPEND;
229
+ #ifndef __APPLE__
230
+ result.second =
231
+ POSIX_FADV_RANDOM;
232
+ #endif
233
+ break;
234
+ case file_mode::write:
235
+ result.first =
236
+ O_RDWR;
237
+ #ifndef __APPLE__
238
+ result.second =
239
+ POSIX_FADV_NORMAL;
240
+ #endif
241
+ break;
242
+ }
243
+ return result;
244
+ }
245
+
246
+ } // nudb
247
+
248
+ #endif
@@ -0,0 +1,209 @@
1
+ //
2
+ // Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
3
+ //
4
+ // Distributed under the Boost Software License, Version 1.0. (See accompanying
5
+ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6
+ //
7
+
8
+ #ifndef NUDB_IMPL_RECOVER_IPP
9
+ #define NUDB_IMPL_RECOVER_IPP
10
+
11
+ #include <nudb/concepts.hpp>
12
+ #include <nudb/file.hpp>
13
+ #include <nudb/type_traits.hpp>
14
+ #include <nudb/detail/bucket.hpp>
15
+ #include <nudb/detail/bulkio.hpp>
16
+ #include <nudb/detail/format.hpp>
17
+ #include <boost/assert.hpp>
18
+ #include <algorithm>
19
+ #include <cstddef>
20
+ #include <string>
21
+
22
+ namespace nudb {
23
+
24
+ template<
25
+ class Hasher,
26
+ class File,
27
+ class... Args>
28
+ void
29
+ recover(
30
+ path_type const& dat_path,
31
+ path_type const& key_path,
32
+ path_type const& log_path,
33
+ error_code& ec,
34
+ Args&&... args)
35
+ {
36
+ static_assert(is_File<File>::value,
37
+ "File requirements not met");
38
+ static_assert(is_Hasher<Hasher>::value,
39
+ "Hasher requirements not met");
40
+ using namespace detail;
41
+
42
+ // Open data file
43
+ File df{args...};
44
+ df.open(file_mode::write, dat_path, ec);
45
+ if(ec)
46
+ return;
47
+ auto const dataFileSize = df.size(ec);
48
+ if(ec)
49
+ return;
50
+ dat_file_header dh;
51
+ read(df, dh, ec);
52
+ if(ec)
53
+ return;
54
+ verify(dh, ec);
55
+ if(ec)
56
+ return;
57
+
58
+ // Open key file
59
+ File kf{args...};
60
+ kf.open(file_mode::write, key_path, ec);
61
+ if(ec)
62
+ return;
63
+ auto const keyFileSize = kf.size(ec);
64
+ if(ec)
65
+ return;
66
+ if(keyFileSize <= key_file_header::size)
67
+ {
68
+ kf.close();
69
+ erase_file(log_path, ec);
70
+ if(ec)
71
+ return;
72
+ File::erase(key_path, ec);
73
+ if(ec)
74
+ return;
75
+ ec = error::no_key_file;
76
+ return;
77
+ }
78
+
79
+ // Open log file
80
+ File lf{args...};
81
+ lf.open(file_mode::append, log_path, ec);
82
+ if(ec == errc::no_such_file_or_directory)
83
+ {
84
+ ec = {};
85
+ return;
86
+ }
87
+ if(ec)
88
+ return;
89
+ auto const logFileSize = lf.size(ec);
90
+ if(ec)
91
+ return;
92
+ // Read log file header
93
+ log_file_header lh;
94
+ read(lf, lh, ec);
95
+ if(ec == error::short_read)
96
+ {
97
+ BOOST_ASSERT(keyFileSize > key_file_header::size);
98
+ ec = {};
99
+ goto clear_log;
100
+ }
101
+ if(ec)
102
+ return;
103
+ verify<Hasher>(lh, ec);
104
+ if(ec)
105
+ return;
106
+ if(lh.key_file_size == 0)
107
+ goto trunc_files;
108
+ {
109
+ // Read key file header
110
+ key_file_header kh;
111
+ read(kf, kh, ec);
112
+ if(ec)
113
+ return;
114
+ verify<Hasher>(kh, ec);
115
+ if(ec)
116
+ return;
117
+ verify<Hasher>(dh, kh, ec);
118
+ if(ec)
119
+ return;
120
+ verify<Hasher>(kh, lh, ec);
121
+ if(ec)
122
+ return;
123
+
124
+ auto const readSize = 1024 * kh.block_size;
125
+ auto const bucketSize = bucket_size(kh.capacity);
126
+ buffer buf{kh.block_size};
127
+ bucket b{kh.block_size, buf.get()};
128
+ bulk_reader<File> r{lf,
129
+ log_file_header::size, logFileSize, readSize};
130
+ while(! r.eof())
131
+ {
132
+ // Log Record
133
+ auto is = r.prepare(field<std::uint64_t>::size, ec);
134
+ // Log file is incomplete, so roll back.
135
+ if(ec == error::short_read)
136
+ {
137
+ ec = {};
138
+ break;
139
+ }
140
+ if(ec)
141
+ return;
142
+ nsize_t n;
143
+ {
144
+ std::uint64_t v;
145
+ // VFALCO This should have been a uint32_t
146
+ read<std::uint64_t>(is, v); // Index
147
+ BOOST_ASSERT(v <= std::numeric_limits<std::uint32_t>::max());
148
+ n = static_cast<nsize_t>(v);
149
+ }
150
+ b.read(r, ec); // Bucket
151
+ if(ec == error::short_read)
152
+ {
153
+ ec = {};
154
+ break;
155
+ }
156
+ if(b.spill() && b.spill() + bucketSize > dataFileSize)
157
+ {
158
+ ec = error::invalid_log_spill;
159
+ return;
160
+ }
161
+ if(n > kh.buckets)
162
+ {
163
+ ec = error::invalid_log_index;
164
+ return;
165
+ }
166
+ b.write(kf, static_cast<noff_t>(n + 1) * kh.block_size, ec);
167
+ if(ec)
168
+ return;
169
+ }
170
+ }
171
+ trunc_files:
172
+ df.trunc(lh.dat_file_size, ec);
173
+ if(ec)
174
+ return;
175
+ df.sync(ec);
176
+ if(ec)
177
+ return;
178
+ if(lh.key_file_size != 0)
179
+ {
180
+ kf.trunc(lh.key_file_size, ec);
181
+ if(ec)
182
+ return;
183
+ kf.sync(ec);
184
+ if(ec)
185
+ return;
186
+ }
187
+ else
188
+ {
189
+ kf.close();
190
+ File::erase(key_path, ec);
191
+ if(ec)
192
+ return;
193
+ }
194
+ clear_log:
195
+ lf.trunc(0, ec);
196
+ if(ec)
197
+ return;
198
+ lf.sync(ec);
199
+ if(ec)
200
+ return;
201
+ lf.close();
202
+ File::erase(log_path, ec);
203
+ if(ec)
204
+ return;
205
+ }
206
+
207
+ } // nudb
208
+
209
+ #endif
@@ -0,0 +1,248 @@
1
+ //
2
+ // Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
3
+ //
4
+ // Distributed under the Boost Software License, Version 1.0. (See accompanying
5
+ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6
+ //
7
+
8
+ #ifndef NUDB_IMPL_REKEY_IPP
9
+ #define NUDB_IMPL_REKEY_IPP
10
+
11
+ #include <nudb/concepts.hpp>
12
+ #include <nudb/create.hpp>
13
+ #include <nudb/detail/bucket.hpp>
14
+ #include <nudb/detail/bulkio.hpp>
15
+ #include <nudb/detail/format.hpp>
16
+ #include <cmath>
17
+
18
+ namespace nudb {
19
+
20
+ // VFALCO Should this delete the key file on an error?
21
+ template<
22
+ class Hasher,
23
+ class File,
24
+ class Progress,
25
+ class... Args
26
+ >
27
+ void
28
+ rekey(
29
+ path_type const& dat_path,
30
+ path_type const& key_path,
31
+ path_type const& log_path,
32
+ std::size_t blockSize,
33
+ float loadFactor,
34
+ std::uint64_t itemCount,
35
+ std::size_t bufferSize,
36
+ error_code& ec,
37
+ Progress&& progress,
38
+ Args&&... args)
39
+ {
40
+ static_assert(is_File<File>::value,
41
+ "File requirements not met");
42
+ static_assert(is_Hasher<Hasher>::value,
43
+ "Hasher requirements not met");
44
+ static_assert(is_Progress<Progress>::value,
45
+ "Progress requirements not met");
46
+ using namespace detail;
47
+ auto const readSize = 1024 * block_size(dat_path);
48
+ auto const writeSize = 16 * block_size(key_path);
49
+
50
+ // Open data file for reading and appending
51
+ File df{args...};
52
+ df.open(file_mode::append, dat_path, ec);
53
+ if(ec)
54
+ return;
55
+ dat_file_header dh;
56
+ read(df, dh, ec);
57
+ if(ec)
58
+ return;
59
+ verify(dh, ec);
60
+ if(ec)
61
+ return;
62
+ auto const dataFileSize = df.size(ec);
63
+ if(ec)
64
+ return;
65
+
66
+ // Make sure log file doesn't exist
67
+ File lf{args...};
68
+ lf.open(file_mode::read, log_path, ec);
69
+ if(! ec)
70
+ ec = error::log_file_exists;
71
+ if(ec != errc::no_such_file_or_directory)
72
+ return;
73
+ ec = {};
74
+
75
+ // Set up key file header
76
+ key_file_header kh;
77
+ kh.version = currentVersion;
78
+ kh.uid = dh.uid;
79
+ kh.appnum = dh.appnum;
80
+ kh.key_size = dh.key_size;
81
+ kh.salt = make_salt();
82
+ kh.pepper = pepper<Hasher>(kh.salt);
83
+ kh.block_size = blockSize;
84
+ kh.load_factor = std::min<std::size_t>(
85
+ static_cast<std::size_t>(65536.0 * loadFactor), 65535);
86
+ kh.buckets = static_cast<std::size_t>(
87
+ std::ceil(itemCount /(
88
+ bucket_capacity(kh.block_size) * loadFactor)));
89
+ kh.modulus = ceil_pow2(kh.buckets);
90
+ // Create key file
91
+ File kf{args...};
92
+ kf.create(file_mode::write, key_path, ec);
93
+ if(ec)
94
+ return;
95
+ // Write key file header
96
+ // Note, file size is less than any valid block_size here
97
+ {
98
+ std::array<std::uint8_t, key_file_header::size> buf;
99
+ ostream os{buf.data(), buf.size()};
100
+ write(os, kh);
101
+ kf.write(0, buf.data(), buf.size(), ec);
102
+ if(ec)
103
+ return;
104
+ kf.sync(ec);
105
+ if(ec)
106
+ return;
107
+ }
108
+
109
+ // Create log file
110
+ lf.create(file_mode::append, log_path, ec);
111
+ if(ec)
112
+ return;
113
+ // Write log file header
114
+ {
115
+ log_file_header lh;
116
+ lh.version = currentVersion; // Version
117
+ lh.uid = kh.uid; // UID
118
+ lh.appnum = kh.appnum; // Appnum
119
+ lh.key_size = kh.key_size; // Key Size
120
+ lh.salt = kh.salt; // Salt
121
+ lh.pepper = pepper<Hasher>(kh.salt); // Pepper
122
+ lh.block_size = kh.block_size; // Block Size
123
+ lh.key_file_size = 0; // Key File Size
124
+ lh.dat_file_size = dataFileSize; // Data File Size
125
+ write(lf, lh, ec);
126
+ if(ec)
127
+ return;
128
+ lf.sync(ec);
129
+ if(ec)
130
+ return;
131
+ }
132
+
133
+ // Create full key file
134
+ buffer buf{kh.block_size};
135
+ {
136
+ // Write key file header
137
+ std::memset(buf.get(), 0, kh.block_size);
138
+ ostream os{buf.get(), kh.block_size};
139
+ write(os, kh);
140
+ kf.write(0, buf.get(), buf.size(), ec);
141
+ if(ec)
142
+ return;
143
+ kf.sync(ec);
144
+ if(ec)
145
+ return;
146
+ // Pre-allocate space for the entire key file
147
+ std::uint8_t zero = 0;
148
+ kf.write(
149
+ static_cast<noff_t>(kh.buckets + 1) * kh.block_size - 1,
150
+ &zero, 1, ec);
151
+ if(ec)
152
+ return;
153
+ kf.sync(ec);
154
+ if(ec)
155
+ return;
156
+ }
157
+
158
+ // Build contiguous sequential sections of the
159
+ // key file using multiple passes over the data.
160
+ //
161
+ auto const chunkSize = std::max<std::size_t>(1,
162
+ bufferSize / kh.block_size);
163
+ // Calculate work required
164
+ auto const passes =
165
+ (kh.buckets + chunkSize - 1) / chunkSize;
166
+ auto const nwork = passes * dataFileSize;
167
+ progress(0, nwork);
168
+
169
+ buf.reserve(chunkSize * kh.block_size);
170
+ bulk_writer<File> dw{df, dataFileSize, writeSize};
171
+ for(nbuck_t b0 = 0; b0 < kh.buckets; b0 += chunkSize)
172
+ {
173
+ auto const b1 = std::min<std::size_t>(b0 + chunkSize, kh.buckets);
174
+ // Buffered range is [b0, b1)
175
+ auto const bn = b1 - b0;
176
+ // Create empty buckets
177
+ for(std::size_t i = 0; i < bn; ++i)
178
+ bucket b{kh.block_size,
179
+ buf.get() + i * kh.block_size, empty};
180
+ // Insert all keys into buckets
181
+ // Iterate Data File
182
+ bulk_reader<File> r{df,
183
+ dat_file_header::size, dataFileSize, readSize};
184
+ while(! r.eof())
185
+ {
186
+ auto const offset = r.offset();
187
+ // Data Record or Spill Record
188
+ nsize_t size;
189
+ auto is = r.prepare(
190
+ field<uint48_t>::size, ec); // Size
191
+ if(ec)
192
+ return;
193
+ progress((b0 / chunkSize) * dataFileSize + r.offset(), nwork);
194
+ read_size48(is, size);
195
+ if(size > 0)
196
+ {
197
+ // Data Record
198
+ is = r.prepare(
199
+ dh.key_size + // Key
200
+ size, ec); // Data
201
+ if(ec)
202
+ return;
203
+ std::uint8_t const* const key =
204
+ is.data(dh.key_size);
205
+ auto const h = hash<Hasher>(
206
+ key, dh.key_size, kh.salt);
207
+ auto const n = bucket_index(
208
+ h, kh.buckets, kh.modulus);
209
+ if(n < b0 || n >= b1)
210
+ continue;
211
+ bucket b{kh.block_size, buf.get() +
212
+ (n - b0) * kh.block_size};
213
+ maybe_spill(b, dw, ec);
214
+ if(ec)
215
+ return;
216
+ b.insert(offset, size, h);
217
+ }
218
+ else
219
+ {
220
+ // VFALCO Should never get here
221
+ // Spill Record
222
+ is = r.prepare(
223
+ field<std::uint16_t>::size, ec);
224
+ if(ec)
225
+ return;
226
+ read<std::uint16_t>(is, size); // Size
227
+ r.prepare(size, ec); // skip
228
+ if(ec)
229
+ return;
230
+ }
231
+ }
232
+ kf.write((b0 + 1) * kh.block_size, buf.get(),
233
+ static_cast<std::size_t>(bn * kh.block_size), ec);
234
+ if(ec)
235
+ return;
236
+ }
237
+ dw.flush(ec);
238
+ if(ec)
239
+ return;
240
+ lf.close();
241
+ File::erase(log_path, ec);
242
+ if(ec)
243
+ return;
244
+ }
245
+
246
+ } // nudb
247
+
248
+ #endif