rrudb 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +1 -0
- data/LICENSE.txt +22 -0
- data/README.md +26 -0
- data/examples/example.rb +39 -0
- data/ext/rudb/NuDB/include/nudb/CMakeLists.txt +104 -0
- data/ext/rudb/NuDB/include/nudb/_experimental/basic_seconds_clock.hpp +200 -0
- data/ext/rudb/NuDB/include/nudb/_experimental/chrono_util.hpp +58 -0
- data/ext/rudb/NuDB/include/nudb/_experimental/test/fail_file.hpp +343 -0
- data/ext/rudb/NuDB/include/nudb/_experimental/test/temp_dir.hpp +73 -0
- data/ext/rudb/NuDB/include/nudb/_experimental/test/test_store.hpp +451 -0
- data/ext/rudb/NuDB/include/nudb/_experimental/test/xor_shift_engine.hpp +105 -0
- data/ext/rudb/NuDB/include/nudb/_experimental/util.hpp +288 -0
- data/ext/rudb/NuDB/include/nudb/basic_store.hpp +461 -0
- data/ext/rudb/NuDB/include/nudb/concepts.hpp +205 -0
- data/ext/rudb/NuDB/include/nudb/context.hpp +144 -0
- data/ext/rudb/NuDB/include/nudb/create.hpp +117 -0
- data/ext/rudb/NuDB/include/nudb/detail/arena.hpp +296 -0
- data/ext/rudb/NuDB/include/nudb/detail/bucket.hpp +473 -0
- data/ext/rudb/NuDB/include/nudb/detail/buffer.hpp +86 -0
- data/ext/rudb/NuDB/include/nudb/detail/bulkio.hpp +196 -0
- data/ext/rudb/NuDB/include/nudb/detail/cache.hpp +236 -0
- data/ext/rudb/NuDB/include/nudb/detail/endian.hpp +93 -0
- data/ext/rudb/NuDB/include/nudb/detail/field.hpp +265 -0
- data/ext/rudb/NuDB/include/nudb/detail/format.hpp +630 -0
- data/ext/rudb/NuDB/include/nudb/detail/gentex.hpp +259 -0
- data/ext/rudb/NuDB/include/nudb/detail/mutex.hpp +26 -0
- data/ext/rudb/NuDB/include/nudb/detail/pool.hpp +243 -0
- data/ext/rudb/NuDB/include/nudb/detail/store_base.hpp +45 -0
- data/ext/rudb/NuDB/include/nudb/detail/stream.hpp +149 -0
- data/ext/rudb/NuDB/include/nudb/detail/xxhash.hpp +328 -0
- data/ext/rudb/NuDB/include/nudb/error.hpp +257 -0
- data/ext/rudb/NuDB/include/nudb/file.hpp +55 -0
- data/ext/rudb/NuDB/include/nudb/impl/basic_store.ipp +785 -0
- data/ext/rudb/NuDB/include/nudb/impl/context.ipp +241 -0
- data/ext/rudb/NuDB/include/nudb/impl/create.ipp +163 -0
- data/ext/rudb/NuDB/include/nudb/impl/error.ipp +175 -0
- data/ext/rudb/NuDB/include/nudb/impl/posix_file.ipp +248 -0
- data/ext/rudb/NuDB/include/nudb/impl/recover.ipp +209 -0
- data/ext/rudb/NuDB/include/nudb/impl/rekey.ipp +248 -0
- data/ext/rudb/NuDB/include/nudb/impl/verify.ipp +634 -0
- data/ext/rudb/NuDB/include/nudb/impl/visit.ipp +96 -0
- data/ext/rudb/NuDB/include/nudb/impl/win32_file.ipp +264 -0
- data/ext/rudb/NuDB/include/nudb/native_file.hpp +76 -0
- data/ext/rudb/NuDB/include/nudb/nudb.hpp +27 -0
- data/ext/rudb/NuDB/include/nudb/posix_file.hpp +228 -0
- data/ext/rudb/NuDB/include/nudb/progress.hpp +32 -0
- data/ext/rudb/NuDB/include/nudb/recover.hpp +73 -0
- data/ext/rudb/NuDB/include/nudb/rekey.hpp +110 -0
- data/ext/rudb/NuDB/include/nudb/store.hpp +27 -0
- data/ext/rudb/NuDB/include/nudb/type_traits.hpp +63 -0
- data/ext/rudb/NuDB/include/nudb/verify.hpp +200 -0
- data/ext/rudb/NuDB/include/nudb/version.hpp +21 -0
- data/ext/rudb/NuDB/include/nudb/visit.hpp +63 -0
- data/ext/rudb/NuDB/include/nudb/win32_file.hpp +246 -0
- data/ext/rudb/NuDB/include/nudb/xxhasher.hpp +45 -0
- data/ext/rudb/extconf.rb +12 -0
- data/ext/rudb/rudb.cpp +234 -0
- data/lib/rudb/version.rb +3 -0
- data/lib/rudb.rb +1 -0
- metadata +104 -0
@@ -0,0 +1,248 @@
|
|
1
|
+
//
|
2
|
+
// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
|
3
|
+
//
|
4
|
+
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
5
|
+
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
6
|
+
//
|
7
|
+
|
8
|
+
#ifndef NUDB_IMPL_POSIX_FILE_IPP
|
9
|
+
#define NUDB_IMPL_POSIX_FILE_IPP
|
10
|
+
|
11
|
+
#include <boost/assert.hpp>
|
12
|
+
#include <limits.h>
|
13
|
+
|
14
|
+
namespace nudb {
|
15
|
+
|
16
|
+
inline
|
17
|
+
posix_file::
|
18
|
+
~posix_file()
|
19
|
+
{
|
20
|
+
close();
|
21
|
+
}
|
22
|
+
|
23
|
+
inline
|
24
|
+
posix_file::
|
25
|
+
posix_file(posix_file &&other)
|
26
|
+
: fd_(other.fd_)
|
27
|
+
{
|
28
|
+
other.fd_ = -1;
|
29
|
+
}
|
30
|
+
|
31
|
+
inline
|
32
|
+
posix_file&
|
33
|
+
posix_file::
|
34
|
+
operator=(posix_file&& other)
|
35
|
+
{
|
36
|
+
if(&other == this)
|
37
|
+
return *this;
|
38
|
+
close();
|
39
|
+
fd_ = other.fd_;
|
40
|
+
other.fd_ = -1;
|
41
|
+
return *this;
|
42
|
+
}
|
43
|
+
|
44
|
+
inline
|
45
|
+
void
|
46
|
+
posix_file::
|
47
|
+
close()
|
48
|
+
{
|
49
|
+
if(fd_ != -1)
|
50
|
+
{
|
51
|
+
::close(fd_);
|
52
|
+
fd_ = -1;
|
53
|
+
}
|
54
|
+
}
|
55
|
+
|
56
|
+
inline
|
57
|
+
void
|
58
|
+
posix_file::
|
59
|
+
create(file_mode mode, path_type const& path, error_code& ec)
|
60
|
+
{
|
61
|
+
auto const result = flags(mode);
|
62
|
+
BOOST_ASSERT(! is_open());
|
63
|
+
fd_ = ::open(path.c_str(), result.first | O_CREAT | O_EXCL, 0644);
|
64
|
+
if(fd_ == -1)
|
65
|
+
return last_err(ec);
|
66
|
+
#ifndef __APPLE__
|
67
|
+
if(::posix_fadvise(fd_, 0, 0, result.second) != 0)
|
68
|
+
return last_err(ec);
|
69
|
+
#endif
|
70
|
+
}
|
71
|
+
|
72
|
+
inline
|
73
|
+
void
|
74
|
+
posix_file::
|
75
|
+
open(file_mode mode, path_type const& path, error_code& ec)
|
76
|
+
{
|
77
|
+
BOOST_ASSERT(! is_open());
|
78
|
+
auto const result = flags(mode);
|
79
|
+
fd_ = ::open(path.c_str(), result.first);
|
80
|
+
if(fd_ == -1)
|
81
|
+
return last_err(ec);
|
82
|
+
#ifndef __APPLE__
|
83
|
+
if(::posix_fadvise(fd_, 0, 0, result.second) != 0)
|
84
|
+
return last_err(ec);
|
85
|
+
#endif
|
86
|
+
}
|
87
|
+
|
88
|
+
inline
|
89
|
+
void
|
90
|
+
posix_file::
|
91
|
+
erase(path_type const& path, error_code& ec)
|
92
|
+
{
|
93
|
+
if(::unlink(path.c_str()) != 0)
|
94
|
+
{
|
95
|
+
int const ev = errno;
|
96
|
+
return err(ev, ec);
|
97
|
+
}
|
98
|
+
}
|
99
|
+
|
100
|
+
inline
|
101
|
+
std::uint64_t
|
102
|
+
posix_file::
|
103
|
+
size(error_code& ec) const
|
104
|
+
{
|
105
|
+
static_assert(sizeof(stat::st_size) == sizeof(std::uint64_t), "");
|
106
|
+
struct stat st;
|
107
|
+
if(::fstat(fd_, &st) != 0)
|
108
|
+
{
|
109
|
+
last_err(ec);
|
110
|
+
return 0;
|
111
|
+
}
|
112
|
+
return st.st_size;
|
113
|
+
}
|
114
|
+
inline
|
115
|
+
void
|
116
|
+
posix_file::
|
117
|
+
read(std::uint64_t offset,
|
118
|
+
void* buffer, std::size_t bytes, error_code& ec)
|
119
|
+
{
|
120
|
+
static_assert(sizeof(off_t) >= sizeof(offset), "");
|
121
|
+
while(bytes > 0)
|
122
|
+
{
|
123
|
+
auto const amount = static_cast<ssize_t>(
|
124
|
+
std::min(bytes, static_cast<std::size_t>(SSIZE_MAX)));
|
125
|
+
auto const n = ::pread(fd_, buffer, amount, offset);
|
126
|
+
if(n == -1)
|
127
|
+
{
|
128
|
+
auto const ev = errno;
|
129
|
+
if(ev == EINTR)
|
130
|
+
continue;
|
131
|
+
return err(ev, ec);
|
132
|
+
}
|
133
|
+
if(n == 0)
|
134
|
+
{
|
135
|
+
ec = error::short_read;
|
136
|
+
return;
|
137
|
+
}
|
138
|
+
offset += n;
|
139
|
+
bytes -= n;
|
140
|
+
buffer = reinterpret_cast<char*>(buffer) + n;
|
141
|
+
}
|
142
|
+
}
|
143
|
+
|
144
|
+
inline
|
145
|
+
void
|
146
|
+
posix_file::
|
147
|
+
write(std::uint64_t offset,
|
148
|
+
void const* buffer, std::size_t bytes, error_code& ec)
|
149
|
+
{
|
150
|
+
static_assert(sizeof(off_t) >= sizeof(offset), "");
|
151
|
+
while(bytes > 0)
|
152
|
+
{
|
153
|
+
auto const amount = static_cast<ssize_t>(
|
154
|
+
std::min(bytes, static_cast<std::size_t>(SSIZE_MAX)));
|
155
|
+
auto const n = ::pwrite(fd_, buffer, amount, offset);
|
156
|
+
if(n == -1)
|
157
|
+
{
|
158
|
+
auto const ev = errno;
|
159
|
+
if(ev == EINTR)
|
160
|
+
continue;
|
161
|
+
return err(ev, ec);
|
162
|
+
}
|
163
|
+
offset += n;
|
164
|
+
bytes -= n;
|
165
|
+
buffer = reinterpret_cast<char const*>(buffer) + n;
|
166
|
+
}
|
167
|
+
}
|
168
|
+
|
169
|
+
inline
|
170
|
+
void
|
171
|
+
posix_file::
|
172
|
+
sync(error_code& ec)
|
173
|
+
{
|
174
|
+
for(;;)
|
175
|
+
{
|
176
|
+
if(::fsync(fd_) == 0)
|
177
|
+
break;
|
178
|
+
auto const ev = errno;
|
179
|
+
if(ev == EINTR)
|
180
|
+
continue;
|
181
|
+
return err(ev, ec);
|
182
|
+
}
|
183
|
+
}
|
184
|
+
|
185
|
+
inline
|
186
|
+
void
|
187
|
+
posix_file::
|
188
|
+
trunc(std::uint64_t length, error_code& ec)
|
189
|
+
{
|
190
|
+
for(;;)
|
191
|
+
{
|
192
|
+
if(::ftruncate(fd_, length) == 0)
|
193
|
+
break;
|
194
|
+
auto const ev = errno;
|
195
|
+
if(ev == EINTR)
|
196
|
+
continue;
|
197
|
+
return err(ev, ec);
|
198
|
+
}
|
199
|
+
}
|
200
|
+
|
201
|
+
inline
|
202
|
+
std::pair<int, int>
|
203
|
+
posix_file::
|
204
|
+
flags(file_mode mode)
|
205
|
+
{
|
206
|
+
std::pair<int, int> result;
|
207
|
+
switch(mode)
|
208
|
+
{
|
209
|
+
case file_mode::scan:
|
210
|
+
result.first =
|
211
|
+
O_RDONLY;
|
212
|
+
#ifndef __APPLE__
|
213
|
+
result.second =
|
214
|
+
POSIX_FADV_SEQUENTIAL;
|
215
|
+
#endif
|
216
|
+
break;
|
217
|
+
case file_mode::read:
|
218
|
+
result.first =
|
219
|
+
O_RDONLY;
|
220
|
+
#ifndef __APPLE__
|
221
|
+
result.second =
|
222
|
+
POSIX_FADV_RANDOM;
|
223
|
+
#endif
|
224
|
+
break;
|
225
|
+
case file_mode::append:
|
226
|
+
result.first =
|
227
|
+
O_RDWR |
|
228
|
+
O_APPEND;
|
229
|
+
#ifndef __APPLE__
|
230
|
+
result.second =
|
231
|
+
POSIX_FADV_RANDOM;
|
232
|
+
#endif
|
233
|
+
break;
|
234
|
+
case file_mode::write:
|
235
|
+
result.first =
|
236
|
+
O_RDWR;
|
237
|
+
#ifndef __APPLE__
|
238
|
+
result.second =
|
239
|
+
POSIX_FADV_NORMAL;
|
240
|
+
#endif
|
241
|
+
break;
|
242
|
+
}
|
243
|
+
return result;
|
244
|
+
}
|
245
|
+
|
246
|
+
} // nudb
|
247
|
+
|
248
|
+
#endif
|
@@ -0,0 +1,209 @@
|
|
1
|
+
//
|
2
|
+
// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
|
3
|
+
//
|
4
|
+
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
5
|
+
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
6
|
+
//
|
7
|
+
|
8
|
+
#ifndef NUDB_IMPL_RECOVER_IPP
|
9
|
+
#define NUDB_IMPL_RECOVER_IPP
|
10
|
+
|
11
|
+
#include <nudb/concepts.hpp>
|
12
|
+
#include <nudb/file.hpp>
|
13
|
+
#include <nudb/type_traits.hpp>
|
14
|
+
#include <nudb/detail/bucket.hpp>
|
15
|
+
#include <nudb/detail/bulkio.hpp>
|
16
|
+
#include <nudb/detail/format.hpp>
|
17
|
+
#include <boost/assert.hpp>
|
18
|
+
#include <algorithm>
|
19
|
+
#include <cstddef>
|
20
|
+
#include <string>
|
21
|
+
|
22
|
+
namespace nudb {
|
23
|
+
|
24
|
+
template<
|
25
|
+
class Hasher,
|
26
|
+
class File,
|
27
|
+
class... Args>
|
28
|
+
void
|
29
|
+
recover(
|
30
|
+
path_type const& dat_path,
|
31
|
+
path_type const& key_path,
|
32
|
+
path_type const& log_path,
|
33
|
+
error_code& ec,
|
34
|
+
Args&&... args)
|
35
|
+
{
|
36
|
+
static_assert(is_File<File>::value,
|
37
|
+
"File requirements not met");
|
38
|
+
static_assert(is_Hasher<Hasher>::value,
|
39
|
+
"Hasher requirements not met");
|
40
|
+
using namespace detail;
|
41
|
+
|
42
|
+
// Open data file
|
43
|
+
File df{args...};
|
44
|
+
df.open(file_mode::write, dat_path, ec);
|
45
|
+
if(ec)
|
46
|
+
return;
|
47
|
+
auto const dataFileSize = df.size(ec);
|
48
|
+
if(ec)
|
49
|
+
return;
|
50
|
+
dat_file_header dh;
|
51
|
+
read(df, dh, ec);
|
52
|
+
if(ec)
|
53
|
+
return;
|
54
|
+
verify(dh, ec);
|
55
|
+
if(ec)
|
56
|
+
return;
|
57
|
+
|
58
|
+
// Open key file
|
59
|
+
File kf{args...};
|
60
|
+
kf.open(file_mode::write, key_path, ec);
|
61
|
+
if(ec)
|
62
|
+
return;
|
63
|
+
auto const keyFileSize = kf.size(ec);
|
64
|
+
if(ec)
|
65
|
+
return;
|
66
|
+
if(keyFileSize <= key_file_header::size)
|
67
|
+
{
|
68
|
+
kf.close();
|
69
|
+
erase_file(log_path, ec);
|
70
|
+
if(ec)
|
71
|
+
return;
|
72
|
+
File::erase(key_path, ec);
|
73
|
+
if(ec)
|
74
|
+
return;
|
75
|
+
ec = error::no_key_file;
|
76
|
+
return;
|
77
|
+
}
|
78
|
+
|
79
|
+
// Open log file
|
80
|
+
File lf{args...};
|
81
|
+
lf.open(file_mode::append, log_path, ec);
|
82
|
+
if(ec == errc::no_such_file_or_directory)
|
83
|
+
{
|
84
|
+
ec = {};
|
85
|
+
return;
|
86
|
+
}
|
87
|
+
if(ec)
|
88
|
+
return;
|
89
|
+
auto const logFileSize = lf.size(ec);
|
90
|
+
if(ec)
|
91
|
+
return;
|
92
|
+
// Read log file header
|
93
|
+
log_file_header lh;
|
94
|
+
read(lf, lh, ec);
|
95
|
+
if(ec == error::short_read)
|
96
|
+
{
|
97
|
+
BOOST_ASSERT(keyFileSize > key_file_header::size);
|
98
|
+
ec = {};
|
99
|
+
goto clear_log;
|
100
|
+
}
|
101
|
+
if(ec)
|
102
|
+
return;
|
103
|
+
verify<Hasher>(lh, ec);
|
104
|
+
if(ec)
|
105
|
+
return;
|
106
|
+
if(lh.key_file_size == 0)
|
107
|
+
goto trunc_files;
|
108
|
+
{
|
109
|
+
// Read key file header
|
110
|
+
key_file_header kh;
|
111
|
+
read(kf, kh, ec);
|
112
|
+
if(ec)
|
113
|
+
return;
|
114
|
+
verify<Hasher>(kh, ec);
|
115
|
+
if(ec)
|
116
|
+
return;
|
117
|
+
verify<Hasher>(dh, kh, ec);
|
118
|
+
if(ec)
|
119
|
+
return;
|
120
|
+
verify<Hasher>(kh, lh, ec);
|
121
|
+
if(ec)
|
122
|
+
return;
|
123
|
+
|
124
|
+
auto const readSize = 1024 * kh.block_size;
|
125
|
+
auto const bucketSize = bucket_size(kh.capacity);
|
126
|
+
buffer buf{kh.block_size};
|
127
|
+
bucket b{kh.block_size, buf.get()};
|
128
|
+
bulk_reader<File> r{lf,
|
129
|
+
log_file_header::size, logFileSize, readSize};
|
130
|
+
while(! r.eof())
|
131
|
+
{
|
132
|
+
// Log Record
|
133
|
+
auto is = r.prepare(field<std::uint64_t>::size, ec);
|
134
|
+
// Log file is incomplete, so roll back.
|
135
|
+
if(ec == error::short_read)
|
136
|
+
{
|
137
|
+
ec = {};
|
138
|
+
break;
|
139
|
+
}
|
140
|
+
if(ec)
|
141
|
+
return;
|
142
|
+
nsize_t n;
|
143
|
+
{
|
144
|
+
std::uint64_t v;
|
145
|
+
// VFALCO This should have been a uint32_t
|
146
|
+
read<std::uint64_t>(is, v); // Index
|
147
|
+
BOOST_ASSERT(v <= std::numeric_limits<std::uint32_t>::max());
|
148
|
+
n = static_cast<nsize_t>(v);
|
149
|
+
}
|
150
|
+
b.read(r, ec); // Bucket
|
151
|
+
if(ec == error::short_read)
|
152
|
+
{
|
153
|
+
ec = {};
|
154
|
+
break;
|
155
|
+
}
|
156
|
+
if(b.spill() && b.spill() + bucketSize > dataFileSize)
|
157
|
+
{
|
158
|
+
ec = error::invalid_log_spill;
|
159
|
+
return;
|
160
|
+
}
|
161
|
+
if(n > kh.buckets)
|
162
|
+
{
|
163
|
+
ec = error::invalid_log_index;
|
164
|
+
return;
|
165
|
+
}
|
166
|
+
b.write(kf, static_cast<noff_t>(n + 1) * kh.block_size, ec);
|
167
|
+
if(ec)
|
168
|
+
return;
|
169
|
+
}
|
170
|
+
}
|
171
|
+
trunc_files:
|
172
|
+
df.trunc(lh.dat_file_size, ec);
|
173
|
+
if(ec)
|
174
|
+
return;
|
175
|
+
df.sync(ec);
|
176
|
+
if(ec)
|
177
|
+
return;
|
178
|
+
if(lh.key_file_size != 0)
|
179
|
+
{
|
180
|
+
kf.trunc(lh.key_file_size, ec);
|
181
|
+
if(ec)
|
182
|
+
return;
|
183
|
+
kf.sync(ec);
|
184
|
+
if(ec)
|
185
|
+
return;
|
186
|
+
}
|
187
|
+
else
|
188
|
+
{
|
189
|
+
kf.close();
|
190
|
+
File::erase(key_path, ec);
|
191
|
+
if(ec)
|
192
|
+
return;
|
193
|
+
}
|
194
|
+
clear_log:
|
195
|
+
lf.trunc(0, ec);
|
196
|
+
if(ec)
|
197
|
+
return;
|
198
|
+
lf.sync(ec);
|
199
|
+
if(ec)
|
200
|
+
return;
|
201
|
+
lf.close();
|
202
|
+
File::erase(log_path, ec);
|
203
|
+
if(ec)
|
204
|
+
return;
|
205
|
+
}
|
206
|
+
|
207
|
+
} // nudb
|
208
|
+
|
209
|
+
#endif
|
@@ -0,0 +1,248 @@
|
|
1
|
+
//
|
2
|
+
// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
|
3
|
+
//
|
4
|
+
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
5
|
+
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
6
|
+
//
|
7
|
+
|
8
|
+
#ifndef NUDB_IMPL_REKEY_IPP
|
9
|
+
#define NUDB_IMPL_REKEY_IPP
|
10
|
+
|
11
|
+
#include <nudb/concepts.hpp>
|
12
|
+
#include <nudb/create.hpp>
|
13
|
+
#include <nudb/detail/bucket.hpp>
|
14
|
+
#include <nudb/detail/bulkio.hpp>
|
15
|
+
#include <nudb/detail/format.hpp>
|
16
|
+
#include <cmath>
|
17
|
+
|
18
|
+
namespace nudb {
|
19
|
+
|
20
|
+
// VFALCO Should this delete the key file on an error?
|
21
|
+
template<
|
22
|
+
class Hasher,
|
23
|
+
class File,
|
24
|
+
class Progress,
|
25
|
+
class... Args
|
26
|
+
>
|
27
|
+
void
|
28
|
+
rekey(
|
29
|
+
path_type const& dat_path,
|
30
|
+
path_type const& key_path,
|
31
|
+
path_type const& log_path,
|
32
|
+
std::size_t blockSize,
|
33
|
+
float loadFactor,
|
34
|
+
std::uint64_t itemCount,
|
35
|
+
std::size_t bufferSize,
|
36
|
+
error_code& ec,
|
37
|
+
Progress&& progress,
|
38
|
+
Args&&... args)
|
39
|
+
{
|
40
|
+
static_assert(is_File<File>::value,
|
41
|
+
"File requirements not met");
|
42
|
+
static_assert(is_Hasher<Hasher>::value,
|
43
|
+
"Hasher requirements not met");
|
44
|
+
static_assert(is_Progress<Progress>::value,
|
45
|
+
"Progress requirements not met");
|
46
|
+
using namespace detail;
|
47
|
+
auto const readSize = 1024 * block_size(dat_path);
|
48
|
+
auto const writeSize = 16 * block_size(key_path);
|
49
|
+
|
50
|
+
// Open data file for reading and appending
|
51
|
+
File df{args...};
|
52
|
+
df.open(file_mode::append, dat_path, ec);
|
53
|
+
if(ec)
|
54
|
+
return;
|
55
|
+
dat_file_header dh;
|
56
|
+
read(df, dh, ec);
|
57
|
+
if(ec)
|
58
|
+
return;
|
59
|
+
verify(dh, ec);
|
60
|
+
if(ec)
|
61
|
+
return;
|
62
|
+
auto const dataFileSize = df.size(ec);
|
63
|
+
if(ec)
|
64
|
+
return;
|
65
|
+
|
66
|
+
// Make sure log file doesn't exist
|
67
|
+
File lf{args...};
|
68
|
+
lf.open(file_mode::read, log_path, ec);
|
69
|
+
if(! ec)
|
70
|
+
ec = error::log_file_exists;
|
71
|
+
if(ec != errc::no_such_file_or_directory)
|
72
|
+
return;
|
73
|
+
ec = {};
|
74
|
+
|
75
|
+
// Set up key file header
|
76
|
+
key_file_header kh;
|
77
|
+
kh.version = currentVersion;
|
78
|
+
kh.uid = dh.uid;
|
79
|
+
kh.appnum = dh.appnum;
|
80
|
+
kh.key_size = dh.key_size;
|
81
|
+
kh.salt = make_salt();
|
82
|
+
kh.pepper = pepper<Hasher>(kh.salt);
|
83
|
+
kh.block_size = blockSize;
|
84
|
+
kh.load_factor = std::min<std::size_t>(
|
85
|
+
static_cast<std::size_t>(65536.0 * loadFactor), 65535);
|
86
|
+
kh.buckets = static_cast<std::size_t>(
|
87
|
+
std::ceil(itemCount /(
|
88
|
+
bucket_capacity(kh.block_size) * loadFactor)));
|
89
|
+
kh.modulus = ceil_pow2(kh.buckets);
|
90
|
+
// Create key file
|
91
|
+
File kf{args...};
|
92
|
+
kf.create(file_mode::write, key_path, ec);
|
93
|
+
if(ec)
|
94
|
+
return;
|
95
|
+
// Write key file header
|
96
|
+
// Note, file size is less than any valid block_size here
|
97
|
+
{
|
98
|
+
std::array<std::uint8_t, key_file_header::size> buf;
|
99
|
+
ostream os{buf.data(), buf.size()};
|
100
|
+
write(os, kh);
|
101
|
+
kf.write(0, buf.data(), buf.size(), ec);
|
102
|
+
if(ec)
|
103
|
+
return;
|
104
|
+
kf.sync(ec);
|
105
|
+
if(ec)
|
106
|
+
return;
|
107
|
+
}
|
108
|
+
|
109
|
+
// Create log file
|
110
|
+
lf.create(file_mode::append, log_path, ec);
|
111
|
+
if(ec)
|
112
|
+
return;
|
113
|
+
// Write log file header
|
114
|
+
{
|
115
|
+
log_file_header lh;
|
116
|
+
lh.version = currentVersion; // Version
|
117
|
+
lh.uid = kh.uid; // UID
|
118
|
+
lh.appnum = kh.appnum; // Appnum
|
119
|
+
lh.key_size = kh.key_size; // Key Size
|
120
|
+
lh.salt = kh.salt; // Salt
|
121
|
+
lh.pepper = pepper<Hasher>(kh.salt); // Pepper
|
122
|
+
lh.block_size = kh.block_size; // Block Size
|
123
|
+
lh.key_file_size = 0; // Key File Size
|
124
|
+
lh.dat_file_size = dataFileSize; // Data File Size
|
125
|
+
write(lf, lh, ec);
|
126
|
+
if(ec)
|
127
|
+
return;
|
128
|
+
lf.sync(ec);
|
129
|
+
if(ec)
|
130
|
+
return;
|
131
|
+
}
|
132
|
+
|
133
|
+
// Create full key file
|
134
|
+
buffer buf{kh.block_size};
|
135
|
+
{
|
136
|
+
// Write key file header
|
137
|
+
std::memset(buf.get(), 0, kh.block_size);
|
138
|
+
ostream os{buf.get(), kh.block_size};
|
139
|
+
write(os, kh);
|
140
|
+
kf.write(0, buf.get(), buf.size(), ec);
|
141
|
+
if(ec)
|
142
|
+
return;
|
143
|
+
kf.sync(ec);
|
144
|
+
if(ec)
|
145
|
+
return;
|
146
|
+
// Pre-allocate space for the entire key file
|
147
|
+
std::uint8_t zero = 0;
|
148
|
+
kf.write(
|
149
|
+
static_cast<noff_t>(kh.buckets + 1) * kh.block_size - 1,
|
150
|
+
&zero, 1, ec);
|
151
|
+
if(ec)
|
152
|
+
return;
|
153
|
+
kf.sync(ec);
|
154
|
+
if(ec)
|
155
|
+
return;
|
156
|
+
}
|
157
|
+
|
158
|
+
// Build contiguous sequential sections of the
|
159
|
+
// key file using multiple passes over the data.
|
160
|
+
//
|
161
|
+
auto const chunkSize = std::max<std::size_t>(1,
|
162
|
+
bufferSize / kh.block_size);
|
163
|
+
// Calculate work required
|
164
|
+
auto const passes =
|
165
|
+
(kh.buckets + chunkSize - 1) / chunkSize;
|
166
|
+
auto const nwork = passes * dataFileSize;
|
167
|
+
progress(0, nwork);
|
168
|
+
|
169
|
+
buf.reserve(chunkSize * kh.block_size);
|
170
|
+
bulk_writer<File> dw{df, dataFileSize, writeSize};
|
171
|
+
for(nbuck_t b0 = 0; b0 < kh.buckets; b0 += chunkSize)
|
172
|
+
{
|
173
|
+
auto const b1 = std::min<std::size_t>(b0 + chunkSize, kh.buckets);
|
174
|
+
// Buffered range is [b0, b1)
|
175
|
+
auto const bn = b1 - b0;
|
176
|
+
// Create empty buckets
|
177
|
+
for(std::size_t i = 0; i < bn; ++i)
|
178
|
+
bucket b{kh.block_size,
|
179
|
+
buf.get() + i * kh.block_size, empty};
|
180
|
+
// Insert all keys into buckets
|
181
|
+
// Iterate Data File
|
182
|
+
bulk_reader<File> r{df,
|
183
|
+
dat_file_header::size, dataFileSize, readSize};
|
184
|
+
while(! r.eof())
|
185
|
+
{
|
186
|
+
auto const offset = r.offset();
|
187
|
+
// Data Record or Spill Record
|
188
|
+
nsize_t size;
|
189
|
+
auto is = r.prepare(
|
190
|
+
field<uint48_t>::size, ec); // Size
|
191
|
+
if(ec)
|
192
|
+
return;
|
193
|
+
progress((b0 / chunkSize) * dataFileSize + r.offset(), nwork);
|
194
|
+
read_size48(is, size);
|
195
|
+
if(size > 0)
|
196
|
+
{
|
197
|
+
// Data Record
|
198
|
+
is = r.prepare(
|
199
|
+
dh.key_size + // Key
|
200
|
+
size, ec); // Data
|
201
|
+
if(ec)
|
202
|
+
return;
|
203
|
+
std::uint8_t const* const key =
|
204
|
+
is.data(dh.key_size);
|
205
|
+
auto const h = hash<Hasher>(
|
206
|
+
key, dh.key_size, kh.salt);
|
207
|
+
auto const n = bucket_index(
|
208
|
+
h, kh.buckets, kh.modulus);
|
209
|
+
if(n < b0 || n >= b1)
|
210
|
+
continue;
|
211
|
+
bucket b{kh.block_size, buf.get() +
|
212
|
+
(n - b0) * kh.block_size};
|
213
|
+
maybe_spill(b, dw, ec);
|
214
|
+
if(ec)
|
215
|
+
return;
|
216
|
+
b.insert(offset, size, h);
|
217
|
+
}
|
218
|
+
else
|
219
|
+
{
|
220
|
+
// VFALCO Should never get here
|
221
|
+
// Spill Record
|
222
|
+
is = r.prepare(
|
223
|
+
field<std::uint16_t>::size, ec);
|
224
|
+
if(ec)
|
225
|
+
return;
|
226
|
+
read<std::uint16_t>(is, size); // Size
|
227
|
+
r.prepare(size, ec); // skip
|
228
|
+
if(ec)
|
229
|
+
return;
|
230
|
+
}
|
231
|
+
}
|
232
|
+
kf.write((b0 + 1) * kh.block_size, buf.get(),
|
233
|
+
static_cast<std::size_t>(bn * kh.block_size), ec);
|
234
|
+
if(ec)
|
235
|
+
return;
|
236
|
+
}
|
237
|
+
dw.flush(ec);
|
238
|
+
if(ec)
|
239
|
+
return;
|
240
|
+
lf.close();
|
241
|
+
File::erase(log_path, ec);
|
242
|
+
if(ec)
|
243
|
+
return;
|
244
|
+
}
|
245
|
+
|
246
|
+
} // nudb
|
247
|
+
|
248
|
+
#endif
|