filiptepper-leveldb-ruby 0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +24 -0
- data/README +72 -0
- data/ext/leveldb/extconf.rb +14 -0
- data/ext/leveldb/leveldb.cc +530 -0
- data/ext/leveldb/platform.rb +83 -0
- data/leveldb/Makefile +191 -0
- data/leveldb/build_detect_platform +160 -0
- data/leveldb/db/builder.cc +88 -0
- data/leveldb/db/builder.h +34 -0
- data/leveldb/db/c.cc +581 -0
- data/leveldb/db/corruption_test.cc +359 -0
- data/leveldb/db/db_bench.cc +970 -0
- data/leveldb/db/db_impl.cc +1448 -0
- data/leveldb/db/db_impl.h +194 -0
- data/leveldb/db/db_iter.cc +299 -0
- data/leveldb/db/db_iter.h +26 -0
- data/leveldb/db/db_test.cc +1901 -0
- data/leveldb/db/dbformat.cc +140 -0
- data/leveldb/db/dbformat.h +227 -0
- data/leveldb/db/dbformat_test.cc +112 -0
- data/leveldb/db/filename.cc +139 -0
- data/leveldb/db/filename.h +80 -0
- data/leveldb/db/filename_test.cc +122 -0
- data/leveldb/db/log_format.h +35 -0
- data/leveldb/db/log_reader.cc +259 -0
- data/leveldb/db/log_reader.h +108 -0
- data/leveldb/db/log_test.cc +500 -0
- data/leveldb/db/log_writer.cc +103 -0
- data/leveldb/db/log_writer.h +48 -0
- data/leveldb/db/memtable.cc +145 -0
- data/leveldb/db/memtable.h +91 -0
- data/leveldb/db/repair.cc +389 -0
- data/leveldb/db/skiplist.h +379 -0
- data/leveldb/db/skiplist_test.cc +378 -0
- data/leveldb/db/snapshot.h +66 -0
- data/leveldb/db/table_cache.cc +121 -0
- data/leveldb/db/table_cache.h +61 -0
- data/leveldb/db/version_edit.cc +266 -0
- data/leveldb/db/version_edit.h +107 -0
- data/leveldb/db/version_edit_test.cc +46 -0
- data/leveldb/db/version_set.cc +1402 -0
- data/leveldb/db/version_set.h +370 -0
- data/leveldb/db/version_set_test.cc +179 -0
- data/leveldb/db/write_batch.cc +147 -0
- data/leveldb/db/write_batch_internal.h +49 -0
- data/leveldb/db/write_batch_test.cc +120 -0
- data/leveldb/helpers/memenv/memenv.cc +374 -0
- data/leveldb/helpers/memenv/memenv.h +20 -0
- data/leveldb/helpers/memenv/memenv_test.cc +232 -0
- data/leveldb/include/leveldb/c.h +275 -0
- data/leveldb/include/leveldb/cache.h +99 -0
- data/leveldb/include/leveldb/comparator.h +63 -0
- data/leveldb/include/leveldb/db.h +161 -0
- data/leveldb/include/leveldb/env.h +323 -0
- data/leveldb/include/leveldb/filter_policy.h +70 -0
- data/leveldb/include/leveldb/iterator.h +100 -0
- data/leveldb/include/leveldb/options.h +195 -0
- data/leveldb/include/leveldb/slice.h +109 -0
- data/leveldb/include/leveldb/status.h +106 -0
- data/leveldb/include/leveldb/table.h +85 -0
- data/leveldb/include/leveldb/table_builder.h +92 -0
- data/leveldb/include/leveldb/write_batch.h +64 -0
- data/leveldb/port/atomic_pointer.h +144 -0
- data/leveldb/port/port.h +21 -0
- data/leveldb/port/port_android.cc +64 -0
- data/leveldb/port/port_android.h +159 -0
- data/leveldb/port/port_example.h +125 -0
- data/leveldb/port/port_posix.cc +50 -0
- data/leveldb/port/port_posix.h +129 -0
- data/leveldb/port/win/stdint.h +24 -0
- data/leveldb/table/block.cc +267 -0
- data/leveldb/table/block.h +44 -0
- data/leveldb/table/block_builder.cc +109 -0
- data/leveldb/table/block_builder.h +57 -0
- data/leveldb/table/filter_block.cc +111 -0
- data/leveldb/table/filter_block.h +68 -0
- data/leveldb/table/filter_block_test.cc +128 -0
- data/leveldb/table/format.cc +145 -0
- data/leveldb/table/format.h +108 -0
- data/leveldb/table/iterator.cc +67 -0
- data/leveldb/table/iterator_wrapper.h +63 -0
- data/leveldb/table/merger.cc +197 -0
- data/leveldb/table/merger.h +26 -0
- data/leveldb/table/table.cc +276 -0
- data/leveldb/table/table_builder.cc +270 -0
- data/leveldb/table/table_test.cc +838 -0
- data/leveldb/table/two_level_iterator.cc +182 -0
- data/leveldb/table/two_level_iterator.h +34 -0
- data/leveldb/util/arena.cc +68 -0
- data/leveldb/util/arena.h +68 -0
- data/leveldb/util/arena_test.cc +68 -0
- data/leveldb/util/bloom.cc +95 -0
- data/leveldb/util/bloom_test.cc +159 -0
- data/leveldb/util/cache.cc +328 -0
- data/leveldb/util/cache_test.cc +186 -0
- data/leveldb/util/coding.cc +194 -0
- data/leveldb/util/coding.h +104 -0
- data/leveldb/util/coding_test.cc +173 -0
- data/leveldb/util/comparator.cc +76 -0
- data/leveldb/util/crc32c.cc +332 -0
- data/leveldb/util/crc32c.h +45 -0
- data/leveldb/util/crc32c_test.cc +72 -0
- data/leveldb/util/env.cc +96 -0
- data/leveldb/util/env_posix.cc +609 -0
- data/leveldb/util/env_test.cc +104 -0
- data/leveldb/util/filter_policy.cc +11 -0
- data/leveldb/util/hash.cc +45 -0
- data/leveldb/util/hash.h +19 -0
- data/leveldb/util/histogram.cc +139 -0
- data/leveldb/util/histogram.h +42 -0
- data/leveldb/util/logging.cc +81 -0
- data/leveldb/util/logging.h +47 -0
- data/leveldb/util/mutexlock.h +39 -0
- data/leveldb/util/options.cc +29 -0
- data/leveldb/util/posix_logger.h +98 -0
- data/leveldb/util/random.h +59 -0
- data/leveldb/util/status.cc +75 -0
- data/leveldb/util/testharness.cc +77 -0
- data/leveldb/util/testharness.h +138 -0
- data/leveldb/util/testutil.cc +51 -0
- data/leveldb/util/testutil.h +53 -0
- data/lib/leveldb.rb +76 -0
- metadata +175 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
4
|
+
|
|
5
|
+
#ifndef STORAGE_LEVELDB_UTIL_CRC32C_H_
|
|
6
|
+
#define STORAGE_LEVELDB_UTIL_CRC32C_H_
|
|
7
|
+
|
|
8
|
+
#include <stddef.h>
|
|
9
|
+
#include <stdint.h>
|
|
10
|
+
|
|
11
|
+
namespace leveldb {
|
|
12
|
+
namespace crc32c {
|
|
13
|
+
|
|
14
|
+
// Return the crc32c of concat(A, data[0,n-1]) where init_crc is the
|
|
15
|
+
// crc32c of some string A. Extend() is often used to maintain the
|
|
16
|
+
// crc32c of a stream of data.
|
|
17
|
+
extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n);
|
|
18
|
+
|
|
19
|
+
// Return the crc32c of data[0,n-1]
|
|
20
|
+
inline uint32_t Value(const char* data, size_t n) {
|
|
21
|
+
return Extend(0, data, n);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
static const uint32_t kMaskDelta = 0xa282ead8ul;
|
|
25
|
+
|
|
26
|
+
// Return a masked representation of crc.
|
|
27
|
+
//
|
|
28
|
+
// Motivation: it is problematic to compute the CRC of a string that
|
|
29
|
+
// contains embedded CRCs. Therefore we recommend that CRCs stored
|
|
30
|
+
// somewhere (e.g., in files) should be masked before being stored.
|
|
31
|
+
inline uint32_t Mask(uint32_t crc) {
|
|
32
|
+
// Rotate right by 15 bits and add a constant.
|
|
33
|
+
return ((crc >> 15) | (crc << 17)) + kMaskDelta;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Return the crc whose masked representation is masked_crc.
|
|
37
|
+
inline uint32_t Unmask(uint32_t masked_crc) {
|
|
38
|
+
uint32_t rot = masked_crc - kMaskDelta;
|
|
39
|
+
return ((rot >> 17) | (rot << 15));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
} // namespace crc32c
|
|
43
|
+
} // namespace leveldb
|
|
44
|
+
|
|
45
|
+
#endif // STORAGE_LEVELDB_UTIL_CRC32C_H_
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
4
|
+
|
|
5
|
+
#include "util/crc32c.h"
|
|
6
|
+
#include "util/testharness.h"
|
|
7
|
+
|
|
8
|
+
namespace leveldb {
|
|
9
|
+
namespace crc32c {
|
|
10
|
+
|
|
11
|
+
class CRC { };
|
|
12
|
+
|
|
13
|
+
TEST(CRC, StandardResults) {
|
|
14
|
+
// From rfc3720 section B.4.
|
|
15
|
+
char buf[32];
|
|
16
|
+
|
|
17
|
+
memset(buf, 0, sizeof(buf));
|
|
18
|
+
ASSERT_EQ(0x8a9136aa, Value(buf, sizeof(buf)));
|
|
19
|
+
|
|
20
|
+
memset(buf, 0xff, sizeof(buf));
|
|
21
|
+
ASSERT_EQ(0x62a8ab43, Value(buf, sizeof(buf)));
|
|
22
|
+
|
|
23
|
+
for (int i = 0; i < 32; i++) {
|
|
24
|
+
buf[i] = i;
|
|
25
|
+
}
|
|
26
|
+
ASSERT_EQ(0x46dd794e, Value(buf, sizeof(buf)));
|
|
27
|
+
|
|
28
|
+
for (int i = 0; i < 32; i++) {
|
|
29
|
+
buf[i] = 31 - i;
|
|
30
|
+
}
|
|
31
|
+
ASSERT_EQ(0x113fdb5c, Value(buf, sizeof(buf)));
|
|
32
|
+
|
|
33
|
+
unsigned char data[48] = {
|
|
34
|
+
0x01, 0xc0, 0x00, 0x00,
|
|
35
|
+
0x00, 0x00, 0x00, 0x00,
|
|
36
|
+
0x00, 0x00, 0x00, 0x00,
|
|
37
|
+
0x00, 0x00, 0x00, 0x00,
|
|
38
|
+
0x14, 0x00, 0x00, 0x00,
|
|
39
|
+
0x00, 0x00, 0x04, 0x00,
|
|
40
|
+
0x00, 0x00, 0x00, 0x14,
|
|
41
|
+
0x00, 0x00, 0x00, 0x18,
|
|
42
|
+
0x28, 0x00, 0x00, 0x00,
|
|
43
|
+
0x00, 0x00, 0x00, 0x00,
|
|
44
|
+
0x02, 0x00, 0x00, 0x00,
|
|
45
|
+
0x00, 0x00, 0x00, 0x00,
|
|
46
|
+
};
|
|
47
|
+
ASSERT_EQ(0xd9963a56, Value(reinterpret_cast<char*>(data), sizeof(data)));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
TEST(CRC, Values) {
|
|
51
|
+
ASSERT_NE(Value("a", 1), Value("foo", 3));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
TEST(CRC, Extend) {
|
|
55
|
+
ASSERT_EQ(Value("hello world", 11),
|
|
56
|
+
Extend(Value("hello ", 6), "world", 5));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
TEST(CRC, Mask) {
|
|
60
|
+
uint32_t crc = Value("foo", 3);
|
|
61
|
+
ASSERT_NE(crc, Mask(crc));
|
|
62
|
+
ASSERT_NE(crc, Mask(Mask(crc)));
|
|
63
|
+
ASSERT_EQ(crc, Unmask(Mask(crc)));
|
|
64
|
+
ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc)))));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
} // namespace crc32c
|
|
68
|
+
} // namespace leveldb
|
|
69
|
+
|
|
70
|
+
int main(int argc, char** argv) {
|
|
71
|
+
return leveldb::test::RunAllTests();
|
|
72
|
+
}
|
data/leveldb/util/env.cc
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
4
|
+
|
|
5
|
+
#include "leveldb/env.h"
|
|
6
|
+
|
|
7
|
+
namespace leveldb {
|
|
8
|
+
|
|
9
|
+
Env::~Env() {
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
SequentialFile::~SequentialFile() {
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
RandomAccessFile::~RandomAccessFile() {
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
WritableFile::~WritableFile() {
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
Logger::~Logger() {
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
FileLock::~FileLock() {
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
void Log(Logger* info_log, const char* format, ...) {
|
|
28
|
+
if (info_log != NULL) {
|
|
29
|
+
va_list ap;
|
|
30
|
+
va_start(ap, format);
|
|
31
|
+
info_log->Logv(format, ap);
|
|
32
|
+
va_end(ap);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
static Status DoWriteStringToFile(Env* env, const Slice& data,
|
|
37
|
+
const std::string& fname,
|
|
38
|
+
bool should_sync) {
|
|
39
|
+
WritableFile* file;
|
|
40
|
+
Status s = env->NewWritableFile(fname, &file);
|
|
41
|
+
if (!s.ok()) {
|
|
42
|
+
return s;
|
|
43
|
+
}
|
|
44
|
+
s = file->Append(data);
|
|
45
|
+
if (s.ok() && should_sync) {
|
|
46
|
+
s = file->Sync();
|
|
47
|
+
}
|
|
48
|
+
if (s.ok()) {
|
|
49
|
+
s = file->Close();
|
|
50
|
+
}
|
|
51
|
+
delete file; // Will auto-close if we did not close above
|
|
52
|
+
if (!s.ok()) {
|
|
53
|
+
env->DeleteFile(fname);
|
|
54
|
+
}
|
|
55
|
+
return s;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
Status WriteStringToFile(Env* env, const Slice& data,
|
|
59
|
+
const std::string& fname) {
|
|
60
|
+
return DoWriteStringToFile(env, data, fname, false);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
Status WriteStringToFileSync(Env* env, const Slice& data,
|
|
64
|
+
const std::string& fname) {
|
|
65
|
+
return DoWriteStringToFile(env, data, fname, true);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
Status ReadFileToString(Env* env, const std::string& fname, std::string* data) {
|
|
69
|
+
data->clear();
|
|
70
|
+
SequentialFile* file;
|
|
71
|
+
Status s = env->NewSequentialFile(fname, &file);
|
|
72
|
+
if (!s.ok()) {
|
|
73
|
+
return s;
|
|
74
|
+
}
|
|
75
|
+
static const int kBufferSize = 8192;
|
|
76
|
+
char* space = new char[kBufferSize];
|
|
77
|
+
while (true) {
|
|
78
|
+
Slice fragment;
|
|
79
|
+
s = file->Read(kBufferSize, &fragment, space);
|
|
80
|
+
if (!s.ok()) {
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
data->append(fragment.data(), fragment.size());
|
|
84
|
+
if (fragment.empty()) {
|
|
85
|
+
break;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
delete[] space;
|
|
89
|
+
delete file;
|
|
90
|
+
return s;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
EnvWrapper::~EnvWrapper() {
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
} // namespace leveldb
|
|
@@ -0,0 +1,609 @@
|
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
4
|
+
|
|
5
|
+
#include <deque>
|
|
6
|
+
#include <dirent.h>
|
|
7
|
+
#include <errno.h>
|
|
8
|
+
#include <fcntl.h>
|
|
9
|
+
#include <pthread.h>
|
|
10
|
+
#include <stdio.h>
|
|
11
|
+
#include <stdlib.h>
|
|
12
|
+
#include <string.h>
|
|
13
|
+
#include <sys/mman.h>
|
|
14
|
+
#include <sys/stat.h>
|
|
15
|
+
#include <sys/time.h>
|
|
16
|
+
#include <sys/types.h>
|
|
17
|
+
#include <time.h>
|
|
18
|
+
#include <unistd.h>
|
|
19
|
+
#if defined(LEVELDB_PLATFORM_ANDROID)
|
|
20
|
+
#include <sys/stat.h>
|
|
21
|
+
#endif
|
|
22
|
+
#include "leveldb/env.h"
|
|
23
|
+
#include "leveldb/slice.h"
|
|
24
|
+
#include "port/port.h"
|
|
25
|
+
#include "util/logging.h"
|
|
26
|
+
#include "util/posix_logger.h"
|
|
27
|
+
|
|
28
|
+
namespace leveldb {
|
|
29
|
+
|
|
30
|
+
namespace {
|
|
31
|
+
|
|
32
|
+
static Status IOError(const std::string& context, int err_number) {
|
|
33
|
+
return Status::IOError(context, strerror(err_number));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
class PosixSequentialFile: public SequentialFile {
|
|
37
|
+
private:
|
|
38
|
+
std::string filename_;
|
|
39
|
+
FILE* file_;
|
|
40
|
+
|
|
41
|
+
public:
|
|
42
|
+
PosixSequentialFile(const std::string& fname, FILE* f)
|
|
43
|
+
: filename_(fname), file_(f) { }
|
|
44
|
+
virtual ~PosixSequentialFile() { fclose(file_); }
|
|
45
|
+
|
|
46
|
+
virtual Status Read(size_t n, Slice* result, char* scratch) {
|
|
47
|
+
Status s;
|
|
48
|
+
size_t r = fread_unlocked(scratch, 1, n, file_);
|
|
49
|
+
*result = Slice(scratch, r);
|
|
50
|
+
if (r < n) {
|
|
51
|
+
if (feof(file_)) {
|
|
52
|
+
// We leave status as ok if we hit the end of the file
|
|
53
|
+
} else {
|
|
54
|
+
// A partial read with an error: return a non-ok status
|
|
55
|
+
s = IOError(filename_, errno);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return s;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
virtual Status Skip(uint64_t n) {
|
|
62
|
+
if (fseek(file_, n, SEEK_CUR)) {
|
|
63
|
+
return IOError(filename_, errno);
|
|
64
|
+
}
|
|
65
|
+
return Status::OK();
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
// pread() based random-access
|
|
70
|
+
class PosixRandomAccessFile: public RandomAccessFile {
|
|
71
|
+
private:
|
|
72
|
+
std::string filename_;
|
|
73
|
+
int fd_;
|
|
74
|
+
|
|
75
|
+
public:
|
|
76
|
+
PosixRandomAccessFile(const std::string& fname, int fd)
|
|
77
|
+
: filename_(fname), fd_(fd) { }
|
|
78
|
+
virtual ~PosixRandomAccessFile() { close(fd_); }
|
|
79
|
+
|
|
80
|
+
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
|
81
|
+
char* scratch) const {
|
|
82
|
+
Status s;
|
|
83
|
+
ssize_t r = pread(fd_, scratch, n, static_cast<off_t>(offset));
|
|
84
|
+
*result = Slice(scratch, (r < 0) ? 0 : r);
|
|
85
|
+
if (r < 0) {
|
|
86
|
+
// An error: return a non-ok status
|
|
87
|
+
s = IOError(filename_, errno);
|
|
88
|
+
}
|
|
89
|
+
return s;
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
// mmap() based random-access
|
|
94
|
+
class PosixMmapReadableFile: public RandomAccessFile {
|
|
95
|
+
private:
|
|
96
|
+
std::string filename_;
|
|
97
|
+
void* mmapped_region_;
|
|
98
|
+
size_t length_;
|
|
99
|
+
|
|
100
|
+
public:
|
|
101
|
+
// base[0,length-1] contains the mmapped contents of the file.
|
|
102
|
+
PosixMmapReadableFile(const std::string& fname, void* base, size_t length)
|
|
103
|
+
: filename_(fname), mmapped_region_(base), length_(length) { }
|
|
104
|
+
virtual ~PosixMmapReadableFile() { munmap(mmapped_region_, length_); }
|
|
105
|
+
|
|
106
|
+
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
|
107
|
+
char* scratch) const {
|
|
108
|
+
Status s;
|
|
109
|
+
if (offset + n > length_) {
|
|
110
|
+
*result = Slice();
|
|
111
|
+
s = IOError(filename_, EINVAL);
|
|
112
|
+
} else {
|
|
113
|
+
*result = Slice(reinterpret_cast<char*>(mmapped_region_) + offset, n);
|
|
114
|
+
}
|
|
115
|
+
return s;
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
// We preallocate up to an extra megabyte and use memcpy to append new
|
|
120
|
+
// data to the file. This is safe since we either properly close the
|
|
121
|
+
// file before reading from it, or for log files, the reading code
|
|
122
|
+
// knows enough to skip zero suffixes.
|
|
123
|
+
class PosixMmapFile : public WritableFile {
|
|
124
|
+
private:
|
|
125
|
+
std::string filename_;
|
|
126
|
+
int fd_;
|
|
127
|
+
size_t page_size_;
|
|
128
|
+
size_t map_size_; // How much extra memory to map at a time
|
|
129
|
+
char* base_; // The mapped region
|
|
130
|
+
char* limit_; // Limit of the mapped region
|
|
131
|
+
char* dst_; // Where to write next (in range [base_,limit_])
|
|
132
|
+
char* last_sync_; // Where have we synced up to
|
|
133
|
+
uint64_t file_offset_; // Offset of base_ in file
|
|
134
|
+
|
|
135
|
+
// Have we done an munmap of unsynced data?
|
|
136
|
+
bool pending_sync_;
|
|
137
|
+
|
|
138
|
+
// Roundup x to a multiple of y
|
|
139
|
+
static size_t Roundup(size_t x, size_t y) {
|
|
140
|
+
return ((x + y - 1) / y) * y;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
size_t TruncateToPageBoundary(size_t s) {
|
|
144
|
+
s -= (s & (page_size_ - 1));
|
|
145
|
+
assert((s % page_size_) == 0);
|
|
146
|
+
return s;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
bool UnmapCurrentRegion() {
|
|
150
|
+
bool result = true;
|
|
151
|
+
if (base_ != NULL) {
|
|
152
|
+
if (last_sync_ < limit_) {
|
|
153
|
+
// Defer syncing this data until next Sync() call, if any
|
|
154
|
+
pending_sync_ = true;
|
|
155
|
+
}
|
|
156
|
+
if (munmap(base_, limit_ - base_) != 0) {
|
|
157
|
+
result = false;
|
|
158
|
+
}
|
|
159
|
+
file_offset_ += limit_ - base_;
|
|
160
|
+
base_ = NULL;
|
|
161
|
+
limit_ = NULL;
|
|
162
|
+
last_sync_ = NULL;
|
|
163
|
+
dst_ = NULL;
|
|
164
|
+
|
|
165
|
+
// Increase the amount we map the next time, but capped at 1MB
|
|
166
|
+
if (map_size_ < (1<<20)) {
|
|
167
|
+
map_size_ *= 2;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return result;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
bool MapNewRegion() {
|
|
174
|
+
assert(base_ == NULL);
|
|
175
|
+
if (ftruncate(fd_, file_offset_ + map_size_) < 0) {
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
178
|
+
void* ptr = mmap(NULL, map_size_, PROT_READ | PROT_WRITE, MAP_SHARED,
|
|
179
|
+
fd_, file_offset_);
|
|
180
|
+
if (ptr == MAP_FAILED) {
|
|
181
|
+
return false;
|
|
182
|
+
}
|
|
183
|
+
base_ = reinterpret_cast<char*>(ptr);
|
|
184
|
+
limit_ = base_ + map_size_;
|
|
185
|
+
dst_ = base_;
|
|
186
|
+
last_sync_ = base_;
|
|
187
|
+
return true;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
public:
|
|
191
|
+
PosixMmapFile(const std::string& fname, int fd, size_t page_size)
|
|
192
|
+
: filename_(fname),
|
|
193
|
+
fd_(fd),
|
|
194
|
+
page_size_(page_size),
|
|
195
|
+
map_size_(Roundup(65536, page_size)),
|
|
196
|
+
base_(NULL),
|
|
197
|
+
limit_(NULL),
|
|
198
|
+
dst_(NULL),
|
|
199
|
+
last_sync_(NULL),
|
|
200
|
+
file_offset_(0),
|
|
201
|
+
pending_sync_(false) {
|
|
202
|
+
assert((page_size & (page_size - 1)) == 0);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
~PosixMmapFile() {
|
|
207
|
+
if (fd_ >= 0) {
|
|
208
|
+
PosixMmapFile::Close();
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
virtual Status Append(const Slice& data) {
|
|
213
|
+
const char* src = data.data();
|
|
214
|
+
size_t left = data.size();
|
|
215
|
+
while (left > 0) {
|
|
216
|
+
assert(base_ <= dst_);
|
|
217
|
+
assert(dst_ <= limit_);
|
|
218
|
+
size_t avail = limit_ - dst_;
|
|
219
|
+
if (avail == 0) {
|
|
220
|
+
if (!UnmapCurrentRegion() ||
|
|
221
|
+
!MapNewRegion()) {
|
|
222
|
+
return IOError(filename_, errno);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
size_t n = (left <= avail) ? left : avail;
|
|
227
|
+
memcpy(dst_, src, n);
|
|
228
|
+
dst_ += n;
|
|
229
|
+
src += n;
|
|
230
|
+
left -= n;
|
|
231
|
+
}
|
|
232
|
+
return Status::OK();
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
virtual Status Close() {
|
|
236
|
+
Status s;
|
|
237
|
+
size_t unused = limit_ - dst_;
|
|
238
|
+
if (!UnmapCurrentRegion()) {
|
|
239
|
+
s = IOError(filename_, errno);
|
|
240
|
+
} else if (unused > 0) {
|
|
241
|
+
// Trim the extra space at the end of the file
|
|
242
|
+
if (ftruncate(fd_, file_offset_ - unused) < 0) {
|
|
243
|
+
s = IOError(filename_, errno);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
if (close(fd_) < 0) {
|
|
248
|
+
if (s.ok()) {
|
|
249
|
+
s = IOError(filename_, errno);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
fd_ = -1;
|
|
254
|
+
base_ = NULL;
|
|
255
|
+
limit_ = NULL;
|
|
256
|
+
return s;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
virtual Status Flush() {
|
|
260
|
+
return Status::OK();
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
virtual Status Sync() {
|
|
264
|
+
Status s;
|
|
265
|
+
|
|
266
|
+
if (pending_sync_) {
|
|
267
|
+
// Some unmapped data was not synced
|
|
268
|
+
pending_sync_ = false;
|
|
269
|
+
if (fdatasync(fd_) < 0) {
|
|
270
|
+
s = IOError(filename_, errno);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
if (dst_ > last_sync_) {
|
|
275
|
+
// Find the beginnings of the pages that contain the first and last
|
|
276
|
+
// bytes to be synced.
|
|
277
|
+
size_t p1 = TruncateToPageBoundary(last_sync_ - base_);
|
|
278
|
+
size_t p2 = TruncateToPageBoundary(dst_ - base_ - 1);
|
|
279
|
+
last_sync_ = dst_;
|
|
280
|
+
if (msync(base_ + p1, p2 - p1 + page_size_, MS_SYNC) < 0) {
|
|
281
|
+
s = IOError(filename_, errno);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
return s;
|
|
286
|
+
}
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
static int LockOrUnlock(int fd, bool lock) {
|
|
290
|
+
errno = 0;
|
|
291
|
+
struct flock f;
|
|
292
|
+
memset(&f, 0, sizeof(f));
|
|
293
|
+
f.l_type = (lock ? F_WRLCK : F_UNLCK);
|
|
294
|
+
f.l_whence = SEEK_SET;
|
|
295
|
+
f.l_start = 0;
|
|
296
|
+
f.l_len = 0; // Lock/unlock entire file
|
|
297
|
+
return fcntl(fd, F_SETLK, &f);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
class PosixFileLock : public FileLock {
|
|
301
|
+
public:
|
|
302
|
+
int fd_;
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
class PosixEnv : public Env {
|
|
306
|
+
public:
|
|
307
|
+
PosixEnv();
|
|
308
|
+
virtual ~PosixEnv() {
|
|
309
|
+
fprintf(stderr, "Destroying Env::Default()\n");
|
|
310
|
+
exit(1);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
virtual Status NewSequentialFile(const std::string& fname,
|
|
314
|
+
SequentialFile** result) {
|
|
315
|
+
FILE* f = fopen(fname.c_str(), "r");
|
|
316
|
+
if (f == NULL) {
|
|
317
|
+
*result = NULL;
|
|
318
|
+
return IOError(fname, errno);
|
|
319
|
+
} else {
|
|
320
|
+
*result = new PosixSequentialFile(fname, f);
|
|
321
|
+
return Status::OK();
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
virtual Status NewRandomAccessFile(const std::string& fname,
|
|
326
|
+
RandomAccessFile** result) {
|
|
327
|
+
*result = NULL;
|
|
328
|
+
Status s;
|
|
329
|
+
int fd = open(fname.c_str(), O_RDONLY);
|
|
330
|
+
if (fd < 0) {
|
|
331
|
+
s = IOError(fname, errno);
|
|
332
|
+
} else if (sizeof(void*) >= 8) {
|
|
333
|
+
// Use mmap when virtual address-space is plentiful.
|
|
334
|
+
uint64_t size;
|
|
335
|
+
s = GetFileSize(fname, &size);
|
|
336
|
+
if (s.ok()) {
|
|
337
|
+
void* base = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
|
|
338
|
+
if (base != MAP_FAILED) {
|
|
339
|
+
*result = new PosixMmapReadableFile(fname, base, size);
|
|
340
|
+
} else {
|
|
341
|
+
s = IOError(fname, errno);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
close(fd);
|
|
345
|
+
} else {
|
|
346
|
+
*result = new PosixRandomAccessFile(fname, fd);
|
|
347
|
+
}
|
|
348
|
+
return s;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
virtual Status NewWritableFile(const std::string& fname,
|
|
352
|
+
WritableFile** result) {
|
|
353
|
+
Status s;
|
|
354
|
+
const int fd = open(fname.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
|
|
355
|
+
if (fd < 0) {
|
|
356
|
+
*result = NULL;
|
|
357
|
+
s = IOError(fname, errno);
|
|
358
|
+
} else {
|
|
359
|
+
*result = new PosixMmapFile(fname, fd, page_size_);
|
|
360
|
+
}
|
|
361
|
+
return s;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
virtual bool FileExists(const std::string& fname) {
|
|
365
|
+
return access(fname.c_str(), F_OK) == 0;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
virtual Status GetChildren(const std::string& dir,
|
|
369
|
+
std::vector<std::string>* result) {
|
|
370
|
+
result->clear();
|
|
371
|
+
DIR* d = opendir(dir.c_str());
|
|
372
|
+
if (d == NULL) {
|
|
373
|
+
return IOError(dir, errno);
|
|
374
|
+
}
|
|
375
|
+
struct dirent* entry;
|
|
376
|
+
while ((entry = readdir(d)) != NULL) {
|
|
377
|
+
result->push_back(entry->d_name);
|
|
378
|
+
}
|
|
379
|
+
closedir(d);
|
|
380
|
+
return Status::OK();
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
virtual Status DeleteFile(const std::string& fname) {
|
|
384
|
+
Status result;
|
|
385
|
+
if (unlink(fname.c_str()) != 0) {
|
|
386
|
+
result = IOError(fname, errno);
|
|
387
|
+
}
|
|
388
|
+
return result;
|
|
389
|
+
};
|
|
390
|
+
|
|
391
|
+
virtual Status CreateDir(const std::string& name) {
|
|
392
|
+
Status result;
|
|
393
|
+
if (mkdir(name.c_str(), 0755) != 0) {
|
|
394
|
+
result = IOError(name, errno);
|
|
395
|
+
}
|
|
396
|
+
return result;
|
|
397
|
+
};
|
|
398
|
+
|
|
399
|
+
virtual Status DeleteDir(const std::string& name) {
|
|
400
|
+
Status result;
|
|
401
|
+
if (rmdir(name.c_str()) != 0) {
|
|
402
|
+
result = IOError(name, errno);
|
|
403
|
+
}
|
|
404
|
+
return result;
|
|
405
|
+
};
|
|
406
|
+
|
|
407
|
+
virtual Status GetFileSize(const std::string& fname, uint64_t* size) {
|
|
408
|
+
Status s;
|
|
409
|
+
struct stat sbuf;
|
|
410
|
+
if (stat(fname.c_str(), &sbuf) != 0) {
|
|
411
|
+
*size = 0;
|
|
412
|
+
s = IOError(fname, errno);
|
|
413
|
+
} else {
|
|
414
|
+
*size = sbuf.st_size;
|
|
415
|
+
}
|
|
416
|
+
return s;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
virtual Status RenameFile(const std::string& src, const std::string& target) {
|
|
420
|
+
Status result;
|
|
421
|
+
if (rename(src.c_str(), target.c_str()) != 0) {
|
|
422
|
+
result = IOError(src, errno);
|
|
423
|
+
}
|
|
424
|
+
return result;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
virtual Status LockFile(const std::string& fname, FileLock** lock) {
|
|
428
|
+
*lock = NULL;
|
|
429
|
+
Status result;
|
|
430
|
+
int fd = open(fname.c_str(), O_RDWR | O_CREAT, 0644);
|
|
431
|
+
if (fd < 0) {
|
|
432
|
+
result = IOError(fname, errno);
|
|
433
|
+
} else if (LockOrUnlock(fd, true) == -1) {
|
|
434
|
+
result = IOError("lock " + fname, errno);
|
|
435
|
+
close(fd);
|
|
436
|
+
} else {
|
|
437
|
+
PosixFileLock* my_lock = new PosixFileLock;
|
|
438
|
+
my_lock->fd_ = fd;
|
|
439
|
+
*lock = my_lock;
|
|
440
|
+
}
|
|
441
|
+
return result;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
virtual Status UnlockFile(FileLock* lock) {
|
|
445
|
+
PosixFileLock* my_lock = reinterpret_cast<PosixFileLock*>(lock);
|
|
446
|
+
Status result;
|
|
447
|
+
if (LockOrUnlock(my_lock->fd_, false) == -1) {
|
|
448
|
+
result = IOError("unlock", errno);
|
|
449
|
+
}
|
|
450
|
+
close(my_lock->fd_);
|
|
451
|
+
delete my_lock;
|
|
452
|
+
return result;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
virtual void Schedule(void (*function)(void*), void* arg);
|
|
456
|
+
|
|
457
|
+
virtual void StartThread(void (*function)(void* arg), void* arg);
|
|
458
|
+
|
|
459
|
+
virtual Status GetTestDirectory(std::string* result) {
|
|
460
|
+
const char* env = getenv("TEST_TMPDIR");
|
|
461
|
+
if (env && env[0] != '\0') {
|
|
462
|
+
*result = env;
|
|
463
|
+
} else {
|
|
464
|
+
char buf[100];
|
|
465
|
+
snprintf(buf, sizeof(buf), "/tmp/leveldbtest-%d", int(geteuid()));
|
|
466
|
+
*result = buf;
|
|
467
|
+
}
|
|
468
|
+
// Directory may already exist
|
|
469
|
+
CreateDir(*result);
|
|
470
|
+
return Status::OK();
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
static uint64_t gettid() {
|
|
474
|
+
pthread_t tid = pthread_self();
|
|
475
|
+
uint64_t thread_id = 0;
|
|
476
|
+
memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid)));
|
|
477
|
+
return thread_id;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
virtual Status NewLogger(const std::string& fname, Logger** result) {
|
|
481
|
+
FILE* f = fopen(fname.c_str(), "w");
|
|
482
|
+
if (f == NULL) {
|
|
483
|
+
*result = NULL;
|
|
484
|
+
return IOError(fname, errno);
|
|
485
|
+
} else {
|
|
486
|
+
*result = new PosixLogger(f, &PosixEnv::gettid);
|
|
487
|
+
return Status::OK();
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
virtual uint64_t NowMicros() {
|
|
492
|
+
struct timeval tv;
|
|
493
|
+
gettimeofday(&tv, NULL);
|
|
494
|
+
return static_cast<uint64_t>(tv.tv_sec) * 1000000 + tv.tv_usec;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
virtual void SleepForMicroseconds(int micros) {
|
|
498
|
+
usleep(micros);
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
private:
|
|
502
|
+
void PthreadCall(const char* label, int result) {
|
|
503
|
+
if (result != 0) {
|
|
504
|
+
fprintf(stderr, "pthread %s: %s\n", label, strerror(result));
|
|
505
|
+
exit(1);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
// BGThread() is the body of the background thread
|
|
510
|
+
void BGThread();
|
|
511
|
+
static void* BGThreadWrapper(void* arg) {
|
|
512
|
+
reinterpret_cast<PosixEnv*>(arg)->BGThread();
|
|
513
|
+
return NULL;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
size_t page_size_;
|
|
517
|
+
pthread_mutex_t mu_;
|
|
518
|
+
pthread_cond_t bgsignal_;
|
|
519
|
+
pthread_t bgthread_;
|
|
520
|
+
bool started_bgthread_;
|
|
521
|
+
|
|
522
|
+
// Entry per Schedule() call
|
|
523
|
+
struct BGItem { void* arg; void (*function)(void*); };
|
|
524
|
+
typedef std::deque<BGItem> BGQueue;
|
|
525
|
+
BGQueue queue_;
|
|
526
|
+
};
|
|
527
|
+
|
|
528
|
+
PosixEnv::PosixEnv() : page_size_(getpagesize()),
|
|
529
|
+
started_bgthread_(false) {
|
|
530
|
+
PthreadCall("mutex_init", pthread_mutex_init(&mu_, NULL));
|
|
531
|
+
PthreadCall("cvar_init", pthread_cond_init(&bgsignal_, NULL));
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
void PosixEnv::Schedule(void (*function)(void*), void* arg) {
|
|
535
|
+
PthreadCall("lock", pthread_mutex_lock(&mu_));
|
|
536
|
+
|
|
537
|
+
// Start background thread if necessary
|
|
538
|
+
if (!started_bgthread_) {
|
|
539
|
+
started_bgthread_ = true;
|
|
540
|
+
PthreadCall(
|
|
541
|
+
"create thread",
|
|
542
|
+
pthread_create(&bgthread_, NULL, &PosixEnv::BGThreadWrapper, this));
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// If the queue is currently empty, the background thread may currently be
|
|
546
|
+
// waiting.
|
|
547
|
+
if (queue_.empty()) {
|
|
548
|
+
PthreadCall("signal", pthread_cond_signal(&bgsignal_));
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
// Add to priority queue
|
|
552
|
+
queue_.push_back(BGItem());
|
|
553
|
+
queue_.back().function = function;
|
|
554
|
+
queue_.back().arg = arg;
|
|
555
|
+
|
|
556
|
+
PthreadCall("unlock", pthread_mutex_unlock(&mu_));
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
void PosixEnv::BGThread() {
|
|
560
|
+
while (true) {
|
|
561
|
+
// Wait until there is an item that is ready to run
|
|
562
|
+
PthreadCall("lock", pthread_mutex_lock(&mu_));
|
|
563
|
+
while (queue_.empty()) {
|
|
564
|
+
PthreadCall("wait", pthread_cond_wait(&bgsignal_, &mu_));
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
void (*function)(void*) = queue_.front().function;
|
|
568
|
+
void* arg = queue_.front().arg;
|
|
569
|
+
queue_.pop_front();
|
|
570
|
+
|
|
571
|
+
PthreadCall("unlock", pthread_mutex_unlock(&mu_));
|
|
572
|
+
(*function)(arg);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
namespace {
|
|
577
|
+
struct StartThreadState {
|
|
578
|
+
void (*user_function)(void*);
|
|
579
|
+
void* arg;
|
|
580
|
+
};
|
|
581
|
+
}
|
|
582
|
+
static void* StartThreadWrapper(void* arg) {
|
|
583
|
+
StartThreadState* state = reinterpret_cast<StartThreadState*>(arg);
|
|
584
|
+
state->user_function(state->arg);
|
|
585
|
+
delete state;
|
|
586
|
+
return NULL;
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
void PosixEnv::StartThread(void (*function)(void* arg), void* arg) {
|
|
590
|
+
pthread_t t;
|
|
591
|
+
StartThreadState* state = new StartThreadState;
|
|
592
|
+
state->user_function = function;
|
|
593
|
+
state->arg = arg;
|
|
594
|
+
PthreadCall("start thread",
|
|
595
|
+
pthread_create(&t, NULL, &StartThreadWrapper, state));
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
} // namespace
|
|
599
|
+
|
|
600
|
+
static pthread_once_t once = PTHREAD_ONCE_INIT;
|
|
601
|
+
static Env* default_env;
|
|
602
|
+
static void InitDefaultEnv() { default_env = new PosixEnv; }
|
|
603
|
+
|
|
604
|
+
Env* Env::Default() {
|
|
605
|
+
pthread_once(&once, InitDefaultEnv);
|
|
606
|
+
return default_env;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
} // namespace leveldb
|