leveldb-ruby 0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README +17 -0
- data/ext/leveldb/extconf.rb +10 -0
- data/ext/leveldb/leveldb.cc +181 -0
- data/leveldb/Makefile +172 -0
- data/leveldb/db/builder.cc +90 -0
- data/leveldb/db/builder.h +36 -0
- data/leveldb/db/corruption_test.cc +354 -0
- data/leveldb/db/db_bench.cc +677 -0
- data/leveldb/db/db_impl.cc +1236 -0
- data/leveldb/db/db_impl.h +180 -0
- data/leveldb/db/db_iter.cc +298 -0
- data/leveldb/db/db_iter.h +26 -0
- data/leveldb/db/db_test.cc +1192 -0
- data/leveldb/db/dbformat.cc +87 -0
- data/leveldb/db/dbformat.h +165 -0
- data/leveldb/db/dbformat_test.cc +112 -0
- data/leveldb/db/filename.cc +135 -0
- data/leveldb/db/filename.h +80 -0
- data/leveldb/db/filename_test.cc +122 -0
- data/leveldb/db/log_format.h +35 -0
- data/leveldb/db/log_reader.cc +254 -0
- data/leveldb/db/log_reader.h +108 -0
- data/leveldb/db/log_test.cc +500 -0
- data/leveldb/db/log_writer.cc +103 -0
- data/leveldb/db/log_writer.h +48 -0
- data/leveldb/db/memtable.cc +108 -0
- data/leveldb/db/memtable.h +85 -0
- data/leveldb/db/repair.cc +384 -0
- data/leveldb/db/skiplist.h +378 -0
- data/leveldb/db/skiplist_test.cc +378 -0
- data/leveldb/db/snapshot.h +66 -0
- data/leveldb/db/table_cache.cc +95 -0
- data/leveldb/db/table_cache.h +50 -0
- data/leveldb/db/version_edit.cc +268 -0
- data/leveldb/db/version_edit.h +106 -0
- data/leveldb/db/version_edit_test.cc +46 -0
- data/leveldb/db/version_set.cc +1060 -0
- data/leveldb/db/version_set.h +306 -0
- data/leveldb/db/write_batch.cc +138 -0
- data/leveldb/db/write_batch_internal.h +45 -0
- data/leveldb/db/write_batch_test.cc +89 -0
- data/leveldb/include/leveldb/cache.h +99 -0
- data/leveldb/include/leveldb/comparator.h +63 -0
- data/leveldb/include/leveldb/db.h +148 -0
- data/leveldb/include/leveldb/env.h +302 -0
- data/leveldb/include/leveldb/iterator.h +100 -0
- data/leveldb/include/leveldb/options.h +198 -0
- data/leveldb/include/leveldb/slice.h +109 -0
- data/leveldb/include/leveldb/status.h +100 -0
- data/leveldb/include/leveldb/table.h +70 -0
- data/leveldb/include/leveldb/table_builder.h +91 -0
- data/leveldb/include/leveldb/write_batch.h +64 -0
- data/leveldb/port/port.h +23 -0
- data/leveldb/port/port_android.cc +64 -0
- data/leveldb/port/port_android.h +150 -0
- data/leveldb/port/port_chromium.cc +80 -0
- data/leveldb/port/port_chromium.h +97 -0
- data/leveldb/port/port_example.h +115 -0
- data/leveldb/port/port_osx.cc +50 -0
- data/leveldb/port/port_osx.h +125 -0
- data/leveldb/port/port_posix.cc +50 -0
- data/leveldb/port/port_posix.h +94 -0
- data/leveldb/port/sha1_portable.cc +298 -0
- data/leveldb/port/sha1_portable.h +25 -0
- data/leveldb/port/sha1_test.cc +39 -0
- data/leveldb/port/win/stdint.h +24 -0
- data/leveldb/table/block.cc +263 -0
- data/leveldb/table/block.h +43 -0
- data/leveldb/table/block_builder.cc +109 -0
- data/leveldb/table/block_builder.h +57 -0
- data/leveldb/table/format.cc +131 -0
- data/leveldb/table/format.h +103 -0
- data/leveldb/table/iterator.cc +67 -0
- data/leveldb/table/iterator_wrapper.h +63 -0
- data/leveldb/table/merger.cc +197 -0
- data/leveldb/table/merger.h +26 -0
- data/leveldb/table/table.cc +175 -0
- data/leveldb/table/table_builder.cc +227 -0
- data/leveldb/table/table_test.cc +845 -0
- data/leveldb/table/two_level_iterator.cc +182 -0
- data/leveldb/table/two_level_iterator.h +34 -0
- data/leveldb/util/arena.cc +68 -0
- data/leveldb/util/arena.h +68 -0
- data/leveldb/util/arena_test.cc +68 -0
- data/leveldb/util/cache.cc +255 -0
- data/leveldb/util/cache_test.cc +169 -0
- data/leveldb/util/coding.cc +194 -0
- data/leveldb/util/coding.h +104 -0
- data/leveldb/util/coding_test.cc +173 -0
- data/leveldb/util/comparator.cc +72 -0
- data/leveldb/util/crc32c.cc +332 -0
- data/leveldb/util/crc32c.h +45 -0
- data/leveldb/util/crc32c_test.cc +72 -0
- data/leveldb/util/env.cc +77 -0
- data/leveldb/util/env_chromium.cc +612 -0
- data/leveldb/util/env_posix.cc +606 -0
- data/leveldb/util/env_test.cc +102 -0
- data/leveldb/util/hash.cc +45 -0
- data/leveldb/util/hash.h +19 -0
- data/leveldb/util/histogram.cc +128 -0
- data/leveldb/util/histogram.h +41 -0
- data/leveldb/util/logging.cc +81 -0
- data/leveldb/util/logging.h +47 -0
- data/leveldb/util/mutexlock.h +39 -0
- data/leveldb/util/options.cc +28 -0
- data/leveldb/util/random.h +59 -0
- data/leveldb/util/status.cc +75 -0
- data/leveldb/util/testharness.cc +65 -0
- data/leveldb/util/testharness.h +129 -0
- data/leveldb/util/testutil.cc +51 -0
- data/leveldb/util/testutil.h +53 -0
- data/lib/leveldb.rb +36 -0
- metadata +183 -0
@@ -0,0 +1,26 @@
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
4
|
+
|
5
|
+
#ifndef STORAGE_LEVELDB_TABLE_MERGER_H_
|
6
|
+
#define STORAGE_LEVELDB_TABLE_MERGER_H_
|
7
|
+
|
8
|
+
namespace leveldb {
|
9
|
+
|
10
|
+
class Comparator;
|
11
|
+
class Iterator;
|
12
|
+
|
13
|
+
// Return an iterator that provided the union of the data in
|
14
|
+
// children[0,n-1]. Takes ownership of the child iterators and
|
15
|
+
// will delete them when the result iterator is deleted.
|
16
|
+
//
|
17
|
+
// The result does no duplicate suppression. I.e., if a particular
|
18
|
+
// key is present in K child iterators, it will be yielded K times.
|
19
|
+
//
|
20
|
+
// REQUIRES: n >= 0
|
21
|
+
extern Iterator* NewMergingIterator(
|
22
|
+
const Comparator* comparator, Iterator** children, int n);
|
23
|
+
|
24
|
+
}
|
25
|
+
|
26
|
+
#endif // STORAGE_LEVELDB_TABLE_MERGER_H_
|
@@ -0,0 +1,175 @@
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
4
|
+
|
5
|
+
#include "leveldb/table.h"
|
6
|
+
|
7
|
+
#include "leveldb/cache.h"
|
8
|
+
#include "leveldb/env.h"
|
9
|
+
#include "table/block.h"
|
10
|
+
#include "table/format.h"
|
11
|
+
#include "table/two_level_iterator.h"
|
12
|
+
#include "util/coding.h"
|
13
|
+
|
14
|
+
namespace leveldb {
|
15
|
+
|
16
|
+
struct Table::Rep {
|
17
|
+
~Rep() {
|
18
|
+
delete index_block;
|
19
|
+
}
|
20
|
+
|
21
|
+
Options options;
|
22
|
+
Status status;
|
23
|
+
RandomAccessFile* file;
|
24
|
+
uint64_t cache_id;
|
25
|
+
|
26
|
+
BlockHandle metaindex_handle; // Handle to metaindex_block: saved from footer
|
27
|
+
Block* index_block;
|
28
|
+
};
|
29
|
+
|
30
|
+
Status Table::Open(const Options& options,
|
31
|
+
RandomAccessFile* file,
|
32
|
+
uint64_t size,
|
33
|
+
Table** table) {
|
34
|
+
*table = NULL;
|
35
|
+
if (size < Footer::kEncodedLength) {
|
36
|
+
return Status::InvalidArgument("file is too short to be an sstable");
|
37
|
+
}
|
38
|
+
|
39
|
+
char footer_space[Footer::kEncodedLength];
|
40
|
+
Slice footer_input;
|
41
|
+
Status s = file->Read(size - Footer::kEncodedLength, Footer::kEncodedLength,
|
42
|
+
&footer_input, footer_space);
|
43
|
+
if (!s.ok()) return s;
|
44
|
+
|
45
|
+
Footer footer;
|
46
|
+
s = footer.DecodeFrom(&footer_input);
|
47
|
+
if (!s.ok()) return s;
|
48
|
+
|
49
|
+
// Read the index block
|
50
|
+
Block* index_block = NULL;
|
51
|
+
if (s.ok()) {
|
52
|
+
s = ReadBlock(file, ReadOptions(), footer.index_handle(), &index_block);
|
53
|
+
}
|
54
|
+
|
55
|
+
if (s.ok()) {
|
56
|
+
// We've successfully read the footer and the index block: we're
|
57
|
+
// ready to serve requests.
|
58
|
+
Rep* rep = new Table::Rep;
|
59
|
+
rep->options = options;
|
60
|
+
rep->file = file;
|
61
|
+
rep->metaindex_handle = footer.metaindex_handle();
|
62
|
+
rep->index_block = index_block;
|
63
|
+
rep->cache_id = (options.block_cache ? options.block_cache->NewId() : 0);
|
64
|
+
*table = new Table(rep);
|
65
|
+
} else {
|
66
|
+
if (index_block) delete index_block;
|
67
|
+
}
|
68
|
+
|
69
|
+
return s;
|
70
|
+
}
|
71
|
+
|
72
|
+
Table::~Table() {
|
73
|
+
delete rep_;
|
74
|
+
}
|
75
|
+
|
76
|
+
static void DeleteBlock(void* arg, void* ignored) {
|
77
|
+
delete reinterpret_cast<Block*>(arg);
|
78
|
+
}
|
79
|
+
|
80
|
+
static void DeleteCachedBlock(const Slice& key, void* value) {
|
81
|
+
Block* block = reinterpret_cast<Block*>(value);
|
82
|
+
delete block;
|
83
|
+
}
|
84
|
+
|
85
|
+
static void ReleaseBlock(void* arg, void* h) {
|
86
|
+
Cache* cache = reinterpret_cast<Cache*>(arg);
|
87
|
+
Cache::Handle* handle = reinterpret_cast<Cache::Handle*>(h);
|
88
|
+
cache->Release(handle);
|
89
|
+
}
|
90
|
+
|
91
|
+
// Convert an index iterator value (i.e., an encoded BlockHandle)
|
92
|
+
// into an iterator over the contents of the corresponding block.
|
93
|
+
Iterator* Table::BlockReader(void* arg,
|
94
|
+
const ReadOptions& options,
|
95
|
+
const Slice& index_value) {
|
96
|
+
Table* table = reinterpret_cast<Table*>(arg);
|
97
|
+
Cache* block_cache = table->rep_->options.block_cache;
|
98
|
+
Block* block = NULL;
|
99
|
+
Cache::Handle* cache_handle = NULL;
|
100
|
+
|
101
|
+
BlockHandle handle;
|
102
|
+
Slice input = index_value;
|
103
|
+
Status s = handle.DecodeFrom(&input);
|
104
|
+
// We intentionally allow extra stuff in index_value so that we
|
105
|
+
// can add more features in the future.
|
106
|
+
|
107
|
+
if (s.ok()) {
|
108
|
+
if (block_cache != NULL) {
|
109
|
+
char cache_key_buffer[16];
|
110
|
+
EncodeFixed64(cache_key_buffer, table->rep_->cache_id);
|
111
|
+
EncodeFixed64(cache_key_buffer+8, handle.offset());
|
112
|
+
Slice key(cache_key_buffer, sizeof(cache_key_buffer));
|
113
|
+
cache_handle = block_cache->Lookup(key);
|
114
|
+
if (cache_handle != NULL) {
|
115
|
+
block = reinterpret_cast<Block*>(block_cache->Value(cache_handle));
|
116
|
+
} else {
|
117
|
+
s = ReadBlock(table->rep_->file, options, handle, &block);
|
118
|
+
if (s.ok() && options.fill_cache) {
|
119
|
+
cache_handle = block_cache->Insert(
|
120
|
+
key, block, block->size(), &DeleteCachedBlock);
|
121
|
+
}
|
122
|
+
}
|
123
|
+
} else {
|
124
|
+
s = ReadBlock(table->rep_->file, options, handle, &block);
|
125
|
+
}
|
126
|
+
}
|
127
|
+
|
128
|
+
Iterator* iter;
|
129
|
+
if (block != NULL) {
|
130
|
+
iter = block->NewIterator(table->rep_->options.comparator);
|
131
|
+
if (cache_handle == NULL) {
|
132
|
+
iter->RegisterCleanup(&DeleteBlock, block, NULL);
|
133
|
+
} else {
|
134
|
+
iter->RegisterCleanup(&ReleaseBlock, block_cache, cache_handle);
|
135
|
+
}
|
136
|
+
} else {
|
137
|
+
iter = NewErrorIterator(s);
|
138
|
+
}
|
139
|
+
return iter;
|
140
|
+
}
|
141
|
+
|
142
|
+
Iterator* Table::NewIterator(const ReadOptions& options) const {
|
143
|
+
return NewTwoLevelIterator(
|
144
|
+
rep_->index_block->NewIterator(rep_->options.comparator),
|
145
|
+
&Table::BlockReader, const_cast<Table*>(this), options);
|
146
|
+
}
|
147
|
+
|
148
|
+
uint64_t Table::ApproximateOffsetOf(const Slice& key) const {
|
149
|
+
Iterator* index_iter =
|
150
|
+
rep_->index_block->NewIterator(rep_->options.comparator);
|
151
|
+
index_iter->Seek(key);
|
152
|
+
uint64_t result;
|
153
|
+
if (index_iter->Valid()) {
|
154
|
+
BlockHandle handle;
|
155
|
+
Slice input = index_iter->value();
|
156
|
+
Status s = handle.DecodeFrom(&input);
|
157
|
+
if (s.ok()) {
|
158
|
+
result = handle.offset();
|
159
|
+
} else {
|
160
|
+
// Strange: we can't decode the block handle in the index block.
|
161
|
+
// We'll just return the offset of the metaindex block, which is
|
162
|
+
// close to the whole file size for this case.
|
163
|
+
result = rep_->metaindex_handle.offset();
|
164
|
+
}
|
165
|
+
} else {
|
166
|
+
// key is past the last key in the file. Approximate the offset
|
167
|
+
// by returning the offset of the metaindex block (which is
|
168
|
+
// right near the end of the file).
|
169
|
+
result = rep_->metaindex_handle.offset();
|
170
|
+
}
|
171
|
+
delete index_iter;
|
172
|
+
return result;
|
173
|
+
}
|
174
|
+
|
175
|
+
}
|
@@ -0,0 +1,227 @@
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
4
|
+
|
5
|
+
#include "leveldb/table_builder.h"
|
6
|
+
|
7
|
+
#include <assert.h>
|
8
|
+
#include <stdio.h>
|
9
|
+
#include "leveldb/comparator.h"
|
10
|
+
#include "leveldb/env.h"
|
11
|
+
#include "table/block_builder.h"
|
12
|
+
#include "table/format.h"
|
13
|
+
#include "util/coding.h"
|
14
|
+
#include "util/crc32c.h"
|
15
|
+
#include "util/logging.h"
|
16
|
+
|
17
|
+
namespace leveldb {
|
18
|
+
|
19
|
+
struct TableBuilder::Rep {
|
20
|
+
Options options;
|
21
|
+
Options index_block_options;
|
22
|
+
WritableFile* file;
|
23
|
+
uint64_t offset;
|
24
|
+
Status status;
|
25
|
+
BlockBuilder data_block;
|
26
|
+
BlockBuilder index_block;
|
27
|
+
std::string last_key;
|
28
|
+
int64_t num_entries;
|
29
|
+
bool closed; // Either Finish() or Abandon() has been called.
|
30
|
+
|
31
|
+
// We do not emit the index entry for a block until we have seen the
|
32
|
+
// first key for the next data block. This allows us to use shorter
|
33
|
+
// keys in the index block. For example, consider a block boundary
|
34
|
+
// between the keys "the quick brown fox" and "the who". We can use
|
35
|
+
// "the r" as the key for the index block entry since it is >= all
|
36
|
+
// entries in the first block and < all entries in subsequent
|
37
|
+
// blocks.
|
38
|
+
//
|
39
|
+
// Invariant: r->pending_index_entry is true only if data_block is empty.
|
40
|
+
bool pending_index_entry;
|
41
|
+
BlockHandle pending_handle; // Handle to add to index block
|
42
|
+
|
43
|
+
std::string compressed_output;
|
44
|
+
|
45
|
+
Rep(const Options& opt, WritableFile* f)
|
46
|
+
: options(opt),
|
47
|
+
index_block_options(opt),
|
48
|
+
file(f),
|
49
|
+
offset(0),
|
50
|
+
data_block(&options),
|
51
|
+
index_block(&index_block_options),
|
52
|
+
num_entries(0),
|
53
|
+
closed(false),
|
54
|
+
pending_index_entry(false) {
|
55
|
+
index_block_options.block_restart_interval = 1;
|
56
|
+
}
|
57
|
+
};
|
58
|
+
|
59
|
+
TableBuilder::TableBuilder(const Options& options, WritableFile* file)
|
60
|
+
: rep_(new Rep(options, file)) {
|
61
|
+
}
|
62
|
+
|
63
|
+
TableBuilder::~TableBuilder() {
|
64
|
+
assert(rep_->closed); // Catch errors where caller forgot to call Finish()
|
65
|
+
delete rep_;
|
66
|
+
}
|
67
|
+
|
68
|
+
Status TableBuilder::ChangeOptions(const Options& options) {
|
69
|
+
// Note: if more fields are added to Options, update
|
70
|
+
// this function to catch changes that should not be allowed to
|
71
|
+
// change in the middle of building a Table.
|
72
|
+
if (options.comparator != rep_->options.comparator) {
|
73
|
+
return Status::InvalidArgument("changing comparator while building table");
|
74
|
+
}
|
75
|
+
|
76
|
+
// Note that any live BlockBuilders point to rep_->options and therefore
|
77
|
+
// will automatically pick up the updated options.
|
78
|
+
rep_->options = options;
|
79
|
+
rep_->index_block_options = options;
|
80
|
+
rep_->index_block_options.block_restart_interval = 1;
|
81
|
+
return Status::OK();
|
82
|
+
}
|
83
|
+
|
84
|
+
void TableBuilder::Add(const Slice& key, const Slice& value) {
|
85
|
+
Rep* r = rep_;
|
86
|
+
assert(!r->closed);
|
87
|
+
if (!ok()) return;
|
88
|
+
if (r->num_entries > 0) {
|
89
|
+
assert(r->options.comparator->Compare(key, Slice(r->last_key)) > 0);
|
90
|
+
}
|
91
|
+
|
92
|
+
if (r->pending_index_entry) {
|
93
|
+
assert(r->data_block.empty());
|
94
|
+
r->options.comparator->FindShortestSeparator(&r->last_key, key);
|
95
|
+
std::string handle_encoding;
|
96
|
+
r->pending_handle.EncodeTo(&handle_encoding);
|
97
|
+
r->index_block.Add(r->last_key, Slice(handle_encoding));
|
98
|
+
r->pending_index_entry = false;
|
99
|
+
}
|
100
|
+
|
101
|
+
r->last_key.assign(key.data(), key.size());
|
102
|
+
r->num_entries++;
|
103
|
+
r->data_block.Add(key, value);
|
104
|
+
|
105
|
+
const size_t estimated_block_size = r->data_block.CurrentSizeEstimate();
|
106
|
+
if (estimated_block_size >= r->options.block_size) {
|
107
|
+
Flush();
|
108
|
+
}
|
109
|
+
}
|
110
|
+
|
111
|
+
void TableBuilder::Flush() {
|
112
|
+
Rep* r = rep_;
|
113
|
+
assert(!r->closed);
|
114
|
+
if (!ok()) return;
|
115
|
+
if (r->data_block.empty()) return;
|
116
|
+
assert(!r->pending_index_entry);
|
117
|
+
WriteBlock(&r->data_block, &r->pending_handle);
|
118
|
+
if (ok()) {
|
119
|
+
r->pending_index_entry = true;
|
120
|
+
r->status = r->file->Flush();
|
121
|
+
}
|
122
|
+
}
|
123
|
+
|
124
|
+
void TableBuilder::WriteBlock(BlockBuilder* block, BlockHandle* handle) {
|
125
|
+
// File format contains a sequence of blocks where each block has:
|
126
|
+
// block_data: uint8[n]
|
127
|
+
// type: uint8
|
128
|
+
// crc: uint32
|
129
|
+
assert(ok());
|
130
|
+
Rep* r = rep_;
|
131
|
+
Slice raw = block->Finish();
|
132
|
+
|
133
|
+
Slice block_contents;
|
134
|
+
CompressionType type = r->options.compression;
|
135
|
+
// TODO(postrelease): Support more compression options: zlib?
|
136
|
+
switch (type) {
|
137
|
+
case kNoCompression:
|
138
|
+
block_contents = raw;
|
139
|
+
break;
|
140
|
+
|
141
|
+
case kSnappyCompression: {
|
142
|
+
std::string* compressed = &r->compressed_output;
|
143
|
+
if (port::Snappy_Compress(raw.data(), raw.size(), compressed) &&
|
144
|
+
compressed->size() < raw.size() - (raw.size() / 8u)) {
|
145
|
+
block_contents = *compressed;
|
146
|
+
} else {
|
147
|
+
// Snappy not supported, or compressed less than 12.5%, so just
|
148
|
+
// store uncompressed form
|
149
|
+
block_contents = raw;
|
150
|
+
type = kNoCompression;
|
151
|
+
}
|
152
|
+
break;
|
153
|
+
}
|
154
|
+
}
|
155
|
+
handle->set_offset(r->offset);
|
156
|
+
handle->set_size(block_contents.size());
|
157
|
+
r->status = r->file->Append(block_contents);
|
158
|
+
if (r->status.ok()) {
|
159
|
+
char trailer[kBlockTrailerSize];
|
160
|
+
trailer[0] = type;
|
161
|
+
uint32_t crc = crc32c::Value(block_contents.data(), block_contents.size());
|
162
|
+
crc = crc32c::Extend(crc, trailer, 1); // Extend crc to cover block type
|
163
|
+
EncodeFixed32(trailer+1, crc32c::Mask(crc));
|
164
|
+
r->status = r->file->Append(Slice(trailer, kBlockTrailerSize));
|
165
|
+
if (r->status.ok()) {
|
166
|
+
r->offset += block_contents.size() + kBlockTrailerSize;
|
167
|
+
}
|
168
|
+
}
|
169
|
+
r->compressed_output.clear();
|
170
|
+
block->Reset();
|
171
|
+
}
|
172
|
+
|
173
|
+
Status TableBuilder::status() const {
|
174
|
+
return rep_->status;
|
175
|
+
}
|
176
|
+
|
177
|
+
Status TableBuilder::Finish() {
|
178
|
+
Rep* r = rep_;
|
179
|
+
Flush();
|
180
|
+
assert(!r->closed);
|
181
|
+
r->closed = true;
|
182
|
+
BlockHandle metaindex_block_handle;
|
183
|
+
BlockHandle index_block_handle;
|
184
|
+
if (ok()) {
|
185
|
+
BlockBuilder meta_index_block(&r->options);
|
186
|
+
// TODO(postrelease): Add stats and other meta blocks
|
187
|
+
WriteBlock(&meta_index_block, &metaindex_block_handle);
|
188
|
+
}
|
189
|
+
if (ok()) {
|
190
|
+
if (r->pending_index_entry) {
|
191
|
+
r->options.comparator->FindShortSuccessor(&r->last_key);
|
192
|
+
std::string handle_encoding;
|
193
|
+
r->pending_handle.EncodeTo(&handle_encoding);
|
194
|
+
r->index_block.Add(r->last_key, Slice(handle_encoding));
|
195
|
+
r->pending_index_entry = false;
|
196
|
+
}
|
197
|
+
WriteBlock(&r->index_block, &index_block_handle);
|
198
|
+
}
|
199
|
+
if (ok()) {
|
200
|
+
Footer footer;
|
201
|
+
footer.set_metaindex_handle(metaindex_block_handle);
|
202
|
+
footer.set_index_handle(index_block_handle);
|
203
|
+
std::string footer_encoding;
|
204
|
+
footer.EncodeTo(&footer_encoding);
|
205
|
+
r->status = r->file->Append(footer_encoding);
|
206
|
+
if (r->status.ok()) {
|
207
|
+
r->offset += footer_encoding.size();
|
208
|
+
}
|
209
|
+
}
|
210
|
+
return r->status;
|
211
|
+
}
|
212
|
+
|
213
|
+
void TableBuilder::Abandon() {
|
214
|
+
Rep* r = rep_;
|
215
|
+
assert(!r->closed);
|
216
|
+
r->closed = true;
|
217
|
+
}
|
218
|
+
|
219
|
+
uint64_t TableBuilder::NumEntries() const {
|
220
|
+
return rep_->num_entries;
|
221
|
+
}
|
222
|
+
|
223
|
+
uint64_t TableBuilder::FileSize() const {
|
224
|
+
return rep_->offset;
|
225
|
+
}
|
226
|
+
|
227
|
+
}
|
@@ -0,0 +1,845 @@
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
4
|
+
|
5
|
+
#include "leveldb/table.h"
|
6
|
+
|
7
|
+
#include <map>
|
8
|
+
#include "db/dbformat.h"
|
9
|
+
#include "db/memtable.h"
|
10
|
+
#include "db/write_batch_internal.h"
|
11
|
+
#include "leveldb/db.h"
|
12
|
+
#include "leveldb/env.h"
|
13
|
+
#include "leveldb/iterator.h"
|
14
|
+
#include "leveldb/table_builder.h"
|
15
|
+
#include "table/block.h"
|
16
|
+
#include "table/block_builder.h"
|
17
|
+
#include "table/format.h"
|
18
|
+
#include "util/random.h"
|
19
|
+
#include "util/testharness.h"
|
20
|
+
#include "util/testutil.h"
|
21
|
+
|
22
|
+
namespace leveldb {
|
23
|
+
|
24
|
+
// Return reverse of "key".
|
25
|
+
// Used to test non-lexicographic comparators.
|
26
|
+
static std::string Reverse(const Slice& key) {
|
27
|
+
std::string str(key.ToString());
|
28
|
+
std::string rev(str.rbegin(), str.rend());
|
29
|
+
return rev;
|
30
|
+
}
|
31
|
+
|
32
|
+
namespace {
|
33
|
+
class ReverseKeyComparator : public Comparator {
|
34
|
+
public:
|
35
|
+
virtual const char* Name() const {
|
36
|
+
return "leveldb.ReverseBytewiseComparator";
|
37
|
+
}
|
38
|
+
|
39
|
+
virtual int Compare(const Slice& a, const Slice& b) const {
|
40
|
+
return BytewiseComparator()->Compare(Reverse(a), Reverse(b));
|
41
|
+
}
|
42
|
+
|
43
|
+
virtual void FindShortestSeparator(
|
44
|
+
std::string* start,
|
45
|
+
const Slice& limit) const {
|
46
|
+
std::string s = Reverse(*start);
|
47
|
+
std::string l = Reverse(limit);
|
48
|
+
BytewiseComparator()->FindShortestSeparator(&s, l);
|
49
|
+
*start = Reverse(s);
|
50
|
+
}
|
51
|
+
|
52
|
+
virtual void FindShortSuccessor(std::string* key) const {
|
53
|
+
std::string s = Reverse(*key);
|
54
|
+
BytewiseComparator()->FindShortSuccessor(&s);
|
55
|
+
*key = Reverse(s);
|
56
|
+
}
|
57
|
+
};
|
58
|
+
}
|
59
|
+
static ReverseKeyComparator reverse_key_comparator;
|
60
|
+
|
61
|
+
static void Increment(const Comparator* cmp, std::string* key) {
|
62
|
+
if (cmp == BytewiseComparator()) {
|
63
|
+
key->push_back('\0');
|
64
|
+
} else {
|
65
|
+
assert(cmp == &reverse_key_comparator);
|
66
|
+
std::string rev = Reverse(*key);
|
67
|
+
rev.push_back('\0');
|
68
|
+
*key = Reverse(rev);
|
69
|
+
}
|
70
|
+
}
|
71
|
+
|
72
|
+
// An STL comparator that uses a Comparator
|
73
|
+
namespace {
|
74
|
+
struct STLLessThan {
|
75
|
+
const Comparator* cmp;
|
76
|
+
|
77
|
+
STLLessThan() : cmp(BytewiseComparator()) { }
|
78
|
+
STLLessThan(const Comparator* c) : cmp(c) { }
|
79
|
+
bool operator()(const std::string& a, const std::string& b) const {
|
80
|
+
return cmp->Compare(Slice(a), Slice(b)) < 0;
|
81
|
+
}
|
82
|
+
};
|
83
|
+
}
|
84
|
+
|
85
|
+
class StringSink: public WritableFile {
|
86
|
+
public:
|
87
|
+
~StringSink() { }
|
88
|
+
|
89
|
+
const std::string& contents() const { return contents_; }
|
90
|
+
|
91
|
+
virtual Status Close() { return Status::OK(); }
|
92
|
+
virtual Status Flush() { return Status::OK(); }
|
93
|
+
virtual Status Sync() { return Status::OK(); }
|
94
|
+
|
95
|
+
virtual Status Append(const Slice& data) {
|
96
|
+
contents_.append(data.data(), data.size());
|
97
|
+
return Status::OK();
|
98
|
+
}
|
99
|
+
|
100
|
+
private:
|
101
|
+
std::string contents_;
|
102
|
+
};
|
103
|
+
|
104
|
+
|
105
|
+
class StringSource: public RandomAccessFile {
|
106
|
+
public:
|
107
|
+
StringSource(const Slice& contents)
|
108
|
+
: contents_(contents.data(), contents.size()) {
|
109
|
+
}
|
110
|
+
|
111
|
+
virtual ~StringSource() { }
|
112
|
+
|
113
|
+
uint64_t Size() const { return contents_.size(); }
|
114
|
+
|
115
|
+
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
116
|
+
char* scratch) const {
|
117
|
+
if (offset > contents_.size()) {
|
118
|
+
return Status::InvalidArgument("invalid Read offset");
|
119
|
+
}
|
120
|
+
if (offset + n > contents_.size()) {
|
121
|
+
n = contents_.size() - offset;
|
122
|
+
}
|
123
|
+
memcpy(scratch, &contents_[offset], n);
|
124
|
+
*result = Slice(scratch, n);
|
125
|
+
return Status::OK();
|
126
|
+
}
|
127
|
+
|
128
|
+
private:
|
129
|
+
std::string contents_;
|
130
|
+
};
|
131
|
+
|
132
|
+
typedef std::map<std::string, std::string, STLLessThan> KVMap;
|
133
|
+
|
134
|
+
// Helper class for tests to unify the interface between
|
135
|
+
// BlockBuilder/TableBuilder and Block/Table.
|
136
|
+
class Constructor {
|
137
|
+
public:
|
138
|
+
explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) { }
|
139
|
+
virtual ~Constructor() { }
|
140
|
+
|
141
|
+
void Add(const std::string& key, const Slice& value) {
|
142
|
+
data_[key] = value.ToString();
|
143
|
+
}
|
144
|
+
|
145
|
+
// Finish constructing the data structure with all the keys that have
|
146
|
+
// been added so far. Returns the keys in sorted order in "*keys"
|
147
|
+
// and stores the key/value pairs in "*kvmap"
|
148
|
+
void Finish(const Options& options,
|
149
|
+
std::vector<std::string>* keys,
|
150
|
+
KVMap* kvmap) {
|
151
|
+
*kvmap = data_;
|
152
|
+
keys->clear();
|
153
|
+
for (KVMap::const_iterator it = data_.begin();
|
154
|
+
it != data_.end();
|
155
|
+
++it) {
|
156
|
+
keys->push_back(it->first);
|
157
|
+
}
|
158
|
+
data_.clear();
|
159
|
+
Status s = FinishImpl(options, *kvmap);
|
160
|
+
ASSERT_TRUE(s.ok()) << s.ToString();
|
161
|
+
}
|
162
|
+
|
163
|
+
// Construct the data structure from the data in "data"
|
164
|
+
virtual Status FinishImpl(const Options& options, const KVMap& data) = 0;
|
165
|
+
|
166
|
+
virtual size_t NumBytes() const = 0;
|
167
|
+
|
168
|
+
virtual Iterator* NewIterator() const = 0;
|
169
|
+
|
170
|
+
virtual const KVMap& data() { return data_; }
|
171
|
+
|
172
|
+
virtual DB* db() const { return NULL; } // Overridden in DBConstructor
|
173
|
+
|
174
|
+
private:
|
175
|
+
KVMap data_;
|
176
|
+
};
|
177
|
+
|
178
|
+
class BlockConstructor: public Constructor {
|
179
|
+
public:
|
180
|
+
explicit BlockConstructor(const Comparator* cmp)
|
181
|
+
: Constructor(cmp),
|
182
|
+
comparator_(cmp),
|
183
|
+
block_size_(-1),
|
184
|
+
block_(NULL) { }
|
185
|
+
~BlockConstructor() {
|
186
|
+
delete block_;
|
187
|
+
}
|
188
|
+
virtual Status FinishImpl(const Options& options, const KVMap& data) {
|
189
|
+
delete block_;
|
190
|
+
block_ = NULL;
|
191
|
+
BlockBuilder builder(&options);
|
192
|
+
|
193
|
+
for (KVMap::const_iterator it = data.begin();
|
194
|
+
it != data.end();
|
195
|
+
++it) {
|
196
|
+
builder.Add(it->first, it->second);
|
197
|
+
}
|
198
|
+
// Open the block
|
199
|
+
Slice block_data = builder.Finish();
|
200
|
+
block_size_ = block_data.size();
|
201
|
+
char* block_data_copy = new char[block_size_];
|
202
|
+
memcpy(block_data_copy, block_data.data(), block_size_);
|
203
|
+
block_ = new Block(block_data_copy, block_size_);
|
204
|
+
return Status::OK();
|
205
|
+
}
|
206
|
+
virtual size_t NumBytes() const { return block_size_; }
|
207
|
+
|
208
|
+
virtual Iterator* NewIterator() const {
|
209
|
+
return block_->NewIterator(comparator_);
|
210
|
+
}
|
211
|
+
|
212
|
+
private:
|
213
|
+
const Comparator* comparator_;
|
214
|
+
int block_size_;
|
215
|
+
Block* block_;
|
216
|
+
|
217
|
+
BlockConstructor();
|
218
|
+
};
|
219
|
+
|
220
|
+
class TableConstructor: public Constructor {
|
221
|
+
public:
|
222
|
+
TableConstructor(const Comparator* cmp)
|
223
|
+
: Constructor(cmp),
|
224
|
+
source_(NULL), table_(NULL) {
|
225
|
+
}
|
226
|
+
~TableConstructor() {
|
227
|
+
Reset();
|
228
|
+
}
|
229
|
+
virtual Status FinishImpl(const Options& options, const KVMap& data) {
|
230
|
+
Reset();
|
231
|
+
StringSink sink;
|
232
|
+
TableBuilder builder(options, &sink);
|
233
|
+
|
234
|
+
for (KVMap::const_iterator it = data.begin();
|
235
|
+
it != data.end();
|
236
|
+
++it) {
|
237
|
+
builder.Add(it->first, it->second);
|
238
|
+
ASSERT_TRUE(builder.status().ok());
|
239
|
+
}
|
240
|
+
Status s = builder.Finish();
|
241
|
+
ASSERT_TRUE(s.ok()) << s.ToString();
|
242
|
+
|
243
|
+
ASSERT_EQ(sink.contents().size(), builder.FileSize());
|
244
|
+
|
245
|
+
// Open the table
|
246
|
+
source_ = new StringSource(sink.contents());
|
247
|
+
Options table_options;
|
248
|
+
table_options.comparator = options.comparator;
|
249
|
+
return Table::Open(table_options, source_, sink.contents().size(), &table_);
|
250
|
+
}
|
251
|
+
virtual size_t NumBytes() const { return source_->Size(); }
|
252
|
+
|
253
|
+
virtual Iterator* NewIterator() const {
|
254
|
+
return table_->NewIterator(ReadOptions());
|
255
|
+
}
|
256
|
+
|
257
|
+
uint64_t ApproximateOffsetOf(const Slice& key) const {
|
258
|
+
return table_->ApproximateOffsetOf(key);
|
259
|
+
}
|
260
|
+
|
261
|
+
private:
|
262
|
+
void Reset() {
|
263
|
+
delete table_;
|
264
|
+
delete source_;
|
265
|
+
table_ = NULL;
|
266
|
+
source_ = NULL;
|
267
|
+
}
|
268
|
+
|
269
|
+
StringSource* source_;
|
270
|
+
Table* table_;
|
271
|
+
|
272
|
+
TableConstructor();
|
273
|
+
};
|
274
|
+
|
275
|
+
// A helper class that converts internal format keys into user keys
|
276
|
+
class KeyConvertingIterator: public Iterator {
|
277
|
+
public:
|
278
|
+
explicit KeyConvertingIterator(Iterator* iter) : iter_(iter) { }
|
279
|
+
virtual ~KeyConvertingIterator() { delete iter_; }
|
280
|
+
virtual bool Valid() const { return iter_->Valid(); }
|
281
|
+
virtual void Seek(const Slice& target) {
|
282
|
+
ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue);
|
283
|
+
std::string encoded;
|
284
|
+
AppendInternalKey(&encoded, ikey);
|
285
|
+
iter_->Seek(encoded);
|
286
|
+
}
|
287
|
+
virtual void SeekToFirst() { iter_->SeekToFirst(); }
|
288
|
+
virtual void SeekToLast() { iter_->SeekToLast(); }
|
289
|
+
virtual void Next() { iter_->Next(); }
|
290
|
+
virtual void Prev() { iter_->Prev(); }
|
291
|
+
|
292
|
+
virtual Slice key() const {
|
293
|
+
assert(Valid());
|
294
|
+
ParsedInternalKey key;
|
295
|
+
if (!ParseInternalKey(iter_->key(), &key)) {
|
296
|
+
status_ = Status::Corruption("malformed internal key");
|
297
|
+
return Slice("corrupted key");
|
298
|
+
}
|
299
|
+
return key.user_key;
|
300
|
+
}
|
301
|
+
|
302
|
+
virtual Slice value() const { return iter_->value(); }
|
303
|
+
virtual Status status() const {
|
304
|
+
return status_.ok() ? iter_->status() : status_;
|
305
|
+
}
|
306
|
+
|
307
|
+
private:
|
308
|
+
mutable Status status_;
|
309
|
+
Iterator* iter_;
|
310
|
+
|
311
|
+
// No copying allowed
|
312
|
+
KeyConvertingIterator(const KeyConvertingIterator&);
|
313
|
+
void operator=(const KeyConvertingIterator&);
|
314
|
+
};
|
315
|
+
|
316
|
+
class MemTableConstructor: public Constructor {
|
317
|
+
public:
|
318
|
+
explicit MemTableConstructor(const Comparator* cmp)
|
319
|
+
: Constructor(cmp),
|
320
|
+
internal_comparator_(cmp) {
|
321
|
+
memtable_ = new MemTable(internal_comparator_);
|
322
|
+
memtable_->Ref();
|
323
|
+
}
|
324
|
+
~MemTableConstructor() {
|
325
|
+
memtable_->Unref();
|
326
|
+
}
|
327
|
+
virtual Status FinishImpl(const Options& options, const KVMap& data) {
|
328
|
+
memtable_->Unref();
|
329
|
+
memtable_ = new MemTable(internal_comparator_);
|
330
|
+
memtable_->Ref();
|
331
|
+
int seq = 1;
|
332
|
+
for (KVMap::const_iterator it = data.begin();
|
333
|
+
it != data.end();
|
334
|
+
++it) {
|
335
|
+
memtable_->Add(seq, kTypeValue, it->first, it->second);
|
336
|
+
seq++;
|
337
|
+
}
|
338
|
+
return Status::OK();
|
339
|
+
}
|
340
|
+
virtual size_t NumBytes() const {
|
341
|
+
return memtable_->ApproximateMemoryUsage();
|
342
|
+
}
|
343
|
+
|
344
|
+
virtual Iterator* NewIterator() const {
|
345
|
+
return new KeyConvertingIterator(memtable_->NewIterator());
|
346
|
+
}
|
347
|
+
|
348
|
+
private:
|
349
|
+
InternalKeyComparator internal_comparator_;
|
350
|
+
MemTable* memtable_;
|
351
|
+
};
|
352
|
+
|
353
|
+
class DBConstructor: public Constructor {
|
354
|
+
public:
|
355
|
+
explicit DBConstructor(const Comparator* cmp)
|
356
|
+
: Constructor(cmp),
|
357
|
+
comparator_(cmp) {
|
358
|
+
db_ = NULL;
|
359
|
+
NewDB();
|
360
|
+
}
|
361
|
+
~DBConstructor() {
|
362
|
+
delete db_;
|
363
|
+
}
|
364
|
+
virtual Status FinishImpl(const Options& options, const KVMap& data) {
|
365
|
+
delete db_;
|
366
|
+
db_ = NULL;
|
367
|
+
NewDB();
|
368
|
+
for (KVMap::const_iterator it = data.begin();
|
369
|
+
it != data.end();
|
370
|
+
++it) {
|
371
|
+
WriteBatch batch;
|
372
|
+
batch.Put(it->first, it->second);
|
373
|
+
ASSERT_TRUE(db_->Write(WriteOptions(), &batch).ok());
|
374
|
+
}
|
375
|
+
return Status::OK();
|
376
|
+
}
|
377
|
+
virtual size_t NumBytes() const {
|
378
|
+
Range r("", "\xff\xff");
|
379
|
+
uint64_t size;
|
380
|
+
db_->GetApproximateSizes(&r, 1, &size);
|
381
|
+
return size;
|
382
|
+
}
|
383
|
+
|
384
|
+
virtual Iterator* NewIterator() const {
|
385
|
+
return db_->NewIterator(ReadOptions());
|
386
|
+
}
|
387
|
+
|
388
|
+
virtual DB* db() const { return db_; }
|
389
|
+
|
390
|
+
private:
|
391
|
+
void NewDB() {
|
392
|
+
std::string name = test::TmpDir() + "/table_testdb";
|
393
|
+
|
394
|
+
Options options;
|
395
|
+
options.comparator = comparator_;
|
396
|
+
Status status = DestroyDB(name, options);
|
397
|
+
ASSERT_TRUE(status.ok()) << status.ToString();
|
398
|
+
|
399
|
+
options.create_if_missing = true;
|
400
|
+
options.error_if_exists = true;
|
401
|
+
options.write_buffer_size = 10000; // Something small to force merging
|
402
|
+
status = DB::Open(options, name, &db_);
|
403
|
+
ASSERT_TRUE(status.ok()) << status.ToString();
|
404
|
+
}
|
405
|
+
|
406
|
+
const Comparator* comparator_;
|
407
|
+
DB* db_;
|
408
|
+
};
|
409
|
+
|
410
|
+
enum TestType {
|
411
|
+
TABLE_TEST,
|
412
|
+
BLOCK_TEST,
|
413
|
+
MEMTABLE_TEST,
|
414
|
+
DB_TEST,
|
415
|
+
};
|
416
|
+
|
417
|
+
struct TestArgs {
|
418
|
+
TestType type;
|
419
|
+
bool reverse_compare;
|
420
|
+
int restart_interval;
|
421
|
+
};
|
422
|
+
|
423
|
+
static const TestArgs kTestArgList[] = {
|
424
|
+
{ TABLE_TEST, false, 16 },
|
425
|
+
{ TABLE_TEST, false, 1 },
|
426
|
+
{ TABLE_TEST, false, 1024 },
|
427
|
+
{ TABLE_TEST, true, 16 },
|
428
|
+
{ TABLE_TEST, true, 1 },
|
429
|
+
{ TABLE_TEST, true, 1024 },
|
430
|
+
|
431
|
+
{ BLOCK_TEST, false, 16 },
|
432
|
+
{ BLOCK_TEST, false, 1 },
|
433
|
+
{ BLOCK_TEST, false, 1024 },
|
434
|
+
{ BLOCK_TEST, true, 16 },
|
435
|
+
{ BLOCK_TEST, true, 1 },
|
436
|
+
{ BLOCK_TEST, true, 1024 },
|
437
|
+
|
438
|
+
// Restart interval does not matter for memtables
|
439
|
+
{ MEMTABLE_TEST, false, 16 },
|
440
|
+
{ MEMTABLE_TEST, true, 16 },
|
441
|
+
|
442
|
+
// Do not bother with restart interval variations for DB
|
443
|
+
{ DB_TEST, false, 16 },
|
444
|
+
{ DB_TEST, true, 16 },
|
445
|
+
};
|
446
|
+
static const int kNumTestArgs = sizeof(kTestArgList) / sizeof(kTestArgList[0]);
|
447
|
+
|
448
|
+
class Harness {
|
449
|
+
public:
|
450
|
+
Harness() : constructor_(NULL) { }
|
451
|
+
|
452
|
+
void Init(const TestArgs& args) {
|
453
|
+
delete constructor_;
|
454
|
+
constructor_ = NULL;
|
455
|
+
options_ = Options();
|
456
|
+
|
457
|
+
options_.block_restart_interval = args.restart_interval;
|
458
|
+
// Use shorter block size for tests to exercise block boundary
|
459
|
+
// conditions more.
|
460
|
+
options_.block_size = 256;
|
461
|
+
if (args.reverse_compare) {
|
462
|
+
options_.comparator = &reverse_key_comparator;
|
463
|
+
}
|
464
|
+
switch (args.type) {
|
465
|
+
case TABLE_TEST:
|
466
|
+
constructor_ = new TableConstructor(options_.comparator);
|
467
|
+
break;
|
468
|
+
case BLOCK_TEST:
|
469
|
+
constructor_ = new BlockConstructor(options_.comparator);
|
470
|
+
break;
|
471
|
+
case MEMTABLE_TEST:
|
472
|
+
constructor_ = new MemTableConstructor(options_.comparator);
|
473
|
+
break;
|
474
|
+
case DB_TEST:
|
475
|
+
constructor_ = new DBConstructor(options_.comparator);
|
476
|
+
break;
|
477
|
+
}
|
478
|
+
}
|
479
|
+
|
480
|
+
~Harness() {
|
481
|
+
delete constructor_;
|
482
|
+
}
|
483
|
+
|
484
|
+
void Add(const std::string& key, const std::string& value) {
|
485
|
+
constructor_->Add(key, value);
|
486
|
+
}
|
487
|
+
|
488
|
+
void Test(Random* rnd) {
|
489
|
+
std::vector<std::string> keys;
|
490
|
+
KVMap data;
|
491
|
+
constructor_->Finish(options_, &keys, &data);
|
492
|
+
|
493
|
+
TestForwardScan(keys, data);
|
494
|
+
TestBackwardScan(keys, data);
|
495
|
+
TestRandomAccess(rnd, keys, data);
|
496
|
+
}
|
497
|
+
|
498
|
+
void TestForwardScan(const std::vector<std::string>& keys,
|
499
|
+
const KVMap& data) {
|
500
|
+
Iterator* iter = constructor_->NewIterator();
|
501
|
+
ASSERT_TRUE(!iter->Valid());
|
502
|
+
iter->SeekToFirst();
|
503
|
+
for (KVMap::const_iterator model_iter = data.begin();
|
504
|
+
model_iter != data.end();
|
505
|
+
++model_iter) {
|
506
|
+
ASSERT_EQ(ToString(data, model_iter), ToString(iter));
|
507
|
+
iter->Next();
|
508
|
+
}
|
509
|
+
ASSERT_TRUE(!iter->Valid());
|
510
|
+
delete iter;
|
511
|
+
}
|
512
|
+
|
513
|
+
void TestBackwardScan(const std::vector<std::string>& keys,
|
514
|
+
const KVMap& data) {
|
515
|
+
Iterator* iter = constructor_->NewIterator();
|
516
|
+
ASSERT_TRUE(!iter->Valid());
|
517
|
+
iter->SeekToLast();
|
518
|
+
for (KVMap::const_reverse_iterator model_iter = data.rbegin();
|
519
|
+
model_iter != data.rend();
|
520
|
+
++model_iter) {
|
521
|
+
ASSERT_EQ(ToString(data, model_iter), ToString(iter));
|
522
|
+
iter->Prev();
|
523
|
+
}
|
524
|
+
ASSERT_TRUE(!iter->Valid());
|
525
|
+
delete iter;
|
526
|
+
}
|
527
|
+
|
528
|
+
void TestRandomAccess(Random* rnd,
|
529
|
+
const std::vector<std::string>& keys,
|
530
|
+
const KVMap& data) {
|
531
|
+
static const bool kVerbose = false;
|
532
|
+
Iterator* iter = constructor_->NewIterator();
|
533
|
+
ASSERT_TRUE(!iter->Valid());
|
534
|
+
KVMap::const_iterator model_iter = data.begin();
|
535
|
+
if (kVerbose) fprintf(stderr, "---\n");
|
536
|
+
for (int i = 0; i < 200; i++) {
|
537
|
+
const int toss = rnd->Uniform(5);
|
538
|
+
switch (toss) {
|
539
|
+
case 0: {
|
540
|
+
if (iter->Valid()) {
|
541
|
+
if (kVerbose) fprintf(stderr, "Next\n");
|
542
|
+
iter->Next();
|
543
|
+
++model_iter;
|
544
|
+
ASSERT_EQ(ToString(data, model_iter), ToString(iter));
|
545
|
+
}
|
546
|
+
break;
|
547
|
+
}
|
548
|
+
|
549
|
+
case 1: {
|
550
|
+
if (kVerbose) fprintf(stderr, "SeekToFirst\n");
|
551
|
+
iter->SeekToFirst();
|
552
|
+
model_iter = data.begin();
|
553
|
+
ASSERT_EQ(ToString(data, model_iter), ToString(iter));
|
554
|
+
break;
|
555
|
+
}
|
556
|
+
|
557
|
+
case 2: {
|
558
|
+
std::string key = PickRandomKey(rnd, keys);
|
559
|
+
model_iter = data.lower_bound(key);
|
560
|
+
if (kVerbose) fprintf(stderr, "Seek '%s'\n",
|
561
|
+
EscapeString(key).c_str());
|
562
|
+
iter->Seek(Slice(key));
|
563
|
+
ASSERT_EQ(ToString(data, model_iter), ToString(iter));
|
564
|
+
break;
|
565
|
+
}
|
566
|
+
|
567
|
+
case 3: {
|
568
|
+
if (iter->Valid()) {
|
569
|
+
if (kVerbose) fprintf(stderr, "Prev\n");
|
570
|
+
iter->Prev();
|
571
|
+
if (model_iter == data.begin()) {
|
572
|
+
model_iter = data.end(); // Wrap around to invalid value
|
573
|
+
} else {
|
574
|
+
--model_iter;
|
575
|
+
}
|
576
|
+
ASSERT_EQ(ToString(data, model_iter), ToString(iter));
|
577
|
+
}
|
578
|
+
break;
|
579
|
+
}
|
580
|
+
|
581
|
+
case 4: {
|
582
|
+
if (kVerbose) fprintf(stderr, "SeekToLast\n");
|
583
|
+
iter->SeekToLast();
|
584
|
+
if (keys.empty()) {
|
585
|
+
model_iter = data.end();
|
586
|
+
} else {
|
587
|
+
std::string last = data.rbegin()->first;
|
588
|
+
model_iter = data.lower_bound(last);
|
589
|
+
}
|
590
|
+
ASSERT_EQ(ToString(data, model_iter), ToString(iter));
|
591
|
+
break;
|
592
|
+
}
|
593
|
+
}
|
594
|
+
}
|
595
|
+
delete iter;
|
596
|
+
}
|
597
|
+
|
598
|
+
std::string ToString(const KVMap& data, const KVMap::const_iterator& it) {
|
599
|
+
if (it == data.end()) {
|
600
|
+
return "END";
|
601
|
+
} else {
|
602
|
+
return "'" + it->first + "->" + it->second + "'";
|
603
|
+
}
|
604
|
+
}
|
605
|
+
|
606
|
+
std::string ToString(const KVMap& data,
|
607
|
+
const KVMap::const_reverse_iterator& it) {
|
608
|
+
if (it == data.rend()) {
|
609
|
+
return "END";
|
610
|
+
} else {
|
611
|
+
return "'" + it->first + "->" + it->second + "'";
|
612
|
+
}
|
613
|
+
}
|
614
|
+
|
615
|
+
std::string ToString(const Iterator* it) {
|
616
|
+
if (!it->Valid()) {
|
617
|
+
return "END";
|
618
|
+
} else {
|
619
|
+
return "'" + it->key().ToString() + "->" + it->value().ToString() + "'";
|
620
|
+
}
|
621
|
+
}
|
622
|
+
|
623
|
+
std::string PickRandomKey(Random* rnd, const std::vector<std::string>& keys) {
|
624
|
+
if (keys.empty()) {
|
625
|
+
return "foo";
|
626
|
+
} else {
|
627
|
+
const int index = rnd->Uniform(keys.size());
|
628
|
+
std::string result = keys[index];
|
629
|
+
switch (rnd->Uniform(3)) {
|
630
|
+
case 0:
|
631
|
+
// Return an existing key
|
632
|
+
break;
|
633
|
+
case 1: {
|
634
|
+
// Attempt to return something smaller than an existing key
|
635
|
+
if (result.size() > 0 && result[result.size()-1] > '\0') {
|
636
|
+
result[result.size()-1]--;
|
637
|
+
}
|
638
|
+
break;
|
639
|
+
}
|
640
|
+
case 2: {
|
641
|
+
// Return something larger than an existing key
|
642
|
+
Increment(options_.comparator, &result);
|
643
|
+
break;
|
644
|
+
}
|
645
|
+
}
|
646
|
+
return result;
|
647
|
+
}
|
648
|
+
}
|
649
|
+
|
650
|
+
// Returns NULL if not running against a DB
|
651
|
+
DB* db() const { return constructor_->db(); }
|
652
|
+
|
653
|
+
private:
|
654
|
+
Options options_;
|
655
|
+
Constructor* constructor_;
|
656
|
+
};
|
657
|
+
|
658
|
+
// Test the empty key
|
659
|
+
TEST(Harness, SimpleEmptyKey) {
|
660
|
+
for (int i = 0; i < kNumTestArgs; i++) {
|
661
|
+
Init(kTestArgList[i]);
|
662
|
+
Random rnd(test::RandomSeed() + 1);
|
663
|
+
Add("", "v");
|
664
|
+
Test(&rnd);
|
665
|
+
}
|
666
|
+
}
|
667
|
+
|
668
|
+
TEST(Harness, SimpleSingle) {
|
669
|
+
for (int i = 0; i < kNumTestArgs; i++) {
|
670
|
+
Init(kTestArgList[i]);
|
671
|
+
Random rnd(test::RandomSeed() + 2);
|
672
|
+
Add("abc", "v");
|
673
|
+
Test(&rnd);
|
674
|
+
}
|
675
|
+
}
|
676
|
+
|
677
|
+
TEST(Harness, SimpleMulti) {
|
678
|
+
for (int i = 0; i < kNumTestArgs; i++) {
|
679
|
+
Init(kTestArgList[i]);
|
680
|
+
Random rnd(test::RandomSeed() + 3);
|
681
|
+
Add("abc", "v");
|
682
|
+
Add("abcd", "v");
|
683
|
+
Add("ac", "v2");
|
684
|
+
Test(&rnd);
|
685
|
+
}
|
686
|
+
}
|
687
|
+
|
688
|
+
TEST(Harness, SimpleSpecialKey) {
|
689
|
+
for (int i = 0; i < kNumTestArgs; i++) {
|
690
|
+
Init(kTestArgList[i]);
|
691
|
+
Random rnd(test::RandomSeed() + 4);
|
692
|
+
Add("\xff\xff", "v3");
|
693
|
+
Test(&rnd);
|
694
|
+
}
|
695
|
+
}
|
696
|
+
|
697
|
+
TEST(Harness, Randomized) {
|
698
|
+
for (int i = 0; i < kNumTestArgs; i++) {
|
699
|
+
Init(kTestArgList[i]);
|
700
|
+
Random rnd(test::RandomSeed() + 5);
|
701
|
+
for (int num_entries = 0; num_entries < 2000;
|
702
|
+
num_entries += (num_entries < 50 ? 1 : 200)) {
|
703
|
+
if ((num_entries % 10) == 0) {
|
704
|
+
fprintf(stderr, "case %d of %d: num_entries = %d\n",
|
705
|
+
(i + 1), int(kNumTestArgs), num_entries);
|
706
|
+
}
|
707
|
+
for (int e = 0; e < num_entries; e++) {
|
708
|
+
std::string v;
|
709
|
+
Add(test::RandomKey(&rnd, rnd.Skewed(4)),
|
710
|
+
test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
|
711
|
+
}
|
712
|
+
Test(&rnd);
|
713
|
+
}
|
714
|
+
}
|
715
|
+
}
|
716
|
+
|
717
|
+
TEST(Harness, RandomizedLongDB) {
|
718
|
+
Random rnd(test::RandomSeed());
|
719
|
+
TestArgs args = { DB_TEST, false, 16 };
|
720
|
+
Init(args);
|
721
|
+
int num_entries = 100000;
|
722
|
+
for (int e = 0; e < num_entries; e++) {
|
723
|
+
std::string v;
|
724
|
+
Add(test::RandomKey(&rnd, rnd.Skewed(4)),
|
725
|
+
test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
|
726
|
+
}
|
727
|
+
Test(&rnd);
|
728
|
+
|
729
|
+
// We must have created enough data to force merging
|
730
|
+
std::string l0_files, l1_files;
|
731
|
+
ASSERT_TRUE(db()->GetProperty("leveldb.num-files-at-level0", &l0_files));
|
732
|
+
ASSERT_TRUE(db()->GetProperty("leveldb.num-files-at-level1", &l1_files));
|
733
|
+
ASSERT_GT(atoi(l0_files.c_str()) + atoi(l1_files.c_str()), 0);
|
734
|
+
|
735
|
+
}
|
736
|
+
|
737
|
+
class MemTableTest { };
|
738
|
+
|
739
|
+
TEST(MemTableTest, Simple) {
|
740
|
+
InternalKeyComparator cmp(BytewiseComparator());
|
741
|
+
MemTable* memtable = new MemTable(cmp);
|
742
|
+
memtable->Ref();
|
743
|
+
WriteBatch batch;
|
744
|
+
WriteBatchInternal::SetSequence(&batch, 100);
|
745
|
+
batch.Put(std::string("k1"), std::string("v1"));
|
746
|
+
batch.Put(std::string("k2"), std::string("v2"));
|
747
|
+
batch.Put(std::string("k3"), std::string("v3"));
|
748
|
+
batch.Put(std::string("largekey"), std::string("vlarge"));
|
749
|
+
ASSERT_TRUE(WriteBatchInternal::InsertInto(&batch, memtable).ok());
|
750
|
+
|
751
|
+
Iterator* iter = memtable->NewIterator();
|
752
|
+
iter->SeekToFirst();
|
753
|
+
while (iter->Valid()) {
|
754
|
+
fprintf(stderr, "key: '%s' -> '%s'\n",
|
755
|
+
iter->key().ToString().c_str(),
|
756
|
+
iter->value().ToString().c_str());
|
757
|
+
iter->Next();
|
758
|
+
}
|
759
|
+
|
760
|
+
delete iter;
|
761
|
+
memtable->Unref();
|
762
|
+
}
|
763
|
+
|
764
|
+
static bool Between(uint64_t val, uint64_t low, uint64_t high) {
|
765
|
+
bool result = (val >= low) && (val <= high);
|
766
|
+
if (!result) {
|
767
|
+
fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
|
768
|
+
(unsigned long long)(val),
|
769
|
+
(unsigned long long)(low),
|
770
|
+
(unsigned long long)(high));
|
771
|
+
}
|
772
|
+
return result;
|
773
|
+
}
|
774
|
+
|
775
|
+
class TableTest { };
|
776
|
+
|
777
|
+
TEST(TableTest, ApproximateOffsetOfPlain) {
|
778
|
+
TableConstructor c(BytewiseComparator());
|
779
|
+
c.Add("k01", "hello");
|
780
|
+
c.Add("k02", "hello2");
|
781
|
+
c.Add("k03", std::string(10000, 'x'));
|
782
|
+
c.Add("k04", std::string(200000, 'x'));
|
783
|
+
c.Add("k05", std::string(300000, 'x'));
|
784
|
+
c.Add("k06", "hello3");
|
785
|
+
c.Add("k07", std::string(100000, 'x'));
|
786
|
+
std::vector<std::string> keys;
|
787
|
+
KVMap kvmap;
|
788
|
+
Options options;
|
789
|
+
options.block_size = 1024;
|
790
|
+
options.compression = kNoCompression;
|
791
|
+
c.Finish(options, &keys, &kvmap);
|
792
|
+
|
793
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
|
794
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
|
795
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01a"), 0, 0));
|
796
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0));
|
797
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 0, 0));
|
798
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 10000, 11000));
|
799
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04a"), 210000, 211000));
|
800
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k05"), 210000, 211000));
|
801
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000));
|
802
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000));
|
803
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 611000));
|
804
|
+
|
805
|
+
}
|
806
|
+
|
807
|
+
static bool SnappyCompressionSupported() {
|
808
|
+
std::string out;
|
809
|
+
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
810
|
+
return port::Snappy_Compress(in.data(), in.size(), &out);
|
811
|
+
}
|
812
|
+
|
813
|
+
TEST(TableTest, ApproximateOffsetOfCompressed) {
|
814
|
+
if (!SnappyCompressionSupported()) {
|
815
|
+
fprintf(stderr, "skipping compression tests\n");
|
816
|
+
return;
|
817
|
+
}
|
818
|
+
|
819
|
+
Random rnd(301);
|
820
|
+
TableConstructor c(BytewiseComparator());
|
821
|
+
std::string tmp;
|
822
|
+
c.Add("k01", "hello");
|
823
|
+
c.Add("k02", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
|
824
|
+
c.Add("k03", "hello3");
|
825
|
+
c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
|
826
|
+
std::vector<std::string> keys;
|
827
|
+
KVMap kvmap;
|
828
|
+
Options options;
|
829
|
+
options.block_size = 1024;
|
830
|
+
options.compression = kSnappyCompression;
|
831
|
+
c.Finish(options, &keys, &kvmap);
|
832
|
+
|
833
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
|
834
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
|
835
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0));
|
836
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 2000, 3000));
|
837
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 2000, 3000));
|
838
|
+
ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 4000, 6000));
|
839
|
+
}
|
840
|
+
|
841
|
+
}
|
842
|
+
|
843
|
+
int main(int argc, char** argv) {
|
844
|
+
return leveldb::test::RunAllTests();
|
845
|
+
}
|