leveldb 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE.txt +22 -0
- data/README.md +95 -0
- data/ext/Rakefile +11 -0
- data/ext/leveldb/LICENSE +27 -0
- data/ext/leveldb/Makefile +206 -0
- data/ext/leveldb/build_config.mk +13 -0
- data/ext/leveldb/db/builder.cc +88 -0
- data/ext/leveldb/db/builder.h +34 -0
- data/ext/leveldb/db/c.cc +595 -0
- data/ext/leveldb/db/c_test.c +390 -0
- data/ext/leveldb/db/corruption_test.cc +359 -0
- data/ext/leveldb/db/db_bench.cc +979 -0
- data/ext/leveldb/db/db_impl.cc +1485 -0
- data/ext/leveldb/db/db_impl.h +203 -0
- data/ext/leveldb/db/db_iter.cc +299 -0
- data/ext/leveldb/db/db_iter.h +26 -0
- data/ext/leveldb/db/db_test.cc +2092 -0
- data/ext/leveldb/db/dbformat.cc +140 -0
- data/ext/leveldb/db/dbformat.h +227 -0
- data/ext/leveldb/db/dbformat_test.cc +112 -0
- data/ext/leveldb/db/filename.cc +139 -0
- data/ext/leveldb/db/filename.h +80 -0
- data/ext/leveldb/db/filename_test.cc +122 -0
- data/ext/leveldb/db/leveldb_main.cc +238 -0
- data/ext/leveldb/db/log_format.h +35 -0
- data/ext/leveldb/db/log_reader.cc +259 -0
- data/ext/leveldb/db/log_reader.h +108 -0
- data/ext/leveldb/db/log_test.cc +500 -0
- data/ext/leveldb/db/log_writer.cc +103 -0
- data/ext/leveldb/db/log_writer.h +48 -0
- data/ext/leveldb/db/memtable.cc +145 -0
- data/ext/leveldb/db/memtable.h +91 -0
- data/ext/leveldb/db/repair.cc +389 -0
- data/ext/leveldb/db/skiplist.h +379 -0
- data/ext/leveldb/db/skiplist_test.cc +378 -0
- data/ext/leveldb/db/snapshot.h +66 -0
- data/ext/leveldb/db/table_cache.cc +121 -0
- data/ext/leveldb/db/table_cache.h +61 -0
- data/ext/leveldb/db/version_edit.cc +266 -0
- data/ext/leveldb/db/version_edit.h +107 -0
- data/ext/leveldb/db/version_edit_test.cc +46 -0
- data/ext/leveldb/db/version_set.cc +1443 -0
- data/ext/leveldb/db/version_set.h +383 -0
- data/ext/leveldb/db/version_set_test.cc +179 -0
- data/ext/leveldb/db/write_batch.cc +147 -0
- data/ext/leveldb/db/write_batch_internal.h +49 -0
- data/ext/leveldb/db/write_batch_test.cc +120 -0
- data/ext/leveldb/doc/bench/db_bench_sqlite3.cc +718 -0
- data/ext/leveldb/doc/bench/db_bench_tree_db.cc +528 -0
- data/ext/leveldb/helpers/memenv/memenv.cc +384 -0
- data/ext/leveldb/helpers/memenv/memenv.h +20 -0
- data/ext/leveldb/helpers/memenv/memenv_test.cc +232 -0
- data/ext/leveldb/include/leveldb/c.h +291 -0
- data/ext/leveldb/include/leveldb/cache.h +99 -0
- data/ext/leveldb/include/leveldb/comparator.h +63 -0
- data/ext/leveldb/include/leveldb/db.h +161 -0
- data/ext/leveldb/include/leveldb/env.h +333 -0
- data/ext/leveldb/include/leveldb/filter_policy.h +70 -0
- data/ext/leveldb/include/leveldb/iterator.h +100 -0
- data/ext/leveldb/include/leveldb/options.h +195 -0
- data/ext/leveldb/include/leveldb/slice.h +109 -0
- data/ext/leveldb/include/leveldb/status.h +106 -0
- data/ext/leveldb/include/leveldb/table.h +85 -0
- data/ext/leveldb/include/leveldb/table_builder.h +92 -0
- data/ext/leveldb/include/leveldb/write_batch.h +64 -0
- data/ext/leveldb/issues/issue178_test.cc +92 -0
- data/ext/leveldb/port/atomic_pointer.h +224 -0
- data/ext/leveldb/port/port.h +19 -0
- data/ext/leveldb/port/port_example.h +135 -0
- data/ext/leveldb/port/port_posix.cc +54 -0
- data/ext/leveldb/port/port_posix.h +157 -0
- data/ext/leveldb/port/thread_annotations.h +59 -0
- data/ext/leveldb/port/win/stdint.h +24 -0
- data/ext/leveldb/table/block.cc +268 -0
- data/ext/leveldb/table/block.h +44 -0
- data/ext/leveldb/table/block_builder.cc +109 -0
- data/ext/leveldb/table/block_builder.h +57 -0
- data/ext/leveldb/table/filter_block.cc +111 -0
- data/ext/leveldb/table/filter_block.h +68 -0
- data/ext/leveldb/table/filter_block_test.cc +128 -0
- data/ext/leveldb/table/format.cc +145 -0
- data/ext/leveldb/table/format.h +108 -0
- data/ext/leveldb/table/iterator.cc +67 -0
- data/ext/leveldb/table/iterator_wrapper.h +63 -0
- data/ext/leveldb/table/merger.cc +197 -0
- data/ext/leveldb/table/merger.h +26 -0
- data/ext/leveldb/table/table.cc +275 -0
- data/ext/leveldb/table/table_builder.cc +270 -0
- data/ext/leveldb/table/table_test.cc +868 -0
- data/ext/leveldb/table/two_level_iterator.cc +182 -0
- data/ext/leveldb/table/two_level_iterator.h +34 -0
- data/ext/leveldb/util/arena.cc +68 -0
- data/ext/leveldb/util/arena.h +68 -0
- data/ext/leveldb/util/arena_test.cc +68 -0
- data/ext/leveldb/util/bloom.cc +95 -0
- data/ext/leveldb/util/bloom_test.cc +160 -0
- data/ext/leveldb/util/cache.cc +325 -0
- data/ext/leveldb/util/cache_test.cc +186 -0
- data/ext/leveldb/util/coding.cc +194 -0
- data/ext/leveldb/util/coding.h +104 -0
- data/ext/leveldb/util/coding_test.cc +196 -0
- data/ext/leveldb/util/comparator.cc +81 -0
- data/ext/leveldb/util/crc32c.cc +332 -0
- data/ext/leveldb/util/crc32c.h +45 -0
- data/ext/leveldb/util/crc32c_test.cc +72 -0
- data/ext/leveldb/util/env.cc +96 -0
- data/ext/leveldb/util/env_posix.cc +698 -0
- data/ext/leveldb/util/env_test.cc +104 -0
- data/ext/leveldb/util/filter_policy.cc +11 -0
- data/ext/leveldb/util/hash.cc +52 -0
- data/ext/leveldb/util/hash.h +19 -0
- data/ext/leveldb/util/histogram.cc +139 -0
- data/ext/leveldb/util/histogram.h +42 -0
- data/ext/leveldb/util/logging.cc +81 -0
- data/ext/leveldb/util/logging.h +47 -0
- data/ext/leveldb/util/mutexlock.h +41 -0
- data/ext/leveldb/util/options.cc +29 -0
- data/ext/leveldb/util/posix_logger.h +98 -0
- data/ext/leveldb/util/random.h +59 -0
- data/ext/leveldb/util/status.cc +75 -0
- data/ext/leveldb/util/testharness.cc +77 -0
- data/ext/leveldb/util/testharness.h +138 -0
- data/ext/leveldb/util/testutil.cc +51 -0
- data/ext/leveldb/util/testutil.h +53 -0
- data/lib/leveldb/version.rb +3 -0
- data/lib/leveldb.rb +1006 -0
- metadata +228 -0
@@ -0,0 +1,383 @@
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
4
|
+
//
|
5
|
+
// The representation of a DBImpl consists of a set of Versions. The
|
6
|
+
// newest version is called "current". Older versions may be kept
|
7
|
+
// around to provide a consistent view to live iterators.
|
8
|
+
//
|
9
|
+
// Each Version keeps track of a set of Table files per level. The
|
10
|
+
// entire set of versions is maintained in a VersionSet.
|
11
|
+
//
|
12
|
+
// Version,VersionSet are thread-compatible, but require external
|
13
|
+
// synchronization on all accesses.
|
14
|
+
|
15
|
+
#ifndef STORAGE_LEVELDB_DB_VERSION_SET_H_
|
16
|
+
#define STORAGE_LEVELDB_DB_VERSION_SET_H_
|
17
|
+
|
18
|
+
#include <map>
|
19
|
+
#include <set>
|
20
|
+
#include <vector>
|
21
|
+
#include "db/dbformat.h"
|
22
|
+
#include "db/version_edit.h"
|
23
|
+
#include "port/port.h"
|
24
|
+
#include "port/thread_annotations.h"
|
25
|
+
|
26
|
+
namespace leveldb {
|
27
|
+
|
28
|
+
namespace log { class Writer; }
|
29
|
+
|
30
|
+
class Compaction;
|
31
|
+
class Iterator;
|
32
|
+
class MemTable;
|
33
|
+
class TableBuilder;
|
34
|
+
class TableCache;
|
35
|
+
class Version;
|
36
|
+
class VersionSet;
|
37
|
+
class WritableFile;
|
38
|
+
|
39
|
+
// Return the smallest index i such that files[i]->largest >= key.
|
40
|
+
// Return files.size() if there is no such file.
|
41
|
+
// REQUIRES: "files" contains a sorted list of non-overlapping files.
|
42
|
+
extern int FindFile(const InternalKeyComparator& icmp,
|
43
|
+
const std::vector<FileMetaData*>& files,
|
44
|
+
const Slice& key);
|
45
|
+
|
46
|
+
// Returns true iff some file in "files" overlaps the user key range
|
47
|
+
// [*smallest,*largest].
|
48
|
+
// smallest==NULL represents a key smaller than all keys in the DB.
|
49
|
+
// largest==NULL represents a key largest than all keys in the DB.
|
50
|
+
// REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges
|
51
|
+
// in sorted order.
|
52
|
+
extern bool SomeFileOverlapsRange(
|
53
|
+
const InternalKeyComparator& icmp,
|
54
|
+
bool disjoint_sorted_files,
|
55
|
+
const std::vector<FileMetaData*>& files,
|
56
|
+
const Slice* smallest_user_key,
|
57
|
+
const Slice* largest_user_key);
|
58
|
+
|
59
|
+
class Version {
|
60
|
+
public:
|
61
|
+
// Append to *iters a sequence of iterators that will
|
62
|
+
// yield the contents of this Version when merged together.
|
63
|
+
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
64
|
+
void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
|
65
|
+
|
66
|
+
// Lookup the value for key. If found, store it in *val and
|
67
|
+
// return OK. Else return a non-OK status. Fills *stats.
|
68
|
+
// REQUIRES: lock is not held
|
69
|
+
struct GetStats {
|
70
|
+
FileMetaData* seek_file;
|
71
|
+
int seek_file_level;
|
72
|
+
};
|
73
|
+
Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
|
74
|
+
GetStats* stats);
|
75
|
+
|
76
|
+
// Adds "stats" into the current state. Returns true if a new
|
77
|
+
// compaction may need to be triggered, false otherwise.
|
78
|
+
// REQUIRES: lock is held
|
79
|
+
bool UpdateStats(const GetStats& stats);
|
80
|
+
|
81
|
+
// Reference count management (so Versions do not disappear out from
|
82
|
+
// under live iterators)
|
83
|
+
void Ref();
|
84
|
+
void Unref();
|
85
|
+
|
86
|
+
void GetOverlappingInputs(
|
87
|
+
int level,
|
88
|
+
const InternalKey* begin, // NULL means before all keys
|
89
|
+
const InternalKey* end, // NULL means after all keys
|
90
|
+
std::vector<FileMetaData*>* inputs);
|
91
|
+
|
92
|
+
// Returns true iff some file in the specified level overlaps
|
93
|
+
// some part of [*smallest_user_key,*largest_user_key].
|
94
|
+
// smallest_user_key==NULL represents a key smaller than all keys in the DB.
|
95
|
+
// largest_user_key==NULL represents a key largest than all keys in the DB.
|
96
|
+
bool OverlapInLevel(int level,
|
97
|
+
const Slice* smallest_user_key,
|
98
|
+
const Slice* largest_user_key);
|
99
|
+
|
100
|
+
// Return the level at which we should place a new memtable compaction
|
101
|
+
// result that covers the range [smallest_user_key,largest_user_key].
|
102
|
+
int PickLevelForMemTableOutput(const Slice& smallest_user_key,
|
103
|
+
const Slice& largest_user_key);
|
104
|
+
|
105
|
+
int NumFiles(int level) const { return files_[level].size(); }
|
106
|
+
|
107
|
+
// Return a human readable string that describes this version's contents.
|
108
|
+
std::string DebugString() const;
|
109
|
+
|
110
|
+
private:
|
111
|
+
friend class Compaction;
|
112
|
+
friend class VersionSet;
|
113
|
+
|
114
|
+
class LevelFileNumIterator;
|
115
|
+
Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const;
|
116
|
+
|
117
|
+
VersionSet* vset_; // VersionSet to which this Version belongs
|
118
|
+
Version* next_; // Next version in linked list
|
119
|
+
Version* prev_; // Previous version in linked list
|
120
|
+
int refs_; // Number of live refs to this version
|
121
|
+
|
122
|
+
// List of files per level
|
123
|
+
std::vector<FileMetaData*> files_[config::kNumLevels];
|
124
|
+
|
125
|
+
// Next file to compact based on seek stats.
|
126
|
+
FileMetaData* file_to_compact_;
|
127
|
+
int file_to_compact_level_;
|
128
|
+
|
129
|
+
// Level that should be compacted next and its compaction score.
|
130
|
+
// Score < 1 means compaction is not strictly needed. These fields
|
131
|
+
// are initialized by Finalize().
|
132
|
+
double compaction_score_;
|
133
|
+
int compaction_level_;
|
134
|
+
|
135
|
+
explicit Version(VersionSet* vset)
|
136
|
+
: vset_(vset), next_(this), prev_(this), refs_(0),
|
137
|
+
file_to_compact_(NULL),
|
138
|
+
file_to_compact_level_(-1),
|
139
|
+
compaction_score_(-1),
|
140
|
+
compaction_level_(-1) {
|
141
|
+
}
|
142
|
+
|
143
|
+
~Version();
|
144
|
+
|
145
|
+
// No copying allowed
|
146
|
+
Version(const Version&);
|
147
|
+
void operator=(const Version&);
|
148
|
+
};
|
149
|
+
|
150
|
+
class VersionSet {
|
151
|
+
public:
|
152
|
+
VersionSet(const std::string& dbname,
|
153
|
+
const Options* options,
|
154
|
+
TableCache* table_cache,
|
155
|
+
const InternalKeyComparator*);
|
156
|
+
~VersionSet();
|
157
|
+
|
158
|
+
// Apply *edit to the current version to form a new descriptor that
|
159
|
+
// is both saved to persistent state and installed as the new
|
160
|
+
// current version. Will release *mu while actually writing to the file.
|
161
|
+
// REQUIRES: *mu is held on entry.
|
162
|
+
// REQUIRES: no other thread concurrently calls LogAndApply()
|
163
|
+
Status LogAndApply(VersionEdit* edit, port::Mutex* mu)
|
164
|
+
EXCLUSIVE_LOCKS_REQUIRED(mu);
|
165
|
+
|
166
|
+
// Recover the last saved descriptor from persistent storage.
|
167
|
+
Status Recover();
|
168
|
+
|
169
|
+
// Return the current version.
|
170
|
+
Version* current() const { return current_; }
|
171
|
+
|
172
|
+
// Return the current manifest file number
|
173
|
+
uint64_t ManifestFileNumber() const { return manifest_file_number_; }
|
174
|
+
|
175
|
+
// Allocate and return a new file number
|
176
|
+
uint64_t NewFileNumber() { return next_file_number_++; }
|
177
|
+
|
178
|
+
// Arrange to reuse "file_number" unless a newer file number has
|
179
|
+
// already been allocated.
|
180
|
+
// REQUIRES: "file_number" was returned by a call to NewFileNumber().
|
181
|
+
void ReuseFileNumber(uint64_t file_number) {
|
182
|
+
if (next_file_number_ == file_number + 1) {
|
183
|
+
next_file_number_ = file_number;
|
184
|
+
}
|
185
|
+
}
|
186
|
+
|
187
|
+
// Return the number of Table files at the specified level.
|
188
|
+
int NumLevelFiles(int level) const;
|
189
|
+
|
190
|
+
// Return the combined file size of all files at the specified level.
|
191
|
+
int64_t NumLevelBytes(int level) const;
|
192
|
+
|
193
|
+
// Return the last sequence number.
|
194
|
+
uint64_t LastSequence() const { return last_sequence_; }
|
195
|
+
|
196
|
+
// Set the last sequence number to s.
|
197
|
+
void SetLastSequence(uint64_t s) {
|
198
|
+
assert(s >= last_sequence_);
|
199
|
+
last_sequence_ = s;
|
200
|
+
}
|
201
|
+
|
202
|
+
// Mark the specified file number as used.
|
203
|
+
void MarkFileNumberUsed(uint64_t number);
|
204
|
+
|
205
|
+
// Return the current log file number.
|
206
|
+
uint64_t LogNumber() const { return log_number_; }
|
207
|
+
|
208
|
+
// Return the log file number for the log file that is currently
|
209
|
+
// being compacted, or zero if there is no such log file.
|
210
|
+
uint64_t PrevLogNumber() const { return prev_log_number_; }
|
211
|
+
|
212
|
+
// Pick level and inputs for a new compaction.
|
213
|
+
// Returns NULL if there is no compaction to be done.
|
214
|
+
// Otherwise returns a pointer to a heap-allocated object that
|
215
|
+
// describes the compaction. Caller should delete the result.
|
216
|
+
Compaction* PickCompaction();
|
217
|
+
|
218
|
+
// Return a compaction object for compacting the range [begin,end] in
|
219
|
+
// the specified level. Returns NULL if there is nothing in that
|
220
|
+
// level that overlaps the specified range. Caller should delete
|
221
|
+
// the result.
|
222
|
+
Compaction* CompactRange(
|
223
|
+
int level,
|
224
|
+
const InternalKey* begin,
|
225
|
+
const InternalKey* end);
|
226
|
+
|
227
|
+
// Return the maximum overlapping data (in bytes) at next level for any
|
228
|
+
// file at a level >= 1.
|
229
|
+
int64_t MaxNextLevelOverlappingBytes();
|
230
|
+
|
231
|
+
// Create an iterator that reads over the compaction inputs for "*c".
|
232
|
+
// The caller should delete the iterator when no longer needed.
|
233
|
+
Iterator* MakeInputIterator(Compaction* c);
|
234
|
+
|
235
|
+
// Returns true iff some level needs a compaction.
|
236
|
+
bool NeedsCompaction() const {
|
237
|
+
Version* v = current_;
|
238
|
+
return (v->compaction_score_ >= 1) || (v->file_to_compact_ != NULL);
|
239
|
+
}
|
240
|
+
|
241
|
+
// Add all files listed in any live version to *live.
|
242
|
+
// May also mutate some internal state.
|
243
|
+
void AddLiveFiles(std::set<uint64_t>* live);
|
244
|
+
|
245
|
+
// Return the approximate offset in the database of the data for
|
246
|
+
// "key" as of version "v".
|
247
|
+
uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key);
|
248
|
+
|
249
|
+
// Return a human-readable short (single-line) summary of the number
|
250
|
+
// of files per level. Uses *scratch as backing store.
|
251
|
+
struct LevelSummaryStorage {
|
252
|
+
char buffer[100];
|
253
|
+
};
|
254
|
+
const char* LevelSummary(LevelSummaryStorage* scratch) const;
|
255
|
+
|
256
|
+
private:
|
257
|
+
class Builder;
|
258
|
+
|
259
|
+
friend class Compaction;
|
260
|
+
friend class Version;
|
261
|
+
|
262
|
+
void Finalize(Version* v);
|
263
|
+
|
264
|
+
void GetRange(const std::vector<FileMetaData*>& inputs,
|
265
|
+
InternalKey* smallest,
|
266
|
+
InternalKey* largest);
|
267
|
+
|
268
|
+
void GetRange2(const std::vector<FileMetaData*>& inputs1,
|
269
|
+
const std::vector<FileMetaData*>& inputs2,
|
270
|
+
InternalKey* smallest,
|
271
|
+
InternalKey* largest);
|
272
|
+
|
273
|
+
void SetupOtherInputs(Compaction* c);
|
274
|
+
|
275
|
+
// Save current contents to *log
|
276
|
+
Status WriteSnapshot(log::Writer* log);
|
277
|
+
|
278
|
+
void AppendVersion(Version* v);
|
279
|
+
|
280
|
+
bool ManifestContains(const std::string& record) const;
|
281
|
+
|
282
|
+
Env* const env_;
|
283
|
+
const std::string dbname_;
|
284
|
+
const Options* const options_;
|
285
|
+
TableCache* const table_cache_;
|
286
|
+
const InternalKeyComparator icmp_;
|
287
|
+
uint64_t next_file_number_;
|
288
|
+
uint64_t manifest_file_number_;
|
289
|
+
uint64_t last_sequence_;
|
290
|
+
uint64_t log_number_;
|
291
|
+
uint64_t prev_log_number_; // 0 or backing store for memtable being compacted
|
292
|
+
|
293
|
+
// Opened lazily
|
294
|
+
WritableFile* descriptor_file_;
|
295
|
+
log::Writer* descriptor_log_;
|
296
|
+
Version dummy_versions_; // Head of circular doubly-linked list of versions.
|
297
|
+
Version* current_; // == dummy_versions_.prev_
|
298
|
+
|
299
|
+
// Per-level key at which the next compaction at that level should start.
|
300
|
+
// Either an empty string, or a valid InternalKey.
|
301
|
+
std::string compact_pointer_[config::kNumLevels];
|
302
|
+
|
303
|
+
// No copying allowed
|
304
|
+
VersionSet(const VersionSet&);
|
305
|
+
void operator=(const VersionSet&);
|
306
|
+
};
|
307
|
+
|
308
|
+
// A Compaction encapsulates information about a compaction.
|
309
|
+
class Compaction {
|
310
|
+
public:
|
311
|
+
~Compaction();
|
312
|
+
|
313
|
+
// Return the level that is being compacted. Inputs from "level"
|
314
|
+
// and "level+1" will be merged to produce a set of "level+1" files.
|
315
|
+
int level() const { return level_; }
|
316
|
+
|
317
|
+
// Return the object that holds the edits to the descriptor done
|
318
|
+
// by this compaction.
|
319
|
+
VersionEdit* edit() { return &edit_; }
|
320
|
+
|
321
|
+
// "which" must be either 0 or 1
|
322
|
+
int num_input_files(int which) const { return inputs_[which].size(); }
|
323
|
+
|
324
|
+
// Return the ith input file at "level()+which" ("which" must be 0 or 1).
|
325
|
+
FileMetaData* input(int which, int i) const { return inputs_[which][i]; }
|
326
|
+
|
327
|
+
// Maximum size of files to build during this compaction.
|
328
|
+
uint64_t MaxOutputFileSize() const { return max_output_file_size_; }
|
329
|
+
|
330
|
+
// Is this a trivial compaction that can be implemented by just
|
331
|
+
// moving a single input file to the next level (no merging or splitting)
|
332
|
+
bool IsTrivialMove() const;
|
333
|
+
|
334
|
+
// Add all inputs to this compaction as delete operations to *edit.
|
335
|
+
void AddInputDeletions(VersionEdit* edit);
|
336
|
+
|
337
|
+
// Returns true if the information we have available guarantees that
|
338
|
+
// the compaction is producing data in "level+1" for which no data exists
|
339
|
+
// in levels greater than "level+1".
|
340
|
+
bool IsBaseLevelForKey(const Slice& user_key);
|
341
|
+
|
342
|
+
// Returns true iff we should stop building the current output
|
343
|
+
// before processing "internal_key".
|
344
|
+
bool ShouldStopBefore(const Slice& internal_key);
|
345
|
+
|
346
|
+
// Release the input version for the compaction, once the compaction
|
347
|
+
// is successful.
|
348
|
+
void ReleaseInputs();
|
349
|
+
|
350
|
+
private:
|
351
|
+
friend class Version;
|
352
|
+
friend class VersionSet;
|
353
|
+
|
354
|
+
explicit Compaction(int level);
|
355
|
+
|
356
|
+
int level_;
|
357
|
+
uint64_t max_output_file_size_;
|
358
|
+
Version* input_version_;
|
359
|
+
VersionEdit edit_;
|
360
|
+
|
361
|
+
// Each compaction reads inputs from "level_" and "level_+1"
|
362
|
+
std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
|
363
|
+
|
364
|
+
// State used to check for number of of overlapping grandparent files
|
365
|
+
// (parent == level_ + 1, grandparent == level_ + 2)
|
366
|
+
std::vector<FileMetaData*> grandparents_;
|
367
|
+
size_t grandparent_index_; // Index in grandparent_starts_
|
368
|
+
bool seen_key_; // Some output key has been seen
|
369
|
+
int64_t overlapped_bytes_; // Bytes of overlap between current output
|
370
|
+
// and grandparent files
|
371
|
+
|
372
|
+
// State for implementing IsBaseLevelForKey
|
373
|
+
|
374
|
+
// level_ptrs_ holds indices into input_version_->levels_: our state
|
375
|
+
// is that we are positioned at one of the file ranges for each
|
376
|
+
// higher level than the ones involved in this compaction (i.e. for
|
377
|
+
// all L >= level_ + 2).
|
378
|
+
size_t level_ptrs_[config::kNumLevels];
|
379
|
+
};
|
380
|
+
|
381
|
+
} // namespace leveldb
|
382
|
+
|
383
|
+
#endif // STORAGE_LEVELDB_DB_VERSION_SET_H_
|
@@ -0,0 +1,179 @@
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
4
|
+
|
5
|
+
#include "db/version_set.h"
|
6
|
+
#include "util/logging.h"
|
7
|
+
#include "util/testharness.h"
|
8
|
+
#include "util/testutil.h"
|
9
|
+
|
10
|
+
namespace leveldb {
|
11
|
+
|
12
|
+
class FindFileTest {
|
13
|
+
public:
|
14
|
+
std::vector<FileMetaData*> files_;
|
15
|
+
bool disjoint_sorted_files_;
|
16
|
+
|
17
|
+
FindFileTest() : disjoint_sorted_files_(true) { }
|
18
|
+
|
19
|
+
~FindFileTest() {
|
20
|
+
for (int i = 0; i < files_.size(); i++) {
|
21
|
+
delete files_[i];
|
22
|
+
}
|
23
|
+
}
|
24
|
+
|
25
|
+
void Add(const char* smallest, const char* largest,
|
26
|
+
SequenceNumber smallest_seq = 100,
|
27
|
+
SequenceNumber largest_seq = 100) {
|
28
|
+
FileMetaData* f = new FileMetaData;
|
29
|
+
f->number = files_.size() + 1;
|
30
|
+
f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
|
31
|
+
f->largest = InternalKey(largest, largest_seq, kTypeValue);
|
32
|
+
files_.push_back(f);
|
33
|
+
}
|
34
|
+
|
35
|
+
int Find(const char* key) {
|
36
|
+
InternalKey target(key, 100, kTypeValue);
|
37
|
+
InternalKeyComparator cmp(BytewiseComparator());
|
38
|
+
return FindFile(cmp, files_, target.Encode());
|
39
|
+
}
|
40
|
+
|
41
|
+
bool Overlaps(const char* smallest, const char* largest) {
|
42
|
+
InternalKeyComparator cmp(BytewiseComparator());
|
43
|
+
Slice s(smallest != NULL ? smallest : "");
|
44
|
+
Slice l(largest != NULL ? largest : "");
|
45
|
+
return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, files_,
|
46
|
+
(smallest != NULL ? &s : NULL),
|
47
|
+
(largest != NULL ? &l : NULL));
|
48
|
+
}
|
49
|
+
};
|
50
|
+
|
51
|
+
TEST(FindFileTest, Empty) {
|
52
|
+
ASSERT_EQ(0, Find("foo"));
|
53
|
+
ASSERT_TRUE(! Overlaps("a", "z"));
|
54
|
+
ASSERT_TRUE(! Overlaps(NULL, "z"));
|
55
|
+
ASSERT_TRUE(! Overlaps("a", NULL));
|
56
|
+
ASSERT_TRUE(! Overlaps(NULL, NULL));
|
57
|
+
}
|
58
|
+
|
59
|
+
TEST(FindFileTest, Single) {
|
60
|
+
Add("p", "q");
|
61
|
+
ASSERT_EQ(0, Find("a"));
|
62
|
+
ASSERT_EQ(0, Find("p"));
|
63
|
+
ASSERT_EQ(0, Find("p1"));
|
64
|
+
ASSERT_EQ(0, Find("q"));
|
65
|
+
ASSERT_EQ(1, Find("q1"));
|
66
|
+
ASSERT_EQ(1, Find("z"));
|
67
|
+
|
68
|
+
ASSERT_TRUE(! Overlaps("a", "b"));
|
69
|
+
ASSERT_TRUE(! Overlaps("z1", "z2"));
|
70
|
+
ASSERT_TRUE(Overlaps("a", "p"));
|
71
|
+
ASSERT_TRUE(Overlaps("a", "q"));
|
72
|
+
ASSERT_TRUE(Overlaps("a", "z"));
|
73
|
+
ASSERT_TRUE(Overlaps("p", "p1"));
|
74
|
+
ASSERT_TRUE(Overlaps("p", "q"));
|
75
|
+
ASSERT_TRUE(Overlaps("p", "z"));
|
76
|
+
ASSERT_TRUE(Overlaps("p1", "p2"));
|
77
|
+
ASSERT_TRUE(Overlaps("p1", "z"));
|
78
|
+
ASSERT_TRUE(Overlaps("q", "q"));
|
79
|
+
ASSERT_TRUE(Overlaps("q", "q1"));
|
80
|
+
|
81
|
+
ASSERT_TRUE(! Overlaps(NULL, "j"));
|
82
|
+
ASSERT_TRUE(! Overlaps("r", NULL));
|
83
|
+
ASSERT_TRUE(Overlaps(NULL, "p"));
|
84
|
+
ASSERT_TRUE(Overlaps(NULL, "p1"));
|
85
|
+
ASSERT_TRUE(Overlaps("q", NULL));
|
86
|
+
ASSERT_TRUE(Overlaps(NULL, NULL));
|
87
|
+
}
|
88
|
+
|
89
|
+
|
90
|
+
TEST(FindFileTest, Multiple) {
|
91
|
+
Add("150", "200");
|
92
|
+
Add("200", "250");
|
93
|
+
Add("300", "350");
|
94
|
+
Add("400", "450");
|
95
|
+
ASSERT_EQ(0, Find("100"));
|
96
|
+
ASSERT_EQ(0, Find("150"));
|
97
|
+
ASSERT_EQ(0, Find("151"));
|
98
|
+
ASSERT_EQ(0, Find("199"));
|
99
|
+
ASSERT_EQ(0, Find("200"));
|
100
|
+
ASSERT_EQ(1, Find("201"));
|
101
|
+
ASSERT_EQ(1, Find("249"));
|
102
|
+
ASSERT_EQ(1, Find("250"));
|
103
|
+
ASSERT_EQ(2, Find("251"));
|
104
|
+
ASSERT_EQ(2, Find("299"));
|
105
|
+
ASSERT_EQ(2, Find("300"));
|
106
|
+
ASSERT_EQ(2, Find("349"));
|
107
|
+
ASSERT_EQ(2, Find("350"));
|
108
|
+
ASSERT_EQ(3, Find("351"));
|
109
|
+
ASSERT_EQ(3, Find("400"));
|
110
|
+
ASSERT_EQ(3, Find("450"));
|
111
|
+
ASSERT_EQ(4, Find("451"));
|
112
|
+
|
113
|
+
ASSERT_TRUE(! Overlaps("100", "149"));
|
114
|
+
ASSERT_TRUE(! Overlaps("251", "299"));
|
115
|
+
ASSERT_TRUE(! Overlaps("451", "500"));
|
116
|
+
ASSERT_TRUE(! Overlaps("351", "399"));
|
117
|
+
|
118
|
+
ASSERT_TRUE(Overlaps("100", "150"));
|
119
|
+
ASSERT_TRUE(Overlaps("100", "200"));
|
120
|
+
ASSERT_TRUE(Overlaps("100", "300"));
|
121
|
+
ASSERT_TRUE(Overlaps("100", "400"));
|
122
|
+
ASSERT_TRUE(Overlaps("100", "500"));
|
123
|
+
ASSERT_TRUE(Overlaps("375", "400"));
|
124
|
+
ASSERT_TRUE(Overlaps("450", "450"));
|
125
|
+
ASSERT_TRUE(Overlaps("450", "500"));
|
126
|
+
}
|
127
|
+
|
128
|
+
TEST(FindFileTest, MultipleNullBoundaries) {
|
129
|
+
Add("150", "200");
|
130
|
+
Add("200", "250");
|
131
|
+
Add("300", "350");
|
132
|
+
Add("400", "450");
|
133
|
+
ASSERT_TRUE(! Overlaps(NULL, "149"));
|
134
|
+
ASSERT_TRUE(! Overlaps("451", NULL));
|
135
|
+
ASSERT_TRUE(Overlaps(NULL, NULL));
|
136
|
+
ASSERT_TRUE(Overlaps(NULL, "150"));
|
137
|
+
ASSERT_TRUE(Overlaps(NULL, "199"));
|
138
|
+
ASSERT_TRUE(Overlaps(NULL, "200"));
|
139
|
+
ASSERT_TRUE(Overlaps(NULL, "201"));
|
140
|
+
ASSERT_TRUE(Overlaps(NULL, "400"));
|
141
|
+
ASSERT_TRUE(Overlaps(NULL, "800"));
|
142
|
+
ASSERT_TRUE(Overlaps("100", NULL));
|
143
|
+
ASSERT_TRUE(Overlaps("200", NULL));
|
144
|
+
ASSERT_TRUE(Overlaps("449", NULL));
|
145
|
+
ASSERT_TRUE(Overlaps("450", NULL));
|
146
|
+
}
|
147
|
+
|
148
|
+
TEST(FindFileTest, OverlapSequenceChecks) {
|
149
|
+
Add("200", "200", 5000, 3000);
|
150
|
+
ASSERT_TRUE(! Overlaps("199", "199"));
|
151
|
+
ASSERT_TRUE(! Overlaps("201", "300"));
|
152
|
+
ASSERT_TRUE(Overlaps("200", "200"));
|
153
|
+
ASSERT_TRUE(Overlaps("190", "200"));
|
154
|
+
ASSERT_TRUE(Overlaps("200", "210"));
|
155
|
+
}
|
156
|
+
|
157
|
+
TEST(FindFileTest, OverlappingFiles) {
|
158
|
+
Add("150", "600");
|
159
|
+
Add("400", "500");
|
160
|
+
disjoint_sorted_files_ = false;
|
161
|
+
ASSERT_TRUE(! Overlaps("100", "149"));
|
162
|
+
ASSERT_TRUE(! Overlaps("601", "700"));
|
163
|
+
ASSERT_TRUE(Overlaps("100", "150"));
|
164
|
+
ASSERT_TRUE(Overlaps("100", "200"));
|
165
|
+
ASSERT_TRUE(Overlaps("100", "300"));
|
166
|
+
ASSERT_TRUE(Overlaps("100", "400"));
|
167
|
+
ASSERT_TRUE(Overlaps("100", "500"));
|
168
|
+
ASSERT_TRUE(Overlaps("375", "400"));
|
169
|
+
ASSERT_TRUE(Overlaps("450", "450"));
|
170
|
+
ASSERT_TRUE(Overlaps("450", "500"));
|
171
|
+
ASSERT_TRUE(Overlaps("450", "700"));
|
172
|
+
ASSERT_TRUE(Overlaps("600", "700"));
|
173
|
+
}
|
174
|
+
|
175
|
+
} // namespace leveldb
|
176
|
+
|
177
|
+
int main(int argc, char** argv) {
|
178
|
+
return leveldb::test::RunAllTests();
|
179
|
+
}
|
@@ -0,0 +1,147 @@
|
|
1
|
+
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
3
|
+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
4
|
+
//
|
5
|
+
// WriteBatch::rep_ :=
|
6
|
+
// sequence: fixed64
|
7
|
+
// count: fixed32
|
8
|
+
// data: record[count]
|
9
|
+
// record :=
|
10
|
+
// kTypeValue varstring varstring |
|
11
|
+
// kTypeDeletion varstring
|
12
|
+
// varstring :=
|
13
|
+
// len: varint32
|
14
|
+
// data: uint8[len]
|
15
|
+
|
16
|
+
#include "leveldb/write_batch.h"
|
17
|
+
|
18
|
+
#include "leveldb/db.h"
|
19
|
+
#include "db/dbformat.h"
|
20
|
+
#include "db/memtable.h"
|
21
|
+
#include "db/write_batch_internal.h"
|
22
|
+
#include "util/coding.h"
|
23
|
+
|
24
|
+
namespace leveldb {
|
25
|
+
|
26
|
+
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
|
27
|
+
static const size_t kHeader = 12;
|
28
|
+
|
29
|
+
WriteBatch::WriteBatch() {
|
30
|
+
Clear();
|
31
|
+
}
|
32
|
+
|
33
|
+
WriteBatch::~WriteBatch() { }
|
34
|
+
|
35
|
+
WriteBatch::Handler::~Handler() { }
|
36
|
+
|
37
|
+
void WriteBatch::Clear() {
|
38
|
+
rep_.clear();
|
39
|
+
rep_.resize(kHeader);
|
40
|
+
}
|
41
|
+
|
42
|
+
Status WriteBatch::Iterate(Handler* handler) const {
|
43
|
+
Slice input(rep_);
|
44
|
+
if (input.size() < kHeader) {
|
45
|
+
return Status::Corruption("malformed WriteBatch (too small)");
|
46
|
+
}
|
47
|
+
|
48
|
+
input.remove_prefix(kHeader);
|
49
|
+
Slice key, value;
|
50
|
+
int found = 0;
|
51
|
+
while (!input.empty()) {
|
52
|
+
found++;
|
53
|
+
char tag = input[0];
|
54
|
+
input.remove_prefix(1);
|
55
|
+
switch (tag) {
|
56
|
+
case kTypeValue:
|
57
|
+
if (GetLengthPrefixedSlice(&input, &key) &&
|
58
|
+
GetLengthPrefixedSlice(&input, &value)) {
|
59
|
+
handler->Put(key, value);
|
60
|
+
} else {
|
61
|
+
return Status::Corruption("bad WriteBatch Put");
|
62
|
+
}
|
63
|
+
break;
|
64
|
+
case kTypeDeletion:
|
65
|
+
if (GetLengthPrefixedSlice(&input, &key)) {
|
66
|
+
handler->Delete(key);
|
67
|
+
} else {
|
68
|
+
return Status::Corruption("bad WriteBatch Delete");
|
69
|
+
}
|
70
|
+
break;
|
71
|
+
default:
|
72
|
+
return Status::Corruption("unknown WriteBatch tag");
|
73
|
+
}
|
74
|
+
}
|
75
|
+
if (found != WriteBatchInternal::Count(this)) {
|
76
|
+
return Status::Corruption("WriteBatch has wrong count");
|
77
|
+
} else {
|
78
|
+
return Status::OK();
|
79
|
+
}
|
80
|
+
}
|
81
|
+
|
82
|
+
int WriteBatchInternal::Count(const WriteBatch* b) {
|
83
|
+
return DecodeFixed32(b->rep_.data() + 8);
|
84
|
+
}
|
85
|
+
|
86
|
+
void WriteBatchInternal::SetCount(WriteBatch* b, int n) {
|
87
|
+
EncodeFixed32(&b->rep_[8], n);
|
88
|
+
}
|
89
|
+
|
90
|
+
SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) {
|
91
|
+
return SequenceNumber(DecodeFixed64(b->rep_.data()));
|
92
|
+
}
|
93
|
+
|
94
|
+
void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) {
|
95
|
+
EncodeFixed64(&b->rep_[0], seq);
|
96
|
+
}
|
97
|
+
|
98
|
+
void WriteBatch::Put(const Slice& key, const Slice& value) {
|
99
|
+
WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
|
100
|
+
rep_.push_back(static_cast<char>(kTypeValue));
|
101
|
+
PutLengthPrefixedSlice(&rep_, key);
|
102
|
+
PutLengthPrefixedSlice(&rep_, value);
|
103
|
+
}
|
104
|
+
|
105
|
+
void WriteBatch::Delete(const Slice& key) {
|
106
|
+
WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
|
107
|
+
rep_.push_back(static_cast<char>(kTypeDeletion));
|
108
|
+
PutLengthPrefixedSlice(&rep_, key);
|
109
|
+
}
|
110
|
+
|
111
|
+
namespace {
|
112
|
+
class MemTableInserter : public WriteBatch::Handler {
|
113
|
+
public:
|
114
|
+
SequenceNumber sequence_;
|
115
|
+
MemTable* mem_;
|
116
|
+
|
117
|
+
virtual void Put(const Slice& key, const Slice& value) {
|
118
|
+
mem_->Add(sequence_, kTypeValue, key, value);
|
119
|
+
sequence_++;
|
120
|
+
}
|
121
|
+
virtual void Delete(const Slice& key) {
|
122
|
+
mem_->Add(sequence_, kTypeDeletion, key, Slice());
|
123
|
+
sequence_++;
|
124
|
+
}
|
125
|
+
};
|
126
|
+
} // namespace
|
127
|
+
|
128
|
+
Status WriteBatchInternal::InsertInto(const WriteBatch* b,
|
129
|
+
MemTable* memtable) {
|
130
|
+
MemTableInserter inserter;
|
131
|
+
inserter.sequence_ = WriteBatchInternal::Sequence(b);
|
132
|
+
inserter.mem_ = memtable;
|
133
|
+
return b->Iterate(&inserter);
|
134
|
+
}
|
135
|
+
|
136
|
+
void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
|
137
|
+
assert(contents.size() >= kHeader);
|
138
|
+
b->rep_.assign(contents.data(), contents.size());
|
139
|
+
}
|
140
|
+
|
141
|
+
void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) {
|
142
|
+
SetCount(dst, Count(dst) + Count(src));
|
143
|
+
assert(src->rep_.size() >= kHeader);
|
144
|
+
dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader);
|
145
|
+
}
|
146
|
+
|
147
|
+
} // namespace leveldb
|