leveldb 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (128) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +22 -0
  3. data/README.md +95 -0
  4. data/ext/Rakefile +11 -0
  5. data/ext/leveldb/LICENSE +27 -0
  6. data/ext/leveldb/Makefile +206 -0
  7. data/ext/leveldb/build_config.mk +13 -0
  8. data/ext/leveldb/db/builder.cc +88 -0
  9. data/ext/leveldb/db/builder.h +34 -0
  10. data/ext/leveldb/db/c.cc +595 -0
  11. data/ext/leveldb/db/c_test.c +390 -0
  12. data/ext/leveldb/db/corruption_test.cc +359 -0
  13. data/ext/leveldb/db/db_bench.cc +979 -0
  14. data/ext/leveldb/db/db_impl.cc +1485 -0
  15. data/ext/leveldb/db/db_impl.h +203 -0
  16. data/ext/leveldb/db/db_iter.cc +299 -0
  17. data/ext/leveldb/db/db_iter.h +26 -0
  18. data/ext/leveldb/db/db_test.cc +2092 -0
  19. data/ext/leveldb/db/dbformat.cc +140 -0
  20. data/ext/leveldb/db/dbformat.h +227 -0
  21. data/ext/leveldb/db/dbformat_test.cc +112 -0
  22. data/ext/leveldb/db/filename.cc +139 -0
  23. data/ext/leveldb/db/filename.h +80 -0
  24. data/ext/leveldb/db/filename_test.cc +122 -0
  25. data/ext/leveldb/db/leveldb_main.cc +238 -0
  26. data/ext/leveldb/db/log_format.h +35 -0
  27. data/ext/leveldb/db/log_reader.cc +259 -0
  28. data/ext/leveldb/db/log_reader.h +108 -0
  29. data/ext/leveldb/db/log_test.cc +500 -0
  30. data/ext/leveldb/db/log_writer.cc +103 -0
  31. data/ext/leveldb/db/log_writer.h +48 -0
  32. data/ext/leveldb/db/memtable.cc +145 -0
  33. data/ext/leveldb/db/memtable.h +91 -0
  34. data/ext/leveldb/db/repair.cc +389 -0
  35. data/ext/leveldb/db/skiplist.h +379 -0
  36. data/ext/leveldb/db/skiplist_test.cc +378 -0
  37. data/ext/leveldb/db/snapshot.h +66 -0
  38. data/ext/leveldb/db/table_cache.cc +121 -0
  39. data/ext/leveldb/db/table_cache.h +61 -0
  40. data/ext/leveldb/db/version_edit.cc +266 -0
  41. data/ext/leveldb/db/version_edit.h +107 -0
  42. data/ext/leveldb/db/version_edit_test.cc +46 -0
  43. data/ext/leveldb/db/version_set.cc +1443 -0
  44. data/ext/leveldb/db/version_set.h +383 -0
  45. data/ext/leveldb/db/version_set_test.cc +179 -0
  46. data/ext/leveldb/db/write_batch.cc +147 -0
  47. data/ext/leveldb/db/write_batch_internal.h +49 -0
  48. data/ext/leveldb/db/write_batch_test.cc +120 -0
  49. data/ext/leveldb/doc/bench/db_bench_sqlite3.cc +718 -0
  50. data/ext/leveldb/doc/bench/db_bench_tree_db.cc +528 -0
  51. data/ext/leveldb/helpers/memenv/memenv.cc +384 -0
  52. data/ext/leveldb/helpers/memenv/memenv.h +20 -0
  53. data/ext/leveldb/helpers/memenv/memenv_test.cc +232 -0
  54. data/ext/leveldb/include/leveldb/c.h +291 -0
  55. data/ext/leveldb/include/leveldb/cache.h +99 -0
  56. data/ext/leveldb/include/leveldb/comparator.h +63 -0
  57. data/ext/leveldb/include/leveldb/db.h +161 -0
  58. data/ext/leveldb/include/leveldb/env.h +333 -0
  59. data/ext/leveldb/include/leveldb/filter_policy.h +70 -0
  60. data/ext/leveldb/include/leveldb/iterator.h +100 -0
  61. data/ext/leveldb/include/leveldb/options.h +195 -0
  62. data/ext/leveldb/include/leveldb/slice.h +109 -0
  63. data/ext/leveldb/include/leveldb/status.h +106 -0
  64. data/ext/leveldb/include/leveldb/table.h +85 -0
  65. data/ext/leveldb/include/leveldb/table_builder.h +92 -0
  66. data/ext/leveldb/include/leveldb/write_batch.h +64 -0
  67. data/ext/leveldb/issues/issue178_test.cc +92 -0
  68. data/ext/leveldb/port/atomic_pointer.h +224 -0
  69. data/ext/leveldb/port/port.h +19 -0
  70. data/ext/leveldb/port/port_example.h +135 -0
  71. data/ext/leveldb/port/port_posix.cc +54 -0
  72. data/ext/leveldb/port/port_posix.h +157 -0
  73. data/ext/leveldb/port/thread_annotations.h +59 -0
  74. data/ext/leveldb/port/win/stdint.h +24 -0
  75. data/ext/leveldb/table/block.cc +268 -0
  76. data/ext/leveldb/table/block.h +44 -0
  77. data/ext/leveldb/table/block_builder.cc +109 -0
  78. data/ext/leveldb/table/block_builder.h +57 -0
  79. data/ext/leveldb/table/filter_block.cc +111 -0
  80. data/ext/leveldb/table/filter_block.h +68 -0
  81. data/ext/leveldb/table/filter_block_test.cc +128 -0
  82. data/ext/leveldb/table/format.cc +145 -0
  83. data/ext/leveldb/table/format.h +108 -0
  84. data/ext/leveldb/table/iterator.cc +67 -0
  85. data/ext/leveldb/table/iterator_wrapper.h +63 -0
  86. data/ext/leveldb/table/merger.cc +197 -0
  87. data/ext/leveldb/table/merger.h +26 -0
  88. data/ext/leveldb/table/table.cc +275 -0
  89. data/ext/leveldb/table/table_builder.cc +270 -0
  90. data/ext/leveldb/table/table_test.cc +868 -0
  91. data/ext/leveldb/table/two_level_iterator.cc +182 -0
  92. data/ext/leveldb/table/two_level_iterator.h +34 -0
  93. data/ext/leveldb/util/arena.cc +68 -0
  94. data/ext/leveldb/util/arena.h +68 -0
  95. data/ext/leveldb/util/arena_test.cc +68 -0
  96. data/ext/leveldb/util/bloom.cc +95 -0
  97. data/ext/leveldb/util/bloom_test.cc +160 -0
  98. data/ext/leveldb/util/cache.cc +325 -0
  99. data/ext/leveldb/util/cache_test.cc +186 -0
  100. data/ext/leveldb/util/coding.cc +194 -0
  101. data/ext/leveldb/util/coding.h +104 -0
  102. data/ext/leveldb/util/coding_test.cc +196 -0
  103. data/ext/leveldb/util/comparator.cc +81 -0
  104. data/ext/leveldb/util/crc32c.cc +332 -0
  105. data/ext/leveldb/util/crc32c.h +45 -0
  106. data/ext/leveldb/util/crc32c_test.cc +72 -0
  107. data/ext/leveldb/util/env.cc +96 -0
  108. data/ext/leveldb/util/env_posix.cc +698 -0
  109. data/ext/leveldb/util/env_test.cc +104 -0
  110. data/ext/leveldb/util/filter_policy.cc +11 -0
  111. data/ext/leveldb/util/hash.cc +52 -0
  112. data/ext/leveldb/util/hash.h +19 -0
  113. data/ext/leveldb/util/histogram.cc +139 -0
  114. data/ext/leveldb/util/histogram.h +42 -0
  115. data/ext/leveldb/util/logging.cc +81 -0
  116. data/ext/leveldb/util/logging.h +47 -0
  117. data/ext/leveldb/util/mutexlock.h +41 -0
  118. data/ext/leveldb/util/options.cc +29 -0
  119. data/ext/leveldb/util/posix_logger.h +98 -0
  120. data/ext/leveldb/util/random.h +59 -0
  121. data/ext/leveldb/util/status.cc +75 -0
  122. data/ext/leveldb/util/testharness.cc +77 -0
  123. data/ext/leveldb/util/testharness.h +138 -0
  124. data/ext/leveldb/util/testutil.cc +51 -0
  125. data/ext/leveldb/util/testutil.h +53 -0
  126. data/lib/leveldb/version.rb +3 -0
  127. data/lib/leveldb.rb +1006 -0
  128. metadata +228 -0
@@ -0,0 +1,1443 @@
1
+ // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2
+ // Use of this source code is governed by a BSD-style license that can be
3
+ // found in the LICENSE file. See the AUTHORS file for names of contributors.
4
+
5
+ #include "db/version_set.h"
6
+
7
+ #include <algorithm>
8
+ #include <stdio.h>
9
+ #include "db/filename.h"
10
+ #include "db/log_reader.h"
11
+ #include "db/log_writer.h"
12
+ #include "db/memtable.h"
13
+ #include "db/table_cache.h"
14
+ #include "leveldb/env.h"
15
+ #include "leveldb/table_builder.h"
16
+ #include "table/merger.h"
17
+ #include "table/two_level_iterator.h"
18
+ #include "util/coding.h"
19
+ #include "util/logging.h"
20
+
21
+ namespace leveldb {
22
+
23
+ static const int kTargetFileSize = 2 * 1048576;
24
+
25
+ // Maximum bytes of overlaps in grandparent (i.e., level+2) before we
26
+ // stop building a single file in a level->level+1 compaction.
27
+ static const int64_t kMaxGrandParentOverlapBytes = 10 * kTargetFileSize;
28
+
29
+ // Maximum number of bytes in all compacted files. We avoid expanding
30
+ // the lower level file set of a compaction if it would make the
31
+ // total compaction cover more than this many bytes.
32
+ static const int64_t kExpandedCompactionByteSizeLimit = 25 * kTargetFileSize;
33
+
34
+ static double MaxBytesForLevel(int level) {
35
+ // Note: the result for level zero is not really used since we set
36
+ // the level-0 compaction threshold based on number of files.
37
+ double result = 10 * 1048576.0; // Result for both level-0 and level-1
38
+ while (level > 1) {
39
+ result *= 10;
40
+ level--;
41
+ }
42
+ return result;
43
+ }
44
+
45
+ static uint64_t MaxFileSizeForLevel(int level) {
46
+ return kTargetFileSize; // We could vary per level to reduce number of files?
47
+ }
48
+
49
+ static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
50
+ int64_t sum = 0;
51
+ for (size_t i = 0; i < files.size(); i++) {
52
+ sum += files[i]->file_size;
53
+ }
54
+ return sum;
55
+ }
56
+
57
+ namespace {
58
+ std::string IntSetToString(const std::set<uint64_t>& s) {
59
+ std::string result = "{";
60
+ for (std::set<uint64_t>::const_iterator it = s.begin();
61
+ it != s.end();
62
+ ++it) {
63
+ result += (result.size() > 1) ? "," : "";
64
+ result += NumberToString(*it);
65
+ }
66
+ result += "}";
67
+ return result;
68
+ }
69
+ } // namespace
70
+
71
+ Version::~Version() {
72
+ assert(refs_ == 0);
73
+
74
+ // Remove from linked list
75
+ prev_->next_ = next_;
76
+ next_->prev_ = prev_;
77
+
78
+ // Drop references to files
79
+ for (int level = 0; level < config::kNumLevels; level++) {
80
+ for (size_t i = 0; i < files_[level].size(); i++) {
81
+ FileMetaData* f = files_[level][i];
82
+ assert(f->refs > 0);
83
+ f->refs--;
84
+ if (f->refs <= 0) {
85
+ delete f;
86
+ }
87
+ }
88
+ }
89
+ }
90
+
91
+ int FindFile(const InternalKeyComparator& icmp,
92
+ const std::vector<FileMetaData*>& files,
93
+ const Slice& key) {
94
+ uint32_t left = 0;
95
+ uint32_t right = files.size();
96
+ while (left < right) {
97
+ uint32_t mid = (left + right) / 2;
98
+ const FileMetaData* f = files[mid];
99
+ if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) {
100
+ // Key at "mid.largest" is < "target". Therefore all
101
+ // files at or before "mid" are uninteresting.
102
+ left = mid + 1;
103
+ } else {
104
+ // Key at "mid.largest" is >= "target". Therefore all files
105
+ // after "mid" are uninteresting.
106
+ right = mid;
107
+ }
108
+ }
109
+ return right;
110
+ }
111
+
112
+ static bool AfterFile(const Comparator* ucmp,
113
+ const Slice* user_key, const FileMetaData* f) {
114
+ // NULL user_key occurs before all keys and is therefore never after *f
115
+ return (user_key != NULL &&
116
+ ucmp->Compare(*user_key, f->largest.user_key()) > 0);
117
+ }
118
+
119
+ static bool BeforeFile(const Comparator* ucmp,
120
+ const Slice* user_key, const FileMetaData* f) {
121
+ // NULL user_key occurs after all keys and is therefore never before *f
122
+ return (user_key != NULL &&
123
+ ucmp->Compare(*user_key, f->smallest.user_key()) < 0);
124
+ }
125
+
126
+ bool SomeFileOverlapsRange(
127
+ const InternalKeyComparator& icmp,
128
+ bool disjoint_sorted_files,
129
+ const std::vector<FileMetaData*>& files,
130
+ const Slice* smallest_user_key,
131
+ const Slice* largest_user_key) {
132
+ const Comparator* ucmp = icmp.user_comparator();
133
+ if (!disjoint_sorted_files) {
134
+ // Need to check against all files
135
+ for (size_t i = 0; i < files.size(); i++) {
136
+ const FileMetaData* f = files[i];
137
+ if (AfterFile(ucmp, smallest_user_key, f) ||
138
+ BeforeFile(ucmp, largest_user_key, f)) {
139
+ // No overlap
140
+ } else {
141
+ return true; // Overlap
142
+ }
143
+ }
144
+ return false;
145
+ }
146
+
147
+ // Binary search over file list
148
+ uint32_t index = 0;
149
+ if (smallest_user_key != NULL) {
150
+ // Find the earliest possible internal key for smallest_user_key
151
+ InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek);
152
+ index = FindFile(icmp, files, small.Encode());
153
+ }
154
+
155
+ if (index >= files.size()) {
156
+ // beginning of range is after all files, so no overlap.
157
+ return false;
158
+ }
159
+
160
+ return !BeforeFile(ucmp, largest_user_key, files[index]);
161
+ }
162
+
163
+ // An internal iterator. For a given version/level pair, yields
164
+ // information about the files in the level. For a given entry, key()
165
+ // is the largest key that occurs in the file, and value() is an
166
+ // 16-byte value containing the file number and file size, both
167
+ // encoded using EncodeFixed64.
168
+ class Version::LevelFileNumIterator : public Iterator {
169
+ public:
170
+ LevelFileNumIterator(const InternalKeyComparator& icmp,
171
+ const std::vector<FileMetaData*>* flist)
172
+ : icmp_(icmp),
173
+ flist_(flist),
174
+ index_(flist->size()) { // Marks as invalid
175
+ }
176
+ virtual bool Valid() const {
177
+ return index_ < flist_->size();
178
+ }
179
+ virtual void Seek(const Slice& target) {
180
+ index_ = FindFile(icmp_, *flist_, target);
181
+ }
182
+ virtual void SeekToFirst() { index_ = 0; }
183
+ virtual void SeekToLast() {
184
+ index_ = flist_->empty() ? 0 : flist_->size() - 1;
185
+ }
186
+ virtual void Next() {
187
+ assert(Valid());
188
+ index_++;
189
+ }
190
+ virtual void Prev() {
191
+ assert(Valid());
192
+ if (index_ == 0) {
193
+ index_ = flist_->size(); // Marks as invalid
194
+ } else {
195
+ index_--;
196
+ }
197
+ }
198
+ Slice key() const {
199
+ assert(Valid());
200
+ return (*flist_)[index_]->largest.Encode();
201
+ }
202
+ Slice value() const {
203
+ assert(Valid());
204
+ EncodeFixed64(value_buf_, (*flist_)[index_]->number);
205
+ EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size);
206
+ return Slice(value_buf_, sizeof(value_buf_));
207
+ }
208
+ virtual Status status() const { return Status::OK(); }
209
+ private:
210
+ const InternalKeyComparator icmp_;
211
+ const std::vector<FileMetaData*>* const flist_;
212
+ uint32_t index_;
213
+
214
+ // Backing store for value(). Holds the file number and size.
215
+ mutable char value_buf_[16];
216
+ };
217
+
218
+ static Iterator* GetFileIterator(void* arg,
219
+ const ReadOptions& options,
220
+ const Slice& file_value) {
221
+ TableCache* cache = reinterpret_cast<TableCache*>(arg);
222
+ if (file_value.size() != 16) {
223
+ return NewErrorIterator(
224
+ Status::Corruption("FileReader invoked with unexpected value"));
225
+ } else {
226
+ return cache->NewIterator(options,
227
+ DecodeFixed64(file_value.data()),
228
+ DecodeFixed64(file_value.data() + 8));
229
+ }
230
+ }
231
+
232
+ Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
233
+ int level) const {
234
+ return NewTwoLevelIterator(
235
+ new LevelFileNumIterator(vset_->icmp_, &files_[level]),
236
+ &GetFileIterator, vset_->table_cache_, options);
237
+ }
238
+
239
+ void Version::AddIterators(const ReadOptions& options,
240
+ std::vector<Iterator*>* iters) {
241
+ // Merge all level zero files together since they may overlap
242
+ for (size_t i = 0; i < files_[0].size(); i++) {
243
+ iters->push_back(
244
+ vset_->table_cache_->NewIterator(
245
+ options, files_[0][i]->number, files_[0][i]->file_size));
246
+ }
247
+
248
+ // For levels > 0, we can use a concatenating iterator that sequentially
249
+ // walks through the non-overlapping files in the level, opening them
250
+ // lazily.
251
+ for (int level = 1; level < config::kNumLevels; level++) {
252
+ if (!files_[level].empty()) {
253
+ iters->push_back(NewConcatenatingIterator(options, level));
254
+ }
255
+ }
256
+ }
257
+
258
+ // Callback from TableCache::Get()
259
+ namespace {
260
+ enum SaverState {
261
+ kNotFound,
262
+ kFound,
263
+ kDeleted,
264
+ kCorrupt,
265
+ };
266
+ struct Saver {
267
+ SaverState state;
268
+ const Comparator* ucmp;
269
+ Slice user_key;
270
+ std::string* value;
271
+ };
272
+ }
273
+ static void SaveValue(void* arg, const Slice& ikey, const Slice& v) {
274
+ Saver* s = reinterpret_cast<Saver*>(arg);
275
+ ParsedInternalKey parsed_key;
276
+ if (!ParseInternalKey(ikey, &parsed_key)) {
277
+ s->state = kCorrupt;
278
+ } else {
279
+ if (s->ucmp->Compare(parsed_key.user_key, s->user_key) == 0) {
280
+ s->state = (parsed_key.type == kTypeValue) ? kFound : kDeleted;
281
+ if (s->state == kFound) {
282
+ s->value->assign(v.data(), v.size());
283
+ }
284
+ }
285
+ }
286
+ }
287
+
288
+ static bool NewestFirst(FileMetaData* a, FileMetaData* b) {
289
+ return a->number > b->number;
290
+ }
291
+
292
+ Status Version::Get(const ReadOptions& options,
293
+ const LookupKey& k,
294
+ std::string* value,
295
+ GetStats* stats) {
296
+ Slice ikey = k.internal_key();
297
+ Slice user_key = k.user_key();
298
+ const Comparator* ucmp = vset_->icmp_.user_comparator();
299
+ Status s;
300
+
301
+ stats->seek_file = NULL;
302
+ stats->seek_file_level = -1;
303
+ FileMetaData* last_file_read = NULL;
304
+ int last_file_read_level = -1;
305
+
306
+ // We can search level-by-level since entries never hop across
307
+ // levels. Therefore we are guaranteed that if we find data
308
+ // in an smaller level, later levels are irrelevant.
309
+ std::vector<FileMetaData*> tmp;
310
+ FileMetaData* tmp2;
311
+ for (int level = 0; level < config::kNumLevels; level++) {
312
+ size_t num_files = files_[level].size();
313
+ if (num_files == 0) continue;
314
+
315
+ // Get the list of files to search in this level
316
+ FileMetaData* const* files = &files_[level][0];
317
+ if (level == 0) {
318
+ // Level-0 files may overlap each other. Find all files that
319
+ // overlap user_key and process them in order from newest to oldest.
320
+ tmp.reserve(num_files);
321
+ for (uint32_t i = 0; i < num_files; i++) {
322
+ FileMetaData* f = files[i];
323
+ if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 &&
324
+ ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
325
+ tmp.push_back(f);
326
+ }
327
+ }
328
+ if (tmp.empty()) continue;
329
+
330
+ std::sort(tmp.begin(), tmp.end(), NewestFirst);
331
+ files = &tmp[0];
332
+ num_files = tmp.size();
333
+ } else {
334
+ // Binary search to find earliest index whose largest key >= ikey.
335
+ uint32_t index = FindFile(vset_->icmp_, files_[level], ikey);
336
+ if (index >= num_files) {
337
+ files = NULL;
338
+ num_files = 0;
339
+ } else {
340
+ tmp2 = files[index];
341
+ if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) {
342
+ // All of "tmp2" is past any data for user_key
343
+ files = NULL;
344
+ num_files = 0;
345
+ } else {
346
+ files = &tmp2;
347
+ num_files = 1;
348
+ }
349
+ }
350
+ }
351
+
352
+ for (uint32_t i = 0; i < num_files; ++i) {
353
+ if (last_file_read != NULL && stats->seek_file == NULL) {
354
+ // We have had more than one seek for this read. Charge the 1st file.
355
+ stats->seek_file = last_file_read;
356
+ stats->seek_file_level = last_file_read_level;
357
+ }
358
+
359
+ FileMetaData* f = files[i];
360
+ last_file_read = f;
361
+ last_file_read_level = level;
362
+
363
+ Saver saver;
364
+ saver.state = kNotFound;
365
+ saver.ucmp = ucmp;
366
+ saver.user_key = user_key;
367
+ saver.value = value;
368
+ s = vset_->table_cache_->Get(options, f->number, f->file_size,
369
+ ikey, &saver, SaveValue);
370
+ if (!s.ok()) {
371
+ return s;
372
+ }
373
+ switch (saver.state) {
374
+ case kNotFound:
375
+ break; // Keep searching in other files
376
+ case kFound:
377
+ return s;
378
+ case kDeleted:
379
+ s = Status::NotFound(Slice()); // Use empty error message for speed
380
+ return s;
381
+ case kCorrupt:
382
+ s = Status::Corruption("corrupted key for ", user_key);
383
+ return s;
384
+ }
385
+ }
386
+ }
387
+
388
+ return Status::NotFound(Slice()); // Use an empty error message for speed
389
+ }
390
+
391
+ bool Version::UpdateStats(const GetStats& stats) {
392
+ FileMetaData* f = stats.seek_file;
393
+ if (f != NULL) {
394
+ f->allowed_seeks--;
395
+ if (f->allowed_seeks <= 0 && file_to_compact_ == NULL) {
396
+ file_to_compact_ = f;
397
+ file_to_compact_level_ = stats.seek_file_level;
398
+ return true;
399
+ }
400
+ }
401
+ return false;
402
+ }
403
+
404
+ void Version::Ref() {
405
+ ++refs_;
406
+ }
407
+
408
+ void Version::Unref() {
409
+ assert(this != &vset_->dummy_versions_);
410
+ assert(refs_ >= 1);
411
+ --refs_;
412
+ if (refs_ == 0) {
413
+ delete this;
414
+ }
415
+ }
416
+
417
+ bool Version::OverlapInLevel(int level,
418
+ const Slice* smallest_user_key,
419
+ const Slice* largest_user_key) {
420
+ return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level],
421
+ smallest_user_key, largest_user_key);
422
+ }
423
+
424
+ int Version::PickLevelForMemTableOutput(
425
+ const Slice& smallest_user_key,
426
+ const Slice& largest_user_key) {
427
+ int level = 0;
428
+ if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) {
429
+ // Push to next level if there is no overlap in next level,
430
+ // and the #bytes overlapping in the level after that are limited.
431
+ InternalKey start(smallest_user_key, kMaxSequenceNumber, kValueTypeForSeek);
432
+ InternalKey limit(largest_user_key, 0, static_cast<ValueType>(0));
433
+ std::vector<FileMetaData*> overlaps;
434
+ while (level < config::kMaxMemCompactLevel) {
435
+ if (OverlapInLevel(level + 1, &smallest_user_key, &largest_user_key)) {
436
+ break;
437
+ }
438
+ GetOverlappingInputs(level + 2, &start, &limit, &overlaps);
439
+ const int64_t sum = TotalFileSize(overlaps);
440
+ if (sum > kMaxGrandParentOverlapBytes) {
441
+ break;
442
+ }
443
+ level++;
444
+ }
445
+ }
446
+ return level;
447
+ }
448
+
449
+ // Store in "*inputs" all files in "level" that overlap [begin,end]
450
+ void Version::GetOverlappingInputs(
451
+ int level,
452
+ const InternalKey* begin,
453
+ const InternalKey* end,
454
+ std::vector<FileMetaData*>* inputs) {
455
+ inputs->clear();
456
+ Slice user_begin, user_end;
457
+ if (begin != NULL) {
458
+ user_begin = begin->user_key();
459
+ }
460
+ if (end != NULL) {
461
+ user_end = end->user_key();
462
+ }
463
+ const Comparator* user_cmp = vset_->icmp_.user_comparator();
464
+ for (size_t i = 0; i < files_[level].size(); ) {
465
+ FileMetaData* f = files_[level][i++];
466
+ const Slice file_start = f->smallest.user_key();
467
+ const Slice file_limit = f->largest.user_key();
468
+ if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) {
469
+ // "f" is completely before specified range; skip it
470
+ } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) {
471
+ // "f" is completely after specified range; skip it
472
+ } else {
473
+ inputs->push_back(f);
474
+ if (level == 0) {
475
+ // Level-0 files may overlap each other. So check if the newly
476
+ // added file has expanded the range. If so, restart search.
477
+ if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) {
478
+ user_begin = file_start;
479
+ inputs->clear();
480
+ i = 0;
481
+ } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) {
482
+ user_end = file_limit;
483
+ inputs->clear();
484
+ i = 0;
485
+ }
486
+ }
487
+ }
488
+ }
489
+ }
490
+
491
+ std::string Version::DebugString() const {
492
+ std::string r;
493
+ for (int level = 0; level < config::kNumLevels; level++) {
494
+ // E.g.,
495
+ // --- level 1 ---
496
+ // 17:123['a' .. 'd']
497
+ // 20:43['e' .. 'g']
498
+ r.append("--- level ");
499
+ AppendNumberTo(&r, level);
500
+ r.append(" ---\n");
501
+ const std::vector<FileMetaData*>& files = files_[level];
502
+ for (size_t i = 0; i < files.size(); i++) {
503
+ r.push_back(' ');
504
+ AppendNumberTo(&r, files[i]->number);
505
+ r.push_back(':');
506
+ AppendNumberTo(&r, files[i]->file_size);
507
+ r.append("[");
508
+ r.append(files[i]->smallest.DebugString());
509
+ r.append(" .. ");
510
+ r.append(files[i]->largest.DebugString());
511
+ r.append("]\n");
512
+ }
513
+ }
514
+ return r;
515
+ }
516
+
517
+ // A helper class so we can efficiently apply a whole sequence
518
+ // of edits to a particular state without creating intermediate
519
+ // Versions that contain full copies of the intermediate state.
520
+ class VersionSet::Builder {
521
+ private:
522
+ // Helper to sort by v->files_[file_number].smallest
523
+ struct BySmallestKey {
524
+ const InternalKeyComparator* internal_comparator;
525
+
526
+ bool operator()(FileMetaData* f1, FileMetaData* f2) const {
527
+ int r = internal_comparator->Compare(f1->smallest, f2->smallest);
528
+ if (r != 0) {
529
+ return (r < 0);
530
+ } else {
531
+ // Break ties by file number
532
+ return (f1->number < f2->number);
533
+ }
534
+ }
535
+ };
536
+
537
+ typedef std::set<FileMetaData*, BySmallestKey> FileSet;
538
+ struct LevelState {
539
+ std::set<uint64_t> deleted_files;
540
+ FileSet* added_files;
541
+ };
542
+
543
+ VersionSet* vset_;
544
+ Version* base_;
545
+ LevelState levels_[config::kNumLevels];
546
+
547
+ public:
548
+ // Initialize a builder with the files from *base and other info from *vset
549
+ Builder(VersionSet* vset, Version* base)
550
+ : vset_(vset),
551
+ base_(base) {
552
+ base_->Ref();
553
+ BySmallestKey cmp;
554
+ cmp.internal_comparator = &vset_->icmp_;
555
+ for (int level = 0; level < config::kNumLevels; level++) {
556
+ levels_[level].added_files = new FileSet(cmp);
557
+ }
558
+ }
559
+
560
+ ~Builder() {
561
+ for (int level = 0; level < config::kNumLevels; level++) {
562
+ const FileSet* added = levels_[level].added_files;
563
+ std::vector<FileMetaData*> to_unref;
564
+ to_unref.reserve(added->size());
565
+ for (FileSet::const_iterator it = added->begin();
566
+ it != added->end(); ++it) {
567
+ to_unref.push_back(*it);
568
+ }
569
+ delete added;
570
+ for (uint32_t i = 0; i < to_unref.size(); i++) {
571
+ FileMetaData* f = to_unref[i];
572
+ f->refs--;
573
+ if (f->refs <= 0) {
574
+ delete f;
575
+ }
576
+ }
577
+ }
578
+ base_->Unref();
579
+ }
580
+
581
+ // Apply all of the edits in *edit to the current state.
582
+ void Apply(VersionEdit* edit) {
583
+ // Update compaction pointers
584
+ for (size_t i = 0; i < edit->compact_pointers_.size(); i++) {
585
+ const int level = edit->compact_pointers_[i].first;
586
+ vset_->compact_pointer_[level] =
587
+ edit->compact_pointers_[i].second.Encode().ToString();
588
+ }
589
+
590
+ // Delete files
591
+ const VersionEdit::DeletedFileSet& del = edit->deleted_files_;
592
+ for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin();
593
+ iter != del.end();
594
+ ++iter) {
595
+ const int level = iter->first;
596
+ const uint64_t number = iter->second;
597
+ levels_[level].deleted_files.insert(number);
598
+ }
599
+
600
+ // Add new files
601
+ for (size_t i = 0; i < edit->new_files_.size(); i++) {
602
+ const int level = edit->new_files_[i].first;
603
+ FileMetaData* f = new FileMetaData(edit->new_files_[i].second);
604
+ f->refs = 1;
605
+
606
+ // We arrange to automatically compact this file after
607
+ // a certain number of seeks. Let's assume:
608
+ // (1) One seek costs 10ms
609
+ // (2) Writing or reading 1MB costs 10ms (100MB/s)
610
+ // (3) A compaction of 1MB does 25MB of IO:
611
+ // 1MB read from this level
612
+ // 10-12MB read from next level (boundaries may be misaligned)
613
+ // 10-12MB written to next level
614
+ // This implies that 25 seeks cost the same as the compaction
615
+ // of 1MB of data. I.e., one seek costs approximately the
616
+ // same as the compaction of 40KB of data. We are a little
617
+ // conservative and allow approximately one seek for every 16KB
618
+ // of data before triggering a compaction.
619
+ f->allowed_seeks = (f->file_size / 16384);
620
+ if (f->allowed_seeks < 100) f->allowed_seeks = 100;
621
+
622
+ levels_[level].deleted_files.erase(f->number);
623
+ levels_[level].added_files->insert(f);
624
+ }
625
+ }
626
+
627
+ // Save the current state in *v.
628
+ void SaveTo(Version* v) {
629
+ BySmallestKey cmp;
630
+ cmp.internal_comparator = &vset_->icmp_;
631
+ for (int level = 0; level < config::kNumLevels; level++) {
632
+ // Merge the set of added files with the set of pre-existing files.
633
+ // Drop any deleted files. Store the result in *v.
634
+ const std::vector<FileMetaData*>& base_files = base_->files_[level];
635
+ std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin();
636
+ std::vector<FileMetaData*>::const_iterator base_end = base_files.end();
637
+ const FileSet* added = levels_[level].added_files;
638
+ v->files_[level].reserve(base_files.size() + added->size());
639
+ for (FileSet::const_iterator added_iter = added->begin();
640
+ added_iter != added->end();
641
+ ++added_iter) {
642
+ // Add all smaller files listed in base_
643
+ for (std::vector<FileMetaData*>::const_iterator bpos
644
+ = std::upper_bound(base_iter, base_end, *added_iter, cmp);
645
+ base_iter != bpos;
646
+ ++base_iter) {
647
+ MaybeAddFile(v, level, *base_iter);
648
+ }
649
+
650
+ MaybeAddFile(v, level, *added_iter);
651
+ }
652
+
653
+ // Add remaining base files
654
+ for (; base_iter != base_end; ++base_iter) {
655
+ MaybeAddFile(v, level, *base_iter);
656
+ }
657
+
658
+ #ifndef NDEBUG
659
+ // Make sure there is no overlap in levels > 0
660
+ if (level > 0) {
661
+ for (uint32_t i = 1; i < v->files_[level].size(); i++) {
662
+ const InternalKey& prev_end = v->files_[level][i-1]->largest;
663
+ const InternalKey& this_begin = v->files_[level][i]->smallest;
664
+ if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) {
665
+ fprintf(stderr, "overlapping ranges in same level %s vs. %s\n",
666
+ prev_end.DebugString().c_str(),
667
+ this_begin.DebugString().c_str());
668
+ abort();
669
+ }
670
+ }
671
+ }
672
+ #endif
673
+ }
674
+ }
675
+
676
+ void MaybeAddFile(Version* v, int level, FileMetaData* f) {
677
+ if (levels_[level].deleted_files.count(f->number) > 0) {
678
+ // File is deleted: do nothing
679
+ } else {
680
+ std::vector<FileMetaData*>* files = &v->files_[level];
681
+ if (level > 0 && !files->empty()) {
682
+ // Must not overlap
683
+ assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest,
684
+ f->smallest) < 0);
685
+ }
686
+ f->refs++;
687
+ files->push_back(f);
688
+ }
689
+ }
690
+ };
691
+
692
+ VersionSet::VersionSet(const std::string& dbname,
693
+ const Options* options,
694
+ TableCache* table_cache,
695
+ const InternalKeyComparator* cmp)
696
+ : env_(options->env),
697
+ dbname_(dbname),
698
+ options_(options),
699
+ table_cache_(table_cache),
700
+ icmp_(*cmp),
701
+ next_file_number_(2),
702
+ manifest_file_number_(0), // Filled by Recover()
703
+ last_sequence_(0),
704
+ log_number_(0),
705
+ prev_log_number_(0),
706
+ descriptor_file_(NULL),
707
+ descriptor_log_(NULL),
708
+ dummy_versions_(this),
709
+ current_(NULL) {
710
+ AppendVersion(new Version(this));
711
+ }
712
+
713
+ VersionSet::~VersionSet() {
714
+ current_->Unref();
715
+ assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty
716
+ delete descriptor_log_;
717
+ delete descriptor_file_;
718
+ }
719
+
720
+ void VersionSet::AppendVersion(Version* v) {
721
+ // Make "v" current
722
+ assert(v->refs_ == 0);
723
+ assert(v != current_);
724
+ if (current_ != NULL) {
725
+ current_->Unref();
726
+ }
727
+ current_ = v;
728
+ v->Ref();
729
+
730
+ // Append to linked list
731
+ v->prev_ = dummy_versions_.prev_;
732
+ v->next_ = &dummy_versions_;
733
+ v->prev_->next_ = v;
734
+ v->next_->prev_ = v;
735
+ }
736
+
737
+ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
738
+ if (edit->has_log_number_) {
739
+ assert(edit->log_number_ >= log_number_);
740
+ assert(edit->log_number_ < next_file_number_);
741
+ } else {
742
+ edit->SetLogNumber(log_number_);
743
+ }
744
+
745
+ if (!edit->has_prev_log_number_) {
746
+ edit->SetPrevLogNumber(prev_log_number_);
747
+ }
748
+
749
+ edit->SetNextFile(next_file_number_);
750
+ edit->SetLastSequence(last_sequence_);
751
+
752
+ Version* v = new Version(this);
753
+ {
754
+ Builder builder(this, current_);
755
+ builder.Apply(edit);
756
+ builder.SaveTo(v);
757
+ }
758
+ Finalize(v);
759
+
760
+ // Initialize new descriptor log file if necessary by creating
761
+ // a temporary file that contains a snapshot of the current version.
762
+ std::string new_manifest_file;
763
+ Status s;
764
+ if (descriptor_log_ == NULL) {
765
+ // No reason to unlock *mu here since we only hit this path in the
766
+ // first call to LogAndApply (when opening the database).
767
+ assert(descriptor_file_ == NULL);
768
+ new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_);
769
+ edit->SetNextFile(next_file_number_);
770
+ s = env_->NewWritableFile(new_manifest_file, &descriptor_file_);
771
+ if (s.ok()) {
772
+ descriptor_log_ = new log::Writer(descriptor_file_);
773
+ s = WriteSnapshot(descriptor_log_);
774
+ }
775
+ }
776
+
777
+ // Unlock during expensive MANIFEST log write
778
+ {
779
+ mu->Unlock();
780
+
781
+ // Write new record to MANIFEST log
782
+ if (s.ok()) {
783
+ std::string record;
784
+ edit->EncodeTo(&record);
785
+ s = descriptor_log_->AddRecord(record);
786
+ if (s.ok()) {
787
+ s = descriptor_file_->Sync();
788
+ }
789
+ if (!s.ok()) {
790
+ Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str());
791
+ if (ManifestContains(record)) {
792
+ Log(options_->info_log,
793
+ "MANIFEST contains log record despite error; advancing to new "
794
+ "version to prevent mismatch between in-memory and logged state");
795
+ s = Status::OK();
796
+ }
797
+ }
798
+ }
799
+
800
+ // If we just created a new descriptor file, install it by writing a
801
+ // new CURRENT file that points to it.
802
+ if (s.ok() && !new_manifest_file.empty()) {
803
+ s = SetCurrentFile(env_, dbname_, manifest_file_number_);
804
+ // No need to double-check MANIFEST in case of error since it
805
+ // will be discarded below.
806
+ }
807
+
808
+ mu->Lock();
809
+ }
810
+
811
+ // Install the new version
812
+ if (s.ok()) {
813
+ AppendVersion(v);
814
+ log_number_ = edit->log_number_;
815
+ prev_log_number_ = edit->prev_log_number_;
816
+ } else {
817
+ delete v;
818
+ if (!new_manifest_file.empty()) {
819
+ delete descriptor_log_;
820
+ delete descriptor_file_;
821
+ descriptor_log_ = NULL;
822
+ descriptor_file_ = NULL;
823
+ env_->DeleteFile(new_manifest_file);
824
+ }
825
+ }
826
+
827
+ return s;
828
+ }
829
+
830
+ Status VersionSet::Recover() {
831
+ struct LogReporter : public log::Reader::Reporter {
832
+ Status* status;
833
+ virtual void Corruption(size_t bytes, const Status& s) {
834
+ if (this->status->ok()) *this->status = s;
835
+ }
836
+ };
837
+
838
+ // Read "CURRENT" file, which contains a pointer to the current manifest file
839
+ std::string current;
840
+ Status s = ReadFileToString(env_, CurrentFileName(dbname_), &current);
841
+ if (!s.ok()) {
842
+ return s;
843
+ }
844
+ if (current.empty() || current[current.size()-1] != '\n') {
845
+ return Status::Corruption("CURRENT file does not end with newline");
846
+ }
847
+ current.resize(current.size() - 1);
848
+
849
+ std::string dscname = dbname_ + "/" + current;
850
+ SequentialFile* file;
851
+ s = env_->NewSequentialFile(dscname, &file);
852
+ if (!s.ok()) {
853
+ return s;
854
+ }
855
+
856
+ bool have_log_number = false;
857
+ bool have_prev_log_number = false;
858
+ bool have_next_file = false;
859
+ bool have_last_sequence = false;
860
+ uint64_t next_file = 0;
861
+ uint64_t last_sequence = 0;
862
+ uint64_t log_number = 0;
863
+ uint64_t prev_log_number = 0;
864
+ Builder builder(this, current_);
865
+
866
+ {
867
+ LogReporter reporter;
868
+ reporter.status = &s;
869
+ log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/);
870
+ Slice record;
871
+ std::string scratch;
872
+ while (reader.ReadRecord(&record, &scratch) && s.ok()) {
873
+ VersionEdit edit;
874
+ s = edit.DecodeFrom(record);
875
+ if (s.ok()) {
876
+ if (edit.has_comparator_ &&
877
+ edit.comparator_ != icmp_.user_comparator()->Name()) {
878
+ s = Status::InvalidArgument(
879
+ edit.comparator_ + " does not match existing comparator ",
880
+ icmp_.user_comparator()->Name());
881
+ }
882
+ }
883
+
884
+ if (s.ok()) {
885
+ builder.Apply(&edit);
886
+ }
887
+
888
+ if (edit.has_log_number_) {
889
+ log_number = edit.log_number_;
890
+ have_log_number = true;
891
+ }
892
+
893
+ if (edit.has_prev_log_number_) {
894
+ prev_log_number = edit.prev_log_number_;
895
+ have_prev_log_number = true;
896
+ }
897
+
898
+ if (edit.has_next_file_number_) {
899
+ next_file = edit.next_file_number_;
900
+ have_next_file = true;
901
+ }
902
+
903
+ if (edit.has_last_sequence_) {
904
+ last_sequence = edit.last_sequence_;
905
+ have_last_sequence = true;
906
+ }
907
+ }
908
+ }
909
+ delete file;
910
+ file = NULL;
911
+
912
+ if (s.ok()) {
913
+ if (!have_next_file) {
914
+ s = Status::Corruption("no meta-nextfile entry in descriptor");
915
+ } else if (!have_log_number) {
916
+ s = Status::Corruption("no meta-lognumber entry in descriptor");
917
+ } else if (!have_last_sequence) {
918
+ s = Status::Corruption("no last-sequence-number entry in descriptor");
919
+ }
920
+
921
+ if (!have_prev_log_number) {
922
+ prev_log_number = 0;
923
+ }
924
+
925
+ MarkFileNumberUsed(prev_log_number);
926
+ MarkFileNumberUsed(log_number);
927
+ }
928
+
929
+ if (s.ok()) {
930
+ Version* v = new Version(this);
931
+ builder.SaveTo(v);
932
+ // Install recovered version
933
+ Finalize(v);
934
+ AppendVersion(v);
935
+ manifest_file_number_ = next_file;
936
+ next_file_number_ = next_file + 1;
937
+ last_sequence_ = last_sequence;
938
+ log_number_ = log_number;
939
+ prev_log_number_ = prev_log_number;
940
+ }
941
+
942
+ return s;
943
+ }
944
+
945
+ void VersionSet::MarkFileNumberUsed(uint64_t number) {
946
+ if (next_file_number_ <= number) {
947
+ next_file_number_ = number + 1;
948
+ }
949
+ }
950
+
951
+ void VersionSet::Finalize(Version* v) {
952
+ // Precomputed best level for next compaction
953
+ int best_level = -1;
954
+ double best_score = -1;
955
+
956
+ for (int level = 0; level < config::kNumLevels-1; level++) {
957
+ double score;
958
+ if (level == 0) {
959
+ // We treat level-0 specially by bounding the number of files
960
+ // instead of number of bytes for two reasons:
961
+ //
962
+ // (1) With larger write-buffer sizes, it is nice not to do too
963
+ // many level-0 compactions.
964
+ //
965
+ // (2) The files in level-0 are merged on every read and
966
+ // therefore we wish to avoid too many files when the individual
967
+ // file size is small (perhaps because of a small write-buffer
968
+ // setting, or very high compression ratios, or lots of
969
+ // overwrites/deletions).
970
+ score = v->files_[level].size() /
971
+ static_cast<double>(config::kL0_CompactionTrigger);
972
+ } else {
973
+ // Compute the ratio of current size to size limit.
974
+ const uint64_t level_bytes = TotalFileSize(v->files_[level]);
975
+ score = static_cast<double>(level_bytes) / MaxBytesForLevel(level);
976
+ }
977
+
978
+ if (score > best_score) {
979
+ best_level = level;
980
+ best_score = score;
981
+ }
982
+ }
983
+
984
+ v->compaction_level_ = best_level;
985
+ v->compaction_score_ = best_score;
986
+ }
987
+
988
+ Status VersionSet::WriteSnapshot(log::Writer* log) {
989
+ // TODO: Break up into multiple records to reduce memory usage on recovery?
990
+
991
+ // Save metadata
992
+ VersionEdit edit;
993
+ edit.SetComparatorName(icmp_.user_comparator()->Name());
994
+
995
+ // Save compaction pointers
996
+ for (int level = 0; level < config::kNumLevels; level++) {
997
+ if (!compact_pointer_[level].empty()) {
998
+ InternalKey key;
999
+ key.DecodeFrom(compact_pointer_[level]);
1000
+ edit.SetCompactPointer(level, key);
1001
+ }
1002
+ }
1003
+
1004
+ // Save files
1005
+ for (int level = 0; level < config::kNumLevels; level++) {
1006
+ const std::vector<FileMetaData*>& files = current_->files_[level];
1007
+ for (size_t i = 0; i < files.size(); i++) {
1008
+ const FileMetaData* f = files[i];
1009
+ edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest);
1010
+ }
1011
+ }
1012
+
1013
+ std::string record;
1014
+ edit.EncodeTo(&record);
1015
+ return log->AddRecord(record);
1016
+ }
1017
+
1018
+ int VersionSet::NumLevelFiles(int level) const {
1019
+ assert(level >= 0);
1020
+ assert(level < config::kNumLevels);
1021
+ return current_->files_[level].size();
1022
+ }
1023
+
1024
+ const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
1025
+ // Update code if kNumLevels changes
1026
+ assert(config::kNumLevels == 7);
1027
+ snprintf(scratch->buffer, sizeof(scratch->buffer),
1028
+ "files[ %d %d %d %d %d %d %d ]",
1029
+ int(current_->files_[0].size()),
1030
+ int(current_->files_[1].size()),
1031
+ int(current_->files_[2].size()),
1032
+ int(current_->files_[3].size()),
1033
+ int(current_->files_[4].size()),
1034
+ int(current_->files_[5].size()),
1035
+ int(current_->files_[6].size()));
1036
+ return scratch->buffer;
1037
+ }
1038
+
1039
+ // Return true iff the manifest contains the specified record.
1040
+ bool VersionSet::ManifestContains(const std::string& record) const {
1041
+ std::string fname = DescriptorFileName(dbname_, manifest_file_number_);
1042
+ Log(options_->info_log, "ManifestContains: checking %s\n", fname.c_str());
1043
+ SequentialFile* file = NULL;
1044
+ Status s = env_->NewSequentialFile(fname, &file);
1045
+ if (!s.ok()) {
1046
+ Log(options_->info_log, "ManifestContains: %s\n", s.ToString().c_str());
1047
+ return false;
1048
+ }
1049
+ log::Reader reader(file, NULL, true/*checksum*/, 0);
1050
+ Slice r;
1051
+ std::string scratch;
1052
+ bool result = false;
1053
+ while (reader.ReadRecord(&r, &scratch)) {
1054
+ if (r == Slice(record)) {
1055
+ result = true;
1056
+ break;
1057
+ }
1058
+ }
1059
+ delete file;
1060
+ Log(options_->info_log, "ManifestContains: result = %d\n", result ? 1 : 0);
1061
+ return result;
1062
+ }
1063
+
1064
+ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
1065
+ uint64_t result = 0;
1066
+ for (int level = 0; level < config::kNumLevels; level++) {
1067
+ const std::vector<FileMetaData*>& files = v->files_[level];
1068
+ for (size_t i = 0; i < files.size(); i++) {
1069
+ if (icmp_.Compare(files[i]->largest, ikey) <= 0) {
1070
+ // Entire file is before "ikey", so just add the file size
1071
+ result += files[i]->file_size;
1072
+ } else if (icmp_.Compare(files[i]->smallest, ikey) > 0) {
1073
+ // Entire file is after "ikey", so ignore
1074
+ if (level > 0) {
1075
+ // Files other than level 0 are sorted by meta->smallest, so
1076
+ // no further files in this level will contain data for
1077
+ // "ikey".
1078
+ break;
1079
+ }
1080
+ } else {
1081
+ // "ikey" falls in the range for this table. Add the
1082
+ // approximate offset of "ikey" within the table.
1083
+ Table* tableptr;
1084
+ Iterator* iter = table_cache_->NewIterator(
1085
+ ReadOptions(), files[i]->number, files[i]->file_size, &tableptr);
1086
+ if (tableptr != NULL) {
1087
+ result += tableptr->ApproximateOffsetOf(ikey.Encode());
1088
+ }
1089
+ delete iter;
1090
+ }
1091
+ }
1092
+ }
1093
+ return result;
1094
+ }
1095
+
1096
+ void VersionSet::AddLiveFiles(std::set<uint64_t>* live) {
1097
+ for (Version* v = dummy_versions_.next_;
1098
+ v != &dummy_versions_;
1099
+ v = v->next_) {
1100
+ for (int level = 0; level < config::kNumLevels; level++) {
1101
+ const std::vector<FileMetaData*>& files = v->files_[level];
1102
+ for (size_t i = 0; i < files.size(); i++) {
1103
+ live->insert(files[i]->number);
1104
+ }
1105
+ }
1106
+ }
1107
+ }
1108
+
1109
+ int64_t VersionSet::NumLevelBytes(int level) const {
1110
+ assert(level >= 0);
1111
+ assert(level < config::kNumLevels);
1112
+ return TotalFileSize(current_->files_[level]);
1113
+ }
1114
+
1115
+ int64_t VersionSet::MaxNextLevelOverlappingBytes() {
1116
+ int64_t result = 0;
1117
+ std::vector<FileMetaData*> overlaps;
1118
+ for (int level = 1; level < config::kNumLevels - 1; level++) {
1119
+ for (size_t i = 0; i < current_->files_[level].size(); i++) {
1120
+ const FileMetaData* f = current_->files_[level][i];
1121
+ current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest,
1122
+ &overlaps);
1123
+ const int64_t sum = TotalFileSize(overlaps);
1124
+ if (sum > result) {
1125
+ result = sum;
1126
+ }
1127
+ }
1128
+ }
1129
+ return result;
1130
+ }
1131
+
1132
+ // Stores the minimal range that covers all entries in inputs in
1133
+ // *smallest, *largest.
1134
+ // REQUIRES: inputs is not empty
1135
+ void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
1136
+ InternalKey* smallest,
1137
+ InternalKey* largest) {
1138
+ assert(!inputs.empty());
1139
+ smallest->Clear();
1140
+ largest->Clear();
1141
+ for (size_t i = 0; i < inputs.size(); i++) {
1142
+ FileMetaData* f = inputs[i];
1143
+ if (i == 0) {
1144
+ *smallest = f->smallest;
1145
+ *largest = f->largest;
1146
+ } else {
1147
+ if (icmp_.Compare(f->smallest, *smallest) < 0) {
1148
+ *smallest = f->smallest;
1149
+ }
1150
+ if (icmp_.Compare(f->largest, *largest) > 0) {
1151
+ *largest = f->largest;
1152
+ }
1153
+ }
1154
+ }
1155
+ }
1156
+
1157
+ // Stores the minimal range that covers all entries in inputs1 and inputs2
1158
+ // in *smallest, *largest.
1159
+ // REQUIRES: inputs is not empty
1160
+ void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1,
1161
+ const std::vector<FileMetaData*>& inputs2,
1162
+ InternalKey* smallest,
1163
+ InternalKey* largest) {
1164
+ std::vector<FileMetaData*> all = inputs1;
1165
+ all.insert(all.end(), inputs2.begin(), inputs2.end());
1166
+ GetRange(all, smallest, largest);
1167
+ }
1168
+
1169
+ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
1170
+ ReadOptions options;
1171
+ options.verify_checksums = options_->paranoid_checks;
1172
+ options.fill_cache = false;
1173
+
1174
+ // Level-0 files have to be merged together. For other levels,
1175
+ // we will make a concatenating iterator per level.
1176
+ // TODO(opt): use concatenating iterator for level-0 if there is no overlap
1177
+ const int space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2);
1178
+ Iterator** list = new Iterator*[space];
1179
+ int num = 0;
1180
+ for (int which = 0; which < 2; which++) {
1181
+ if (!c->inputs_[which].empty()) {
1182
+ if (c->level() + which == 0) {
1183
+ const std::vector<FileMetaData*>& files = c->inputs_[which];
1184
+ for (size_t i = 0; i < files.size(); i++) {
1185
+ list[num++] = table_cache_->NewIterator(
1186
+ options, files[i]->number, files[i]->file_size);
1187
+ }
1188
+ } else {
1189
+ // Create concatenating iterator for the files from this level
1190
+ list[num++] = NewTwoLevelIterator(
1191
+ new Version::LevelFileNumIterator(icmp_, &c->inputs_[which]),
1192
+ &GetFileIterator, table_cache_, options);
1193
+ }
1194
+ }
1195
+ }
1196
+ assert(num <= space);
1197
+ Iterator* result = NewMergingIterator(&icmp_, list, num);
1198
+ delete[] list;
1199
+ return result;
1200
+ }
1201
+
1202
+ Compaction* VersionSet::PickCompaction() {
1203
+ Compaction* c;
1204
+ int level;
1205
+
1206
+ // We prefer compactions triggered by too much data in a level over
1207
+ // the compactions triggered by seeks.
1208
+ const bool size_compaction = (current_->compaction_score_ >= 1);
1209
+ const bool seek_compaction = (current_->file_to_compact_ != NULL);
1210
+ if (size_compaction) {
1211
+ level = current_->compaction_level_;
1212
+ assert(level >= 0);
1213
+ assert(level+1 < config::kNumLevels);
1214
+ c = new Compaction(level);
1215
+
1216
+ // Pick the first file that comes after compact_pointer_[level]
1217
+ for (size_t i = 0; i < current_->files_[level].size(); i++) {
1218
+ FileMetaData* f = current_->files_[level][i];
1219
+ if (compact_pointer_[level].empty() ||
1220
+ icmp_.Compare(f->largest.Encode(), compact_pointer_[level]) > 0) {
1221
+ c->inputs_[0].push_back(f);
1222
+ break;
1223
+ }
1224
+ }
1225
+ if (c->inputs_[0].empty()) {
1226
+ // Wrap-around to the beginning of the key space
1227
+ c->inputs_[0].push_back(current_->files_[level][0]);
1228
+ }
1229
+ } else if (seek_compaction) {
1230
+ level = current_->file_to_compact_level_;
1231
+ c = new Compaction(level);
1232
+ c->inputs_[0].push_back(current_->file_to_compact_);
1233
+ } else {
1234
+ return NULL;
1235
+ }
1236
+
1237
+ c->input_version_ = current_;
1238
+ c->input_version_->Ref();
1239
+
1240
+ // Files in level 0 may overlap each other, so pick up all overlapping ones
1241
+ if (level == 0) {
1242
+ InternalKey smallest, largest;
1243
+ GetRange(c->inputs_[0], &smallest, &largest);
1244
+ // Note that the next call will discard the file we placed in
1245
+ // c->inputs_[0] earlier and replace it with an overlapping set
1246
+ // which will include the picked file.
1247
+ current_->GetOverlappingInputs(0, &smallest, &largest, &c->inputs_[0]);
1248
+ assert(!c->inputs_[0].empty());
1249
+ }
1250
+
1251
+ SetupOtherInputs(c);
1252
+
1253
+ return c;
1254
+ }
1255
+
1256
+ void VersionSet::SetupOtherInputs(Compaction* c) {
1257
+ const int level = c->level();
1258
+ InternalKey smallest, largest;
1259
+ GetRange(c->inputs_[0], &smallest, &largest);
1260
+
1261
+ current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]);
1262
+
1263
+ // Get entire range covered by compaction
1264
+ InternalKey all_start, all_limit;
1265
+ GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
1266
+
1267
+ // See if we can grow the number of inputs in "level" without
1268
+ // changing the number of "level+1" files we pick up.
1269
+ if (!c->inputs_[1].empty()) {
1270
+ std::vector<FileMetaData*> expanded0;
1271
+ current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0);
1272
+ const int64_t inputs0_size = TotalFileSize(c->inputs_[0]);
1273
+ const int64_t inputs1_size = TotalFileSize(c->inputs_[1]);
1274
+ const int64_t expanded0_size = TotalFileSize(expanded0);
1275
+ if (expanded0.size() > c->inputs_[0].size() &&
1276
+ inputs1_size + expanded0_size < kExpandedCompactionByteSizeLimit) {
1277
+ InternalKey new_start, new_limit;
1278
+ GetRange(expanded0, &new_start, &new_limit);
1279
+ std::vector<FileMetaData*> expanded1;
1280
+ current_->GetOverlappingInputs(level+1, &new_start, &new_limit,
1281
+ &expanded1);
1282
+ if (expanded1.size() == c->inputs_[1].size()) {
1283
+ Log(options_->info_log,
1284
+ "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n",
1285
+ level,
1286
+ int(c->inputs_[0].size()),
1287
+ int(c->inputs_[1].size()),
1288
+ long(inputs0_size), long(inputs1_size),
1289
+ int(expanded0.size()),
1290
+ int(expanded1.size()),
1291
+ long(expanded0_size), long(inputs1_size));
1292
+ smallest = new_start;
1293
+ largest = new_limit;
1294
+ c->inputs_[0] = expanded0;
1295
+ c->inputs_[1] = expanded1;
1296
+ GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
1297
+ }
1298
+ }
1299
+ }
1300
+
1301
+ // Compute the set of grandparent files that overlap this compaction
1302
+ // (parent == level+1; grandparent == level+2)
1303
+ if (level + 2 < config::kNumLevels) {
1304
+ current_->GetOverlappingInputs(level + 2, &all_start, &all_limit,
1305
+ &c->grandparents_);
1306
+ }
1307
+
1308
+ if (false) {
1309
+ Log(options_->info_log, "Compacting %d '%s' .. '%s'",
1310
+ level,
1311
+ smallest.DebugString().c_str(),
1312
+ largest.DebugString().c_str());
1313
+ }
1314
+
1315
+ // Update the place where we will do the next compaction for this level.
1316
+ // We update this immediately instead of waiting for the VersionEdit
1317
+ // to be applied so that if the compaction fails, we will try a different
1318
+ // key range next time.
1319
+ compact_pointer_[level] = largest.Encode().ToString();
1320
+ c->edit_.SetCompactPointer(level, largest);
1321
+ }
1322
+
1323
+ Compaction* VersionSet::CompactRange(
1324
+ int level,
1325
+ const InternalKey* begin,
1326
+ const InternalKey* end) {
1327
+ std::vector<FileMetaData*> inputs;
1328
+ current_->GetOverlappingInputs(level, begin, end, &inputs);
1329
+ if (inputs.empty()) {
1330
+ return NULL;
1331
+ }
1332
+
1333
+ // Avoid compacting too much in one shot in case the range is large.
1334
+ // But we cannot do this for level-0 since level-0 files can overlap
1335
+ // and we must not pick one file and drop another older file if the
1336
+ // two files overlap.
1337
+ if (level > 0) {
1338
+ const uint64_t limit = MaxFileSizeForLevel(level);
1339
+ uint64_t total = 0;
1340
+ for (size_t i = 0; i < inputs.size(); i++) {
1341
+ uint64_t s = inputs[i]->file_size;
1342
+ total += s;
1343
+ if (total >= limit) {
1344
+ inputs.resize(i + 1);
1345
+ break;
1346
+ }
1347
+ }
1348
+ }
1349
+
1350
+ Compaction* c = new Compaction(level);
1351
+ c->input_version_ = current_;
1352
+ c->input_version_->Ref();
1353
+ c->inputs_[0] = inputs;
1354
+ SetupOtherInputs(c);
1355
+ return c;
1356
+ }
1357
+
1358
+ Compaction::Compaction(int level)
1359
+ : level_(level),
1360
+ max_output_file_size_(MaxFileSizeForLevel(level)),
1361
+ input_version_(NULL),
1362
+ grandparent_index_(0),
1363
+ seen_key_(false),
1364
+ overlapped_bytes_(0) {
1365
+ for (int i = 0; i < config::kNumLevels; i++) {
1366
+ level_ptrs_[i] = 0;
1367
+ }
1368
+ }
1369
+
1370
+ Compaction::~Compaction() {
1371
+ if (input_version_ != NULL) {
1372
+ input_version_->Unref();
1373
+ }
1374
+ }
1375
+
1376
+ bool Compaction::IsTrivialMove() const {
1377
+ // Avoid a move if there is lots of overlapping grandparent data.
1378
+ // Otherwise, the move could create a parent file that will require
1379
+ // a very expensive merge later on.
1380
+ return (num_input_files(0) == 1 &&
1381
+ num_input_files(1) == 0 &&
1382
+ TotalFileSize(grandparents_) <= kMaxGrandParentOverlapBytes);
1383
+ }
1384
+
1385
+ void Compaction::AddInputDeletions(VersionEdit* edit) {
1386
+ for (int which = 0; which < 2; which++) {
1387
+ for (size_t i = 0; i < inputs_[which].size(); i++) {
1388
+ edit->DeleteFile(level_ + which, inputs_[which][i]->number);
1389
+ }
1390
+ }
1391
+ }
1392
+
1393
+ bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
1394
+ // Maybe use binary search to find right entry instead of linear search?
1395
+ const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
1396
+ for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) {
1397
+ const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
1398
+ for (; level_ptrs_[lvl] < files.size(); ) {
1399
+ FileMetaData* f = files[level_ptrs_[lvl]];
1400
+ if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
1401
+ // We've advanced far enough
1402
+ if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
1403
+ // Key falls in this file's range, so definitely not base level
1404
+ return false;
1405
+ }
1406
+ break;
1407
+ }
1408
+ level_ptrs_[lvl]++;
1409
+ }
1410
+ }
1411
+ return true;
1412
+ }
1413
+
1414
+ bool Compaction::ShouldStopBefore(const Slice& internal_key) {
1415
+ // Scan to find earliest grandparent file that contains key.
1416
+ const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
1417
+ while (grandparent_index_ < grandparents_.size() &&
1418
+ icmp->Compare(internal_key,
1419
+ grandparents_[grandparent_index_]->largest.Encode()) > 0) {
1420
+ if (seen_key_) {
1421
+ overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
1422
+ }
1423
+ grandparent_index_++;
1424
+ }
1425
+ seen_key_ = true;
1426
+
1427
+ if (overlapped_bytes_ > kMaxGrandParentOverlapBytes) {
1428
+ // Too much overlap for current output; start new output
1429
+ overlapped_bytes_ = 0;
1430
+ return true;
1431
+ } else {
1432
+ return false;
1433
+ }
1434
+ }
1435
+
1436
+ void Compaction::ReleaseInputs() {
1437
+ if (input_version_ != NULL) {
1438
+ input_version_->Unref();
1439
+ input_version_ = NULL;
1440
+ }
1441
+ }
1442
+
1443
+ } // namespace leveldb