leveldb 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (128) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +22 -0
  3. data/README.md +95 -0
  4. data/ext/Rakefile +11 -0
  5. data/ext/leveldb/LICENSE +27 -0
  6. data/ext/leveldb/Makefile +206 -0
  7. data/ext/leveldb/build_config.mk +13 -0
  8. data/ext/leveldb/db/builder.cc +88 -0
  9. data/ext/leveldb/db/builder.h +34 -0
  10. data/ext/leveldb/db/c.cc +595 -0
  11. data/ext/leveldb/db/c_test.c +390 -0
  12. data/ext/leveldb/db/corruption_test.cc +359 -0
  13. data/ext/leveldb/db/db_bench.cc +979 -0
  14. data/ext/leveldb/db/db_impl.cc +1485 -0
  15. data/ext/leveldb/db/db_impl.h +203 -0
  16. data/ext/leveldb/db/db_iter.cc +299 -0
  17. data/ext/leveldb/db/db_iter.h +26 -0
  18. data/ext/leveldb/db/db_test.cc +2092 -0
  19. data/ext/leveldb/db/dbformat.cc +140 -0
  20. data/ext/leveldb/db/dbformat.h +227 -0
  21. data/ext/leveldb/db/dbformat_test.cc +112 -0
  22. data/ext/leveldb/db/filename.cc +139 -0
  23. data/ext/leveldb/db/filename.h +80 -0
  24. data/ext/leveldb/db/filename_test.cc +122 -0
  25. data/ext/leveldb/db/leveldb_main.cc +238 -0
  26. data/ext/leveldb/db/log_format.h +35 -0
  27. data/ext/leveldb/db/log_reader.cc +259 -0
  28. data/ext/leveldb/db/log_reader.h +108 -0
  29. data/ext/leveldb/db/log_test.cc +500 -0
  30. data/ext/leveldb/db/log_writer.cc +103 -0
  31. data/ext/leveldb/db/log_writer.h +48 -0
  32. data/ext/leveldb/db/memtable.cc +145 -0
  33. data/ext/leveldb/db/memtable.h +91 -0
  34. data/ext/leveldb/db/repair.cc +389 -0
  35. data/ext/leveldb/db/skiplist.h +379 -0
  36. data/ext/leveldb/db/skiplist_test.cc +378 -0
  37. data/ext/leveldb/db/snapshot.h +66 -0
  38. data/ext/leveldb/db/table_cache.cc +121 -0
  39. data/ext/leveldb/db/table_cache.h +61 -0
  40. data/ext/leveldb/db/version_edit.cc +266 -0
  41. data/ext/leveldb/db/version_edit.h +107 -0
  42. data/ext/leveldb/db/version_edit_test.cc +46 -0
  43. data/ext/leveldb/db/version_set.cc +1443 -0
  44. data/ext/leveldb/db/version_set.h +383 -0
  45. data/ext/leveldb/db/version_set_test.cc +179 -0
  46. data/ext/leveldb/db/write_batch.cc +147 -0
  47. data/ext/leveldb/db/write_batch_internal.h +49 -0
  48. data/ext/leveldb/db/write_batch_test.cc +120 -0
  49. data/ext/leveldb/doc/bench/db_bench_sqlite3.cc +718 -0
  50. data/ext/leveldb/doc/bench/db_bench_tree_db.cc +528 -0
  51. data/ext/leveldb/helpers/memenv/memenv.cc +384 -0
  52. data/ext/leveldb/helpers/memenv/memenv.h +20 -0
  53. data/ext/leveldb/helpers/memenv/memenv_test.cc +232 -0
  54. data/ext/leveldb/include/leveldb/c.h +291 -0
  55. data/ext/leveldb/include/leveldb/cache.h +99 -0
  56. data/ext/leveldb/include/leveldb/comparator.h +63 -0
  57. data/ext/leveldb/include/leveldb/db.h +161 -0
  58. data/ext/leveldb/include/leveldb/env.h +333 -0
  59. data/ext/leveldb/include/leveldb/filter_policy.h +70 -0
  60. data/ext/leveldb/include/leveldb/iterator.h +100 -0
  61. data/ext/leveldb/include/leveldb/options.h +195 -0
  62. data/ext/leveldb/include/leveldb/slice.h +109 -0
  63. data/ext/leveldb/include/leveldb/status.h +106 -0
  64. data/ext/leveldb/include/leveldb/table.h +85 -0
  65. data/ext/leveldb/include/leveldb/table_builder.h +92 -0
  66. data/ext/leveldb/include/leveldb/write_batch.h +64 -0
  67. data/ext/leveldb/issues/issue178_test.cc +92 -0
  68. data/ext/leveldb/port/atomic_pointer.h +224 -0
  69. data/ext/leveldb/port/port.h +19 -0
  70. data/ext/leveldb/port/port_example.h +135 -0
  71. data/ext/leveldb/port/port_posix.cc +54 -0
  72. data/ext/leveldb/port/port_posix.h +157 -0
  73. data/ext/leveldb/port/thread_annotations.h +59 -0
  74. data/ext/leveldb/port/win/stdint.h +24 -0
  75. data/ext/leveldb/table/block.cc +268 -0
  76. data/ext/leveldb/table/block.h +44 -0
  77. data/ext/leveldb/table/block_builder.cc +109 -0
  78. data/ext/leveldb/table/block_builder.h +57 -0
  79. data/ext/leveldb/table/filter_block.cc +111 -0
  80. data/ext/leveldb/table/filter_block.h +68 -0
  81. data/ext/leveldb/table/filter_block_test.cc +128 -0
  82. data/ext/leveldb/table/format.cc +145 -0
  83. data/ext/leveldb/table/format.h +108 -0
  84. data/ext/leveldb/table/iterator.cc +67 -0
  85. data/ext/leveldb/table/iterator_wrapper.h +63 -0
  86. data/ext/leveldb/table/merger.cc +197 -0
  87. data/ext/leveldb/table/merger.h +26 -0
  88. data/ext/leveldb/table/table.cc +275 -0
  89. data/ext/leveldb/table/table_builder.cc +270 -0
  90. data/ext/leveldb/table/table_test.cc +868 -0
  91. data/ext/leveldb/table/two_level_iterator.cc +182 -0
  92. data/ext/leveldb/table/two_level_iterator.h +34 -0
  93. data/ext/leveldb/util/arena.cc +68 -0
  94. data/ext/leveldb/util/arena.h +68 -0
  95. data/ext/leveldb/util/arena_test.cc +68 -0
  96. data/ext/leveldb/util/bloom.cc +95 -0
  97. data/ext/leveldb/util/bloom_test.cc +160 -0
  98. data/ext/leveldb/util/cache.cc +325 -0
  99. data/ext/leveldb/util/cache_test.cc +186 -0
  100. data/ext/leveldb/util/coding.cc +194 -0
  101. data/ext/leveldb/util/coding.h +104 -0
  102. data/ext/leveldb/util/coding_test.cc +196 -0
  103. data/ext/leveldb/util/comparator.cc +81 -0
  104. data/ext/leveldb/util/crc32c.cc +332 -0
  105. data/ext/leveldb/util/crc32c.h +45 -0
  106. data/ext/leveldb/util/crc32c_test.cc +72 -0
  107. data/ext/leveldb/util/env.cc +96 -0
  108. data/ext/leveldb/util/env_posix.cc +698 -0
  109. data/ext/leveldb/util/env_test.cc +104 -0
  110. data/ext/leveldb/util/filter_policy.cc +11 -0
  111. data/ext/leveldb/util/hash.cc +52 -0
  112. data/ext/leveldb/util/hash.h +19 -0
  113. data/ext/leveldb/util/histogram.cc +139 -0
  114. data/ext/leveldb/util/histogram.h +42 -0
  115. data/ext/leveldb/util/logging.cc +81 -0
  116. data/ext/leveldb/util/logging.h +47 -0
  117. data/ext/leveldb/util/mutexlock.h +41 -0
  118. data/ext/leveldb/util/options.cc +29 -0
  119. data/ext/leveldb/util/posix_logger.h +98 -0
  120. data/ext/leveldb/util/random.h +59 -0
  121. data/ext/leveldb/util/status.cc +75 -0
  122. data/ext/leveldb/util/testharness.cc +77 -0
  123. data/ext/leveldb/util/testharness.h +138 -0
  124. data/ext/leveldb/util/testutil.cc +51 -0
  125. data/ext/leveldb/util/testutil.h +53 -0
  126. data/lib/leveldb/version.rb +3 -0
  127. data/lib/leveldb.rb +1006 -0
  128. metadata +228 -0
@@ -0,0 +1,359 @@
1
+ // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2
+ // Use of this source code is governed by a BSD-style license that can be
3
+ // found in the LICENSE file. See the AUTHORS file for names of contributors.
4
+
5
+ #include "leveldb/db.h"
6
+
7
+ #include <errno.h>
8
+ #include <fcntl.h>
9
+ #include <sys/stat.h>
10
+ #include <sys/types.h>
11
+ #include "leveldb/cache.h"
12
+ #include "leveldb/env.h"
13
+ #include "leveldb/table.h"
14
+ #include "leveldb/write_batch.h"
15
+ #include "db/db_impl.h"
16
+ #include "db/filename.h"
17
+ #include "db/log_format.h"
18
+ #include "db/version_set.h"
19
+ #include "util/logging.h"
20
+ #include "util/testharness.h"
21
+ #include "util/testutil.h"
22
+
23
+ namespace leveldb {
24
+
25
+ static const int kValueSize = 1000;
26
+
27
+ class CorruptionTest {
28
+ public:
29
+ test::ErrorEnv env_;
30
+ std::string dbname_;
31
+ Cache* tiny_cache_;
32
+ Options options_;
33
+ DB* db_;
34
+
35
+ CorruptionTest() {
36
+ tiny_cache_ = NewLRUCache(100);
37
+ options_.env = &env_;
38
+ dbname_ = test::TmpDir() + "/db_test";
39
+ DestroyDB(dbname_, options_);
40
+
41
+ db_ = NULL;
42
+ options_.create_if_missing = true;
43
+ Reopen();
44
+ options_.create_if_missing = false;
45
+ }
46
+
47
+ ~CorruptionTest() {
48
+ delete db_;
49
+ DestroyDB(dbname_, Options());
50
+ delete tiny_cache_;
51
+ }
52
+
53
+ Status TryReopen(Options* options = NULL) {
54
+ delete db_;
55
+ db_ = NULL;
56
+ Options opt = (options ? *options : options_);
57
+ opt.env = &env_;
58
+ opt.block_cache = tiny_cache_;
59
+ return DB::Open(opt, dbname_, &db_);
60
+ }
61
+
62
+ void Reopen(Options* options = NULL) {
63
+ ASSERT_OK(TryReopen(options));
64
+ }
65
+
66
+ void RepairDB() {
67
+ delete db_;
68
+ db_ = NULL;
69
+ ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
70
+ }
71
+
72
+ void Build(int n) {
73
+ std::string key_space, value_space;
74
+ WriteBatch batch;
75
+ for (int i = 0; i < n; i++) {
76
+ //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
77
+ Slice key = Key(i, &key_space);
78
+ batch.Clear();
79
+ batch.Put(key, Value(i, &value_space));
80
+ ASSERT_OK(db_->Write(WriteOptions(), &batch));
81
+ }
82
+ }
83
+
84
+ void Check(int min_expected, int max_expected) {
85
+ int next_expected = 0;
86
+ int missed = 0;
87
+ int bad_keys = 0;
88
+ int bad_values = 0;
89
+ int correct = 0;
90
+ std::string value_space;
91
+ Iterator* iter = db_->NewIterator(ReadOptions());
92
+ for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
93
+ uint64_t key;
94
+ Slice in(iter->key());
95
+ if (!ConsumeDecimalNumber(&in, &key) ||
96
+ !in.empty() ||
97
+ key < next_expected) {
98
+ bad_keys++;
99
+ continue;
100
+ }
101
+ missed += (key - next_expected);
102
+ next_expected = key + 1;
103
+ if (iter->value() != Value(key, &value_space)) {
104
+ bad_values++;
105
+ } else {
106
+ correct++;
107
+ }
108
+ }
109
+ delete iter;
110
+
111
+ fprintf(stderr,
112
+ "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n",
113
+ min_expected, max_expected, correct, bad_keys, bad_values, missed);
114
+ ASSERT_LE(min_expected, correct);
115
+ ASSERT_GE(max_expected, correct);
116
+ }
117
+
118
+ void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
119
+ // Pick file to corrupt
120
+ std::vector<std::string> filenames;
121
+ ASSERT_OK(env_.GetChildren(dbname_, &filenames));
122
+ uint64_t number;
123
+ FileType type;
124
+ std::string fname;
125
+ int picked_number = -1;
126
+ for (int i = 0; i < filenames.size(); i++) {
127
+ if (ParseFileName(filenames[i], &number, &type) &&
128
+ type == filetype &&
129
+ int(number) > picked_number) { // Pick latest file
130
+ fname = dbname_ + "/" + filenames[i];
131
+ picked_number = number;
132
+ }
133
+ }
134
+ ASSERT_TRUE(!fname.empty()) << filetype;
135
+
136
+ struct stat sbuf;
137
+ if (stat(fname.c_str(), &sbuf) != 0) {
138
+ const char* msg = strerror(errno);
139
+ ASSERT_TRUE(false) << fname << ": " << msg;
140
+ }
141
+
142
+ if (offset < 0) {
143
+ // Relative to end of file; make it absolute
144
+ if (-offset > sbuf.st_size) {
145
+ offset = 0;
146
+ } else {
147
+ offset = sbuf.st_size + offset;
148
+ }
149
+ }
150
+ if (offset > sbuf.st_size) {
151
+ offset = sbuf.st_size;
152
+ }
153
+ if (offset + bytes_to_corrupt > sbuf.st_size) {
154
+ bytes_to_corrupt = sbuf.st_size - offset;
155
+ }
156
+
157
+ // Do it
158
+ std::string contents;
159
+ Status s = ReadFileToString(Env::Default(), fname, &contents);
160
+ ASSERT_TRUE(s.ok()) << s.ToString();
161
+ for (int i = 0; i < bytes_to_corrupt; i++) {
162
+ contents[i + offset] ^= 0x80;
163
+ }
164
+ s = WriteStringToFile(Env::Default(), contents, fname);
165
+ ASSERT_TRUE(s.ok()) << s.ToString();
166
+ }
167
+
168
+ int Property(const std::string& name) {
169
+ std::string property;
170
+ int result;
171
+ if (db_->GetProperty(name, &property) &&
172
+ sscanf(property.c_str(), "%d", &result) == 1) {
173
+ return result;
174
+ } else {
175
+ return -1;
176
+ }
177
+ }
178
+
179
+ // Return the ith key
180
+ Slice Key(int i, std::string* storage) {
181
+ char buf[100];
182
+ snprintf(buf, sizeof(buf), "%016d", i);
183
+ storage->assign(buf, strlen(buf));
184
+ return Slice(*storage);
185
+ }
186
+
187
+ // Return the value to associate with the specified key
188
+ Slice Value(int k, std::string* storage) {
189
+ Random r(k);
190
+ return test::RandomString(&r, kValueSize, storage);
191
+ }
192
+ };
193
+
194
+ TEST(CorruptionTest, Recovery) {
195
+ Build(100);
196
+ Check(100, 100);
197
+ Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
198
+ Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block
199
+ Reopen();
200
+
201
+ // The 64 records in the first two log blocks are completely lost.
202
+ Check(36, 36);
203
+ }
204
+
205
+ TEST(CorruptionTest, RecoverWriteError) {
206
+ env_.writable_file_error_ = true;
207
+ Status s = TryReopen();
208
+ ASSERT_TRUE(!s.ok());
209
+ }
210
+
211
+ TEST(CorruptionTest, NewFileErrorDuringWrite) {
212
+ // Do enough writing to force minor compaction
213
+ env_.writable_file_error_ = true;
214
+ const int num = 3 + (Options().write_buffer_size / kValueSize);
215
+ std::string value_storage;
216
+ Status s;
217
+ for (int i = 0; s.ok() && i < num; i++) {
218
+ WriteBatch batch;
219
+ batch.Put("a", Value(100, &value_storage));
220
+ s = db_->Write(WriteOptions(), &batch);
221
+ }
222
+ ASSERT_TRUE(!s.ok());
223
+ ASSERT_GE(env_.num_writable_file_errors_, 1);
224
+ env_.writable_file_error_ = false;
225
+ Reopen();
226
+ }
227
+
228
+ TEST(CorruptionTest, TableFile) {
229
+ Build(100);
230
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
231
+ dbi->TEST_CompactMemTable();
232
+ dbi->TEST_CompactRange(0, NULL, NULL);
233
+ dbi->TEST_CompactRange(1, NULL, NULL);
234
+
235
+ Corrupt(kTableFile, 100, 1);
236
+ Check(99, 99);
237
+ }
238
+
239
+ TEST(CorruptionTest, TableFileIndexData) {
240
+ Build(10000); // Enough to build multiple Tables
241
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
242
+ dbi->TEST_CompactMemTable();
243
+
244
+ Corrupt(kTableFile, -2000, 500);
245
+ Reopen();
246
+ Check(5000, 9999);
247
+ }
248
+
249
+ TEST(CorruptionTest, MissingDescriptor) {
250
+ Build(1000);
251
+ RepairDB();
252
+ Reopen();
253
+ Check(1000, 1000);
254
+ }
255
+
256
+ TEST(CorruptionTest, SequenceNumberRecovery) {
257
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
258
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
259
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3"));
260
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4"));
261
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5"));
262
+ RepairDB();
263
+ Reopen();
264
+ std::string v;
265
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
266
+ ASSERT_EQ("v5", v);
267
+ // Write something. If sequence number was not recovered properly,
268
+ // it will be hidden by an earlier write.
269
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6"));
270
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
271
+ ASSERT_EQ("v6", v);
272
+ Reopen();
273
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
274
+ ASSERT_EQ("v6", v);
275
+ }
276
+
277
+ TEST(CorruptionTest, CorruptedDescriptor) {
278
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
279
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
280
+ dbi->TEST_CompactMemTable();
281
+ dbi->TEST_CompactRange(0, NULL, NULL);
282
+
283
+ Corrupt(kDescriptorFile, 0, 1000);
284
+ Status s = TryReopen();
285
+ ASSERT_TRUE(!s.ok());
286
+
287
+ RepairDB();
288
+ Reopen();
289
+ std::string v;
290
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
291
+ ASSERT_EQ("hello", v);
292
+ }
293
+
294
+ TEST(CorruptionTest, CompactionInputError) {
295
+ Build(10);
296
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
297
+ dbi->TEST_CompactMemTable();
298
+ const int last = config::kMaxMemCompactLevel;
299
+ ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
300
+
301
+ Corrupt(kTableFile, 100, 1);
302
+ Check(9, 9);
303
+
304
+ // Force compactions by writing lots of values
305
+ Build(10000);
306
+ Check(10000, 10000);
307
+ }
308
+
309
+ TEST(CorruptionTest, CompactionInputErrorParanoid) {
310
+ Options options;
311
+ options.paranoid_checks = true;
312
+ options.write_buffer_size = 1048576;
313
+ Reopen(&options);
314
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
315
+
316
+ // Fill levels >= 1 so memtable compaction outputs to level 1
317
+ for (int level = 1; level < config::kNumLevels; level++) {
318
+ dbi->Put(WriteOptions(), "", "begin");
319
+ dbi->Put(WriteOptions(), "~", "end");
320
+ dbi->TEST_CompactMemTable();
321
+ }
322
+
323
+ Build(10);
324
+ dbi->TEST_CompactMemTable();
325
+ ASSERT_EQ(1, Property("leveldb.num-files-at-level0"));
326
+
327
+ Corrupt(kTableFile, 100, 1);
328
+ Check(9, 9);
329
+
330
+ // Write must eventually fail because of corrupted table
331
+ Status s;
332
+ std::string tmp1, tmp2;
333
+ for (int i = 0; i < 10000 && s.ok(); i++) {
334
+ s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2));
335
+ }
336
+ ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
337
+ }
338
+
339
+ TEST(CorruptionTest, UnrelatedKeys) {
340
+ Build(10);
341
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
342
+ dbi->TEST_CompactMemTable();
343
+ Corrupt(kTableFile, 100, 1);
344
+
345
+ std::string tmp1, tmp2;
346
+ ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
347
+ std::string v;
348
+ ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
349
+ ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
350
+ dbi->TEST_CompactMemTable();
351
+ ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
352
+ ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
353
+ }
354
+
355
+ } // namespace leveldb
356
+
357
+ int main(int argc, char** argv) {
358
+ return leveldb::test::RunAllTests();
359
+ }