leveldb-ruby 0.10 → 0.11

Sign up to get free protection for your applications and to get access to all the features.
Files changed (116) hide show
  1. data/ext/leveldb/extconf.rb +2 -6
  2. data/ext/leveldb/leveldb.cc +377 -33
  3. data/ext/leveldb/platform.rb +83 -0
  4. data/leveldb/Makefile +29 -28
  5. data/leveldb/build_detect_platform +23 -4
  6. data/leveldb/db/builder.cc +1 -1
  7. data/leveldb/db/builder.h +1 -1
  8. data/leveldb/db/corruption_test.cc +1 -1
  9. data/leveldb/db/db_bench.cc +2 -2
  10. data/leveldb/db/db_impl.cc +8 -16
  11. data/leveldb/db/db_impl.h +1 -1
  12. data/leveldb/db/db_iter.cc +1 -1
  13. data/leveldb/db/db_iter.h +1 -1
  14. data/leveldb/db/db_test.cc +180 -9
  15. data/leveldb/db/dbformat.cc +9 -7
  16. data/leveldb/db/dbformat.h +2 -2
  17. data/leveldb/db/dbformat_test.cc +1 -1
  18. data/leveldb/db/filename.cc +6 -2
  19. data/leveldb/db/filename.h +1 -1
  20. data/leveldb/db/filename_test.cc +1 -1
  21. data/leveldb/db/log_format.h +2 -2
  22. data/leveldb/db/log_reader.cc +2 -2
  23. data/leveldb/db/log_reader.h +2 -2
  24. data/leveldb/db/log_test.cc +2 -2
  25. data/leveldb/db/log_writer.cc +2 -2
  26. data/leveldb/db/log_writer.h +2 -2
  27. data/leveldb/db/memtable.cc +1 -1
  28. data/leveldb/db/memtable.h +1 -1
  29. data/leveldb/db/repair.cc +2 -2
  30. data/leveldb/db/skiplist.h +1 -1
  31. data/leveldb/db/skiplist_test.cc +1 -1
  32. data/leveldb/db/snapshot.h +1 -1
  33. data/leveldb/db/table_cache.cc +1 -1
  34. data/leveldb/db/table_cache.h +1 -1
  35. data/leveldb/db/version_edit.cc +1 -1
  36. data/leveldb/db/version_edit.h +1 -1
  37. data/leveldb/db/version_edit_test.cc +1 -1
  38. data/leveldb/db/version_set.cc +39 -14
  39. data/leveldb/db/version_set.h +1 -1
  40. data/leveldb/db/version_set_test.cc +1 -1
  41. data/leveldb/db/write_batch.cc +2 -2
  42. data/leveldb/db/write_batch_internal.h +1 -1
  43. data/leveldb/db/write_batch_test.cc +1 -1
  44. data/leveldb/helpers/memenv/memenv.cc +374 -0
  45. data/leveldb/helpers/memenv/memenv.h +20 -0
  46. data/leveldb/helpers/memenv/memenv_test.cc +232 -0
  47. data/leveldb/include/leveldb/cache.h +1 -1
  48. data/leveldb/include/leveldb/comparator.h +1 -1
  49. data/leveldb/include/leveldb/db.h +1 -1
  50. data/leveldb/include/leveldb/env.h +1 -1
  51. data/leveldb/include/leveldb/iterator.h +1 -1
  52. data/leveldb/include/leveldb/options.h +1 -1
  53. data/leveldb/include/leveldb/slice.h +1 -1
  54. data/leveldb/include/leveldb/status.h +1 -1
  55. data/leveldb/include/leveldb/table.h +1 -1
  56. data/leveldb/include/leveldb/table_builder.h +1 -1
  57. data/leveldb/include/leveldb/write_batch.h +1 -1
  58. data/leveldb/port/atomic_pointer.h +2 -2
  59. data/leveldb/port/port_android.cc +2 -2
  60. data/leveldb/port/port_android.h +2 -2
  61. data/leveldb/port/port_example.h +2 -2
  62. data/leveldb/port/port_posix.cc +2 -2
  63. data/leveldb/port/port_posix.h +11 -3
  64. data/leveldb/table/block.cc +1 -1
  65. data/leveldb/table/block.h +1 -1
  66. data/leveldb/table/block_builder.cc +1 -1
  67. data/leveldb/table/block_builder.h +1 -1
  68. data/leveldb/table/format.cc +1 -1
  69. data/leveldb/table/format.h +1 -1
  70. data/leveldb/table/iterator.cc +2 -2
  71. data/leveldb/table/merger.cc +2 -2
  72. data/leveldb/table/merger.h +1 -1
  73. data/leveldb/table/table.cc +1 -1
  74. data/leveldb/table/table_builder.cc +1 -1
  75. data/leveldb/table/table_test.cc +3 -3
  76. data/leveldb/table/two_level_iterator.cc +2 -2
  77. data/leveldb/table/two_level_iterator.h +1 -1
  78. data/leveldb/util/arena.cc +1 -1
  79. data/leveldb/util/arena.h +1 -1
  80. data/leveldb/util/arena_test.cc +1 -1
  81. data/leveldb/util/cache.cc +1 -1
  82. data/leveldb/util/cache_test.cc +1 -1
  83. data/leveldb/util/coding.cc +1 -1
  84. data/leveldb/util/coding.h +1 -1
  85. data/leveldb/util/coding_test.cc +1 -1
  86. data/leveldb/util/comparator.cc +7 -4
  87. data/leveldb/util/crc32c.cc +2 -2
  88. data/leveldb/util/crc32c.h +2 -2
  89. data/leveldb/util/crc32c_test.cc +2 -2
  90. data/leveldb/util/env.cc +17 -3
  91. data/leveldb/util/env_posix.cc +2 -2
  92. data/leveldb/util/env_test.cc +13 -11
  93. data/leveldb/util/hash.cc +1 -1
  94. data/leveldb/util/histogram.cc +1 -1
  95. data/leveldb/util/histogram.h +1 -1
  96. data/leveldb/util/logging.cc +1 -1
  97. data/leveldb/util/logging.h +1 -1
  98. data/leveldb/util/mutexlock.h +1 -1
  99. data/leveldb/util/options.cc +1 -1
  100. data/leveldb/util/posix_logger.h +1 -1
  101. data/leveldb/util/random.h +1 -1
  102. data/leveldb/util/status.cc +1 -1
  103. data/leveldb/util/testharness.cc +2 -2
  104. data/leveldb/util/testharness.h +2 -2
  105. data/leveldb/util/testutil.cc +2 -2
  106. data/leveldb/util/testutil.h +2 -2
  107. data/lib/leveldb.rb +34 -2
  108. metadata +7 -13
  109. data/leveldb/port/port_chromium.cc +0 -80
  110. data/leveldb/port/port_chromium.h +0 -97
  111. data/leveldb/port/port_osx.cc +0 -50
  112. data/leveldb/port/port_osx.h +0 -125
  113. data/leveldb/port/sha1_portable.cc +0 -298
  114. data/leveldb/port/sha1_portable.h +0 -25
  115. data/leveldb/port/sha1_test.cc +0 -39
  116. data/leveldb/util/env_chromium.cc +0 -612
@@ -73,9 +73,10 @@ void InternalKeyComparator::FindShortestSeparator(
73
73
  Slice user_limit = ExtractUserKey(limit);
74
74
  std::string tmp(user_start.data(), user_start.size());
75
75
  user_comparator_->FindShortestSeparator(&tmp, user_limit);
76
- if (user_comparator_->Compare(*start, tmp) < 0) {
77
- // User key has become larger. Tack on the earliest possible
78
- // number to the shortened user key.
76
+ if (tmp.size() < user_start.size() &&
77
+ user_comparator_->Compare(user_start, tmp) < 0) {
78
+ // User key has become shorter physically, but larger logically.
79
+ // Tack on the earliest possible number to the shortened user key.
79
80
  PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
80
81
  assert(this->Compare(*start, tmp) < 0);
81
82
  assert(this->Compare(tmp, limit) < 0);
@@ -87,9 +88,10 @@ void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
87
88
  Slice user_key = ExtractUserKey(*key);
88
89
  std::string tmp(user_key.data(), user_key.size());
89
90
  user_comparator_->FindShortSuccessor(&tmp);
90
- if (user_comparator_->Compare(user_key, tmp) < 0) {
91
- // User key has become larger. Tack on the earliest possible
92
- // number to the shortened user key.
91
+ if (tmp.size() < user_key.size() &&
92
+ user_comparator_->Compare(user_key, tmp) < 0) {
93
+ // User key has become shorter physically, but larger logically.
94
+ // Tack on the earliest possible number to the shortened user key.
93
95
  PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
94
96
  assert(this->Compare(*key, tmp) < 0);
95
97
  key->swap(tmp);
@@ -115,4 +117,4 @@ LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
115
117
  end_ = dst;
116
118
  }
117
119
 
118
- }
120
+ } // namespace leveldb
@@ -37,7 +37,7 @@ static const int kL0_StopWritesTrigger = 12;
37
37
  // space if the same key space is being repeatedly overwritten.
38
38
  static const int kMaxMemCompactLevel = 2;
39
39
 
40
- }
40
+ } // namespace config
41
41
 
42
42
  class InternalKey;
43
43
 
@@ -210,6 +210,6 @@ inline LookupKey::~LookupKey() {
210
210
  if (start_ != space_) delete[] start_;
211
211
  }
212
212
 
213
- }
213
+ } // namespace leveldb
214
214
 
215
215
  #endif // STORAGE_LEVELDB_DB_FORMAT_H_
@@ -105,7 +105,7 @@ TEST(FormatTest, InternalKeyShortestSuccessor) {
105
105
  ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
106
106
  }
107
107
 
108
- }
108
+ } // namespace leveldb
109
109
 
110
110
  int main(int argc, char** argv) {
111
111
  return leveldb::test::RunAllTests();
@@ -11,6 +11,10 @@
11
11
 
12
12
  namespace leveldb {
13
13
 
14
+ // A utility routine: write "data" to the named file and Sync() it.
15
+ extern Status WriteStringToFileSync(Env* env, const Slice& data,
16
+ const std::string& fname);
17
+
14
18
  static std::string MakeFileName(const std::string& name, uint64_t number,
15
19
  const char* suffix) {
16
20
  char buf[100];
@@ -122,7 +126,7 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
122
126
  assert(contents.starts_with(dbname + "/"));
123
127
  contents.remove_prefix(dbname.size() + 1);
124
128
  std::string tmp = TempFileName(dbname, descriptor_number);
125
- Status s = WriteStringToFile(env, contents.ToString() + "\n", tmp);
129
+ Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp);
126
130
  if (s.ok()) {
127
131
  s = env->RenameFile(tmp, CurrentFileName(dbname));
128
132
  }
@@ -132,4 +136,4 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
132
136
  return s;
133
137
  }
134
138
 
135
- }
139
+ } // namespace leveldb
@@ -75,6 +75,6 @@ extern Status SetCurrentFile(Env* env, const std::string& dbname,
75
75
  uint64_t descriptor_number);
76
76
 
77
77
 
78
- }
78
+ } // namespace leveldb
79
79
 
80
80
  #endif // STORAGE_LEVELDB_DB_FILENAME_H_
@@ -115,7 +115,7 @@ TEST(FileNameTest, Construction) {
115
115
  ASSERT_EQ(kTempFile, type);
116
116
  }
117
117
 
118
- }
118
+ } // namespace leveldb
119
119
 
120
120
  int main(int argc, char** argv) {
121
121
  return leveldb::test::RunAllTests();
@@ -29,7 +29,7 @@ static const int kBlockSize = 32768;
29
29
  // Header is checksum (4 bytes), type (1 byte), length (2 bytes).
30
30
  static const int kHeaderSize = 4 + 1 + 2;
31
31
 
32
- }
33
- }
32
+ } // namespace log
33
+ } // namespace leveldb
34
34
 
35
35
  #endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_
@@ -255,5 +255,5 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
255
255
  }
256
256
  }
257
257
 
258
- }
259
- }
258
+ } // namespace log
259
+ } // namespace leveldb
@@ -102,7 +102,7 @@ class Reader {
102
102
  void operator=(const Reader&);
103
103
  };
104
104
 
105
- }
106
- }
105
+ } // namespace log
106
+ } // namespace leveldb
107
107
 
108
108
  #endif // STORAGE_LEVELDB_DB_LOG_READER_H_
@@ -492,8 +492,8 @@ TEST(LogTest, ReadPastEnd) {
492
492
  CheckOffsetPastEndReturnsNoRecords(5);
493
493
  }
494
494
 
495
- }
496
- }
495
+ } // namespace log
496
+ } // namespace leveldb
497
497
 
498
498
  int main(int argc, char** argv) {
499
499
  return leveldb::test::RunAllTests();
@@ -99,5 +99,5 @@ Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
99
99
  return s;
100
100
  }
101
101
 
102
- }
103
- }
102
+ } // namespace log
103
+ } // namespace leveldb
@@ -42,7 +42,7 @@ class Writer {
42
42
  void operator=(const Writer&);
43
43
  };
44
44
 
45
- }
46
- }
45
+ } // namespace log
46
+ } // namespace leveldb
47
47
 
48
48
  #endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_
@@ -142,4 +142,4 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
142
142
  return false;
143
143
  }
144
144
 
145
- }
145
+ } // namespace leveldb
@@ -86,6 +86,6 @@ class MemTable {
86
86
  void operator=(const MemTable&);
87
87
  };
88
88
 
89
- }
89
+ } // namespace leveldb
90
90
 
91
91
  #endif // STORAGE_LEVELDB_DB_MEMTABLE_H_
@@ -377,11 +377,11 @@ class Repairer {
377
377
  fname.c_str(), s.ToString().c_str());
378
378
  }
379
379
  };
380
- }
380
+ } // namespace
381
381
 
382
382
  Status RepairDB(const std::string& dbname, const Options& options) {
383
383
  Repairer repairer(dbname, options);
384
384
  return repairer.Run();
385
385
  }
386
386
 
387
- }
387
+ } // namespace leveldb
@@ -375,4 +375,4 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const {
375
375
  }
376
376
  }
377
377
 
378
- }
378
+ } // namespace leveldb
@@ -371,7 +371,7 @@ TEST(SkipTest, Concurrent3) { RunConcurrent(3); }
371
371
  TEST(SkipTest, Concurrent4) { RunConcurrent(4); }
372
372
  TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
373
373
 
374
- }
374
+ } // namespace leveldb
375
375
 
376
376
  int main(int argc, char** argv) {
377
377
  return leveldb::test::RunAllTests();
@@ -61,6 +61,6 @@ class SnapshotList {
61
61
  SnapshotImpl list_;
62
62
  };
63
63
 
64
- }
64
+ } // namespace leveldb
65
65
 
66
66
  #endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_
@@ -92,4 +92,4 @@ void TableCache::Evict(uint64_t file_number) {
92
92
  cache_->Erase(Slice(buf, sizeof(buf)));
93
93
  }
94
94
 
95
- }
95
+ } // namespace leveldb
@@ -45,6 +45,6 @@ class TableCache {
45
45
  Cache* cache_;
46
46
  };
47
47
 
48
- }
48
+ } // namespace leveldb
49
49
 
50
50
  #endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_
@@ -263,4 +263,4 @@ std::string VersionEdit::DebugString() const {
263
263
  return r;
264
264
  }
265
265
 
266
- }
266
+ } // namespace leveldb
@@ -102,6 +102,6 @@ class VersionEdit {
102
102
  std::vector< std::pair<int, FileMetaData> > new_files_;
103
103
  };
104
104
 
105
- }
105
+ } // namespace leveldb
106
106
 
107
107
  #endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_
@@ -39,7 +39,7 @@ TEST(VersionEditTest, EncodeDecode) {
39
39
  TestEncodeDecode(edit);
40
40
  }
41
41
 
42
- }
42
+ } // namespace leveldb
43
43
 
44
44
  int main(int argc, char** argv) {
45
45
  return leveldb::test::RunAllTests();
@@ -26,6 +26,11 @@ static const int kTargetFileSize = 2 * 1048576;
26
26
  // stop building a single file in a level->level+1 compaction.
27
27
  static const int64_t kMaxGrandParentOverlapBytes = 10 * kTargetFileSize;
28
28
 
29
+ // Maximum number of bytes in all compacted files. We avoid expanding
30
+ // the lower level file set of a compaction if it would make the
31
+ // total compaction cover more than this many bytes.
32
+ static const int64_t kExpandedCompactionByteSizeLimit = 25 * kTargetFileSize;
33
+
29
34
  static double MaxBytesForLevel(int level) {
30
35
  // Note: the result for level zero is not really used since we set
31
36
  // the level-0 compaction threshold based on number of files.
@@ -61,7 +66,7 @@ std::string IntSetToString(const std::set<uint64_t>& s) {
61
66
  result += "}";
62
67
  return result;
63
68
  }
64
- }
69
+ } // namespace
65
70
 
66
71
  Version::~Version() {
67
72
  assert(refs_ == 0);
@@ -253,7 +258,8 @@ void Version::AddIterators(const ReadOptions& options,
253
258
  // If "*iter" points at a value or deletion for user_key, store
254
259
  // either the value, or a NotFound error and return true.
255
260
  // Else return false.
256
- static bool GetValue(Iterator* iter, const Slice& user_key,
261
+ static bool GetValue(const Comparator* cmp,
262
+ Iterator* iter, const Slice& user_key,
257
263
  std::string* value,
258
264
  Status* s) {
259
265
  if (!iter->Valid()) {
@@ -264,7 +270,7 @@ static bool GetValue(Iterator* iter, const Slice& user_key,
264
270
  *s = Status::Corruption("corrupted key for ", user_key);
265
271
  return true;
266
272
  }
267
- if (parsed_key.user_key != user_key) {
273
+ if (cmp->Compare(parsed_key.user_key, user_key) != 0) {
268
274
  return false;
269
275
  }
270
276
  switch (parsed_key.type) {
@@ -360,7 +366,7 @@ Status Version::Get(const ReadOptions& options,
360
366
  f->number,
361
367
  f->file_size);
362
368
  iter->Seek(ikey);
363
- const bool done = GetValue(iter, user_key, value, &s);
369
+ const bool done = GetValue(ucmp, iter, user_key, value, &s);
364
370
  if (!iter->status().ok()) {
365
371
  s = iter->status();
366
372
  delete iter;
@@ -450,16 +456,29 @@ void Version::GetOverlappingInputs(
450
456
  user_end = end->user_key();
451
457
  }
452
458
  const Comparator* user_cmp = vset_->icmp_.user_comparator();
453
- for (size_t i = 0; i < files_[level].size(); i++) {
454
- FileMetaData* f = files_[level][i];
455
- if (begin != NULL &&
456
- user_cmp->Compare(f->largest.user_key(), user_begin) < 0) {
459
+ for (size_t i = 0; i < files_[level].size(); ) {
460
+ FileMetaData* f = files_[level][i++];
461
+ const Slice file_start = f->smallest.user_key();
462
+ const Slice file_limit = f->largest.user_key();
463
+ if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) {
457
464
  // "f" is completely before specified range; skip it
458
- } else if (end != NULL &&
459
- user_cmp->Compare(f->smallest.user_key(), user_end) > 0) {
465
+ } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) {
460
466
  // "f" is completely after specified range; skip it
461
467
  } else {
462
468
  inputs->push_back(f);
469
+ if (level == 0) {
470
+ // Level-0 files may overlap each other. So check if the newly
471
+ // added file has expanded the range. If so, restart search.
472
+ if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) {
473
+ user_begin = file_start;
474
+ inputs->clear();
475
+ i = 0;
476
+ } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) {
477
+ user_end = file_limit;
478
+ inputs->clear();
479
+ i = 0;
480
+ }
481
+ }
463
482
  }
464
483
  }
465
484
  }
@@ -1209,7 +1228,11 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
1209
1228
  if (!c->inputs_[1].empty()) {
1210
1229
  std::vector<FileMetaData*> expanded0;
1211
1230
  current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0);
1212
- if (expanded0.size() > c->inputs_[0].size()) {
1231
+ const int64_t inputs0_size = TotalFileSize(c->inputs_[0]);
1232
+ const int64_t inputs1_size = TotalFileSize(c->inputs_[1]);
1233
+ const int64_t expanded0_size = TotalFileSize(expanded0);
1234
+ if (expanded0.size() > c->inputs_[0].size() &&
1235
+ inputs1_size + expanded0_size < kExpandedCompactionByteSizeLimit) {
1213
1236
  InternalKey new_start, new_limit;
1214
1237
  GetRange(expanded0, &new_start, &new_limit);
1215
1238
  std::vector<FileMetaData*> expanded1;
@@ -1217,12 +1240,14 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
1217
1240
  &expanded1);
1218
1241
  if (expanded1.size() == c->inputs_[1].size()) {
1219
1242
  Log(options_->info_log,
1220
- "Expanding@%d %d+%d to %d+%d\n",
1243
+ "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n",
1221
1244
  level,
1222
1245
  int(c->inputs_[0].size()),
1223
1246
  int(c->inputs_[1].size()),
1247
+ long(inputs0_size), long(inputs1_size),
1224
1248
  int(expanded0.size()),
1225
- int(expanded1.size()));
1249
+ int(expanded1.size()),
1250
+ long(expanded0_size), long(inputs1_size));
1226
1251
  smallest = new_start;
1227
1252
  largest = new_limit;
1228
1253
  c->inputs_[0] = expanded0;
@@ -1369,4 +1394,4 @@ void Compaction::ReleaseInputs() {
1369
1394
  }
1370
1395
  }
1371
1396
 
1372
- }
1397
+ } // namespace leveldb
@@ -365,6 +365,6 @@ class Compaction {
365
365
  size_t level_ptrs_[config::kNumLevels];
366
366
  };
367
367
 
368
- }
368
+ } // namespace leveldb
369
369
 
370
370
  #endif // STORAGE_LEVELDB_DB_VERSION_SET_H_
@@ -172,7 +172,7 @@ TEST(FindFileTest, OverlappingFiles) {
172
172
  ASSERT_TRUE(Overlaps("600", "700"));
173
173
  }
174
174
 
175
- }
175
+ } // namespace leveldb
176
176
 
177
177
  int main(int argc, char** argv) {
178
178
  return leveldb::test::RunAllTests();
@@ -120,7 +120,7 @@ class MemTableInserter : public WriteBatch::Handler {
120
120
  sequence_++;
121
121
  }
122
122
  };
123
- }
123
+ } // namespace
124
124
 
125
125
  Status WriteBatchInternal::InsertInto(const WriteBatch* b,
126
126
  MemTable* memtable) {
@@ -135,4 +135,4 @@ void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
135
135
  b->rep_.assign(contents.data(), contents.size());
136
136
  }
137
137
 
138
- }
138
+ } // namespace leveldb
@@ -41,7 +41,7 @@ class WriteBatchInternal {
41
41
  static Status InsertInto(const WriteBatch* batch, MemTable* memtable);
42
42
  };
43
43
 
44
- }
44
+ } // namespace leveldb
45
45
 
46
46
 
47
47
  #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
@@ -82,7 +82,7 @@ TEST(WriteBatchTest, Corruption) {
82
82
  PrintContents(&batch));
83
83
  }
84
84
 
85
- }
85
+ } // namespace leveldb
86
86
 
87
87
  int main(int argc, char** argv) {
88
88
  return leveldb::test::RunAllTests();
@@ -0,0 +1,374 @@
1
+ // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2
+ // Use of this source code is governed by a BSD-style license that can be
3
+ // found in the LICENSE file. See the AUTHORS file for names of contributors.
4
+
5
+ #include "helpers/memenv/memenv.h"
6
+
7
+ #include "leveldb/env.h"
8
+ #include "leveldb/status.h"
9
+ #include "port/port.h"
10
+ #include "util/mutexlock.h"
11
+ #include <map>
12
+ #include <string.h>
13
+ #include <string>
14
+ #include <vector>
15
+
16
+ namespace leveldb {
17
+
18
+ namespace {
19
+
20
+ class FileState {
21
+ public:
22
+ // FileStates are reference counted. The initial reference count is zero
23
+ // and the caller must call Ref() at least once.
24
+ FileState() : refs_(0), size_(0) {}
25
+
26
+ // Increase the reference count.
27
+ void Ref() {
28
+ MutexLock lock(&refs_mutex_);
29
+ ++refs_;
30
+ }
31
+
32
+ // Decrease the reference count. Delete if this is the last reference.
33
+ void Unref() {
34
+ bool do_delete = false;
35
+
36
+ {
37
+ MutexLock lock(&refs_mutex_);
38
+ --refs_;
39
+ assert(refs_ >= 0);
40
+ if (refs_ <= 0) {
41
+ do_delete = true;
42
+ }
43
+ }
44
+
45
+ if (do_delete) {
46
+ delete this;
47
+ }
48
+ }
49
+
50
+ uint64_t Size() const { return size_; }
51
+
52
+ Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
53
+ if (offset > size_) {
54
+ return Status::IOError("Offset greater than file size.");
55
+ }
56
+ const uint64_t available = size_ - offset;
57
+ if (n > available) {
58
+ n = available;
59
+ }
60
+ if (n == 0) {
61
+ *result = Slice();
62
+ return Status::OK();
63
+ }
64
+
65
+ size_t block = offset / kBlockSize;
66
+ size_t block_offset = offset % kBlockSize;
67
+
68
+ if (n <= kBlockSize - block_offset) {
69
+ // The requested bytes are all in the first block.
70
+ *result = Slice(blocks_[block] + block_offset, n);
71
+ return Status::OK();
72
+ }
73
+
74
+ size_t bytes_to_copy = n;
75
+ char* dst = scratch;
76
+
77
+ while (bytes_to_copy > 0) {
78
+ size_t avail = kBlockSize - block_offset;
79
+ if (avail > bytes_to_copy) {
80
+ avail = bytes_to_copy;
81
+ }
82
+ memcpy(dst, blocks_[block] + block_offset, avail);
83
+
84
+ bytes_to_copy -= avail;
85
+ dst += avail;
86
+ block++;
87
+ block_offset = 0;
88
+ }
89
+
90
+ *result = Slice(scratch, n);
91
+ return Status::OK();
92
+ }
93
+
94
+ Status Append(const Slice& data) {
95
+ const char* src = data.data();
96
+ size_t src_len = data.size();
97
+
98
+ while (src_len > 0) {
99
+ size_t avail;
100
+ size_t offset = size_ % kBlockSize;
101
+
102
+ if (offset != 0) {
103
+ // There is some room in the last block.
104
+ avail = kBlockSize - offset;
105
+ } else {
106
+ // No room in the last block; push new one.
107
+ blocks_.push_back(new char[kBlockSize]);
108
+ avail = kBlockSize;
109
+ }
110
+
111
+ if (avail > src_len) {
112
+ avail = src_len;
113
+ }
114
+ memcpy(blocks_.back() + offset, src, avail);
115
+ src_len -= avail;
116
+ src += avail;
117
+ size_ += avail;
118
+ }
119
+
120
+ return Status::OK();
121
+ }
122
+
123
+ private:
124
+ // Private since only Unref() should be used to delete it.
125
+ ~FileState() {
126
+ for (std::vector<char*>::iterator i = blocks_.begin(); i != blocks_.end();
127
+ ++i) {
128
+ delete [] *i;
129
+ }
130
+ }
131
+
132
+ // No copying allowed.
133
+ FileState(const FileState&);
134
+ void operator=(const FileState&);
135
+
136
+ port::Mutex refs_mutex_;
137
+ int refs_; // Protected by refs_mutex_;
138
+
139
+ // The following fields are not protected by any mutex. They are only mutable
140
+ // while the file is being written, and concurrent access is not allowed
141
+ // to writable files.
142
+ std::vector<char*> blocks_;
143
+ uint64_t size_;
144
+
145
+ enum { kBlockSize = 8 * 1024 };
146
+ };
147
+
148
+ class SequentialFileImpl : public SequentialFile {
149
+ public:
150
+ explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) {
151
+ file_->Ref();
152
+ }
153
+
154
+ ~SequentialFileImpl() {
155
+ file_->Unref();
156
+ }
157
+
158
+ virtual Status Read(size_t n, Slice* result, char* scratch) {
159
+ Status s = file_->Read(pos_, n, result, scratch);
160
+ if (s.ok()) {
161
+ pos_ += result->size();
162
+ }
163
+ return s;
164
+ }
165
+
166
+ virtual Status Skip(uint64_t n) {
167
+ if (pos_ > file_->Size()) {
168
+ return Status::IOError("pos_ > file_->Size()");
169
+ }
170
+ const size_t available = file_->Size() - pos_;
171
+ if (n > available) {
172
+ n = available;
173
+ }
174
+ pos_ += n;
175
+ return Status::OK();
176
+ }
177
+
178
+ private:
179
+ FileState* file_;
180
+ size_t pos_;
181
+ };
182
+
183
+ class RandomAccessFileImpl : public RandomAccessFile {
184
+ public:
185
+ explicit RandomAccessFileImpl(FileState* file) : file_(file) {
186
+ file_->Ref();
187
+ }
188
+
189
+ ~RandomAccessFileImpl() {
190
+ file_->Unref();
191
+ }
192
+
193
+ virtual Status Read(uint64_t offset, size_t n, Slice* result,
194
+ char* scratch) const {
195
+ return file_->Read(offset, n, result, scratch);
196
+ }
197
+
198
+ private:
199
+ FileState* file_;
200
+ };
201
+
202
+ class WritableFileImpl : public WritableFile {
203
+ public:
204
+ WritableFileImpl(FileState* file) : file_(file) {
205
+ file_->Ref();
206
+ }
207
+
208
+ ~WritableFileImpl() {
209
+ file_->Unref();
210
+ }
211
+
212
+ virtual Status Append(const Slice& data) {
213
+ return file_->Append(data);
214
+ }
215
+
216
+ virtual Status Close() { return Status::OK(); }
217
+ virtual Status Flush() { return Status::OK(); }
218
+ virtual Status Sync() { return Status::OK(); }
219
+
220
+ private:
221
+ FileState* file_;
222
+ };
223
+
224
+ class InMemoryEnv : public EnvWrapper {
225
+ public:
226
+ explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
227
+
228
+ virtual ~InMemoryEnv() {
229
+ for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
230
+ i->second->Unref();
231
+ }
232
+ }
233
+
234
+ // Partial implementation of the Env interface.
235
+ virtual Status NewSequentialFile(const std::string& fname,
236
+ SequentialFile** result) {
237
+ MutexLock lock(&mutex_);
238
+ if (file_map_.find(fname) == file_map_.end()) {
239
+ *result = NULL;
240
+ return Status::IOError(fname, "File not found");
241
+ }
242
+
243
+ *result = new SequentialFileImpl(file_map_[fname]);
244
+ return Status::OK();
245
+ }
246
+
247
+ virtual Status NewRandomAccessFile(const std::string& fname,
248
+ RandomAccessFile** result) {
249
+ MutexLock lock(&mutex_);
250
+ if (file_map_.find(fname) == file_map_.end()) {
251
+ *result = NULL;
252
+ return Status::IOError(fname, "File not found");
253
+ }
254
+
255
+ *result = new RandomAccessFileImpl(file_map_[fname]);
256
+ return Status::OK();
257
+ }
258
+
259
+ virtual Status NewWritableFile(const std::string& fname,
260
+ WritableFile** result) {
261
+ MutexLock lock(&mutex_);
262
+ if (file_map_.find(fname) != file_map_.end()) {
263
+ DeleteFileInternal(fname);
264
+ }
265
+
266
+ FileState* file = new FileState();
267
+ file->Ref();
268
+ file_map_[fname] = file;
269
+
270
+ *result = new WritableFileImpl(file);
271
+ return Status::OK();
272
+ }
273
+
274
+ virtual bool FileExists(const std::string& fname) {
275
+ MutexLock lock(&mutex_);
276
+ return file_map_.find(fname) != file_map_.end();
277
+ }
278
+
279
+ virtual Status GetChildren(const std::string& dir,
280
+ std::vector<std::string>* result) {
281
+ MutexLock lock(&mutex_);
282
+ result->clear();
283
+
284
+ for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
285
+ const std::string& filename = i->first;
286
+
287
+ if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' &&
288
+ Slice(filename).starts_with(Slice(dir))) {
289
+ result->push_back(filename.substr(dir.size() + 1));
290
+ }
291
+ }
292
+
293
+ return Status::OK();
294
+ }
295
+
296
+ void DeleteFileInternal(const std::string& fname) {
297
+ if (file_map_.find(fname) == file_map_.end()) {
298
+ return;
299
+ }
300
+
301
+ file_map_[fname]->Unref();
302
+ file_map_.erase(fname);
303
+ }
304
+
305
+ virtual Status DeleteFile(const std::string& fname) {
306
+ MutexLock lock(&mutex_);
307
+ if (file_map_.find(fname) == file_map_.end()) {
308
+ return Status::IOError(fname, "File not found");
309
+ }
310
+
311
+ DeleteFileInternal(fname);
312
+ return Status::OK();
313
+ }
314
+
315
+ virtual Status CreateDir(const std::string& dirname) {
316
+ return Status::OK();
317
+ }
318
+
319
+ virtual Status DeleteDir(const std::string& dirname) {
320
+ return Status::OK();
321
+ }
322
+
323
+ virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) {
324
+ MutexLock lock(&mutex_);
325
+ if (file_map_.find(fname) == file_map_.end()) {
326
+ return Status::IOError(fname, "File not found");
327
+ }
328
+
329
+ *file_size = file_map_[fname]->Size();
330
+ return Status::OK();
331
+ }
332
+
333
+ virtual Status RenameFile(const std::string& src,
334
+ const std::string& target) {
335
+ MutexLock lock(&mutex_);
336
+ if (file_map_.find(src) == file_map_.end()) {
337
+ return Status::IOError(src, "File not found");
338
+ }
339
+
340
+ DeleteFileInternal(target);
341
+ file_map_[target] = file_map_[src];
342
+ file_map_.erase(src);
343
+ return Status::OK();
344
+ }
345
+
346
+ virtual Status LockFile(const std::string& fname, FileLock** lock) {
347
+ *lock = new FileLock;
348
+ return Status::OK();
349
+ }
350
+
351
+ virtual Status UnlockFile(FileLock* lock) {
352
+ delete lock;
353
+ return Status::OK();
354
+ }
355
+
356
+ virtual Status GetTestDirectory(std::string* path) {
357
+ *path = "/test";
358
+ return Status::OK();
359
+ }
360
+
361
+ private:
362
+ // Map from filenames to FileState objects, representing a simple file system.
363
+ typedef std::map<std::string, FileState*> FileSystem;
364
+ port::Mutex mutex_;
365
+ FileSystem file_map_; // Protected by mutex_.
366
+ };
367
+
368
+ } // namespace
369
+
370
+ Env* NewMemEnv(Env* base_env) {
371
+ return new InMemoryEnv(base_env);
372
+ }
373
+
374
+ } // namespace leveldb