leveldb-ruby 0.10 → 0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. data/ext/leveldb/extconf.rb +2 -6
  2. data/ext/leveldb/leveldb.cc +377 -33
  3. data/ext/leveldb/platform.rb +83 -0
  4. data/leveldb/Makefile +29 -28
  5. data/leveldb/build_detect_platform +23 -4
  6. data/leveldb/db/builder.cc +1 -1
  7. data/leveldb/db/builder.h +1 -1
  8. data/leveldb/db/corruption_test.cc +1 -1
  9. data/leveldb/db/db_bench.cc +2 -2
  10. data/leveldb/db/db_impl.cc +8 -16
  11. data/leveldb/db/db_impl.h +1 -1
  12. data/leveldb/db/db_iter.cc +1 -1
  13. data/leveldb/db/db_iter.h +1 -1
  14. data/leveldb/db/db_test.cc +180 -9
  15. data/leveldb/db/dbformat.cc +9 -7
  16. data/leveldb/db/dbformat.h +2 -2
  17. data/leveldb/db/dbformat_test.cc +1 -1
  18. data/leveldb/db/filename.cc +6 -2
  19. data/leveldb/db/filename.h +1 -1
  20. data/leveldb/db/filename_test.cc +1 -1
  21. data/leveldb/db/log_format.h +2 -2
  22. data/leveldb/db/log_reader.cc +2 -2
  23. data/leveldb/db/log_reader.h +2 -2
  24. data/leveldb/db/log_test.cc +2 -2
  25. data/leveldb/db/log_writer.cc +2 -2
  26. data/leveldb/db/log_writer.h +2 -2
  27. data/leveldb/db/memtable.cc +1 -1
  28. data/leveldb/db/memtable.h +1 -1
  29. data/leveldb/db/repair.cc +2 -2
  30. data/leveldb/db/skiplist.h +1 -1
  31. data/leveldb/db/skiplist_test.cc +1 -1
  32. data/leveldb/db/snapshot.h +1 -1
  33. data/leveldb/db/table_cache.cc +1 -1
  34. data/leveldb/db/table_cache.h +1 -1
  35. data/leveldb/db/version_edit.cc +1 -1
  36. data/leveldb/db/version_edit.h +1 -1
  37. data/leveldb/db/version_edit_test.cc +1 -1
  38. data/leveldb/db/version_set.cc +39 -14
  39. data/leveldb/db/version_set.h +1 -1
  40. data/leveldb/db/version_set_test.cc +1 -1
  41. data/leveldb/db/write_batch.cc +2 -2
  42. data/leveldb/db/write_batch_internal.h +1 -1
  43. data/leveldb/db/write_batch_test.cc +1 -1
  44. data/leveldb/helpers/memenv/memenv.cc +374 -0
  45. data/leveldb/helpers/memenv/memenv.h +20 -0
  46. data/leveldb/helpers/memenv/memenv_test.cc +232 -0
  47. data/leveldb/include/leveldb/cache.h +1 -1
  48. data/leveldb/include/leveldb/comparator.h +1 -1
  49. data/leveldb/include/leveldb/db.h +1 -1
  50. data/leveldb/include/leveldb/env.h +1 -1
  51. data/leveldb/include/leveldb/iterator.h +1 -1
  52. data/leveldb/include/leveldb/options.h +1 -1
  53. data/leveldb/include/leveldb/slice.h +1 -1
  54. data/leveldb/include/leveldb/status.h +1 -1
  55. data/leveldb/include/leveldb/table.h +1 -1
  56. data/leveldb/include/leveldb/table_builder.h +1 -1
  57. data/leveldb/include/leveldb/write_batch.h +1 -1
  58. data/leveldb/port/atomic_pointer.h +2 -2
  59. data/leveldb/port/port_android.cc +2 -2
  60. data/leveldb/port/port_android.h +2 -2
  61. data/leveldb/port/port_example.h +2 -2
  62. data/leveldb/port/port_posix.cc +2 -2
  63. data/leveldb/port/port_posix.h +11 -3
  64. data/leveldb/table/block.cc +1 -1
  65. data/leveldb/table/block.h +1 -1
  66. data/leveldb/table/block_builder.cc +1 -1
  67. data/leveldb/table/block_builder.h +1 -1
  68. data/leveldb/table/format.cc +1 -1
  69. data/leveldb/table/format.h +1 -1
  70. data/leveldb/table/iterator.cc +2 -2
  71. data/leveldb/table/merger.cc +2 -2
  72. data/leveldb/table/merger.h +1 -1
  73. data/leveldb/table/table.cc +1 -1
  74. data/leveldb/table/table_builder.cc +1 -1
  75. data/leveldb/table/table_test.cc +3 -3
  76. data/leveldb/table/two_level_iterator.cc +2 -2
  77. data/leveldb/table/two_level_iterator.h +1 -1
  78. data/leveldb/util/arena.cc +1 -1
  79. data/leveldb/util/arena.h +1 -1
  80. data/leveldb/util/arena_test.cc +1 -1
  81. data/leveldb/util/cache.cc +1 -1
  82. data/leveldb/util/cache_test.cc +1 -1
  83. data/leveldb/util/coding.cc +1 -1
  84. data/leveldb/util/coding.h +1 -1
  85. data/leveldb/util/coding_test.cc +1 -1
  86. data/leveldb/util/comparator.cc +7 -4
  87. data/leveldb/util/crc32c.cc +2 -2
  88. data/leveldb/util/crc32c.h +2 -2
  89. data/leveldb/util/crc32c_test.cc +2 -2
  90. data/leveldb/util/env.cc +17 -3
  91. data/leveldb/util/env_posix.cc +2 -2
  92. data/leveldb/util/env_test.cc +13 -11
  93. data/leveldb/util/hash.cc +1 -1
  94. data/leveldb/util/histogram.cc +1 -1
  95. data/leveldb/util/histogram.h +1 -1
  96. data/leveldb/util/logging.cc +1 -1
  97. data/leveldb/util/logging.h +1 -1
  98. data/leveldb/util/mutexlock.h +1 -1
  99. data/leveldb/util/options.cc +1 -1
  100. data/leveldb/util/posix_logger.h +1 -1
  101. data/leveldb/util/random.h +1 -1
  102. data/leveldb/util/status.cc +1 -1
  103. data/leveldb/util/testharness.cc +2 -2
  104. data/leveldb/util/testharness.h +2 -2
  105. data/leveldb/util/testutil.cc +2 -2
  106. data/leveldb/util/testutil.h +2 -2
  107. data/lib/leveldb.rb +34 -2
  108. metadata +7 -13
  109. data/leveldb/port/port_chromium.cc +0 -80
  110. data/leveldb/port/port_chromium.h +0 -97
  111. data/leveldb/port/port_osx.cc +0 -50
  112. data/leveldb/port/port_osx.h +0 -125
  113. data/leveldb/port/sha1_portable.cc +0 -298
  114. data/leveldb/port/sha1_portable.h +0 -25
  115. data/leveldb/port/sha1_test.cc +0 -39
  116. data/leveldb/util/env_chromium.cc +0 -612
@@ -73,9 +73,10 @@ void InternalKeyComparator::FindShortestSeparator(
73
73
  Slice user_limit = ExtractUserKey(limit);
74
74
  std::string tmp(user_start.data(), user_start.size());
75
75
  user_comparator_->FindShortestSeparator(&tmp, user_limit);
76
- if (user_comparator_->Compare(*start, tmp) < 0) {
77
- // User key has become larger. Tack on the earliest possible
78
- // number to the shortened user key.
76
+ if (tmp.size() < user_start.size() &&
77
+ user_comparator_->Compare(user_start, tmp) < 0) {
78
+ // User key has become shorter physically, but larger logically.
79
+ // Tack on the earliest possible number to the shortened user key.
79
80
  PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
80
81
  assert(this->Compare(*start, tmp) < 0);
81
82
  assert(this->Compare(tmp, limit) < 0);
@@ -87,9 +88,10 @@ void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
87
88
  Slice user_key = ExtractUserKey(*key);
88
89
  std::string tmp(user_key.data(), user_key.size());
89
90
  user_comparator_->FindShortSuccessor(&tmp);
90
- if (user_comparator_->Compare(user_key, tmp) < 0) {
91
- // User key has become larger. Tack on the earliest possible
92
- // number to the shortened user key.
91
+ if (tmp.size() < user_key.size() &&
92
+ user_comparator_->Compare(user_key, tmp) < 0) {
93
+ // User key has become shorter physically, but larger logically.
94
+ // Tack on the earliest possible number to the shortened user key.
93
95
  PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
94
96
  assert(this->Compare(*key, tmp) < 0);
95
97
  key->swap(tmp);
@@ -115,4 +117,4 @@ LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
115
117
  end_ = dst;
116
118
  }
117
119
 
118
- }
120
+ } // namespace leveldb
@@ -37,7 +37,7 @@ static const int kL0_StopWritesTrigger = 12;
37
37
  // space if the same key space is being repeatedly overwritten.
38
38
  static const int kMaxMemCompactLevel = 2;
39
39
 
40
- }
40
+ } // namespace config
41
41
 
42
42
  class InternalKey;
43
43
 
@@ -210,6 +210,6 @@ inline LookupKey::~LookupKey() {
210
210
  if (start_ != space_) delete[] start_;
211
211
  }
212
212
 
213
- }
213
+ } // namespace leveldb
214
214
 
215
215
  #endif // STORAGE_LEVELDB_DB_FORMAT_H_
@@ -105,7 +105,7 @@ TEST(FormatTest, InternalKeyShortestSuccessor) {
105
105
  ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
106
106
  }
107
107
 
108
- }
108
+ } // namespace leveldb
109
109
 
110
110
  int main(int argc, char** argv) {
111
111
  return leveldb::test::RunAllTests();
@@ -11,6 +11,10 @@
11
11
 
12
12
  namespace leveldb {
13
13
 
14
+ // A utility routine: write "data" to the named file and Sync() it.
15
+ extern Status WriteStringToFileSync(Env* env, const Slice& data,
16
+ const std::string& fname);
17
+
14
18
  static std::string MakeFileName(const std::string& name, uint64_t number,
15
19
  const char* suffix) {
16
20
  char buf[100];
@@ -122,7 +126,7 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
122
126
  assert(contents.starts_with(dbname + "/"));
123
127
  contents.remove_prefix(dbname.size() + 1);
124
128
  std::string tmp = TempFileName(dbname, descriptor_number);
125
- Status s = WriteStringToFile(env, contents.ToString() + "\n", tmp);
129
+ Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp);
126
130
  if (s.ok()) {
127
131
  s = env->RenameFile(tmp, CurrentFileName(dbname));
128
132
  }
@@ -132,4 +136,4 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
132
136
  return s;
133
137
  }
134
138
 
135
- }
139
+ } // namespace leveldb
@@ -75,6 +75,6 @@ extern Status SetCurrentFile(Env* env, const std::string& dbname,
75
75
  uint64_t descriptor_number);
76
76
 
77
77
 
78
- }
78
+ } // namespace leveldb
79
79
 
80
80
  #endif // STORAGE_LEVELDB_DB_FILENAME_H_
@@ -115,7 +115,7 @@ TEST(FileNameTest, Construction) {
115
115
  ASSERT_EQ(kTempFile, type);
116
116
  }
117
117
 
118
- }
118
+ } // namespace leveldb
119
119
 
120
120
  int main(int argc, char** argv) {
121
121
  return leveldb::test::RunAllTests();
@@ -29,7 +29,7 @@ static const int kBlockSize = 32768;
29
29
  // Header is checksum (4 bytes), type (1 byte), length (2 bytes).
30
30
  static const int kHeaderSize = 4 + 1 + 2;
31
31
 
32
- }
33
- }
32
+ } // namespace log
33
+ } // namespace leveldb
34
34
 
35
35
  #endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_
@@ -255,5 +255,5 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
255
255
  }
256
256
  }
257
257
 
258
- }
259
- }
258
+ } // namespace log
259
+ } // namespace leveldb
@@ -102,7 +102,7 @@ class Reader {
102
102
  void operator=(const Reader&);
103
103
  };
104
104
 
105
- }
106
- }
105
+ } // namespace log
106
+ } // namespace leveldb
107
107
 
108
108
  #endif // STORAGE_LEVELDB_DB_LOG_READER_H_
@@ -492,8 +492,8 @@ TEST(LogTest, ReadPastEnd) {
492
492
  CheckOffsetPastEndReturnsNoRecords(5);
493
493
  }
494
494
 
495
- }
496
- }
495
+ } // namespace log
496
+ } // namespace leveldb
497
497
 
498
498
  int main(int argc, char** argv) {
499
499
  return leveldb::test::RunAllTests();
@@ -99,5 +99,5 @@ Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
99
99
  return s;
100
100
  }
101
101
 
102
- }
103
- }
102
+ } // namespace log
103
+ } // namespace leveldb
@@ -42,7 +42,7 @@ class Writer {
42
42
  void operator=(const Writer&);
43
43
  };
44
44
 
45
- }
46
- }
45
+ } // namespace log
46
+ } // namespace leveldb
47
47
 
48
48
  #endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_
@@ -142,4 +142,4 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
142
142
  return false;
143
143
  }
144
144
 
145
- }
145
+ } // namespace leveldb
@@ -86,6 +86,6 @@ class MemTable {
86
86
  void operator=(const MemTable&);
87
87
  };
88
88
 
89
- }
89
+ } // namespace leveldb
90
90
 
91
91
  #endif // STORAGE_LEVELDB_DB_MEMTABLE_H_
@@ -377,11 +377,11 @@ class Repairer {
377
377
  fname.c_str(), s.ToString().c_str());
378
378
  }
379
379
  };
380
- }
380
+ } // namespace
381
381
 
382
382
  Status RepairDB(const std::string& dbname, const Options& options) {
383
383
  Repairer repairer(dbname, options);
384
384
  return repairer.Run();
385
385
  }
386
386
 
387
- }
387
+ } // namespace leveldb
@@ -375,4 +375,4 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const {
375
375
  }
376
376
  }
377
377
 
378
- }
378
+ } // namespace leveldb
@@ -371,7 +371,7 @@ TEST(SkipTest, Concurrent3) { RunConcurrent(3); }
371
371
  TEST(SkipTest, Concurrent4) { RunConcurrent(4); }
372
372
  TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
373
373
 
374
- }
374
+ } // namespace leveldb
375
375
 
376
376
  int main(int argc, char** argv) {
377
377
  return leveldb::test::RunAllTests();
@@ -61,6 +61,6 @@ class SnapshotList {
61
61
  SnapshotImpl list_;
62
62
  };
63
63
 
64
- }
64
+ } // namespace leveldb
65
65
 
66
66
  #endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_
@@ -92,4 +92,4 @@ void TableCache::Evict(uint64_t file_number) {
92
92
  cache_->Erase(Slice(buf, sizeof(buf)));
93
93
  }
94
94
 
95
- }
95
+ } // namespace leveldb
@@ -45,6 +45,6 @@ class TableCache {
45
45
  Cache* cache_;
46
46
  };
47
47
 
48
- }
48
+ } // namespace leveldb
49
49
 
50
50
  #endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_
@@ -263,4 +263,4 @@ std::string VersionEdit::DebugString() const {
263
263
  return r;
264
264
  }
265
265
 
266
- }
266
+ } // namespace leveldb
@@ -102,6 +102,6 @@ class VersionEdit {
102
102
  std::vector< std::pair<int, FileMetaData> > new_files_;
103
103
  };
104
104
 
105
- }
105
+ } // namespace leveldb
106
106
 
107
107
  #endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_
@@ -39,7 +39,7 @@ TEST(VersionEditTest, EncodeDecode) {
39
39
  TestEncodeDecode(edit);
40
40
  }
41
41
 
42
- }
42
+ } // namespace leveldb
43
43
 
44
44
  int main(int argc, char** argv) {
45
45
  return leveldb::test::RunAllTests();
@@ -26,6 +26,11 @@ static const int kTargetFileSize = 2 * 1048576;
26
26
  // stop building a single file in a level->level+1 compaction.
27
27
  static const int64_t kMaxGrandParentOverlapBytes = 10 * kTargetFileSize;
28
28
 
29
+ // Maximum number of bytes in all compacted files. We avoid expanding
30
+ // the lower level file set of a compaction if it would make the
31
+ // total compaction cover more than this many bytes.
32
+ static const int64_t kExpandedCompactionByteSizeLimit = 25 * kTargetFileSize;
33
+
29
34
  static double MaxBytesForLevel(int level) {
30
35
  // Note: the result for level zero is not really used since we set
31
36
  // the level-0 compaction threshold based on number of files.
@@ -61,7 +66,7 @@ std::string IntSetToString(const std::set<uint64_t>& s) {
61
66
  result += "}";
62
67
  return result;
63
68
  }
64
- }
69
+ } // namespace
65
70
 
66
71
  Version::~Version() {
67
72
  assert(refs_ == 0);
@@ -253,7 +258,8 @@ void Version::AddIterators(const ReadOptions& options,
253
258
  // If "*iter" points at a value or deletion for user_key, store
254
259
  // either the value, or a NotFound error and return true.
255
260
  // Else return false.
256
- static bool GetValue(Iterator* iter, const Slice& user_key,
261
+ static bool GetValue(const Comparator* cmp,
262
+ Iterator* iter, const Slice& user_key,
257
263
  std::string* value,
258
264
  Status* s) {
259
265
  if (!iter->Valid()) {
@@ -264,7 +270,7 @@ static bool GetValue(Iterator* iter, const Slice& user_key,
264
270
  *s = Status::Corruption("corrupted key for ", user_key);
265
271
  return true;
266
272
  }
267
- if (parsed_key.user_key != user_key) {
273
+ if (cmp->Compare(parsed_key.user_key, user_key) != 0) {
268
274
  return false;
269
275
  }
270
276
  switch (parsed_key.type) {
@@ -360,7 +366,7 @@ Status Version::Get(const ReadOptions& options,
360
366
  f->number,
361
367
  f->file_size);
362
368
  iter->Seek(ikey);
363
- const bool done = GetValue(iter, user_key, value, &s);
369
+ const bool done = GetValue(ucmp, iter, user_key, value, &s);
364
370
  if (!iter->status().ok()) {
365
371
  s = iter->status();
366
372
  delete iter;
@@ -450,16 +456,29 @@ void Version::GetOverlappingInputs(
450
456
  user_end = end->user_key();
451
457
  }
452
458
  const Comparator* user_cmp = vset_->icmp_.user_comparator();
453
- for (size_t i = 0; i < files_[level].size(); i++) {
454
- FileMetaData* f = files_[level][i];
455
- if (begin != NULL &&
456
- user_cmp->Compare(f->largest.user_key(), user_begin) < 0) {
459
+ for (size_t i = 0; i < files_[level].size(); ) {
460
+ FileMetaData* f = files_[level][i++];
461
+ const Slice file_start = f->smallest.user_key();
462
+ const Slice file_limit = f->largest.user_key();
463
+ if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) {
457
464
  // "f" is completely before specified range; skip it
458
- } else if (end != NULL &&
459
- user_cmp->Compare(f->smallest.user_key(), user_end) > 0) {
465
+ } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) {
460
466
  // "f" is completely after specified range; skip it
461
467
  } else {
462
468
  inputs->push_back(f);
469
+ if (level == 0) {
470
+ // Level-0 files may overlap each other. So check if the newly
471
+ // added file has expanded the range. If so, restart search.
472
+ if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) {
473
+ user_begin = file_start;
474
+ inputs->clear();
475
+ i = 0;
476
+ } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) {
477
+ user_end = file_limit;
478
+ inputs->clear();
479
+ i = 0;
480
+ }
481
+ }
463
482
  }
464
483
  }
465
484
  }
@@ -1209,7 +1228,11 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
1209
1228
  if (!c->inputs_[1].empty()) {
1210
1229
  std::vector<FileMetaData*> expanded0;
1211
1230
  current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0);
1212
- if (expanded0.size() > c->inputs_[0].size()) {
1231
+ const int64_t inputs0_size = TotalFileSize(c->inputs_[0]);
1232
+ const int64_t inputs1_size = TotalFileSize(c->inputs_[1]);
1233
+ const int64_t expanded0_size = TotalFileSize(expanded0);
1234
+ if (expanded0.size() > c->inputs_[0].size() &&
1235
+ inputs1_size + expanded0_size < kExpandedCompactionByteSizeLimit) {
1213
1236
  InternalKey new_start, new_limit;
1214
1237
  GetRange(expanded0, &new_start, &new_limit);
1215
1238
  std::vector<FileMetaData*> expanded1;
@@ -1217,12 +1240,14 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
1217
1240
  &expanded1);
1218
1241
  if (expanded1.size() == c->inputs_[1].size()) {
1219
1242
  Log(options_->info_log,
1220
- "Expanding@%d %d+%d to %d+%d\n",
1243
+ "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n",
1221
1244
  level,
1222
1245
  int(c->inputs_[0].size()),
1223
1246
  int(c->inputs_[1].size()),
1247
+ long(inputs0_size), long(inputs1_size),
1224
1248
  int(expanded0.size()),
1225
- int(expanded1.size()));
1249
+ int(expanded1.size()),
1250
+ long(expanded0_size), long(inputs1_size));
1226
1251
  smallest = new_start;
1227
1252
  largest = new_limit;
1228
1253
  c->inputs_[0] = expanded0;
@@ -1369,4 +1394,4 @@ void Compaction::ReleaseInputs() {
1369
1394
  }
1370
1395
  }
1371
1396
 
1372
- }
1397
+ } // namespace leveldb
@@ -365,6 +365,6 @@ class Compaction {
365
365
  size_t level_ptrs_[config::kNumLevels];
366
366
  };
367
367
 
368
- }
368
+ } // namespace leveldb
369
369
 
370
370
  #endif // STORAGE_LEVELDB_DB_VERSION_SET_H_
@@ -172,7 +172,7 @@ TEST(FindFileTest, OverlappingFiles) {
172
172
  ASSERT_TRUE(Overlaps("600", "700"));
173
173
  }
174
174
 
175
- }
175
+ } // namespace leveldb
176
176
 
177
177
  int main(int argc, char** argv) {
178
178
  return leveldb::test::RunAllTests();
@@ -120,7 +120,7 @@ class MemTableInserter : public WriteBatch::Handler {
120
120
  sequence_++;
121
121
  }
122
122
  };
123
- }
123
+ } // namespace
124
124
 
125
125
  Status WriteBatchInternal::InsertInto(const WriteBatch* b,
126
126
  MemTable* memtable) {
@@ -135,4 +135,4 @@ void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
135
135
  b->rep_.assign(contents.data(), contents.size());
136
136
  }
137
137
 
138
- }
138
+ } // namespace leveldb
@@ -41,7 +41,7 @@ class WriteBatchInternal {
41
41
  static Status InsertInto(const WriteBatch* batch, MemTable* memtable);
42
42
  };
43
43
 
44
- }
44
+ } // namespace leveldb
45
45
 
46
46
 
47
47
  #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
@@ -82,7 +82,7 @@ TEST(WriteBatchTest, Corruption) {
82
82
  PrintContents(&batch));
83
83
  }
84
84
 
85
- }
85
+ } // namespace leveldb
86
86
 
87
87
  int main(int argc, char** argv) {
88
88
  return leveldb::test::RunAllTests();
@@ -0,0 +1,374 @@
1
+ // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2
+ // Use of this source code is governed by a BSD-style license that can be
3
+ // found in the LICENSE file. See the AUTHORS file for names of contributors.
4
+
5
+ #include "helpers/memenv/memenv.h"
6
+
7
+ #include "leveldb/env.h"
8
+ #include "leveldb/status.h"
9
+ #include "port/port.h"
10
+ #include "util/mutexlock.h"
11
+ #include <map>
12
+ #include <string.h>
13
+ #include <string>
14
+ #include <vector>
15
+
16
+ namespace leveldb {
17
+
18
+ namespace {
19
+
20
+ class FileState {
21
+ public:
22
+ // FileStates are reference counted. The initial reference count is zero
23
+ // and the caller must call Ref() at least once.
24
+ FileState() : refs_(0), size_(0) {}
25
+
26
+ // Increase the reference count.
27
+ void Ref() {
28
+ MutexLock lock(&refs_mutex_);
29
+ ++refs_;
30
+ }
31
+
32
+ // Decrease the reference count. Delete if this is the last reference.
33
+ void Unref() {
34
+ bool do_delete = false;
35
+
36
+ {
37
+ MutexLock lock(&refs_mutex_);
38
+ --refs_;
39
+ assert(refs_ >= 0);
40
+ if (refs_ <= 0) {
41
+ do_delete = true;
42
+ }
43
+ }
44
+
45
+ if (do_delete) {
46
+ delete this;
47
+ }
48
+ }
49
+
50
+ uint64_t Size() const { return size_; }
51
+
52
+ Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
53
+ if (offset > size_) {
54
+ return Status::IOError("Offset greater than file size.");
55
+ }
56
+ const uint64_t available = size_ - offset;
57
+ if (n > available) {
58
+ n = available;
59
+ }
60
+ if (n == 0) {
61
+ *result = Slice();
62
+ return Status::OK();
63
+ }
64
+
65
+ size_t block = offset / kBlockSize;
66
+ size_t block_offset = offset % kBlockSize;
67
+
68
+ if (n <= kBlockSize - block_offset) {
69
+ // The requested bytes are all in the first block.
70
+ *result = Slice(blocks_[block] + block_offset, n);
71
+ return Status::OK();
72
+ }
73
+
74
+ size_t bytes_to_copy = n;
75
+ char* dst = scratch;
76
+
77
+ while (bytes_to_copy > 0) {
78
+ size_t avail = kBlockSize - block_offset;
79
+ if (avail > bytes_to_copy) {
80
+ avail = bytes_to_copy;
81
+ }
82
+ memcpy(dst, blocks_[block] + block_offset, avail);
83
+
84
+ bytes_to_copy -= avail;
85
+ dst += avail;
86
+ block++;
87
+ block_offset = 0;
88
+ }
89
+
90
+ *result = Slice(scratch, n);
91
+ return Status::OK();
92
+ }
93
+
94
+ Status Append(const Slice& data) {
95
+ const char* src = data.data();
96
+ size_t src_len = data.size();
97
+
98
+ while (src_len > 0) {
99
+ size_t avail;
100
+ size_t offset = size_ % kBlockSize;
101
+
102
+ if (offset != 0) {
103
+ // There is some room in the last block.
104
+ avail = kBlockSize - offset;
105
+ } else {
106
+ // No room in the last block; push new one.
107
+ blocks_.push_back(new char[kBlockSize]);
108
+ avail = kBlockSize;
109
+ }
110
+
111
+ if (avail > src_len) {
112
+ avail = src_len;
113
+ }
114
+ memcpy(blocks_.back() + offset, src, avail);
115
+ src_len -= avail;
116
+ src += avail;
117
+ size_ += avail;
118
+ }
119
+
120
+ return Status::OK();
121
+ }
122
+
123
+ private:
124
+ // Private since only Unref() should be used to delete it.
125
+ ~FileState() {
126
+ for (std::vector<char*>::iterator i = blocks_.begin(); i != blocks_.end();
127
+ ++i) {
128
+ delete [] *i;
129
+ }
130
+ }
131
+
132
+ // No copying allowed.
133
+ FileState(const FileState&);
134
+ void operator=(const FileState&);
135
+
136
+ port::Mutex refs_mutex_;
137
+ int refs_; // Protected by refs_mutex_;
138
+
139
+ // The following fields are not protected by any mutex. They are only mutable
140
+ // while the file is being written, and concurrent access is not allowed
141
+ // to writable files.
142
+ std::vector<char*> blocks_;
143
+ uint64_t size_;
144
+
145
+ enum { kBlockSize = 8 * 1024 };
146
+ };
147
+
148
+ class SequentialFileImpl : public SequentialFile {
149
+ public:
150
+ explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) {
151
+ file_->Ref();
152
+ }
153
+
154
+ ~SequentialFileImpl() {
155
+ file_->Unref();
156
+ }
157
+
158
+ virtual Status Read(size_t n, Slice* result, char* scratch) {
159
+ Status s = file_->Read(pos_, n, result, scratch);
160
+ if (s.ok()) {
161
+ pos_ += result->size();
162
+ }
163
+ return s;
164
+ }
165
+
166
+ virtual Status Skip(uint64_t n) {
167
+ if (pos_ > file_->Size()) {
168
+ return Status::IOError("pos_ > file_->Size()");
169
+ }
170
+ const size_t available = file_->Size() - pos_;
171
+ if (n > available) {
172
+ n = available;
173
+ }
174
+ pos_ += n;
175
+ return Status::OK();
176
+ }
177
+
178
+ private:
179
+ FileState* file_;
180
+ size_t pos_;
181
+ };
182
+
183
+ class RandomAccessFileImpl : public RandomAccessFile {
184
+ public:
185
+ explicit RandomAccessFileImpl(FileState* file) : file_(file) {
186
+ file_->Ref();
187
+ }
188
+
189
+ ~RandomAccessFileImpl() {
190
+ file_->Unref();
191
+ }
192
+
193
+ virtual Status Read(uint64_t offset, size_t n, Slice* result,
194
+ char* scratch) const {
195
+ return file_->Read(offset, n, result, scratch);
196
+ }
197
+
198
+ private:
199
+ FileState* file_;
200
+ };
201
+
202
+ class WritableFileImpl : public WritableFile {
203
+ public:
204
+ WritableFileImpl(FileState* file) : file_(file) {
205
+ file_->Ref();
206
+ }
207
+
208
+ ~WritableFileImpl() {
209
+ file_->Unref();
210
+ }
211
+
212
+ virtual Status Append(const Slice& data) {
213
+ return file_->Append(data);
214
+ }
215
+
216
+ virtual Status Close() { return Status::OK(); }
217
+ virtual Status Flush() { return Status::OK(); }
218
+ virtual Status Sync() { return Status::OK(); }
219
+
220
+ private:
221
+ FileState* file_;
222
+ };
223
+
224
+ class InMemoryEnv : public EnvWrapper {
225
+ public:
226
+ explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
227
+
228
+ virtual ~InMemoryEnv() {
229
+ for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
230
+ i->second->Unref();
231
+ }
232
+ }
233
+
234
+ // Partial implementation of the Env interface.
235
+ virtual Status NewSequentialFile(const std::string& fname,
236
+ SequentialFile** result) {
237
+ MutexLock lock(&mutex_);
238
+ if (file_map_.find(fname) == file_map_.end()) {
239
+ *result = NULL;
240
+ return Status::IOError(fname, "File not found");
241
+ }
242
+
243
+ *result = new SequentialFileImpl(file_map_[fname]);
244
+ return Status::OK();
245
+ }
246
+
247
+ virtual Status NewRandomAccessFile(const std::string& fname,
248
+ RandomAccessFile** result) {
249
+ MutexLock lock(&mutex_);
250
+ if (file_map_.find(fname) == file_map_.end()) {
251
+ *result = NULL;
252
+ return Status::IOError(fname, "File not found");
253
+ }
254
+
255
+ *result = new RandomAccessFileImpl(file_map_[fname]);
256
+ return Status::OK();
257
+ }
258
+
259
+ virtual Status NewWritableFile(const std::string& fname,
260
+ WritableFile** result) {
261
+ MutexLock lock(&mutex_);
262
+ if (file_map_.find(fname) != file_map_.end()) {
263
+ DeleteFileInternal(fname);
264
+ }
265
+
266
+ FileState* file = new FileState();
267
+ file->Ref();
268
+ file_map_[fname] = file;
269
+
270
+ *result = new WritableFileImpl(file);
271
+ return Status::OK();
272
+ }
273
+
274
+ virtual bool FileExists(const std::string& fname) {
275
+ MutexLock lock(&mutex_);
276
+ return file_map_.find(fname) != file_map_.end();
277
+ }
278
+
279
+ virtual Status GetChildren(const std::string& dir,
280
+ std::vector<std::string>* result) {
281
+ MutexLock lock(&mutex_);
282
+ result->clear();
283
+
284
+ for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
285
+ const std::string& filename = i->first;
286
+
287
+ if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' &&
288
+ Slice(filename).starts_with(Slice(dir))) {
289
+ result->push_back(filename.substr(dir.size() + 1));
290
+ }
291
+ }
292
+
293
+ return Status::OK();
294
+ }
295
+
296
+ void DeleteFileInternal(const std::string& fname) {
297
+ if (file_map_.find(fname) == file_map_.end()) {
298
+ return;
299
+ }
300
+
301
+ file_map_[fname]->Unref();
302
+ file_map_.erase(fname);
303
+ }
304
+
305
+ virtual Status DeleteFile(const std::string& fname) {
306
+ MutexLock lock(&mutex_);
307
+ if (file_map_.find(fname) == file_map_.end()) {
308
+ return Status::IOError(fname, "File not found");
309
+ }
310
+
311
+ DeleteFileInternal(fname);
312
+ return Status::OK();
313
+ }
314
+
315
+ virtual Status CreateDir(const std::string& dirname) {
316
+ return Status::OK();
317
+ }
318
+
319
+ virtual Status DeleteDir(const std::string& dirname) {
320
+ return Status::OK();
321
+ }
322
+
323
+ virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) {
324
+ MutexLock lock(&mutex_);
325
+ if (file_map_.find(fname) == file_map_.end()) {
326
+ return Status::IOError(fname, "File not found");
327
+ }
328
+
329
+ *file_size = file_map_[fname]->Size();
330
+ return Status::OK();
331
+ }
332
+
333
+ virtual Status RenameFile(const std::string& src,
334
+ const std::string& target) {
335
+ MutexLock lock(&mutex_);
336
+ if (file_map_.find(src) == file_map_.end()) {
337
+ return Status::IOError(src, "File not found");
338
+ }
339
+
340
+ DeleteFileInternal(target);
341
+ file_map_[target] = file_map_[src];
342
+ file_map_.erase(src);
343
+ return Status::OK();
344
+ }
345
+
346
+ virtual Status LockFile(const std::string& fname, FileLock** lock) {
347
+ *lock = new FileLock;
348
+ return Status::OK();
349
+ }
350
+
351
+ virtual Status UnlockFile(FileLock* lock) {
352
+ delete lock;
353
+ return Status::OK();
354
+ }
355
+
356
+ virtual Status GetTestDirectory(std::string* path) {
357
+ *path = "/test";
358
+ return Status::OK();
359
+ }
360
+
361
+ private:
362
+ // Map from filenames to FileState objects, representing a simple file system.
363
+ typedef std::map<std::string, FileState*> FileSystem;
364
+ port::Mutex mutex_;
365
+ FileSystem file_map_; // Protected by mutex_.
366
+ };
367
+
368
+ } // namespace
369
+
370
+ Env* NewMemEnv(Env* base_env) {
371
+ return new InMemoryEnv(base_env);
372
+ }
373
+
374
+ } // namespace leveldb