leveldb-ruby 0.7 → 0.8

data/leveldb/db/dbformat.cc

@@ -31,6 +31,18 @@ std::string ParsedInternalKey::DebugString() const {
   return result;
 }
 
+std::string InternalKey::DebugString() const {
+  std::string result;
+  ParsedInternalKey parsed;
+  if (ParseInternalKey(rep_, &parsed)) {
+    result = parsed.DebugString();
+  } else {
+    result = "(bad)";
+    result.append(EscapeString(rep_));
+  }
+  return result;
+}
+
 const char* InternalKeyComparator::Name() const {
   return "leveldb.InternalKeyComparator";
 }
@@ -84,4 +96,23 @@ void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
   }
 }
 
+LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
+  size_t usize = user_key.size();
+  size_t needed = usize + 13;  // A conservative estimate
+  char* dst;
+  if (needed <= sizeof(space_)) {
+    dst = space_;
+  } else {
+    dst = new char[needed];
+  }
+  start_ = dst;
+  dst = EncodeVarint32(dst, usize + 8);
+  kstart_ = dst;
+  memcpy(dst, user_key.data(), usize);
+  dst += usize;
+  EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek));
+  dst += 8;
+  end_ = dst;
+}
+
 }

data/leveldb/db/dbformat.h

@@ -29,6 +29,14 @@ static const int kL0_SlowdownWritesTrigger = 8;
 // Maximum number of level-0 files.  We stop writes at this point.
 static const int kL0_StopWritesTrigger = 12;
 
+// Maximum level to which a new compacted memtable is pushed if it
+// does not create overlap.  We try to push to level 2 to avoid the
+// relatively expensive level 0=>1 compactions and to avoid some
+// expensive manifest file operations.  We do not push all the way to
+// the largest level since that can generate a lot of wasted disk
+// space if the same key space is being repeatedly overwritten.
+static const int kMaxMemCompactLevel = 2;
+
 }
 
 class InternalKey;
@@ -38,7 +46,7 @@ class InternalKey;
 // data structures.
 enum ValueType {
   kTypeDeletion = 0x0,
-  kTypeValue = 0x1,
+  kTypeValue = 0x1
 };
 // kValueTypeForSeek defines the ValueType that should be passed when
 // constructing a ParsedInternalKey object for seeking to a particular
@@ -141,6 +149,8 @@ class InternalKey {
   }
 
   void Clear() { rep_.clear(); }
+
+  std::string DebugString() const;
 };
 
 inline int InternalKeyComparator::Compare(
@@ -160,6 +170,46 @@ inline bool ParseInternalKey(const Slice& internal_key,
   return (c <= static_cast<unsigned char>(kTypeValue));
 }
 
+// A helper class useful for DBImpl::Get()
+class LookupKey {
+ public:
+  // Initialize *this for looking up user_key at a snapshot with
+  // the specified sequence number.
+  LookupKey(const Slice& user_key, SequenceNumber sequence);
+
+  ~LookupKey();
+
+  // Return a key suitable for lookup in a MemTable.
+  Slice memtable_key() const { return Slice(start_, end_ - start_); }
+
+  // Return an internal key (suitable for passing to an internal iterator)
+  Slice internal_key() const { return Slice(kstart_, end_ - kstart_); }
+
+  // Return the user key
+  Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); }
+
+ private:
+  // We construct a char array of the form:
+  //    klength  varint32               <-- start_
+  //    userkey  char[klength]          <-- kstart_
+  //    tag      uint64
+  //                                    <-- end_
+  // The array is a suitable MemTable key.
+  // The suffix starting with "userkey" can be used as an InternalKey.
+  const char* start_;
+  const char* kstart_;
+  const char* end_;
+  char space_[200];      // Avoid allocation for short keys
+
+  // No copying allowed
+  LookupKey(const LookupKey&);
+  void operator=(const LookupKey&);
+};
+
+inline LookupKey::~LookupKey() {
+  if (start_ != space_) delete[] start_;
+}
+
 }
 
 #endif  // STORAGE_LEVELDB_DB_FORMAT_H_

data/leveldb/db/filename.h

@@ -24,7 +24,7 @@ enum FileType {
   kDescriptorFile,
   kCurrentFile,
   kTempFile,
-  kInfoLogFile,  // Either the current one, or an old one
+  kInfoLogFile  // Either the current one, or an old one
 };
 
 // Return the name of the log file with the specified number

data/leveldb/db/log_format.h

@@ -20,7 +20,7 @@ enum RecordType {
   // For fragments
   kFirstType = 2,
   kMiddleType = 3,
-  kLastType = 4,
+  kLastType = 4
 };
 static const int kMaxRecordType = kLastType;
 

data/leveldb/db/log_reader.cc

@@ -4,6 +4,7 @@
 
 #include "db/log_reader.h"
 
+#include <stdio.h>
 #include "leveldb/env.h"
 #include "util/coding.h"
 #include "util/crc32c.h"
@@ -72,7 +73,8 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
   Slice fragment;
   while (true) {
     uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
-    switch (ReadPhysicalRecord(&fragment)) {
+    const unsigned int record_type = ReadPhysicalRecord(&fragment);
+    switch (record_type) {
       case kFullType:
         if (in_fragmented_record) {
           // Handle bug in earlier versions of log::Writer where
@@ -144,13 +146,16 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
         }
         break;
 
-      default:
+      default: {
+        char buf[40];
+        snprintf(buf, sizeof(buf), "unknown record type %u", record_type);
         ReportCorruption(
             (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
-            "unknown record type");
+            buf);
         in_fragmented_record = false;
         scratch->clear();
         break;
+      }
     }
   }
   return false;
@@ -212,16 +217,16 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
       return kBadRecord;
     }
 
+    if (type == kZeroType && length == 0) {
+      // Skip zero length record without reporting any drops since
+      // such records are produced by the mmap based writing code in
+      // env_posix.cc that preallocates file regions.
+      buffer_.clear();
+      return kBadRecord;
+    }
+
     // Check crc
     if (checksum_) {
-      if (type == kZeroType && length == 0) {
-        // Skip zero length record without reporting any drops since
-        // such records are produced by the mmap based writing code in
-        // env_posix.cc that preallocates file regions.
-        buffer_.clear();
-        return kBadRecord;
-      }
-
       uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header));
       uint32_t actual_crc = crc32c::Value(header + 6, 1 + length);
       if (actual_crc != expected_crc) {

data/leveldb/db/memtable.cc

@@ -105,4 +105,41 @@ void MemTable::Add(SequenceNumber s, ValueType type,
   table_.Insert(buf);
 }
 
+bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
+  Slice memkey = key.memtable_key();
+  Table::Iterator iter(&table_);
+  iter.Seek(memkey.data());
+  if (iter.Valid()) {
+    // entry format is:
+    //    klength  varint32
+    //    userkey  char[klength]
+    //    tag      uint64
+    //    vlength  varint32
+    //    value    char[vlength]
+    // Check that it belongs to same user key.  We do not check the
+    // sequence number since the Seek() call above should have skipped
+    // all entries with overly large sequence numbers.
+    const char* entry = iter.key();
+    uint32_t key_length;
+    const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
+    if (comparator_.comparator.user_comparator()->Compare(
+            Slice(key_ptr, key_length - 8),
+            key.user_key()) == 0) {
+      // Correct user key
+      const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
+      switch (static_cast<ValueType>(tag & 0xff)) {
+        case kTypeValue: {
+          Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
+          value->assign(v.data(), v.size());
+          return true;
+        }
+        case kTypeDeletion:
+          *s = Status::NotFound(Slice());
+          return true;
+      }
+    }
+  }
+  return false;
+}
+
 }

data/leveldb/db/memtable.h

@@ -57,6 +57,12 @@ class MemTable {
            const Slice& key,
            const Slice& value);
 
+  // If memtable contains a value for key, store it in *value and return true.
+  // If memtable contains a deletion for key, store a NotFound() error
+  // in *status and return true.
+  // Else, return false.
+  bool Get(const LookupKey& key, std::string* value, Status* s);
+
  private:
   ~MemTable();  // Private since only Unref() should be used to delete it
 

data/leveldb/db/repair.cc

@@ -50,6 +50,7 @@ class Repairer {
         icmp_(options.comparator),
         options_(SanitizeOptions(dbname, &icmp_, options)),
         owns_info_log_(options_.info_log != options.info_log),
+        owns_cache_(options_.block_cache != options.block_cache),
         next_file_number_(1) {
     // TableCache can be small since we expect each table to be opened once.
     table_cache_ = new TableCache(dbname_, &options_, 10);
@@ -60,6 +61,9 @@ class Repairer {
     if (owns_info_log_) {
       delete options_.info_log;
     }
+    if (owns_cache_) {
+      delete options_.block_cache;
+    }
   }
 
   Status Run() {
@@ -74,7 +78,7 @@ class Repairer {
       for (size_t i = 0; i < tables_.size(); i++) {
         bytes += tables_[i].meta.file_size;
       }
-      Log(env_, options_.info_log,
+      Log(options_.info_log,
           "**** Repaired leveldb %s; "
           "recovered %d files; %llu bytes. "
           "Some data may have been lost. "
@@ -97,6 +101,7 @@ class Repairer {
   InternalKeyComparator const icmp_;
   Options const options_;
   bool owns_info_log_;
+  bool owns_cache_;
   TableCache* table_cache_;
   VersionEdit edit_;
 
@@ -144,7 +149,7 @@ class Repairer {
       std::string logname = LogFileName(dbname_, logs_[i]);
       Status status = ConvertLogToTable(logs_[i]);
       if (!status.ok()) {
-        Log(env_, options_.info_log, "Log #%llu: ignoring conversion error: %s",
+        Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
             (unsigned long long) logs_[i],
             status.ToString().c_str());
       }
@@ -155,11 +160,11 @@ class Repairer {
   Status ConvertLogToTable(uint64_t log) {
     struct LogReporter : public log::Reader::Reporter {
       Env* env;
-      WritableFile* info_log;
+      Logger* info_log;
       uint64_t lognum;
       virtual void Corruption(size_t bytes, const Status& s) {
         // We print error messages for corruption, but continue repairing.
-        Log(env, info_log, "Log #%llu: dropping %d bytes; %s",
+        Log(info_log, "Log #%llu: dropping %d bytes; %s",
             (unsigned long long) lognum,
             static_cast<int>(bytes),
             s.ToString().c_str());
@@ -204,7 +209,7 @@ class Repairer {
       if (status.ok()) {
         counter += WriteBatchInternal::Count(&batch);
       } else {
-        Log(env_, options_.info_log, "Log #%llu: ignoring %s",
+        Log(options_.info_log, "Log #%llu: ignoring %s",
             (unsigned long long) log,
             status.ToString().c_str());
         status = Status::OK();  // Keep going with rest of file
@@ -212,14 +217,12 @@ class Repairer {
     }
     delete lfile;
 
-    // We ignore any version edits generated by the conversion to a Table
+    // Do not record a version edit for this conversion to a Table
     // since ExtractMetaData() will also generate edits.
-    VersionEdit skipped;
     FileMetaData meta;
     meta.number = next_file_number_++;
     Iterator* iter = mem->NewIterator();
-    status = BuildTable(dbname_, env_, options_, table_cache_, iter,
-                        &meta, &skipped);
+    status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
     delete iter;
     mem->Unref();
     mem = NULL;
@@ -228,7 +231,7 @@ class Repairer {
         table_numbers_.push_back(meta.number);
       }
     }
-    Log(env_, options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
+    Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
         (unsigned long long) log,
         counter,
         (unsigned long long) meta.number,
@@ -244,7 +247,7 @@ class Repairer {
       Status status = ScanTable(&t);
       if (!status.ok()) {
         std::string fname = TableFileName(dbname_, table_numbers_[i]);
-        Log(env_, options_.info_log, "Table #%llu: ignoring %s",
+        Log(options_.info_log, "Table #%llu: ignoring %s",
             (unsigned long long) table_numbers_[i],
             status.ToString().c_str());
         ArchiveFile(fname);
@@ -267,7 +270,7 @@ class Repairer {
       for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
         Slice key = iter->key();
         if (!ParseInternalKey(key, &parsed)) {
-          Log(env_, options_.info_log, "Table #%llu: unparsable key %s",
+          Log(options_.info_log, "Table #%llu: unparsable key %s",
               (unsigned long long) t->meta.number,
               EscapeString(key).c_str());
           continue;
@@ -288,7 +291,7 @@ class Repairer {
       }
       delete iter;
     }
-    Log(env_, options_.info_log, "Table #%llu: %d entries %s",
+    Log(options_.info_log, "Table #%llu: %d entries %s",
         (unsigned long long) t->meta.number,
         counter,
         status.ToString().c_str());
@@ -370,7 +373,7 @@ class Repairer {
     new_file.append("/");
     new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
     Status s = env_->RenameFile(fname, new_file);
-    Log(env_, options_.info_log, "Archiving %s: %s\n",
+    Log(options_.info_log, "Archiving %s: %s\n",
         fname.c_str(), s.ToString().c_str());
   }
 };

data/leveldb/db/skiplist_test.cc

@@ -238,14 +238,14 @@ class ConcurrentTest {
         current = MakeKey(K, 0);
       } else {
         current = iter.key();
-        ASSERT_TRUE(IsValidKey(current)) << std::hex << current;
+        ASSERT_TRUE(IsValidKey(current)) << current;
       }
       ASSERT_LE(pos, current) << "should not go backwards";
 
       // Verify that everything in [pos,current) was not present in
       // initial_state.
       while (pos < current) {
-        ASSERT_LT(key(pos), K) << std::hex << pos;
+        ASSERT_LT(key(pos), K) << pos;
 
         // Note that generation 0 is never inserted, so it is ok if
         // <*,0,*> is missing.

data/leveldb/db/version_edit.cc

@@ -20,7 +20,7 @@ enum Tag {
   kDeletedFile          = 6,
   kNewFile              = 7,
   // 8 was used for large value refs
-  kPrevLogNumber        = 9,
+  kPrevLogNumber        = 9
 };
 
 void VersionEdit::Clear() {
@@ -235,9 +235,8 @@ std::string VersionEdit::DebugString() const {
   for (size_t i = 0; i < compact_pointers_.size(); i++) {
     r.append("\n  CompactPointer: ");
     AppendNumberTo(&r, compact_pointers_[i].first);
-    r.append(" '");
-    AppendEscapedStringTo(&r, compact_pointers_[i].second.Encode());
-    r.append("'");
+    r.append(" ");
+    r.append(compact_pointers_[i].second.DebugString());
   }
   for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
        iter != deleted_files_.end();
@@ -255,11 +254,10 @@ std::string VersionEdit::DebugString() const {
     AppendNumberTo(&r, f.number);
     r.append(" ");
     AppendNumberTo(&r, f.file_size);
-    r.append(" '");
-    AppendEscapedStringTo(&r, f.smallest.Encode());
-    r.append("' .. '");
-    AppendEscapedStringTo(&r, f.largest.Encode());
-    r.append("'");
+    r.append(" ");
+    r.append(f.smallest.DebugString());
+    r.append(" .. ");
+    r.append(f.largest.DebugString());
   }
   r.append("\n}\n");
   return r;

data/leveldb/db/version_edit.h

@@ -16,12 +16,13 @@ class VersionSet;
 
 struct FileMetaData {
   int refs;
+  int allowed_seeks;          // Seeks allowed until compaction
   uint64_t number;
   uint64_t file_size;         // File size in bytes
   InternalKey smallest;       // Smallest internal key served by table
   InternalKey largest;        // Largest internal key served by table
 
-  FileMetaData() : refs(0), file_size(0) { }
+  FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { }
 };
 
 class VersionEdit {