leveldb-ruby 0.7 → 0.8

data/leveldb/util/cache_test.cc

@@ -32,7 +32,7 @@ class CacheTest {
     current_->deleted_values_.push_back(DecodeValue(v));
   }
 
-  static const int kCacheSize = 100;
+  static const int kCacheSize = 1000;
   std::vector<int> deleted_keys_;
   std::vector<int> deleted_values_;
   Cache* cache_;
@@ -137,23 +137,40 @@ TEST(CacheTest, EvictionPolicy) {
   Insert(200, 201);
 
   // Frequently used entry must be kept around
-  for (int i = 0; i < kCacheSize; i++) {
+  for (int i = 0; i < kCacheSize + 100; i++) {
     Insert(1000+i, 2000+i);
     ASSERT_EQ(2000+i, Lookup(1000+i));
     ASSERT_EQ(101, Lookup(100));
   }
   ASSERT_EQ(101, Lookup(100));
-  ASSERT_EQ(2, deleted_keys_.size());
-  ASSERT_EQ(200, deleted_keys_[0]);
-  ASSERT_EQ(201, deleted_values_[0]);
+  ASSERT_EQ(-1, Lookup(200));
 }
 
-TEST(CacheTest, HeavyEntry) {
-  Insert(100, 101);
-  Insert(200, 201, kCacheSize);
-  ASSERT_EQ(1, deleted_keys_.size());
-  ASSERT_EQ(100, deleted_keys_[0]);
-  ASSERT_EQ(101, deleted_values_[0]);
+TEST(CacheTest, HeavyEntries) {
+  // Add a bunch of light and heavy entries and then count the combined
+  // size of items still in the cache, which must be approximately the
+  // same as the total capacity.
+  const int kLight = 1;
+  const int kHeavy = 10;
+  int added = 0;
+  int index = 0;
+  while (added < 2*kCacheSize) {
+    const int weight = (index & 1) ? kLight : kHeavy;
+    Insert(index, 1000+index, weight);
+    added += weight;
+    index++;
+  }
+
+  int cached_weight = 0;
+  for (int i = 0; i < index; i++) {
+    const int weight = (i & 1 ? kLight : kHeavy);
+    int r = Lookup(i);
+    if (r >= 0) {
+      cached_weight += weight;
+      ASSERT_EQ(1000+i, r);
+    }
+  }
+  ASSERT_LE(cached_weight, kCacheSize + kCacheSize/10);
 }
 
 TEST(CacheTest, NewId) {

data/leveldb/util/coding.h

@@ -62,10 +62,10 @@ inline uint32_t DecodeFixed32(const char* ptr) {
     memcpy(&result, ptr, sizeof(result));  // gcc optimizes this to a plain load
     return result;
   } else {
-    return ((static_cast<uint32_t>(ptr[0]))
-            | (static_cast<uint32_t>(ptr[1]) << 8)
-            | (static_cast<uint32_t>(ptr[2]) << 16)
-            | (static_cast<uint32_t>(ptr[3]) << 24));
+    return ((static_cast<uint32_t>(static_cast<unsigned char>(ptr[0])))
+        | (static_cast<uint32_t>(static_cast<unsigned char>(ptr[1])) << 8)
+        | (static_cast<uint32_t>(static_cast<unsigned char>(ptr[2])) << 16)
+        | (static_cast<uint32_t>(static_cast<unsigned char>(ptr[3])) << 24));
   }
 }
 

data/leveldb/util/comparator.cc

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#include <algorithm>
 #include <stdint.h>
 #include "leveldb/comparator.h"
 #include "leveldb/slice.h"

data/leveldb/util/env.cc

@@ -18,14 +18,19 @@ RandomAccessFile::~RandomAccessFile() {
 WritableFile::~WritableFile() {
 }
 
+Logger::~Logger() {
+}
+
 FileLock::~FileLock() {
 }
 
-void Log(Env* env, WritableFile* info_log, const char* format, ...) {
-  va_list ap;
-  va_start(ap, format);
-  env->Logv(info_log, format, ap);
-  va_end(ap);
+void Log(Logger* info_log, const char* format, ...) {
+  if (info_log != NULL) {
+    va_list ap;
+    va_start(ap, format);
+    info_log->Logv(format, ap);
+    va_end(ap);
+  }
 }
 
 Status WriteStringToFile(Env* env, const Slice& data,

data/leveldb/util/env_posix.cc

@@ -23,11 +23,16 @@
 #include "leveldb/slice.h"
 #include "port/port.h"
 #include "util/logging.h"
+#include "util/posix_logger.h"
 
 namespace leveldb {
 
 namespace {
 
+static Status IOError(const std::string& context, int err_number) {
+  return Status::IOError(context, strerror(err_number));
+}
+
 class PosixSequentialFile: public SequentialFile {
  private:
   std::string filename_;
@@ -47,7 +52,7 @@ class PosixSequentialFile: public SequentialFile {
         // We leave status as ok if we hit the end of the file
       } else {
         // A partial read with an error: return a non-ok status
-        s = Status::IOError(filename_, strerror(errno));
+        s = IOError(filename_, errno);
       }
     }
     return s;
@@ -55,7 +60,7 @@ class PosixSequentialFile: public SequentialFile {
 
   virtual Status Skip(uint64_t n) {
     if (fseek(file_, n, SEEK_CUR)) {
-      return Status::IOError(filename_, strerror(errno));
+      return IOError(filename_, errno);
     }
     return Status::OK();
   }
@@ -78,7 +83,7 @@ class PosixRandomAccessFile: public RandomAccessFile {
     *result = Slice(scratch, (r < 0) ? 0 : r);
     if (r < 0) {
       // An error: return a non-ok status
-      s = Status::IOError(filename_, strerror(errno));
+      s = IOError(filename_, errno);
     }
     return s;
   }
@@ -114,13 +119,16 @@ class PosixMmapFile : public WritableFile {
     return s;
   }
 
-  void UnmapCurrentRegion() {
+  bool UnmapCurrentRegion() {
+    bool result = true;
     if (base_ != NULL) {
       if (last_sync_ < limit_) {
         // Defer syncing this data until next Sync() call, if any
         pending_sync_ = true;
       }
-      munmap(base_, limit_ - base_);
+      if (munmap(base_, limit_ - base_) != 0) {
+        result = false;
+      }
       file_offset_ += limit_ - base_;
       base_ = NULL;
       limit_ = NULL;
@@ -132,6 +140,7 @@ class PosixMmapFile : public WritableFile {
         map_size_ *= 2;
       }
     }
+    return result;
   }
 
   bool MapNewRegion() {
@@ -181,8 +190,10 @@ class PosixMmapFile : public WritableFile {
       assert(dst_ <= limit_);
       size_t avail = limit_ - dst_;
       if (avail == 0) {
-        UnmapCurrentRegion();
-        MapNewRegion();
+        if (!UnmapCurrentRegion() ||
+            !MapNewRegion()) {
+          return IOError(filename_, errno);
+        }
       }
 
       size_t n = (left <= avail) ? left : avail;
@@ -197,17 +208,18 @@ class PosixMmapFile : public WritableFile {
   virtual Status Close() {
     Status s;
     size_t unused = limit_ - dst_;
-    UnmapCurrentRegion();
-    if (unused > 0) {
+    if (!UnmapCurrentRegion()) {
+      s = IOError(filename_, errno);
+    } else if (unused > 0) {
       // Trim the extra space at the end of the file
       if (ftruncate(fd_, file_offset_ - unused) < 0) {
-        s = Status::IOError(filename_, strerror(errno));
+        s = IOError(filename_, errno);
       }
     }
 
     if (close(fd_) < 0) {
       if (s.ok()) {
-        s = Status::IOError(filename_, strerror(errno));
+        s = IOError(filename_, errno);
       }
     }
 
@@ -228,7 +240,7 @@ class PosixMmapFile : public WritableFile {
       // Some unmapped data was not synced
       pending_sync_ = false;
       if (fdatasync(fd_) < 0) {
-        s = Status::IOError(filename_, strerror(errno));
+        s = IOError(filename_, errno);
       }
     }
 
@@ -239,7 +251,7 @@ class PosixMmapFile : public WritableFile {
       size_t p2 = TruncateToPageBoundary(dst_ - base_ - 1);
       last_sync_ = dst_;
       if (msync(base_ + p1, p2 - p1 + page_size_, MS_SYNC) < 0) {
-        s = Status::IOError(filename_, strerror(errno));
+        s = IOError(filename_, errno);
       }
     }
 
@@ -276,7 +288,7 @@ class PosixEnv : public Env {
     FILE* f = fopen(fname.c_str(), "r");
     if (f == NULL) {
       *result = NULL;
-      return Status::IOError(fname, strerror(errno));
+      return IOError(fname, errno);
     } else {
       *result = new PosixSequentialFile(fname, f);
       return Status::OK();
@@ -288,7 +300,7 @@ class PosixEnv : public Env {
     int fd = open(fname.c_str(), O_RDONLY);
     if (fd < 0) {
       *result = NULL;
-      return Status::IOError(fname, strerror(errno));
+      return IOError(fname, errno);
     }
     *result = new PosixRandomAccessFile(fname, fd);
     return Status::OK();
@@ -300,7 +312,7 @@ class PosixEnv : public Env {
     const int fd = open(fname.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
     if (fd < 0) {
       *result = NULL;
-      s = Status::IOError(fname, strerror(errno));
+      s = IOError(fname, errno);
     } else {
       *result = new PosixMmapFile(fname, fd, page_size_);
     }
@@ -316,7 +328,7 @@ class PosixEnv : public Env {
     result->clear();
     DIR* d = opendir(dir.c_str());
     if (d == NULL) {
-      return Status::IOError(dir, strerror(errno));
+      return IOError(dir, errno);
     }
     struct dirent* entry;
     while ((entry = readdir(d)) != NULL) {
@@ -329,7 +341,7 @@ class PosixEnv : public Env {
   virtual Status DeleteFile(const std::string& fname) {
     Status result;
     if (unlink(fname.c_str()) != 0) {
-      result = Status::IOError(fname, strerror(errno));
+      result = IOError(fname, errno);
     }
     return result;
   };
@@ -337,7 +349,7 @@ class PosixEnv : public Env {
   virtual Status CreateDir(const std::string& name) {
     Status result;
     if (mkdir(name.c_str(), 0755) != 0) {
-      result = Status::IOError(name, strerror(errno));
+      result = IOError(name, errno);
     }
     return result;
   };
@@ -345,7 +357,7 @@ class PosixEnv : public Env {
   virtual Status DeleteDir(const std::string& name) {
     Status result;
     if (rmdir(name.c_str()) != 0) {
-      result = Status::IOError(name, strerror(errno));
+      result = IOError(name, errno);
     }
     return result;
   };
@@ -355,7 +367,7 @@ class PosixEnv : public Env {
     struct stat sbuf;
     if (stat(fname.c_str(), &sbuf) != 0) {
       *size = 0;
-      s = Status::IOError(fname, strerror(errno));
+      s = IOError(fname, errno);
     } else {
       *size = sbuf.st_size;
     }
@@ -365,7 +377,7 @@ class PosixEnv : public Env {
   virtual Status RenameFile(const std::string& src, const std::string& target) {
     Status result;
     if (rename(src.c_str(), target.c_str()) != 0) {
-      result = Status::IOError(src, strerror(errno));
+      result = IOError(src, errno);
     }
     return result;
   }
@@ -375,9 +387,9 @@ class PosixEnv : public Env {
     Status result;
     int fd = open(fname.c_str(), O_RDWR | O_CREAT, 0644);
     if (fd < 0) {
-      result = Status::IOError(fname, strerror(errno));
+      result = IOError(fname, errno);
     } else if (LockOrUnlock(fd, true) == -1) {
-      result = Status::IOError("lock " + fname, strerror(errno));
+      result = IOError("lock " + fname, errno);
       close(fd);
     } else {
       PosixFileLock* my_lock = new PosixFileLock;
@@ -391,7 +403,7 @@ class PosixEnv : public Env {
     PosixFileLock* my_lock = reinterpret_cast<PosixFileLock*>(lock);
     Status result;
     if (LockOrUnlock(my_lock->fd_, false) == -1) {
-      result = Status::IOError(strerror(errno));
+      result = IOError("unlock", errno);
     }
     close(my_lock->fd_);
     delete my_lock;
@@ -416,72 +428,21 @@ class PosixEnv : public Env {
     return Status::OK();
   }
 
-  virtual void Logv(WritableFile* info_log, const char* format, va_list ap) {
+  static uint64_t gettid() {
     pthread_t tid = pthread_self();
     uint64_t thread_id = 0;
     memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid)));
+    return thread_id;
+  }
 
-    // We try twice: the first time with a fixed-size stack allocated buffer,
-    // and the second time with a much larger dynamically allocated buffer.
-    char buffer[500];
-    for (int iter = 0; iter < 2; iter++) {
-      char* base;
-      int bufsize;
-      if (iter == 0) {
-        bufsize = sizeof(buffer);
-        base = buffer;
-      } else {
-        bufsize = 30000;
-        base = new char[bufsize];
-      }
-      char* p = base;
-      char* limit = base + bufsize;
-
-      struct timeval now_tv;
-      gettimeofday(&now_tv, NULL);
-      const time_t seconds = now_tv.tv_sec;
-      struct tm t;
-      localtime_r(&seconds, &t);
-      p += snprintf(p, limit - p,
-                    "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ",
-                    t.tm_year + 1900,
-                    t.tm_mon + 1,
-                    t.tm_mday,
-                    t.tm_hour,
-                    t.tm_min,
-                    t.tm_sec,
-                    static_cast<int>(now_tv.tv_usec),
-                    static_cast<long long unsigned int>(thread_id));
-
-      // Print the message
-      if (p < limit) {
-        va_list backup_ap;
-        va_copy(backup_ap, ap);
-        p += vsnprintf(p, limit - p, format, backup_ap);
-        va_end(backup_ap);
-      }
-
-      // Truncate to available space if necessary
-      if (p >= limit) {
-        if (iter == 0) {
-          continue;       // Try again with larger buffer
-        } else {
-          p = limit - 1;
-        }
-      }
-
-      // Add newline if necessary
-      if (p == base || p[-1] != '\n') {
-        *p++ = '\n';
-      }
-
-      assert(p <= limit);
-      info_log->Append(Slice(base, p - base));
-      info_log->Flush();
-      if (base != buffer) {
-        delete[] base;
-      }
-      break;
+  virtual Status NewLogger(const std::string& fname, Logger** result) {
+    FILE* f = fopen(fname.c_str(), "w");
+    if (f == NULL) {
+      *result = NULL;
+      return IOError(fname, errno);
+    } else {
+      *result = new PosixLogger(f, &PosixEnv::gettid);
+      return Status::OK();
     }
   }
 

data/leveldb/util/histogram.cc

@@ -55,6 +55,17 @@ void Histogram::Add(double value) {
   sum_squares_ += (value * value);
 }
 
+void Histogram::Merge(const Histogram& other) {
+  if (other.min_ < min_) min_ = other.min_;
+  if (other.max_ > max_) max_ = other.max_;
+  num_ += other.num_;
+  sum_ += other.sum_;
+  sum_squares_ += other.sum_squares_;
+  for (int b = 0; b < kNumBuckets; b++) {
+    buckets_[b] += other.buckets_[b];
+  }
+}
+
 double Histogram::Median() const {
   return Percentile(50.0);
 }

data/leveldb/util/histogram.h

@@ -16,6 +16,7 @@ class Histogram {
 
   void Clear();
   void Add(double value);
+  void Merge(const Histogram& other);
 
   std::string ToString() const;
 

data/leveldb/util/posix_logger.h

@@ -0,0 +1,98 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Logger implementation that can be shared by all environments
+// where enough posix functionality is available.
+
+#ifndef STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_
+#define STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_
+
+#include <algorithm>
+#include <stdio.h>
+#include <sys/time.h>
+#include <time.h>
+#include "leveldb/env.h"
+
+namespace leveldb {
+
+class PosixLogger : public Logger {
+ private:
+  FILE* file_;
+  uint64_t (*gettid_)();  // Return the thread id for the current thread
+ public:
+  PosixLogger(FILE* f, uint64_t (*gettid)()) : file_(f), gettid_(gettid) { }
+  virtual ~PosixLogger() {
+    fclose(file_);
+  }
+  virtual void Logv(const char* format, va_list ap) {
+    const uint64_t thread_id = (*gettid_)();
+
+    // We try twice: the first time with a fixed-size stack allocated buffer,
+    // and the second time with a much larger dynamically allocated buffer.
+    char buffer[500];
+    for (int iter = 0; iter < 2; iter++) {
+      char* base;
+      int bufsize;
+      if (iter == 0) {
+        bufsize = sizeof(buffer);
+        base = buffer;
+      } else {
+        bufsize = 30000;
+        base = new char[bufsize];
+      }
+      char* p = base;
+      char* limit = base + bufsize;
+
+      struct timeval now_tv;
+      gettimeofday(&now_tv, NULL);
+      const time_t seconds = now_tv.tv_sec;
+      struct tm t;
+      localtime_r(&seconds, &t);
+      p += snprintf(p, limit - p,
+                    "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ",
+                    t.tm_year + 1900,
+                    t.tm_mon + 1,
+                    t.tm_mday,
+                    t.tm_hour,
+                    t.tm_min,
+                    t.tm_sec,
+                    static_cast<int>(now_tv.tv_usec),
+                    static_cast<long long unsigned int>(thread_id));
+
+      // Print the message
+      if (p < limit) {
+        va_list backup_ap;
+        va_copy(backup_ap, ap);
+        p += vsnprintf(p, limit - p, format, backup_ap);
+        va_end(backup_ap);
+      }
+
+      // Truncate to available space if necessary
+      if (p >= limit) {
+        if (iter == 0) {
+          continue;       // Try again with larger buffer
+        } else {
+          p = limit - 1;
+        }
+      }
+
+      // Add newline if necessary
+      if (p == base || p[-1] != '\n') {
+        *p++ = '\n';
+      }
+
+      assert(p <= limit);
+      fwrite(base, 1, p - base, file_);
+      fflush(file_);
+      if (base != buffer) {
+        delete[] base;
+      }
+      break;
+    }
+  }
+};
+
+}
+
+#endif  // STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_