leveldb-ruby 0.14 → 0.15

data/leveldb/db/db_impl.cc

@@ -35,6 +35,17 @@
 
 namespace leveldb {
 
+// Information kept for every waiting writer
+struct DBImpl::Writer {
+  Status status;
+  WriteBatch* batch;
+  bool sync;
+  bool done;
+  port::CondVar cv;
+
+  explicit Writer(port::Mutex* mu) : cv(mu) { }
+};
+
 struct DBImpl::CompactionState {
   Compaction* const compaction;
 
@@ -76,12 +87,14 @@ static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
 }
 Options SanitizeOptions(const std::string& dbname,
                         const InternalKeyComparator* icmp,
+                        const InternalFilterPolicy* ipolicy,
                         const Options& src) {
   Options result = src;
   result.comparator = icmp;
-  ClipToRange(&result.max_open_files,           20,     50000);
-  ClipToRange(&result.write_buffer_size,        64<<10, 1<<30);
-  ClipToRange(&result.block_size,               1<<10,  4<<20);
+  result.filter_policy = (src.filter_policy != NULL) ? ipolicy : NULL;
+  ClipToRange(&result.max_open_files,            20,     50000);
+  ClipToRange(&result.write_buffer_size,         64<<10, 1<<30);
+  ClipToRange(&result.block_size,                1<<10,  4<<20);
   if (result.info_log == NULL) {
     // Open a log file in the same directory as the db
     src.env->CreateDir(dbname);  // In case it does not exist
@@ -101,7 +114,9 @@ Options SanitizeOptions(const std::string& dbname,
 DBImpl::DBImpl(const Options& options, const std::string& dbname)
     : env_(options.env),
       internal_comparator_(options.comparator),
-      options_(SanitizeOptions(dbname, &internal_comparator_, options)),
+      internal_filter_policy_(options.filter_policy),
+      options_(SanitizeOptions(
+          dbname, &internal_comparator_, &internal_filter_policy_, options)),
       owns_info_log_(options_.info_log != options.info_log),
       owns_cache_(options_.block_cache != options.block_cache),
       dbname_(dbname),
@@ -113,8 +128,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
       logfile_(NULL),
       logfile_number_(0),
       log_(NULL),
-      logger_(NULL),
-      logger_cv_(&mutex_),
+      tmp_batch_(new WriteBatch),
       bg_compaction_scheduled_(false),
       manual_compaction_(NULL) {
   mem_->Ref();
@@ -144,6 +158,7 @@ DBImpl::~DBImpl() {
   delete versions_;
   if (mem_ != NULL) mem_->Unref();
   if (imm_ != NULL) imm_->Unref();
+  delete tmp_batch_;
   delete log_;
   delete logfile_;
   delete table_cache_;
@@ -554,13 +569,11 @@ void DBImpl::TEST_CompactRange(int level, const Slice* begin,const Slice* end) {
 }
 
 Status DBImpl::TEST_CompactMemTable() {
-  MutexLock l(&mutex_);
-  LoggerId self;
-  AcquireLoggingResponsibility(&self);
-  Status s = MakeRoomForWrite(true /* force compaction */);
-  ReleaseLoggingResponsibility(&self);
+  // NULL batch means just wait for earlier writes to be done
+  Status s = Write(WriteOptions(), NULL);
   if (s.ok()) {
     // Wait until the compaction completes
+    MutexLock l(&mutex_);
     while (imm_ != NULL && bg_error_.ok()) {
       bg_cv_.Wait();
     }
@@ -595,8 +608,21 @@ void DBImpl::BackgroundCall() {
   MutexLock l(&mutex_);
   assert(bg_compaction_scheduled_);
   if (!shutting_down_.Acquire_Load()) {
-    BackgroundCompaction();
+    Status s = BackgroundCompaction();
+    if (!s.ok()) {
+      // Wait a little bit before retrying background compaction in
+      // case this is an environmental problem and we do not want to
+      // chew up resources for failed compactions for the duration of
+      // the problem.
+      bg_cv_.SignalAll();  // In case a waiter can proceed despite the error
+      Log(options_.info_log, "Waiting after background compaction error: %s",
+          s.ToString().c_str());
+      mutex_.Unlock();
+      env_->SleepForMicroseconds(1000000);
+      mutex_.Lock();
+    }
   }
+
   bg_compaction_scheduled_ = false;
 
   // Previous compaction may have produced too many files in a level,
@@ -605,12 +631,11 @@ void DBImpl::BackgroundCall() {
   bg_cv_.SignalAll();
 }
 
-void DBImpl::BackgroundCompaction() {
+Status DBImpl::BackgroundCompaction() {
   mutex_.AssertHeld();
 
   if (imm_ != NULL) {
-    CompactMemTable();
-    return;
+    return CompactMemTable();
   }
 
   Compaction* c;
@@ -685,6 +710,7 @@ void DBImpl::BackgroundCompaction() {
     }
     manual_compaction_ = NULL;
   }
+  return status;
 }
 
 void DBImpl::CleanupCompaction(CompactionState* compact) {
@@ -1094,38 +1120,35 @@ Status DBImpl::Delete(const WriteOptions& options, const Slice& key) {
   return DB::Delete(options, key);
 }
 
-// There is at most one thread that is the current logger.  This call
-// waits until preceding logger(s) have finished and becomes the
-// current logger.
-void DBImpl::AcquireLoggingResponsibility(LoggerId* self) {
-  while (logger_ != NULL) {
-    logger_cv_.Wait();
-  }
-  logger_ = self;
-}
-
-void DBImpl::ReleaseLoggingResponsibility(LoggerId* self) {
-  assert(logger_ == self);
-  logger_ = NULL;
-  logger_cv_.SignalAll();
-}
+Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
+  Writer w(&mutex_);
+  w.batch = my_batch;
+  w.sync = options.sync;
+  w.done = false;
 
-Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
-  Status status;
   MutexLock l(&mutex_);
-  LoggerId self;
-  AcquireLoggingResponsibility(&self);
-  status = MakeRoomForWrite(false);  // May temporarily release lock and wait
+  writers_.push_back(&w);
+  while (!w.done && &w != writers_.front()) {
+    w.cv.Wait();
+  }
+  if (w.done) {
+    return w.status;
+  }
+
+  // May temporarily unlock and wait.
+  Status status = MakeRoomForWrite(my_batch == NULL);
   uint64_t last_sequence = versions_->LastSequence();
-  if (status.ok()) {
+  Writer* last_writer = &w;
+  if (status.ok() && my_batch != NULL) {  // NULL batch is for compactions
+    WriteBatch* updates = BuildBatchGroup(&last_writer);
     WriteBatchInternal::SetSequence(updates, last_sequence + 1);
     last_sequence += WriteBatchInternal::Count(updates);
 
-    // Add to log and apply to memtable.  We can release the lock during
-    // this phase since the "logger_" flag protects against concurrent
-    // loggers and concurrent writes into mem_.
+    // Add to log and apply to memtable.  We can release the lock
+    // during this phase since &w is currently responsible for logging
+    // and protects against concurrent loggers and concurrent writes
+    // into mem_.
     {
-      assert(logger_ == &self);
       mutex_.Unlock();
       status = log_->AddRecord(WriteBatchInternal::Contents(updates));
       if (status.ok() && options.sync) {
@@ -1135,20 +1158,85 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
         status = WriteBatchInternal::InsertInto(updates, mem_);
       }
       mutex_.Lock();
-      assert(logger_ == &self);
     }
+    if (updates == tmp_batch_) tmp_batch_->Clear();
 
     versions_->SetLastSequence(last_sequence);
   }
-  ReleaseLoggingResponsibility(&self);
+
+  while (true) {
+    Writer* ready = writers_.front();
+    writers_.pop_front();
+    if (ready != &w) {
+      ready->status = status;
+      ready->done = true;
+      ready->cv.Signal();
+    }
+    if (ready == last_writer) break;
+  }
+
+  // Notify new head of write queue
+  if (!writers_.empty()) {
+    writers_.front()->cv.Signal();
+  }
+
   return status;
 }
 
+// REQUIRES: Writer list must be non-empty
+// REQUIRES: First writer must have a non-NULL batch
+WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) {
+  assert(!writers_.empty());
+  Writer* first = writers_.front();
+  WriteBatch* result = first->batch;
+  assert(result != NULL);
+
+  size_t size = WriteBatchInternal::ByteSize(first->batch);
+
+  // Allow the group to grow up to a maximum size, but if the
+  // original write is small, limit the growth so we do not slow
+  // down the small write too much.
+  size_t max_size = 1 << 20;
+  if (size <= (128<<10)) {
+    max_size = size + (128<<10);
+  }
+
+  *last_writer = first;
+  std::deque<Writer*>::iterator iter = writers_.begin();
+  ++iter;  // Advance past "first"
+  for (; iter != writers_.end(); ++iter) {
+    Writer* w = *iter;
+    if (w->sync && !first->sync) {
+      // Do not include a sync write into a batch handled by a non-sync write.
+      break;
+    }
+
+    if (w->batch != NULL) {
+      size += WriteBatchInternal::ByteSize(w->batch);
+      if (size > max_size) {
+        // Do not make batch too big
+        break;
+      }
+
+      // Append to *reuslt
+      if (result == first->batch) {
+        // Switch to temporary batch instead of disturbing caller's batch
+        result = tmp_batch_;
+        assert(WriteBatchInternal::Count(result) == 0);
+        WriteBatchInternal::Append(result, first->batch);
+      }
+      WriteBatchInternal::Append(result, w->batch);
+    }
+    *last_writer = w;
+  }
+  return result;
+}
+
 // REQUIRES: mutex_ is held
-// REQUIRES: this thread is the current logger
+// REQUIRES: this thread is currently at the front of the writer queue
 Status DBImpl::MakeRoomForWrite(bool force) {
   mutex_.AssertHeld();
-  assert(logger_ != NULL);
+  assert(!writers_.empty());
   bool allow_delay = !force;
   Status s;
   while (true) {
@@ -1188,6 +1276,8 @@ Status DBImpl::MakeRoomForWrite(bool force) {
       WritableFile* lfile = NULL;
       s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile);
       if (!s.ok()) {
+        // Avoid chewing through file number space in a tight loop.
+        versions_->ReuseFileNumber(new_log_number);
         break;
       }
       delete log_;
@@ -1356,7 +1446,7 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
     FileType type;
     for (size_t i = 0; i < filenames.size(); i++) {
       if (ParseFileName(filenames[i], &number, &type) &&
-          filenames[i] != lockname) {  // Lock file will be deleted at end
+          type != kDBLockFile) {  // Lock file will be deleted at end
         Status del = env->DeleteFile(dbname + "/" + filenames[i]);
         if (result.ok() && !del.ok()) {
           result = del;

data/leveldb/db/db_impl.h

@@ -5,6 +5,7 @@
 #ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_
 #define STORAGE_LEVELDB_DB_DB_IMPL_H_
 
+#include <deque>
 #include <set>
 #include "db/dbformat.h"
 #include "db/log_writer.h"
@@ -59,6 +60,8 @@ class DBImpl : public DB {
 
  private:
   friend class DB;
+  struct CompactionState;
+  struct Writer;
 
   Iterator* NewInternalIterator(const ReadOptions&,
                                 SequenceNumber* latest_snapshot);
@@ -85,19 +88,13 @@ class DBImpl : public DB {
 
   Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base);
 
-  // Only thread is allowed to log at a time.
-  struct LoggerId { };          // Opaque identifier for logging thread
-  void AcquireLoggingResponsibility(LoggerId* self);
-  void ReleaseLoggingResponsibility(LoggerId* self);
-
   Status MakeRoomForWrite(bool force /* compact even if there is room? */);
-
-  struct CompactionState;
+  WriteBatch* BuildBatchGroup(Writer** last_writer);
 
   void MaybeScheduleCompaction();
   static void BGWork(void* db);
   void BackgroundCall();
-  void BackgroundCompaction();
+  Status BackgroundCompaction();
   void CleanupCompaction(CompactionState* compact);
   Status DoCompactionWork(CompactionState* compact);
 
@@ -108,6 +105,7 @@ class DBImpl : public DB {
   // Constant after construction
   Env* const env_;
   const InternalKeyComparator internal_comparator_;
+  const InternalFilterPolicy internal_filter_policy_;
   const Options options_;  // options_.comparator == &internal_comparator_
   bool owns_info_log_;
   bool owns_cache_;
@@ -129,8 +127,11 @@ class DBImpl : public DB {
   WritableFile* logfile_;
   uint64_t logfile_number_;
   log::Writer* log_;
-  LoggerId* logger_;            // NULL, or the id of the current logging thread
-  port::CondVar logger_cv_;     // For threads waiting to log
+
+  // Queue of writers.
+  std::deque<Writer*> writers_;
+  WriteBatch* tmp_batch_;
+
   SnapshotList snapshots_;
 
   // Set of table files to protect from deletion because they are
@@ -185,6 +186,7 @@ class DBImpl : public DB {
 // it is not equal to src.info_log.
 extern Options SanitizeOptions(const std::string& db,
                                const InternalKeyComparator* icmp,
+                               const InternalFilterPolicy* ipolicy,
                                const Options& src);
 
 }  // namespace leveldb

data/leveldb/db/db_test.cc

@@ -3,12 +3,15 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "leveldb/db.h"
+#include "leveldb/filter_policy.h"
 #include "db/db_impl.h"
 #include "db/filename.h"
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
+#include "leveldb/cache.h"
 #include "leveldb/env.h"
 #include "leveldb/table.h"
+#include "util/hash.h"
 #include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/testharness.h"
@@ -22,6 +25,28 @@ static std::string RandomString(Random* rnd, int len) {
   return r;
 }
 
+namespace {
+class AtomicCounter {
+ private:
+  port::Mutex mu_;
+  int count_;
+ public:
+  AtomicCounter() : count_(0) { }
+  void Increment() {
+    MutexLock l(&mu_);
+    count_++;
+  }
+  int Read() {
+    MutexLock l(&mu_);
+    return count_;
+  }
+  void Reset() {
+    MutexLock l(&mu_);
+    count_ = 0;
+  }
+};
+}
+
 // Special Env used to delay background operations
 class SpecialEnv : public EnvWrapper {
  public:
@@ -31,9 +56,19 @@ class SpecialEnv : public EnvWrapper {
   // Simulate no-space errors while this pointer is non-NULL.
   port::AtomicPointer no_space_;
 
+  // Simulate non-writable file system while this pointer is non-NULL
+  port::AtomicPointer non_writable_;
+
+  bool count_random_reads_;
+  AtomicCounter random_read_counter_;
+
+  AtomicCounter sleep_counter_;
+
   explicit SpecialEnv(Env* base) : EnvWrapper(base) {
     delay_sstable_sync_.Release_Store(NULL);
     no_space_.Release_Store(NULL);
+    non_writable_.Release_Store(NULL);
+    count_random_reads_ = false;
   }
 
   Status NewWritableFile(const std::string& f, WritableFile** r) {
@@ -66,6 +101,10 @@ class SpecialEnv : public EnvWrapper {
       }
     };
 
+    if (non_writable_.Acquire_Load() != NULL) {
+      return Status::IOError("simulated write error");
+    }
+
     Status s = target()->NewWritableFile(f, r);
     if (s.ok()) {
       if (strstr(f.c_str(), ".sst") != NULL) {
@@ -74,9 +113,50 @@ class SpecialEnv : public EnvWrapper {
     }
     return s;
   }
+
+  Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) {
+    class CountingFile : public RandomAccessFile {
+     private:
+      RandomAccessFile* target_;
+      AtomicCounter* counter_;
+     public:
+      CountingFile(RandomAccessFile* target, AtomicCounter* counter)
+          : target_(target), counter_(counter) {
+      }
+      virtual ~CountingFile() { delete target_; }
+      virtual Status Read(uint64_t offset, size_t n, Slice* result,
+                          char* scratch) const {
+        counter_->Increment();
+        return target_->Read(offset, n, result, scratch);
+      }
+    };
+
+    Status s = target()->NewRandomAccessFile(f, r);
+    if (s.ok() && count_random_reads_) {
+      *r = new CountingFile(*r, &random_read_counter_);
+    }
+    return s;
+  }
+
+  virtual void SleepForMicroseconds(int micros) {
+    sleep_counter_.Increment();
+    target()->SleepForMicroseconds(micros);
+  }
 };
 
 class DBTest {
+ private:
+  const FilterPolicy* filter_policy_;
+
+  // Sequence of option configurations to try
+  enum OptionConfig {
+    kDefault,
+    kFilter,
+    kUncompressed,
+    kEnd
+  };
+  int option_config_;
+
  public:
   std::string dbname_;
   SpecialEnv* env_;
@@ -84,7 +164,9 @@ class DBTest {
 
   Options last_options_;
 
-  DBTest() : env_(new SpecialEnv(Env::Default())) {
+  DBTest() : option_config_(kDefault),
+             env_(new SpecialEnv(Env::Default())) {
+    filter_policy_ = NewBloomFilterPolicy(10);
     dbname_ = test::TmpDir() + "/db_test";
     DestroyDB(dbname_, Options());
     db_ = NULL;
@@ -95,6 +177,35 @@ class DBTest {
     delete db_;
     DestroyDB(dbname_, Options());
     delete env_;
+    delete filter_policy_;
+  }
+
+  // Switch to a fresh database with the next option configuration to
+  // test.  Return false if there are no more configurations to test.
+  bool ChangeOptions() {
+    option_config_++;
+    if (option_config_ >= kEnd) {
+      return false;
+    } else {
+      DestroyAndReopen();
+      return true;
+    }
+  }
+
+  // Return the current option configuration.
+  Options CurrentOptions() {
+    Options options;
+    switch (option_config_) {
+      case kFilter:
+        options.filter_policy = filter_policy_;
+        break;
+      case kUncompressed:
+        options.compression = kNoCompression;
+        break;
+      default:
+        break;
+    }
+    return options;
   }
 
   DBImpl* dbfull() {
@@ -105,6 +216,11 @@ class DBTest {
     ASSERT_OK(TryReopen(options));
   }
 
+  void Close() {
+    delete db_;
+    db_ = NULL;
+  }
+
   void DestroyAndReopen(Options* options = NULL) {
     delete db_;
     db_ = NULL;
@@ -119,6 +235,7 @@ class DBTest {
     if (options != NULL) {
       opts = *options;
     } else {
+      opts = CurrentOptions();
       opts.create_if_missing = true;
     }
     last_options_ = opts;
@@ -189,8 +306,7 @@ class DBTest {
         if (!ParseInternalKey(iter->key(), &ikey)) {
           result += "CORRUPTED";
         } else {
-          if (last_options_.comparator->Compare(
-                  ikey.user_key, user_key) != 0) {
+          if (last_options_.comparator->Compare(ikey.user_key, user_key) != 0) {
             break;
           }
           if (!first) {
@@ -314,135 +430,157 @@ class DBTest {
 };
 
 TEST(DBTest, Empty) {
-  ASSERT_TRUE(db_ != NULL);
-  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  do {
+    ASSERT_TRUE(db_ != NULL);
+    ASSERT_EQ("NOT_FOUND", Get("foo"));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, ReadWrite) {
-  ASSERT_OK(Put("foo", "v1"));
-  ASSERT_EQ("v1", Get("foo"));
-  ASSERT_OK(Put("bar", "v2"));
-  ASSERT_OK(Put("foo", "v3"));
-  ASSERT_EQ("v3", Get("foo"));
-  ASSERT_EQ("v2", Get("bar"));
+  do {
+    ASSERT_OK(Put("foo", "v1"));
+    ASSERT_EQ("v1", Get("foo"));
+    ASSERT_OK(Put("bar", "v2"));
+    ASSERT_OK(Put("foo", "v3"));
+    ASSERT_EQ("v3", Get("foo"));
+    ASSERT_EQ("v2", Get("bar"));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, PutDeleteGet) {
-  ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
-  ASSERT_EQ("v1", Get("foo"));
-  ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
-  ASSERT_EQ("v2", Get("foo"));
-  ASSERT_OK(db_->Delete(WriteOptions(), "foo"));
-  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  do {
+    ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
+    ASSERT_EQ("v1", Get("foo"));
+    ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
+    ASSERT_EQ("v2", Get("foo"));
+    ASSERT_OK(db_->Delete(WriteOptions(), "foo"));
+    ASSERT_EQ("NOT_FOUND", Get("foo"));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, GetFromImmutableLayer) {
-  Options options;
-  options.env = env_;
-  options.write_buffer_size = 100000;  // Small write buffer
-  Reopen(&options);
+  do {
+    Options options = CurrentOptions();
+    options.env = env_;
+    options.write_buffer_size = 100000;  // Small write buffer
+    Reopen(&options);
 
-  ASSERT_OK(Put("foo", "v1"));
-  ASSERT_EQ("v1", Get("foo"));
+    ASSERT_OK(Put("foo", "v1"));
+    ASSERT_EQ("v1", Get("foo"));
 
-  env_->delay_sstable_sync_.Release_Store(env_);   // Block sync calls
-  Put("k1", std::string(100000, 'x'));             // Fill memtable
-  Put("k2", std::string(100000, 'y'));             // Trigger compaction
-  ASSERT_EQ("v1", Get("foo"));
-  env_->delay_sstable_sync_.Release_Store(NULL);   // Release sync calls
+    env_->delay_sstable_sync_.Release_Store(env_);   // Block sync calls
+    Put("k1", std::string(100000, 'x'));             // Fill memtable
+    Put("k2", std::string(100000, 'y'));             // Trigger compaction
+    ASSERT_EQ("v1", Get("foo"));
+    env_->delay_sstable_sync_.Release_Store(NULL);   // Release sync calls
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, GetFromVersions) {
-  ASSERT_OK(Put("foo", "v1"));
-  dbfull()->TEST_CompactMemTable();
-  ASSERT_EQ("v1", Get("foo"));
+  do {
+    ASSERT_OK(Put("foo", "v1"));
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_EQ("v1", Get("foo"));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, GetSnapshot) {
-  // Try with both a short key and a long key
-  for (int i = 0; i < 2; i++) {
-    std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
-    ASSERT_OK(Put(key, "v1"));
-    const Snapshot* s1 = db_->GetSnapshot();
-    ASSERT_OK(Put(key, "v2"));
-    ASSERT_EQ("v2", Get(key));
-    ASSERT_EQ("v1", Get(key, s1));
-    dbfull()->TEST_CompactMemTable();
-    ASSERT_EQ("v2", Get(key));
-    ASSERT_EQ("v1", Get(key, s1));
-    db_->ReleaseSnapshot(s1);
-  }
+  do {
+    // Try with both a short key and a long key
+    for (int i = 0; i < 2; i++) {
+      std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
+      ASSERT_OK(Put(key, "v1"));
+      const Snapshot* s1 = db_->GetSnapshot();
+      ASSERT_OK(Put(key, "v2"));
+      ASSERT_EQ("v2", Get(key));
+      ASSERT_EQ("v1", Get(key, s1));
+      dbfull()->TEST_CompactMemTable();
+      ASSERT_EQ("v2", Get(key));
+      ASSERT_EQ("v1", Get(key, s1));
+      db_->ReleaseSnapshot(s1);
+    }
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, GetLevel0Ordering) {
-  // Check that we process level-0 files in correct order.  The code
-  // below generates two level-0 files where the earlier one comes
-  // before the later one in the level-0 file list since the earlier
-  // one has a smaller "smallest" key.
-  ASSERT_OK(Put("bar", "b"));
-  ASSERT_OK(Put("foo", "v1"));
-  dbfull()->TEST_CompactMemTable();
-  ASSERT_OK(Put("foo", "v2"));
-  dbfull()->TEST_CompactMemTable();
-  ASSERT_EQ("v2", Get("foo"));
+  do {
+    // Check that we process level-0 files in correct order.  The code
+    // below generates two level-0 files where the earlier one comes
+    // before the later one in the level-0 file list since the earlier
+    // one has a smaller "smallest" key.
+    ASSERT_OK(Put("bar", "b"));
+    ASSERT_OK(Put("foo", "v1"));
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_OK(Put("foo", "v2"));
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_EQ("v2", Get("foo"));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, GetOrderedByLevels) {
-  ASSERT_OK(Put("foo", "v1"));
-  Compact("a", "z");
-  ASSERT_EQ("v1", Get("foo"));
-  ASSERT_OK(Put("foo", "v2"));
-  ASSERT_EQ("v2", Get("foo"));
-  dbfull()->TEST_CompactMemTable();
-  ASSERT_EQ("v2", Get("foo"));
+  do {
+    ASSERT_OK(Put("foo", "v1"));
+    Compact("a", "z");
+    ASSERT_EQ("v1", Get("foo"));
+    ASSERT_OK(Put("foo", "v2"));
+    ASSERT_EQ("v2", Get("foo"));
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_EQ("v2", Get("foo"));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, GetPicksCorrectFile) {
-  // Arrange to have multiple files in a non-level-0 level.
-  ASSERT_OK(Put("a", "va"));
-  Compact("a", "b");
-  ASSERT_OK(Put("x", "vx"));
-  Compact("x", "y");
-  ASSERT_OK(Put("f", "vf"));
-  Compact("f", "g");
-  ASSERT_EQ("va", Get("a"));
-  ASSERT_EQ("vf", Get("f"));
-  ASSERT_EQ("vx", Get("x"));
+  do {
+    // Arrange to have multiple files in a non-level-0 level.
+    ASSERT_OK(Put("a", "va"));
+    Compact("a", "b");
+    ASSERT_OK(Put("x", "vx"));
+    Compact("x", "y");
+    ASSERT_OK(Put("f", "vf"));
+    Compact("f", "g");
+    ASSERT_EQ("va", Get("a"));
+    ASSERT_EQ("vf", Get("f"));
+    ASSERT_EQ("vx", Get("x"));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, GetEncountersEmptyLevel) {
-  // Arrange for the following to happen:
-  //   * sstable A in level 0
-  //   * nothing in level 1
-  //   * sstable B in level 2
-  // Then do enough Get() calls to arrange for an automatic compaction
-  // of sstable A.  A bug would cause the compaction to be marked as
-  // occuring at level 1 (instead of the correct level 0).
-
-  // Step 1: First place sstables in levels 0 and 2
-  int compaction_count = 0;
-  while (NumTableFilesAtLevel(0) == 0 ||
-         NumTableFilesAtLevel(2) == 0) {
-    ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2";
-    compaction_count++;
-    Put("a", "begin");
-    Put("z", "end");
-    dbfull()->TEST_CompactMemTable();
-  }
+  do {
+    // Arrange for the following to happen:
+    //   * sstable A in level 0
+    //   * nothing in level 1
+    //   * sstable B in level 2
+    // Then do enough Get() calls to arrange for an automatic compaction
+    // of sstable A.  A bug would cause the compaction to be marked as
+    // occuring at level 1 (instead of the correct level 0).
+
+    // Step 1: First place sstables in levels 0 and 2
+    int compaction_count = 0;
+    while (NumTableFilesAtLevel(0) == 0 ||
+           NumTableFilesAtLevel(2) == 0) {
+      ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2";
+      compaction_count++;
+      Put("a", "begin");
+      Put("z", "end");
+      dbfull()->TEST_CompactMemTable();
+    }
 
-  // Step 2: clear level 1 if necessary.
-  dbfull()->TEST_CompactRange(1, NULL, NULL);
-  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
-  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
-  ASSERT_EQ(NumTableFilesAtLevel(2), 1);
+    // Step 2: clear level 1 if necessary.
+    dbfull()->TEST_CompactRange(1, NULL, NULL);
+    ASSERT_EQ(NumTableFilesAtLevel(0), 1);
+    ASSERT_EQ(NumTableFilesAtLevel(1), 0);
+    ASSERT_EQ(NumTableFilesAtLevel(2), 1);
 
-  // Step 3: read until level 0 compaction disappears.
-  int read_count = 0;
-  while (NumTableFilesAtLevel(0) > 0) {
-    ASSERT_LE(read_count, 10000) << "did not trigger level 0 compaction";
-    read_count++;
-    ASSERT_EQ("NOT_FOUND", Get("missing"));
-  }
+    // Step 3: read a bunch of times
+    for (int i = 0; i < 1000; i++) {
+      ASSERT_EQ("NOT_FOUND", Get("missing"));
+    }
+
+    // Step 4: Wait for compaction to finish
+    env_->SleepForMicroseconds(1000000);
+
+    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, IterEmpty) {
@@ -620,69 +758,77 @@ TEST(DBTest, IterSmallAndLargeMix) {
 }
 
 TEST(DBTest, IterMultiWithDelete) {
-  ASSERT_OK(Put("a", "va"));
-  ASSERT_OK(Put("b", "vb"));
-  ASSERT_OK(Put("c", "vc"));
-  ASSERT_OK(Delete("b"));
-  ASSERT_EQ("NOT_FOUND", Get("b"));
+  do {
+    ASSERT_OK(Put("a", "va"));
+    ASSERT_OK(Put("b", "vb"));
+    ASSERT_OK(Put("c", "vc"));
+    ASSERT_OK(Delete("b"));
+    ASSERT_EQ("NOT_FOUND", Get("b"));
 
-  Iterator* iter = db_->NewIterator(ReadOptions());
-  iter->Seek("c");
-  ASSERT_EQ(IterStatus(iter), "c->vc");
-  iter->Prev();
-  ASSERT_EQ(IterStatus(iter), "a->va");
-  delete iter;
+    Iterator* iter = db_->NewIterator(ReadOptions());
+    iter->Seek("c");
+    ASSERT_EQ(IterStatus(iter), "c->vc");
+    iter->Prev();
+    ASSERT_EQ(IterStatus(iter), "a->va");
+    delete iter;
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, Recover) {
-  ASSERT_OK(Put("foo", "v1"));
-  ASSERT_OK(Put("baz", "v5"));
+  do {
+    ASSERT_OK(Put("foo", "v1"));
+    ASSERT_OK(Put("baz", "v5"));
 
-  Reopen();
-  ASSERT_EQ("v1", Get("foo"));
+    Reopen();
+    ASSERT_EQ("v1", Get("foo"));
 
-  ASSERT_EQ("v1", Get("foo"));
-  ASSERT_EQ("v5", Get("baz"));
-  ASSERT_OK(Put("bar", "v2"));
-  ASSERT_OK(Put("foo", "v3"));
+    ASSERT_EQ("v1", Get("foo"));
+    ASSERT_EQ("v5", Get("baz"));
+    ASSERT_OK(Put("bar", "v2"));
+    ASSERT_OK(Put("foo", "v3"));
 
-  Reopen();
-  ASSERT_EQ("v3", Get("foo"));
-  ASSERT_OK(Put("foo", "v4"));
-  ASSERT_EQ("v4", Get("foo"));
-  ASSERT_EQ("v2", Get("bar"));
-  ASSERT_EQ("v5", Get("baz"));
+    Reopen();
+    ASSERT_EQ("v3", Get("foo"));
+    ASSERT_OK(Put("foo", "v4"));
+    ASSERT_EQ("v4", Get("foo"));
+    ASSERT_EQ("v2", Get("bar"));
+    ASSERT_EQ("v5", Get("baz"));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, RecoveryWithEmptyLog) {
-  ASSERT_OK(Put("foo", "v1"));
-  ASSERT_OK(Put("foo", "v2"));
-  Reopen();
-  Reopen();
-  ASSERT_OK(Put("foo", "v3"));
-  Reopen();
-  ASSERT_EQ("v3", Get("foo"));
+  do {
+    ASSERT_OK(Put("foo", "v1"));
+    ASSERT_OK(Put("foo", "v2"));
+    Reopen();
+    Reopen();
+    ASSERT_OK(Put("foo", "v3"));
+    Reopen();
+    ASSERT_EQ("v3", Get("foo"));
+  } while (ChangeOptions());
 }
 
 // Check that writes done during a memtable compaction are recovered
 // if the database is shutdown during the memtable compaction.
 TEST(DBTest, RecoverDuringMemtableCompaction) {
-  Options options;
-  options.env = env_;
-  options.write_buffer_size = 1000000;
-  Reopen(&options);
+  do {
+    Options options = CurrentOptions();
+    options.env = env_;
+    options.write_buffer_size = 1000000;
+    Reopen(&options);
 
-  // Trigger a long memtable compaction and reopen the database during it
-  ASSERT_OK(Put("foo", "v1"));                         // Goes to 1st log file
-  ASSERT_OK(Put("big1", std::string(10000000, 'x')));  // Fills memtable
-  ASSERT_OK(Put("big2", std::string(1000, 'y')));      // Triggers compaction
-  ASSERT_OK(Put("bar", "v2"));                         // Goes to new log file
+    // Trigger a long memtable compaction and reopen the database during it
+    ASSERT_OK(Put("foo", "v1"));                         // Goes to 1st log file
+    ASSERT_OK(Put("big1", std::string(10000000, 'x')));  // Fills memtable
+    ASSERT_OK(Put("big2", std::string(1000, 'y')));      // Triggers compaction
+    ASSERT_OK(Put("bar", "v2"));                         // Goes to new log file
 
-  Reopen(&options);
-  ASSERT_EQ("v1", Get("foo"));
-  ASSERT_EQ("v2", Get("bar"));
-  ASSERT_EQ(std::string(10000000, 'x'), Get("big1"));
-  ASSERT_EQ(std::string(1000, 'y'), Get("big2"));
+    Reopen(&options);
+    ASSERT_EQ("v1", Get("foo"));
+    ASSERT_EQ("v2", Get("bar"));
+    ASSERT_EQ(std::string(10000000, 'x'), Get("big1"));
+    ASSERT_EQ(std::string(1000, 'y'), Get("big2"));
+  } while (ChangeOptions());
 }
 
 static std::string Key(int i) {
@@ -692,7 +838,7 @@ static std::string Key(int i) {
 }
 
 TEST(DBTest, MinorCompactionsHappen) {
-  Options options;
+  Options options = CurrentOptions();
   options.write_buffer_size = 10000;
   Reopen(&options);
 
@@ -718,7 +864,7 @@ TEST(DBTest, MinorCompactionsHappen) {
 
 TEST(DBTest, RecoverWithLargeLog) {
   {
-    Options options;
+    Options options = CurrentOptions();
     Reopen(&options);
     ASSERT_OK(Put("big1", std::string(200000, '1')));
     ASSERT_OK(Put("big2", std::string(200000, '2')));
@@ -729,7 +875,7 @@ TEST(DBTest, RecoverWithLargeLog) {
 
   // Make sure that if we re-open with a small write buffer size that
   // we flush table files in the middle of a large log file.
-  Options options;
+  Options options = CurrentOptions();
   options.write_buffer_size = 100000;
   Reopen(&options);
   ASSERT_EQ(NumTableFilesAtLevel(0), 3);
@@ -741,7 +887,7 @@ TEST(DBTest, RecoverWithLargeLog) {
 }
 
 TEST(DBTest, CompactionsGenerateMultipleFiles) {
-  Options options;
+  Options options = CurrentOptions();
   options.write_buffer_size = 100000000;        // Large write buffer
   Reopen(&options);
 
@@ -767,7 +913,7 @@ TEST(DBTest, CompactionsGenerateMultipleFiles) {
 }
 
 TEST(DBTest, RepeatedWritesToSameKey) {
-  Options options;
+  Options options = CurrentOptions();
   options.env = env_;
   options.write_buffer_size = 100000;  // Small write buffer
   Reopen(&options);
@@ -786,7 +932,7 @@ TEST(DBTest, RepeatedWritesToSameKey) {
 }
 
 TEST(DBTest, SparseMerge) {
-  Options options;
+  Options options = CurrentOptions();
   options.compression = kNoCompression;
   Reopen(&options);
 
@@ -837,87 +983,91 @@ static bool Between(uint64_t val, uint64_t low, uint64_t high) {
 }
 
 TEST(DBTest, ApproximateSizes) {
-  Options options;
-  options.write_buffer_size = 100000000;        // Large write buffer
-  options.compression = kNoCompression;
-  DestroyAndReopen();
+  do {
+    Options options = CurrentOptions();
+    options.write_buffer_size = 100000000;        // Large write buffer
+    options.compression = kNoCompression;
+    DestroyAndReopen();
 
-  ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
-  Reopen(&options);
-  ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
+    ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
+    Reopen(&options);
+    ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
 
-  // Write 8MB (80 values, each 100K)
-  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
-  const int N = 80;
-  Random rnd(301);
-  for (int i = 0; i < N; i++) {
-    ASSERT_OK(Put(Key(i), RandomString(&rnd, 100000)));
-  }
+    // Write 8MB (80 values, each 100K)
+    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
+    const int N = 80;
+    static const int S1 = 100000;
+    static const int S2 = 105000;  // Allow some expansion from metadata
+    Random rnd(301);
+    for (int i = 0; i < N; i++) {
+      ASSERT_OK(Put(Key(i), RandomString(&rnd, S1)));
+    }
 
-  // 0 because GetApproximateSizes() does not account for memtable space
-  ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
+    // 0 because GetApproximateSizes() does not account for memtable space
+    ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
 
-  // Check sizes across recovery by reopening a few times
-  for (int run = 0; run < 3; run++) {
-    Reopen(&options);
+    // Check sizes across recovery by reopening a few times
+    for (int run = 0; run < 3; run++) {
+      Reopen(&options);
 
-    for (int compact_start = 0; compact_start < N; compact_start += 10) {
-      for (int i = 0; i < N; i += 10) {
-        ASSERT_TRUE(Between(Size("", Key(i)), 100000*i, 100000*i + 10000));
-        ASSERT_TRUE(Between(Size("", Key(i)+".suffix"),
-                            100000 * (i+1), 100000 * (i+1) + 10000));
-        ASSERT_TRUE(Between(Size(Key(i), Key(i+10)),
-                            100000 * 10, 100000 * 10 + 10000));
+      for (int compact_start = 0; compact_start < N; compact_start += 10) {
+        for (int i = 0; i < N; i += 10) {
+          ASSERT_TRUE(Between(Size("", Key(i)), S1*i, S2*i));
+          ASSERT_TRUE(Between(Size("", Key(i)+".suffix"), S1*(i+1), S2*(i+1)));
+          ASSERT_TRUE(Between(Size(Key(i), Key(i+10)), S1*10, S2*10));
+        }
+        ASSERT_TRUE(Between(Size("", Key(50)), S1*50, S2*50));
+        ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), S1*50, S2*50));
+
+        std::string cstart_str = Key(compact_start);
+        std::string cend_str = Key(compact_start + 9);
+        Slice cstart = cstart_str;
+        Slice cend = cend_str;
+        dbfull()->TEST_CompactRange(0, &cstart, &cend);
       }
-      ASSERT_TRUE(Between(Size("", Key(50)), 5000000, 5010000));
-      ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), 5100000, 5110000));
-
-      std::string cstart_str = Key(compact_start);
-      std::string cend_str = Key(compact_start + 9);
-      Slice cstart = cstart_str;
-      Slice cend = cend_str;
-      dbfull()->TEST_CompactRange(0, &cstart, &cend);
-    }
 
-    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
-    ASSERT_GT(NumTableFilesAtLevel(1), 0);
-  }
+      ASSERT_EQ(NumTableFilesAtLevel(0), 0);
+      ASSERT_GT(NumTableFilesAtLevel(1), 0);
+    }
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
-  Options options;
-  options.compression = kNoCompression;
-  Reopen();
-
-  Random rnd(301);
-  std::string big1 = RandomString(&rnd, 100000);
-  ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000)));
-  ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000)));
-  ASSERT_OK(Put(Key(2), big1));
-  ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000)));
-  ASSERT_OK(Put(Key(4), big1));
-  ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000)));
-  ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
-  ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
-
-  // Check sizes across recovery by reopening a few times
-  for (int run = 0; run < 3; run++) {
-    Reopen(&options);
-
-    ASSERT_TRUE(Between(Size("", Key(0)), 0, 0));
-    ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000));
-    ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000));
-    ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000));
-    ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000));
-    ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000));
-    ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000));
-    ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000));
-    ASSERT_TRUE(Between(Size("", Key(8)), 550000, 551000));
-
-    ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000));
+  do {
+    Options options = CurrentOptions();
+    options.compression = kNoCompression;
+    Reopen();
 
-    dbfull()->TEST_CompactRange(0, NULL, NULL);
-  }
+    Random rnd(301);
+    std::string big1 = RandomString(&rnd, 100000);
+    ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000)));
+    ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000)));
+    ASSERT_OK(Put(Key(2), big1));
+    ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000)));
+    ASSERT_OK(Put(Key(4), big1));
+    ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000)));
+    ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
+    ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
+
+    // Check sizes across recovery by reopening a few times
+    for (int run = 0; run < 3; run++) {
+      Reopen(&options);
+
+      ASSERT_TRUE(Between(Size("", Key(0)), 0, 0));
+      ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000));
+      ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000));
+      ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000));
+      ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000));
+      ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000));
+      ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000));
+      ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000));
+      ASSERT_TRUE(Between(Size("", Key(8)), 550000, 560000));
+
+      ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000));
+
+      dbfull()->TEST_CompactRange(0, NULL, NULL);
+    }
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, IteratorPinsRef) {
@@ -943,59 +1093,63 @@ TEST(DBTest, IteratorPinsRef) {
 }
 
 TEST(DBTest, Snapshot) {
-  Put("foo", "v1");
-  const Snapshot* s1 = db_->GetSnapshot();
-  Put("foo", "v2");
-  const Snapshot* s2 = db_->GetSnapshot();
-  Put("foo", "v3");
-  const Snapshot* s3 = db_->GetSnapshot();
-
-  Put("foo", "v4");
-  ASSERT_EQ("v1", Get("foo", s1));
-  ASSERT_EQ("v2", Get("foo", s2));
-  ASSERT_EQ("v3", Get("foo", s3));
-  ASSERT_EQ("v4", Get("foo"));
-
-  db_->ReleaseSnapshot(s3);
-  ASSERT_EQ("v1", Get("foo", s1));
-  ASSERT_EQ("v2", Get("foo", s2));
-  ASSERT_EQ("v4", Get("foo"));
-
-  db_->ReleaseSnapshot(s1);
-  ASSERT_EQ("v2", Get("foo", s2));
-  ASSERT_EQ("v4", Get("foo"));
-
-  db_->ReleaseSnapshot(s2);
-  ASSERT_EQ("v4", Get("foo"));
-}
+  do {
+    Put("foo", "v1");
+    const Snapshot* s1 = db_->GetSnapshot();
+    Put("foo", "v2");
+    const Snapshot* s2 = db_->GetSnapshot();
+    Put("foo", "v3");
+    const Snapshot* s3 = db_->GetSnapshot();
+
+    Put("foo", "v4");
+    ASSERT_EQ("v1", Get("foo", s1));
+    ASSERT_EQ("v2", Get("foo", s2));
+    ASSERT_EQ("v3", Get("foo", s3));
+    ASSERT_EQ("v4", Get("foo"));
+
+    db_->ReleaseSnapshot(s3);
+    ASSERT_EQ("v1", Get("foo", s1));
+    ASSERT_EQ("v2", Get("foo", s2));
+    ASSERT_EQ("v4", Get("foo"));
 
-TEST(DBTest, HiddenValuesAreRemoved) {
-  Random rnd(301);
-  FillLevels("a", "z");
+    db_->ReleaseSnapshot(s1);
+    ASSERT_EQ("v2", Get("foo", s2));
+    ASSERT_EQ("v4", Get("foo"));
 
-  std::string big = RandomString(&rnd, 50000);
-  Put("foo", big);
-  Put("pastfoo", "v");
-  const Snapshot* snapshot = db_->GetSnapshot();
-  Put("foo", "tiny");
-  Put("pastfoo2", "v2");        // Advance sequence number one more
+    db_->ReleaseSnapshot(s2);
+    ASSERT_EQ("v4", Get("foo"));
+  } while (ChangeOptions());
+}
 
-  ASSERT_OK(dbfull()->TEST_CompactMemTable());
-  ASSERT_GT(NumTableFilesAtLevel(0), 0);
-
-  ASSERT_EQ(big, Get("foo", snapshot));
-  ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000));
-  db_->ReleaseSnapshot(snapshot);
-  ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
-  Slice x("x");
-  dbfull()->TEST_CompactRange(0, NULL, &x);
-  ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
-  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
-  ASSERT_GE(NumTableFilesAtLevel(1), 1);
-  dbfull()->TEST_CompactRange(1, NULL, &x);
-  ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
+TEST(DBTest, HiddenValuesAreRemoved) {
+  do {
+    Random rnd(301);
+    FillLevels("a", "z");
+
+    std::string big = RandomString(&rnd, 50000);
+    Put("foo", big);
+    Put("pastfoo", "v");
+    const Snapshot* snapshot = db_->GetSnapshot();
+    Put("foo", "tiny");
+    Put("pastfoo2", "v2");        // Advance sequence number one more
+
+    ASSERT_OK(dbfull()->TEST_CompactMemTable());
+    ASSERT_GT(NumTableFilesAtLevel(0), 0);
+
+    ASSERT_EQ(big, Get("foo", snapshot));
+    ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000));
+    db_->ReleaseSnapshot(snapshot);
+    ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
+    Slice x("x");
+    dbfull()->TEST_CompactRange(0, NULL, &x);
+    ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
+    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
+    ASSERT_GE(NumTableFilesAtLevel(1), 1);
+    dbfull()->TEST_CompactRange(1, NULL, &x);
+    ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
 
-  ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
+    ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, DeletionMarkers1) {
@@ -1054,85 +1208,87 @@ TEST(DBTest, DeletionMarkers2) {
 }
 
 TEST(DBTest, OverlapInLevel0) {
-  ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config";
+  do {
+    ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config";
 
-  // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
-  ASSERT_OK(Put("100", "v100"));
-  ASSERT_OK(Put("999", "v999"));
-  dbfull()->TEST_CompactMemTable();
-  ASSERT_OK(Delete("100"));
-  ASSERT_OK(Delete("999"));
-  dbfull()->TEST_CompactMemTable();
-  ASSERT_EQ("0,1,1", FilesPerLevel());
-
-  // Make files spanning the following ranges in level-0:
-  //  files[0]  200 .. 900
-  //  files[1]  300 .. 500
-  // Note that files are sorted by smallest key.
-  ASSERT_OK(Put("300", "v300"));
-  ASSERT_OK(Put("500", "v500"));
-  dbfull()->TEST_CompactMemTable();
-  ASSERT_OK(Put("200", "v200"));
-  ASSERT_OK(Put("600", "v600"));
-  ASSERT_OK(Put("900", "v900"));
-  dbfull()->TEST_CompactMemTable();
-  ASSERT_EQ("2,1,1", FilesPerLevel());
+    // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
+    ASSERT_OK(Put("100", "v100"));
+    ASSERT_OK(Put("999", "v999"));
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_OK(Delete("100"));
+    ASSERT_OK(Delete("999"));
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_EQ("0,1,1", FilesPerLevel());
+
+    // Make files spanning the following ranges in level-0:
+    //  files[0]  200 .. 900
+    //  files[1]  300 .. 500
+    // Note that files are sorted by smallest key.
+    ASSERT_OK(Put("300", "v300"));
+    ASSERT_OK(Put("500", "v500"));
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_OK(Put("200", "v200"));
+    ASSERT_OK(Put("600", "v600"));
+    ASSERT_OK(Put("900", "v900"));
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_EQ("2,1,1", FilesPerLevel());
 
-  // Compact away the placeholder files we created initially
-  dbfull()->TEST_CompactRange(1, NULL, NULL);
-  dbfull()->TEST_CompactRange(2, NULL, NULL);
-  ASSERT_EQ("2", FilesPerLevel());
+    // Compact away the placeholder files we created initially
+    dbfull()->TEST_CompactRange(1, NULL, NULL);
+    dbfull()->TEST_CompactRange(2, NULL, NULL);
+    ASSERT_EQ("2", FilesPerLevel());
 
-  // Do a memtable compaction.  Before bug-fix, the compaction would
-  // not detect the overlap with level-0 files and would incorrectly place
-  // the deletion in a deeper level.
-  ASSERT_OK(Delete("600"));
-  dbfull()->TEST_CompactMemTable();
-  ASSERT_EQ("3", FilesPerLevel());
-  ASSERT_EQ("NOT_FOUND", Get("600"));
+    // Do a memtable compaction.  Before bug-fix, the compaction would
+    // not detect the overlap with level-0 files and would incorrectly place
+    // the deletion in a deeper level.
+    ASSERT_OK(Delete("600"));
+    dbfull()->TEST_CompactMemTable();
+    ASSERT_EQ("3", FilesPerLevel());
+    ASSERT_EQ("NOT_FOUND", Get("600"));
+  } while (ChangeOptions());
 }
 
 TEST(DBTest, L0_CompactionBug_Issue44_a) {
- Reopen();
- ASSERT_OK(Put("b", "v"));
- Reopen();
- ASSERT_OK(Delete("b"));
- ASSERT_OK(Delete("a"));
- Reopen();
- ASSERT_OK(Delete("a"));
- Reopen();
- ASSERT_OK(Put("a", "v"));
- Reopen();
- Reopen();
- ASSERT_EQ("(a->v)", Contents());
- env_->SleepForMicroseconds(1000000);  // Wait for compaction to finish
- ASSERT_EQ("(a->v)", Contents());
+  Reopen();
+  ASSERT_OK(Put("b", "v"));
+  Reopen();
+  ASSERT_OK(Delete("b"));
+  ASSERT_OK(Delete("a"));
+  Reopen();
+  ASSERT_OK(Delete("a"));
+  Reopen();
+  ASSERT_OK(Put("a", "v"));
+  Reopen();
+  Reopen();
+  ASSERT_EQ("(a->v)", Contents());
+  env_->SleepForMicroseconds(1000000);  // Wait for compaction to finish
+  ASSERT_EQ("(a->v)", Contents());
 }
 
 TEST(DBTest, L0_CompactionBug_Issue44_b) {
- Reopen();
- Put("","");
- Reopen();
- Delete("e");
- Put("","");
- Reopen();
- Put("c", "cv");
- Reopen();
- Put("","");
- Reopen();
- Put("","");
- env_->SleepForMicroseconds(1000000);  // Wait for compaction to finish
- Reopen();
- Put("d","dv");
- Reopen();
- Put("","");
- Reopen();
- Delete("d");
- Delete("b");
- Reopen();
- ASSERT_EQ("(->)(c->cv)", Contents());
- env_->SleepForMicroseconds(1000000);  // Wait for compaction to finish
- ASSERT_EQ("(->)(c->cv)", Contents());
+  Reopen();
+  Put("","");
+  Reopen();
+  Delete("e");
+  Put("","");
+  Reopen();
+  Put("c", "cv");
+  Reopen();
+  Put("","");
+  Reopen();
+  Put("","");
+  env_->SleepForMicroseconds(1000000);  // Wait for compaction to finish
+  Reopen();
+  Put("d","dv");
+  Reopen();
+  Put("","");
+  Reopen();
+  Delete("d");
+  Delete("b");
+  Reopen();
+  ASSERT_EQ("(->)(c->cv)", Contents());
+  env_->SleepForMicroseconds(1000000);  // Wait for compaction to finish
+  ASSERT_EQ("(->)(c->cv)", Contents());
 }
 
 TEST(DBTest, ComparatorCheck) {
@@ -1150,7 +1306,7 @@ TEST(DBTest, ComparatorCheck) {
     }
   };
   NewComparator cmp;
-  Options new_options;
+  Options new_options = CurrentOptions();
   new_options.comparator = &cmp;
   Status s = TryReopen(&new_options);
   ASSERT_TRUE(!s.ok());
@@ -1185,9 +1341,10 @@ TEST(DBTest, CustomComparator) {
     }
   };
   NumberComparator cmp;
-  Options new_options;
+  Options new_options = CurrentOptions();
   new_options.create_if_missing = true;
   new_options.comparator = &cmp;
+  new_options.filter_policy = NULL;     // Cannot use bloom filters
   new_options.write_buffer_size = 1000;  // Compact more often
   DestroyAndReopen(&new_options);
   ASSERT_OK(Put("[10]", "ten"));
@@ -1197,6 +1354,8 @@ TEST(DBTest, CustomComparator) {
     ASSERT_EQ("ten", Get("[0xa]"));
     ASSERT_EQ("twenty", Get("[20]"));
     ASSERT_EQ("twenty", Get("[0x14]"));
+    ASSERT_EQ("NOT_FOUND", Get("[15]"));
+    ASSERT_EQ("NOT_FOUND", Get("[0xf]"));
     Compact("[0]", "[9999]");
   }
 
@@ -1285,7 +1444,7 @@ TEST(DBTest, DBOpen_Options) {
 
 // Check that number of files does not grow when we are out of space
 TEST(DBTest, NoSpace) {
-  Options options;
+  Options options = CurrentOptions();
   options.env = env_;
   Reopen(&options);
 
@@ -1294,13 +1453,37 @@ TEST(DBTest, NoSpace) {
   Compact("a", "z");
   const int num_files = CountFiles();
   env_->no_space_.Release_Store(env_);   // Force out-of-space errors
-  for (int i = 0; i < 10; i++) {
+  env_->sleep_counter_.Reset();
+  for (int i = 0; i < 5; i++) {
     for (int level = 0; level < config::kNumLevels-1; level++) {
       dbfull()->TEST_CompactRange(level, NULL, NULL);
     }
   }
   env_->no_space_.Release_Store(NULL);
-  ASSERT_LT(CountFiles(), num_files + 5);
+  ASSERT_LT(CountFiles(), num_files + 3);
+
+  // Check that compaction attempts slept after errors
+  ASSERT_GE(env_->sleep_counter_.Read(), 5);
+}
+
+TEST(DBTest, NonWritableFileSystem) {
+  Options options = CurrentOptions();
+  options.write_buffer_size = 1000;
+  options.env = env_;
+  Reopen(&options);
+  ASSERT_OK(Put("foo", "v1"));
+  env_->non_writable_.Release_Store(env_);  // Force errors for new files
+  std::string big(100000, 'x');
+  int errors = 0;
+  for (int i = 0; i < 20; i++) {
+    fprintf(stderr, "iter %d; errors %d\n", i, errors);
+    if (!Put("foo", big).ok()) {
+      errors++;
+      env_->SleepForMicroseconds(100000);
+    }
+  }
+  ASSERT_GT(errors, 0);
+  env_->non_writable_.Release_Store(NULL);
 }
 
 TEST(DBTest, FilesDeletedAfterCompaction) {
@@ -1314,6 +1497,53 @@ TEST(DBTest, FilesDeletedAfterCompaction) {
   ASSERT_EQ(CountFiles(), num_files);
 }
 
+TEST(DBTest, BloomFilter) {
+  env_->count_random_reads_ = true;
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.block_cache = NewLRUCache(0);  // Prevent cache hits
+  options.filter_policy = NewBloomFilterPolicy(10);
+  Reopen(&options);
+
+  // Populate multiple layers
+  const int N = 10000;
+  for (int i = 0; i < N; i++) {
+    ASSERT_OK(Put(Key(i), Key(i)));
+  }
+  Compact("a", "z");
+  for (int i = 0; i < N; i += 100) {
+    ASSERT_OK(Put(Key(i), Key(i)));
+  }
+  dbfull()->TEST_CompactMemTable();
+
+  // Prevent auto compactions triggered by seeks
+  env_->delay_sstable_sync_.Release_Store(env_);
+
+  // Lookup present keys.  Should rarely read from small sstable.
+  env_->random_read_counter_.Reset();
+  for (int i = 0; i < N; i++) {
+    ASSERT_EQ(Key(i), Get(Key(i)));
+  }
+  int reads = env_->random_read_counter_.Read();
+  fprintf(stderr, "%d present => %d reads\n", N, reads);
+  ASSERT_GE(reads, N);
+  ASSERT_LE(reads, N + 2*N/100);
+
+  // Lookup present keys.  Should rarely read from either sstable.
+  env_->random_read_counter_.Reset();
+  for (int i = 0; i < N; i++) {
+    ASSERT_EQ("NOT_FOUND", Get(Key(i) + ".missing"));
+  }
+  reads = env_->random_read_counter_.Read();
+  fprintf(stderr, "%d missing => %d reads\n", N, reads);
+  ASSERT_LE(reads, 3*N/100);
+
+  env_->delay_sstable_sync_.Release_Store(NULL);
+  Close();
+  delete options.block_cache;
+  delete options.filter_policy;
+}
+
 // Multi-threaded test:
 namespace {
 
@@ -1381,33 +1611,35 @@ static void MTThreadBody(void* arg) {
 }  // namespace
 
 TEST(DBTest, MultiThreaded) {
-  // Initialize state
-  MTState mt;
-  mt.test = this;
-  mt.stop.Release_Store(0);
-  for (int id = 0; id < kNumThreads; id++) {
-    mt.counter[id].Release_Store(0);
-    mt.thread_done[id].Release_Store(0);
-  }
-
-  // Start threads
-  MTThread thread[kNumThreads];
-  for (int id = 0; id < kNumThreads; id++) {
-    thread[id].state = &mt;
-    thread[id].id = id;
-    env_->StartThread(MTThreadBody, &thread[id]);
-  }
-
-  // Let them run for a while
-  env_->SleepForMicroseconds(kTestSeconds * 1000000);
-
-  // Stop the threads and wait for them to finish
-  mt.stop.Release_Store(&mt);
-  for (int id = 0; id < kNumThreads; id++) {
-    while (mt.thread_done[id].Acquire_Load() == NULL) {
-      env_->SleepForMicroseconds(100000);
+  do {
+    // Initialize state
+    MTState mt;
+    mt.test = this;
+    mt.stop.Release_Store(0);
+    for (int id = 0; id < kNumThreads; id++) {
+      mt.counter[id].Release_Store(0);
+      mt.thread_done[id].Release_Store(0);
     }
-  }
+
+    // Start threads
+    MTThread thread[kNumThreads];
+    for (int id = 0; id < kNumThreads; id++) {
+      thread[id].state = &mt;
+      thread[id].id = id;
+      env_->StartThread(MTThreadBody, &thread[id]);
+    }
+
+    // Let them run for a while
+    env_->SleepForMicroseconds(kTestSeconds * 1000000);
+
+    // Stop the threads and wait for them to finish
+    mt.stop.Release_Store(&mt);
+    for (int id = 0; id < kNumThreads; id++) {
+      while (mt.thread_done[id].Acquire_Load() == NULL) {
+        env_->SleepForMicroseconds(100000);
+      }
+    }
+  } while (ChangeOptions());
 }
 
 namespace {
@@ -1573,70 +1805,73 @@ static bool CompareIterators(int step,
 
 TEST(DBTest, Randomized) {
   Random rnd(test::RandomSeed());
-  ModelDB model(last_options_);
-  const int N = 10000;
-  const Snapshot* model_snap = NULL;
-  const Snapshot* db_snap = NULL;
-  std::string k, v;
-  for (int step = 0; step < N; step++) {
-    if (step % 100 == 0) {
-      fprintf(stderr, "Step %d of %d\n", step, N);
-    }
-    int p = rnd.Uniform(100);
-    if (p < 45) {                               // Put
-      k = RandomKey(&rnd);
-      v = RandomString(&rnd,
-                       rnd.OneIn(20)
-                       ? 100 + rnd.Uniform(100)
-                       : rnd.Uniform(8));
-      ASSERT_OK(model.Put(WriteOptions(), k, v));
-      ASSERT_OK(db_->Put(WriteOptions(), k, v));
-
-    } else if (p < 90) {                        // Delete
-      k = RandomKey(&rnd);
-      ASSERT_OK(model.Delete(WriteOptions(), k));
-      ASSERT_OK(db_->Delete(WriteOptions(), k));
-
-
-    } else {                                    // Multi-element batch
-      WriteBatch b;
-      const int num = rnd.Uniform(8);
-      for (int i = 0; i < num; i++) {
-        if (i == 0 || !rnd.OneIn(10)) {
-          k = RandomKey(&rnd);
-        } else {
-          // Periodically re-use the same key from the previous iter, so
-          // we have multiple entries in the write batch for the same key
-        }
-        if (rnd.OneIn(2)) {
-          v = RandomString(&rnd, rnd.Uniform(10));
-          b.Put(k, v);
-        } else {
-          b.Delete(k);
+  do {
+    ModelDB model(CurrentOptions());
+    const int N = 10000;
+    const Snapshot* model_snap = NULL;
+    const Snapshot* db_snap = NULL;
+    std::string k, v;
+    for (int step = 0; step < N; step++) {
+      if (step % 100 == 0) {
+        fprintf(stderr, "Step %d of %d\n", step, N);
+      }
+      // TODO(sanjay): Test Get() works
+      int p = rnd.Uniform(100);
+      if (p < 45) {                               // Put
+        k = RandomKey(&rnd);
+        v = RandomString(&rnd,
+                         rnd.OneIn(20)
+                         ? 100 + rnd.Uniform(100)
+                         : rnd.Uniform(8));
+        ASSERT_OK(model.Put(WriteOptions(), k, v));
+        ASSERT_OK(db_->Put(WriteOptions(), k, v));
+
+      } else if (p < 90) {                        // Delete
+        k = RandomKey(&rnd);
+        ASSERT_OK(model.Delete(WriteOptions(), k));
+        ASSERT_OK(db_->Delete(WriteOptions(), k));
+
+
+      } else {                                    // Multi-element batch
+        WriteBatch b;
+        const int num = rnd.Uniform(8);
+        for (int i = 0; i < num; i++) {
+          if (i == 0 || !rnd.OneIn(10)) {
+            k = RandomKey(&rnd);
+          } else {
+            // Periodically re-use the same key from the previous iter, so
+            // we have multiple entries in the write batch for the same key
+          }
+          if (rnd.OneIn(2)) {
+            v = RandomString(&rnd, rnd.Uniform(10));
+            b.Put(k, v);
+          } else {
+            b.Delete(k);
+          }
         }
+        ASSERT_OK(model.Write(WriteOptions(), &b));
+        ASSERT_OK(db_->Write(WriteOptions(), &b));
       }
-      ASSERT_OK(model.Write(WriteOptions(), &b));
-      ASSERT_OK(db_->Write(WriteOptions(), &b));
-    }
 
-    if ((step % 100) == 0) {
-      ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
-      ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
-      // Save a snapshot from each DB this time that we'll use next
-      // time we compare things, to make sure the current state is
-      // preserved with the snapshot
-      if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
-      if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
+      if ((step % 100) == 0) {
+        ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
+        ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
+        // Save a snapshot from each DB this time that we'll use next
+        // time we compare things, to make sure the current state is
+        // preserved with the snapshot
+        if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
+        if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
 
-      Reopen();
-      ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
+        Reopen();
+        ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
 
-      model_snap = model.GetSnapshot();
-      db_snap = db_->GetSnapshot();
+        model_snap = model.GetSnapshot();
+        db_snap = db_->GetSnapshot();
+      }
     }
-  }
-  if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
-  if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
+    if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
+    if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
+  } while (ChangeOptions());
 }
 
 std::string MakeKey(unsigned int num) {