@nxtedition/rocksdb 7.0.39 → 7.0.42

package/deps/rocksdb/rocksdb/db/perf_context_test.cc

@@ -390,7 +390,7 @@ void ProfileQueries(bool enabled_time = false) {
     EXPECT_GT(hist_write_scheduling_time.Average(), 0);
 
 #ifndef NDEBUG
-    ASSERT_GT(total_db_mutex_nanos, 2000U);
+    ASSERT_LT(total_db_mutex_nanos, 100U);
 #endif
   }
 

package/deps/rocksdb/rocksdb/db/table_cache.cc

@@ -527,6 +527,27 @@ Status TableCache::GetTableProperties(
   return s;
 }
 
+Status TableCache::ApproximateKeyAnchors(
+    const ReadOptions& ro, const InternalKeyComparator& internal_comparator,
+    const FileDescriptor& fd, std::vector<TableReader::Anchor>& anchors) {
+  Status s;
+  TableReader* t = fd.table_reader;
+  Cache::Handle* handle = nullptr;
+  if (t == nullptr) {
+    s = FindTable(ro, file_options_, internal_comparator, fd, &handle);
+    if (s.ok()) {
+      t = GetTableReaderFromHandle(handle);
+    }
+  }
+  if (s.ok() && t != nullptr) {
+    s = t->ApproximateKeyAnchors(ro, anchors);
+  }
+  if (handle != nullptr) {
+    ReleaseHandle(handle);
+  }
+  return s;
+}
+
 size_t TableCache::GetMemoryUsageByTableReader(
     const FileOptions& file_options,
     const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,

package/deps/rocksdb/rocksdb/db/table_cache.h

@@ -165,6 +165,11 @@ class TableCache {
       const std::shared_ptr<const SliceTransform>& prefix_extractor = nullptr,
       bool no_io = false);
 
+  Status ApproximateKeyAnchors(const ReadOptions& ro,
+                               const InternalKeyComparator& internal_comparator,
+                               const FileDescriptor& file_meta,
+                               std::vector<TableReader::Anchor>& anchors);
+
   // Return total memory usage of the table reader of the file.
   // 0 if table reader of the file is not loaded.
   size_t GetMemoryUsageByTableReader(

package/deps/rocksdb/rocksdb/db/version_set.cc

@@ -3209,7 +3209,7 @@ void SortFileByRoundRobin(const InternalKeyComparator& icmp,
   }
 
   bool should_move_files =
-      compact_cursor->at(level).Valid() && temp->size() > 1;
+      compact_cursor->at(level).size() > 0 && temp->size() > 1;
 
   // The iterator points to the Fsize with smallest key larger than or equal to
   // the given cursor
@@ -3225,7 +3225,8 @@ void SortFileByRoundRobin(const InternalKeyComparator& icmp,
           return icmp.Compare(cursor, f.file->smallest) > 0;
         });
 
-    should_move_files = current_file_iter != temp->end();
+    should_move_files =
+        current_file_iter != temp->end() && current_file_iter != temp->begin();
   }
   if (should_move_files) {
     // Construct a local temporary vector

package/deps/rocksdb/rocksdb/db/version_set.h

@@ -146,10 +146,12 @@ class VersionStorageInfo {
   }
 
   // REQUIRES: lock is held
-  // Update the compact cursor and advance the file index so that it can point
-  // to the next cursor
-  const InternalKey& GetNextCompactCursor(int level) {
-    int cmp_idx = next_file_to_compact_by_size_[level] + 1;
+  // Update the compact cursor and advance the file index using increment
+  // so that it can point to the next cursor (increment means the number of
+  // input files in this level of the last compaction)
+  const InternalKey& GetNextCompactCursor(int level, size_t increment) {
+    int cmp_idx = next_file_to_compact_by_size_[level] + (int)increment;
+    assert(cmp_idx <= (int)files_by_compaction_pri_[level].size());
     // TODO(zichen): may need to update next_file_to_compact_by_size_
     // for parallel compaction.
     InternalKey new_cursor;

package/deps/rocksdb/rocksdb/db/version_set_test.cc

@@ -1978,6 +1978,7 @@ TEST_F(VersionSetTest, WalCreateAfterClose) {
 
 TEST_F(VersionSetTest, AddWalWithSmallerSize) {
   NewDB();
+  assert(versions_);
 
   constexpr WalNumber kLogNumber = 10;
   constexpr uint64_t kSizeInBytes = 111;
@@ -1990,6 +1991,9 @@ TEST_F(VersionSetTest, AddWalWithSmallerSize) {
 
     ASSERT_OK(LogAndApplyToDefaultCF(edit));
   }
+  // Copy for future comparison.
+  const std::map<WalNumber, WalMetadata> wals1 =
+      versions_->GetWalSet().GetWals();
 
   {
     // Add the same WAL with smaller synced size.
@@ -1998,13 +2002,11 @@ TEST_F(VersionSetTest, AddWalWithSmallerSize) {
     edit.AddWal(kLogNumber, wal);
 
     Status s = LogAndApplyToDefaultCF(edit);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(
-        s.ToString().find(
-            "WAL 10 must not have smaller synced size than previous one") !=
-        std::string::npos)
-        << s.ToString();
+    ASSERT_OK(s);
   }
+  const std::map<WalNumber, WalMetadata> wals2 =
+      versions_->GetWalSet().GetWals();
+  ASSERT_EQ(wals1, wals2);
 }
 
 TEST_F(VersionSetTest, DeleteWalsBeforeNonExistingWalNumber) {

package/deps/rocksdb/rocksdb/db/wal_edit.cc

@@ -112,26 +112,33 @@ Status WalSet::AddWal(const WalAddition& wal) {
 
   auto it = wals_.lower_bound(wal.GetLogNumber());
   bool existing = it != wals_.end() && it->first == wal.GetLogNumber();
-  if (existing && !wal.GetMetadata().HasSyncedSize()) {
-    std::stringstream ss;
-    ss << "WAL " << wal.GetLogNumber() << " is created more than once";
-    return Status::Corruption("WalSet::AddWal", ss.str());
+
+  if (!existing) {
+    wals_.insert(it, {wal.GetLogNumber(), wal.GetMetadata()});
+    return Status::OK();
   }
-  // If the WAL has synced size, it must >= the previous size.
-  if (wal.GetMetadata().HasSyncedSize() && existing &&
-      it->second.HasSyncedSize() &&
-      wal.GetMetadata().GetSyncedSizeInBytes() <
-          it->second.GetSyncedSizeInBytes()) {
+
+  assert(existing);
+  if (!wal.GetMetadata().HasSyncedSize()) {
     std::stringstream ss;
-    ss << "WAL " << wal.GetLogNumber()
-       << " must not have smaller synced size than previous one";
+    ss << "WAL " << wal.GetLogNumber() << " is created more than once";
     return Status::Corruption("WalSet::AddWal", ss.str());
   }
-  if (existing) {
-    it->second.SetSyncedSizeInBytes(wal.GetMetadata().GetSyncedSizeInBytes());
-  } else {
-    wals_.insert(it, {wal.GetLogNumber(), wal.GetMetadata()});
+
+  assert(wal.GetMetadata().HasSyncedSize());
+  if (it->second.HasSyncedSize() && wal.GetMetadata().GetSyncedSizeInBytes() <=
+                                        it->second.GetSyncedSizeInBytes()) {
+    // This is possible because version edits with different synced WAL sizes
+    // for the same WAL can be committed out-of-order. For example, thread
+    // 1 synces the first 10 bytes of 1.log, while thread 2 synces the first 20
+    // bytes of 1.log. It's possible that thread 1 calls LogAndApply() after
+    // thread 2.
+    // In this case, just return ok.
+    return Status::OK();
   }
+
+  // Update synced size for the given WAL.
+  it->second.SetSyncedSizeInBytes(wal.GetMetadata().GetSyncedSizeInBytes());
   return Status::OK();
 }
 

package/deps/rocksdb/rocksdb/db/wal_edit.h

@@ -42,6 +42,8 @@ class WalMetadata {
   uint64_t GetSyncedSizeInBytes() const { return synced_size_bytes_; }
 
  private:
+  friend bool operator==(const WalMetadata& lhs, const WalMetadata& rhs);
+  friend bool operator!=(const WalMetadata& lhs, const WalMetadata& rhs);
   // The size of WAL is unknown, used when the WAL is not synced yet or is
   // empty.
   constexpr static uint64_t kUnknownWalSize =
@@ -51,6 +53,14 @@ class WalMetadata {
   uint64_t synced_size_bytes_ = kUnknownWalSize;
 };
 
+inline bool operator==(const WalMetadata& lhs, const WalMetadata& rhs) {
+  return lhs.synced_size_bytes_ == rhs.synced_size_bytes_;
+}
+
+inline bool operator!=(const WalMetadata& lhs, const WalMetadata& rhs) {
+  return !(lhs == rhs);
+}
+
 // These tags are persisted to MANIFEST, so it's part of the user API.
 enum class WalAdditionTag : uint32_t {
   // Indicates that there are no more tags.

package/deps/rocksdb/rocksdb/db/wal_edit_test.cc

@@ -54,12 +54,11 @@ TEST(WalSet, SmallerSyncedSize) {
   constexpr uint64_t kBytes = 100;
   WalSet wals;
   ASSERT_OK(wals.AddWal(WalAddition(kNumber, WalMetadata(kBytes))));
+  const auto wals1 = wals.GetWals();
   Status s = wals.AddWal(WalAddition(kNumber, WalMetadata(0)));
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(
-      s.ToString().find(
-          "WAL 100 must not have smaller synced size than previous one") !=
-      std::string::npos);
+  const auto wals2 = wals.GetWals();
+  ASSERT_OK(s);
+  ASSERT_EQ(wals1, wals2);
 }
 
 TEST(WalSet, CreateTwice) {

package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc

@@ -148,42 +148,6 @@ void DbVerificationThread(void* v) {
   }
 }
 
-void TimestampedSnapshotsThread(void* v) {
-  assert(FLAGS_create_timestamped_snapshot_one_in > 0);
-  auto* thread = reinterpret_cast<ThreadState*>(v);
-  assert(thread);
-  SharedState* shared = thread->shared;
-  assert(shared);
-  StressTest* stress_test = shared->GetStressTest();
-  assert(stress_test);
-  while (true) {
-    {
-      MutexLock l(shared->GetMutex());
-      if (shared->ShouldStopBgThread()) {
-        shared->IncBgThreadsFinished();
-        if (shared->BgThreadsFinished()) {
-          shared->GetCondVar()->SignalAll();
-        }
-        return;
-      }
-    }
-
-    uint64_t now = db_stress_env->NowNanos();
-    std::pair<Status, std::shared_ptr<const Snapshot>> res =
-        stress_test->CreateTimestampedSnapshot(now);
-    if (res.first.ok()) {
-      assert(res.second);
-      assert(res.second->GetTimestamp() == now);
-    } else {
-      assert(!res.second);
-    }
-    constexpr uint64_t time_diff = static_cast<uint64_t>(1000) * 1000 * 1000;
-    stress_test->ReleaseOldTimestampedSnapshots(now - time_diff);
-
-    db_stress_env->SleepForMicroseconds(1000 * 1000);
-  }
-}
-
 void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz) {
   if (!FLAGS_verbose) {
     return;

package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc

@@ -84,10 +84,6 @@ bool RunStressTest(StressTest* stress) {
     shared.IncBgThreads();
   }
 
-  if (FLAGS_create_timestamped_snapshot_one_in > 0) {
-    shared.IncBgThreads();
-  }
-
   std::vector<ThreadState*> threads(n);
   for (uint32_t i = 0; i < n; i++) {
     threads[i] = new ThreadState(i, &shared);
@@ -105,12 +101,6 @@ bool RunStressTest(StressTest* stress) {
                                &continuous_verification_thread);
   }
 
-  ThreadState timestamped_snapshots_thread(0, &shared);
-  if (FLAGS_create_timestamped_snapshot_one_in > 0) {
-    db_stress_env->StartThread(TimestampedSnapshotsThread,
-                               &timestamped_snapshots_thread);
-  }
-
   // Each thread goes through the following states:
   // initializing -> wait for others to init -> read/populate/depopulate
   // wait for others to operate -> verify -> done
@@ -179,8 +169,7 @@ bool RunStressTest(StressTest* stress) {
   stress->PrintStatistics();
 
   if (FLAGS_compaction_thread_pool_adjust_interval > 0 ||
-      FLAGS_continuous_verification_interval > 0 ||
-      FLAGS_create_timestamped_snapshot_one_in > 0) {
+      FLAGS_continuous_verification_interval > 0) {
     MutexLock l(shared.GetMutex());
     shared.SetShouldStopBgThread();
     while (!shared.BgThreadsFinished()) {

package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc

@@ -421,35 +421,6 @@ void StressTest::PrintStatistics() {
   }
 }
 
-void StressTest::ReleaseOldTimestampedSnapshots(uint64_t ts) {
-#ifndef ROCKSDB_LITE
-  if (!txn_db_) {
-    return;
-  }
-  assert(txn_db_);
-  txn_db_->ReleaseTimestampedSnapshotsOlderThan(ts);
-#else
-  (void)ts;
-  fprintf(stderr, "timestamped snapshots not supported in LITE mode\n");
-  exit(1);
-#endif  // ROCKSDB_LITE
-}
-
-std::pair<Status, std::shared_ptr<const Snapshot>>
-StressTest::CreateTimestampedSnapshot(uint64_t ts) {
-#ifndef ROCKSDB_LITE
-  if (!txn_db_) {
-    return std::make_pair(Status::InvalidArgument(), nullptr);
-  }
-  assert(txn_db_);
-  return txn_db_->CreateTimestampedSnapshot(ts);
-#else
-  (void)ts;
-  fprintf(stderr, "timestamped snapshots not supported in LITE mode\n");
-  exit(1);
-#endif  // ROCKSDB_LITE
-}
-
 // Currently PreloadDb has to be single-threaded.
 void StressTest::PreloadDbAndReopenAsReadOnly(int64_t number_of_keys,
                                               SharedState* shared) {
@@ -594,6 +565,7 @@ Status StressTest::CommitTxn(Transaction* txn, ThreadState* thread) {
   if (!FLAGS_use_txn) {
     return Status::InvalidArgument("CommitTxn when FLAGS_use_txn is not set");
   }
+  assert(txn_db_);
   Status s = txn->Prepare();
   std::shared_ptr<const Snapshot> timestamped_snapshot;
   if (s.ok()) {
@@ -602,10 +574,32 @@ Status StressTest::CommitTxn(Transaction* txn, ThreadState* thread) {
       uint64_t ts = db_stress_env->NowNanos();
       s = txn->CommitAndTryCreateSnapshot(/*notifier=*/nullptr, ts,
                                           &timestamped_snapshot);
+
+      std::pair<Status, std::shared_ptr<const Snapshot>> res;
+      if (thread->tid == 0) {
+        uint64_t now = db_stress_env->NowNanos();
+        res = txn_db_->CreateTimestampedSnapshot(now);
+        if (res.first.ok()) {
+          assert(res.second);
+          assert(res.second->GetTimestamp() == now);
+          if (timestamped_snapshot) {
+            assert(res.second->GetTimestamp() >
+                   timestamped_snapshot->GetTimestamp());
+          }
+        } else {
+          assert(!res.second);
+        }
+      }
     } else {
       s = txn->Commit();
     }
   }
+  if (thread && FLAGS_create_timestamped_snapshot_one_in > 0 &&
+      thread->rand.OneInOpt(50000)) {
+    uint64_t now = db_stress_env->NowNanos();
+    constexpr uint64_t time_diff = static_cast<uint64_t>(1000) * 1000 * 1000;
+    txn_db_->ReleaseTimestampedSnapshotsOlderThan(now - time_diff);
+  }
   delete txn;
   return s;
 }

package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h

@@ -43,11 +43,6 @@ class StressTest {
 
   void PrintStatistics();
 
-  void ReleaseOldTimestampedSnapshots(uint64_t ts);
-
-  std::pair<Status, std::shared_ptr<const Snapshot>> CreateTimestampedSnapshot(
-      uint64_t ts);
-
  protected:
   Status AssertSame(DB* db, ColumnFamilyHandle* cf,
                     ThreadState::SnapshotState& snap_state);

package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc

@@ -1382,6 +1382,13 @@ Status MultiOpsTxnsStressTest::CommitAndCreateTimestampedSnapshotIfNeeded(
   } else {
     s = txn.Commit();
   }
+  assert(txn_db_);
+  if (FLAGS_create_timestamped_snapshot_one_in > 0 &&
+      thread->rand.OneInOpt(50000)) {
+    uint64_t now = db_stress_env->NowNanos();
+    constexpr uint64_t time_diff = static_cast<uint64_t>(1000) * 1000 * 1000;
+    txn_db_->ReleaseTimestampedSnapshotsOlderThan(now - time_diff);
+  }
   return s;
 }
 

package/deps/rocksdb/rocksdb/env/env_test.cc

@@ -969,10 +969,6 @@ TEST_P(EnvPosixTestWithParam, ReserveThreads) {
 }
 
 #if (defined OS_LINUX || defined OS_WIN)
-// Travis doesn't support fallocate or getting unique ID from files for whatever
-// reason.
-#ifndef TRAVIS
-
 namespace {
 bool IsSingleVarint(const std::string& s) {
   Slice slice(s);
@@ -1780,7 +1776,6 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
     ASSERT_OK(env_->DeleteFile(fname));
     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace();
 }
-#endif  // not TRAVIS
 #endif  // OS_LINUX || OS_WIN
 
 class TestLogger : public Logger {

package/deps/rocksdb/rocksdb/env/io_posix.cc

@@ -1354,8 +1354,7 @@ IOStatus PosixWritableFile::Close(const IOOptions& /*opts*/,
     // but it will be nice to log these errors.
     int dummy __attribute__((__unused__));
     dummy = ftruncate(fd_, filesize_);
-#if defined(ROCKSDB_FALLOCATE_PRESENT) && defined(FALLOC_FL_PUNCH_HOLE) && \
-    !defined(TRAVIS)
+#if defined(ROCKSDB_FALLOCATE_PRESENT) && defined(FALLOC_FL_PUNCH_HOLE)
     // in some file systems, ftruncate only trims trailing space if the
     // new file size is smaller than the current size. Calling fallocate
     // with FALLOC_FL_PUNCH_HOLE flag to explicitly release these unused
@@ -1367,11 +1366,6 @@ IOStatus PosixWritableFile::Close(const IOOptions& /*opts*/,
     //   tmpfs (since Linux 3.5)
     // We ignore error since failure of this operation does not affect
     // correctness.
-    // TRAVIS - this code does not work on TRAVIS filesystems.
-    // the FALLOC_FL_KEEP_SIZE option is expected to not change the size
-    // of the file, but it does. Simple strace report will show that.
-    // While we work with Travis-CI team to figure out if this is a
-    // quirk of Docker/AUFS, we will comment this out.
     struct stat file_stats;
     int result = fstat(fd_, &file_stats);
     // After ftruncate, we check whether ftruncate has the correct behavior.