@nxtedition/rocksdb 8.1.17 → 8.2.0-alpha.2

package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc

@@ -101,12 +101,9 @@ class NonBatchedOpsStressTest : public StressTest {
             if (diff > 0) {
               s = Status::NotFound();
             } else if (diff == 0) {
-              const WideColumns expected_columns = GenerateExpectedWideColumns(
-                  GetValueBase(iter->value()), iter->value());
-              if (iter->columns() != expected_columns) {
+              if (!VerifyWideColumns(iter->value(), iter->columns())) {
                 VerificationAbort(shared, static_cast<int>(cf), i,
-                                  iter->value(), iter->columns(),
-                                  expected_columns);
+                                  iter->value(), iter->columns());
               }
 
               from_db = iter->value().ToString();
@@ -159,26 +156,24 @@ class NonBatchedOpsStressTest : public StressTest {
           }
 
           const std::string key = Key(i);
-          PinnableWideColumns columns;
+          PinnableWideColumns result;
 
           Status s =
-              db_->GetEntity(options, column_families_[cf], key, &columns);
+              db_->GetEntity(options, column_families_[cf], key, &result);
 
           std::string from_db;
 
           if (s.ok()) {
-            const WideColumns& columns_from_db = columns.columns();
+            const WideColumns& columns = result.columns();
 
-            if (!columns_from_db.empty() &&
-                columns_from_db[0].name() == kDefaultWideColumnName) {
-              from_db = columns_from_db[0].value().ToString();
+            if (!columns.empty() &&
+                columns.front().name() == kDefaultWideColumnName) {
+              from_db = columns.front().value().ToString();
             }
 
-            const WideColumns expected_columns =
-                GenerateExpectedWideColumns(GetValueBase(from_db), from_db);
-            if (columns_from_db != expected_columns) {
+            if (!VerifyWideColumns(columns)) {
               VerificationAbort(shared, static_cast<int>(cf), i, from_db,
-                                columns_from_db, expected_columns);
+                                columns);
             }
           }
 
@@ -256,18 +251,16 @@ class NonBatchedOpsStressTest : public StressTest {
             std::string from_db;
 
             if (statuses[j].ok()) {
-              const WideColumns& columns_from_db = results[j].columns();
+              const WideColumns& columns = results[j].columns();
 
-              if (!columns_from_db.empty() &&
-                  columns_from_db[0].name() == kDefaultWideColumnName) {
-                from_db = columns_from_db[0].value().ToString();
+              if (!columns.empty() &&
+                  columns.front().name() == kDefaultWideColumnName) {
+                from_db = columns.front().value().ToString();
               }
 
-              const WideColumns expected_columns =
-                  GenerateExpectedWideColumns(GetValueBase(from_db), from_db);
-              if (columns_from_db != expected_columns) {
+              if (!VerifyWideColumns(columns)) {
                 VerificationAbort(shared, static_cast<int>(cf), i, from_db,
-                                  columns_from_db, expected_columns);
+                                  columns);
               }
             }
 
@@ -492,6 +485,11 @@ class NonBatchedOpsStressTest : public StressTest {
     ReadOptions read_opts_copy = read_opts;
     std::string read_ts_str;
     Slice read_ts_slice;
+    if (FLAGS_user_timestamp_size > 0) {
+      read_ts_str = GetNowNanos();
+      read_ts_slice = read_ts_str;
+      read_opts_copy.timestamp = &read_ts_slice;
+    }
     bool read_older_ts = MaybeUseOlderTimestampForPointLookup(
         thread, read_ts_str, read_ts_slice, read_opts_copy);
 
@@ -514,7 +512,7 @@ class NonBatchedOpsStressTest : public StressTest {
       // found case
       thread->stats.AddGets(1, 1);
       // we only have the latest expected state
-      if (!FLAGS_skip_verifydb && !read_opts_copy.timestamp &&
+      if (!FLAGS_skip_verifydb && !read_older_ts &&
           thread->shared->Get(rand_column_families[0], rand_keys[0]) ==
               SharedState::DELETION_SENTINEL) {
         thread->shared->SetVerificationFailure();
@@ -751,6 +749,104 @@ class NonBatchedOpsStressTest : public StressTest {
     return statuses;
   }
 
+  void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts,
+                     const std::vector<int>& rand_column_families,
+                     const std::vector<int64_t>& rand_keys) override {
+    if (fault_fs_guard) {
+      fault_fs_guard->EnableErrorInjection();
+      SharedState::ignore_read_error = false;
+    }
+
+    assert(thread);
+
+    SharedState* const shared = thread->shared;
+    assert(shared);
+
+    assert(!rand_column_families.empty());
+    assert(!rand_keys.empty());
+
+    std::unique_ptr<MutexLock> lock(new MutexLock(
+        shared->GetMutexForKey(rand_column_families[0], rand_keys[0])));
+
+    assert(rand_column_families[0] >= 0);
+    assert(rand_column_families[0] < static_cast<int>(column_families_.size()));
+
+    ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]];
+    assert(cfh);
+
+    const std::string key = Key(rand_keys[0]);
+
+    PinnableWideColumns from_db;
+
+    const Status s = db_->GetEntity(read_opts, cfh, key, &from_db);
+
+    int error_count = 0;
+
+    if (fault_fs_guard) {
+      error_count = fault_fs_guard->GetAndResetErrorCount();
+    }
+
+    if (s.ok()) {
+      if (fault_fs_guard) {
+        if (error_count && !SharedState::ignore_read_error) {
+          // Grab mutex so multiple threads don't try to print the
+          // stack trace at the same time
+          MutexLock l(shared->GetMutex());
+          fprintf(stderr, "Didn't get expected error from GetEntity\n");
+          fprintf(stderr, "Call stack that injected the fault\n");
+          fault_fs_guard->PrintFaultBacktrace();
+          std::terminate();
+        }
+      }
+
+      thread->stats.AddGets(1, 1);
+
+      if (!FLAGS_skip_verifydb) {
+        const WideColumns& columns = from_db.columns();
+
+        if (!VerifyWideColumns(columns)) {
+          shared->SetVerificationFailure();
+          fprintf(stderr,
+                  "error : inconsistent columns returned by GetEntity for key "
+                  "%s: %s\n",
+                  StringToHex(key).c_str(), WideColumnsToHex(columns).c_str());
+        } else if (shared->Get(rand_column_families[0], rand_keys[0]) ==
+                   SharedState::DELETION_SENTINEL) {
+          shared->SetVerificationFailure();
+          fprintf(
+              stderr,
+              "error : inconsistent values for key %s: GetEntity returns %s, "
+              "expected state does not have the key.\n",
+              StringToHex(key).c_str(), WideColumnsToHex(columns).c_str());
+        }
+      }
+    } else if (s.IsNotFound()) {
+      thread->stats.AddGets(1, 0);
+
+      if (!FLAGS_skip_verifydb) {
+        auto expected = shared->Get(rand_column_families[0], rand_keys[0]);
+        if (expected != SharedState::DELETION_SENTINEL &&
+            expected != SharedState::UNKNOWN_SENTINEL) {
+          shared->SetVerificationFailure();
+          fprintf(stderr,
+                  "error : inconsistent values for key %s: expected state has "
+                  "the key, GetEntity returns NotFound.\n",
+                  StringToHex(key).c_str());
+        }
+      }
+    } else {
+      if (error_count == 0) {
+        thread->stats.AddErrors(1);
+      } else {
+        thread->stats.AddVerifiedErrors(1);
+      }
+    }
+
+    if (fault_fs_guard) {
+      fault_fs_guard->DisableErrorInjection();
+    }
+  }
+
   Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts,
                         const std::vector<int>& rand_column_families,
                         const std::vector<int64_t>& rand_keys) override {
@@ -805,12 +901,9 @@ class NonBatchedOpsStressTest : public StressTest {
         }
       }
 
-      const WideColumns expected_columns = GenerateExpectedWideColumns(
-          GetValueBase(iter->value()), iter->value());
-      if (iter->columns() != expected_columns) {
-        s = Status::Corruption(
-            "Value and columns inconsistent",
-            DebugString(iter->value(), iter->columns(), expected_columns));
+      if (!VerifyWideColumns(iter->value(), iter->columns())) {
+        s = Status::Corruption("Value and columns inconsistent",
+                               DebugString(iter->value(), iter->columns()));
         break;
       }
     }
@@ -1263,17 +1356,15 @@ class NonBatchedOpsStressTest : public StressTest {
       assert(iter);
       assert(iter->Valid());
 
-      const WideColumns expected_columns = GenerateExpectedWideColumns(
-          GetValueBase(iter->value()), iter->value());
-      if (iter->columns() != expected_columns) {
+      if (!VerifyWideColumns(iter->value(), iter->columns())) {
         shared->SetVerificationFailure();
 
         fprintf(stderr,
                 "Verification failed for key %s: "
-                "Value and columns inconsistent: %s\n",
+                "Value and columns inconsistent: value: %s, columns: %s\n",
                 Slice(iter->key()).ToString(/* hex */ true).c_str(),
-                DebugString(iter->value(), iter->columns(), expected_columns)
-                    .c_str());
+                iter->value().ToString(/* hex */ true).c_str(),
+                WideColumnsToHex(iter->columns()).c_str());
         fprintf(stderr, "Column family: %s, op_logs: %s\n",
                 cfh->GetName().c_str(), op_logs.c_str());
 

package/deps/rocksdb/rocksdb/env/fs_posix.cc

@@ -1183,6 +1183,14 @@ class PosixFileSystem : public FileSystem {
 #endif
   }
 
+  bool use_async_io() override {
+#if defined(ROCKSDB_IOURING_PRESENT)
+    return IsIOUringEnabled();
+#else
+    return false;
+#endif
+  }
+
 #if defined(ROCKSDB_IOURING_PRESENT)
   // io_uring instance
   std::unique_ptr<ThreadLocalPtr> thread_local_io_urings_;

package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc

@@ -162,6 +162,9 @@ Status FilePrefetchBuffer::Prefetch(const IOOptions& opts,
 
   Status s = Read(opts, reader, rate_limiter_priority, read_len, chunk_len,
                   rounddown_offset, curr_);
+  if (usage_ == FilePrefetchBufferUsage::kTableOpenPrefetchTail && s.ok()) {
+    RecordInHistogram(stats_, TABLE_OPEN_PREFETCH_TAIL_READ_BYTES, read_len);
+  }
   return s;
 }
 
@@ -609,6 +612,22 @@ bool FilePrefetchBuffer::TryReadFromCache(const IOOptions& opts,
                                           Slice* result, Status* status,
                                           Env::IOPriority rate_limiter_priority,
                                           bool for_compaction /* = false */) {
+  bool ret = TryReadFromCacheUntracked(opts, reader, offset, n, result, status,
+                                       rate_limiter_priority, for_compaction);
+  if (usage_ == FilePrefetchBufferUsage::kTableOpenPrefetchTail && enable_) {
+    if (ret) {
+      RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_HIT);
+    } else {
+      RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_MISS);
+    }
+  }
+  return ret;
+}
+
+bool FilePrefetchBuffer::TryReadFromCacheUntracked(
+    const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset,
+    size_t n, Slice* result, Status* status,
+    Env::IOPriority rate_limiter_priority, bool for_compaction /* = false */) {
   if (track_min_offset_ && offset < min_offset_read_) {
     min_offset_read_ = static_cast<size_t>(offset);
   }
@@ -666,6 +685,22 @@ bool FilePrefetchBuffer::TryReadFromCacheAsync(
     const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset,
     size_t n, Slice* result, Status* status,
     Env::IOPriority rate_limiter_priority) {
+  bool ret = TryReadFromCacheAsyncUntracked(opts, reader, offset, n, result,
+                                            status, rate_limiter_priority);
+  if (usage_ == FilePrefetchBufferUsage::kTableOpenPrefetchTail && enable_) {
+    if (ret) {
+      RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_HIT);
+    } else {
+      RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_MISS);
+    }
+  }
+  return ret;
+}
+
+bool FilePrefetchBuffer::TryReadFromCacheAsyncUntracked(
+    const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset,
+    size_t n, Slice* result, Status* status,
+    Env::IOPriority rate_limiter_priority) {
   if (track_min_offset_ && offset < min_offset_read_) {
     min_offset_read_ = static_cast<size_t>(offset);
   }

package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h

@@ -54,6 +54,11 @@ struct BufferInfo {
   uint32_t pos_ = 0;
 };
 
+enum class FilePrefetchBufferUsage {
+  kTableOpenPrefetchTail,
+  kUnknown,
+};
+
 // FilePrefetchBuffer is a smart buffer to store and read data from a file.
 class FilePrefetchBuffer {
  public:
@@ -78,13 +83,13 @@ class FilePrefetchBuffer {
   // and max_readahead_size are passed in.
   // A user can construct a FilePrefetchBuffer without any arguments, but use
   // `Prefetch` to load data into the buffer.
-  FilePrefetchBuffer(size_t readahead_size = 0, size_t max_readahead_size = 0,
-                     bool enable = true, bool track_min_offset = false,
-                     bool implicit_auto_readahead = false,
-                     uint64_t num_file_reads = 0,
-                     uint64_t num_file_reads_for_auto_readahead = 0,
-                     FileSystem* fs = nullptr, SystemClock* clock = nullptr,
-                     Statistics* stats = nullptr)
+  FilePrefetchBuffer(
+      size_t readahead_size = 0, size_t max_readahead_size = 0,
+      bool enable = true, bool track_min_offset = false,
+      bool implicit_auto_readahead = false, uint64_t num_file_reads = 0,
+      uint64_t num_file_reads_for_auto_readahead = 0, FileSystem* fs = nullptr,
+      SystemClock* clock = nullptr, Statistics* stats = nullptr,
+      FilePrefetchBufferUsage usage = FilePrefetchBufferUsage::kUnknown)
       : curr_(0),
         readahead_size_(readahead_size),
         initial_auto_readahead_size_(readahead_size),
@@ -100,7 +105,8 @@ class FilePrefetchBuffer {
         explicit_prefetch_submitted_(false),
         fs_(fs),
         clock_(clock),
-        stats_(stats) {
+        stats_(stats),
+        usage_(usage) {
     assert((num_file_reads_ >= num_file_reads_for_auto_readahead_ + 1) ||
            (num_file_reads_ == 0));
     // If ReadOptions.async_io is enabled, data is asynchronously filled in
@@ -403,6 +409,19 @@ class FilePrefetchBuffer {
                                bool& copy_to_third_buffer, uint64_t& tmp_offset,
                                size_t& tmp_length);
 
+  bool TryReadFromCacheUntracked(const IOOptions& opts,
+                                 RandomAccessFileReader* reader,
+                                 uint64_t offset, size_t n, Slice* result,
+                                 Status* s,
+                                 Env::IOPriority rate_limiter_priority,
+                                 bool for_compaction = false);
+
+  bool TryReadFromCacheAsyncUntracked(const IOOptions& opts,
+                                      RandomAccessFileReader* reader,
+                                      uint64_t offset, size_t n, Slice* result,
+                                      Status* status,
+                                      Env::IOPriority rate_limiter_priority);
+
   std::vector<BufferInfo> bufs_;
   // curr_ represents the index for bufs_ indicating which buffer is being
   // consumed currently.
@@ -442,5 +461,7 @@ class FilePrefetchBuffer {
   FileSystem* fs_;
   SystemClock* clock_;
   Statistics* stats_;
+
+  FilePrefetchBufferUsage usage_;
 };
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/file/file_util.cc

@@ -135,7 +135,7 @@ IOStatus GenerateOneFileChecksum(
     FileChecksumGenFactory* checksum_factory,
     const std::string& requested_checksum_func_name, std::string* file_checksum,
     std::string* file_checksum_func_name,
-    size_t verify_checksums_readahead_size, bool allow_mmap_reads,
+    size_t verify_checksums_readahead_size, bool /*allow_mmap_reads*/,
     std::shared_ptr<IOTracer>& io_tracer, RateLimiter* rate_limiter,
     Env::IOPriority rate_limiter_priority) {
   if (checksum_factory == nullptr) {
@@ -196,10 +196,12 @@ IOStatus GenerateOneFileChecksum(
   size_t readahead_size = (verify_checksums_readahead_size != 0)
                               ? verify_checksums_readahead_size
                               : default_max_read_ahead_size;
-
-  FilePrefetchBuffer prefetch_buffer(readahead_size /* readahead_size */,
-                                     readahead_size /* max_readahead_size */,
-                                     !allow_mmap_reads /* enable */);
+  std::unique_ptr<char[]> buf;
+  if (reader->use_direct_io()) {
+    size_t alignment = reader->file()->GetRequiredBufferAlignment();
+    readahead_size = (readahead_size + alignment - 1) & ~(alignment - 1);
+  }
+  buf.reset(new char[readahead_size]);
 
   Slice slice;
   uint64_t offset = 0;
@@ -207,11 +209,11 @@ IOStatus GenerateOneFileChecksum(
   while (size > 0) {
     size_t bytes_to_read =
         static_cast<size_t>(std::min(uint64_t{readahead_size}, size));
-    if (!prefetch_buffer.TryReadFromCache(
-            opts, reader.get(), offset, bytes_to_read, &slice,
-            nullptr /* status */, rate_limiter_priority,
-            false /* for_compaction */)) {
-      return IOStatus::Corruption("file read failed");
+    io_s = reader->Read(opts, offset, bytes_to_read, &slice, buf.get(), nullptr,
+                        rate_limiter_priority);
+    if (!io_s.ok()) {
+      return IOStatus::Corruption("file read failed with error: " +
+                                  io_s.ToString());
     }
     if (slice.size() == 0) {
       return IOStatus::Corruption("file too small");
@@ -219,6 +221,8 @@ IOStatus GenerateOneFileChecksum(
     checksum_generator->Update(slice.data(), slice.size());
     size -= slice.size();
     offset += slice.size();
+
+    TEST_SYNC_POINT("GenerateOneFileChecksum::Chunk:0");
   }
   checksum_generator->Finalize();
   *file_checksum = checksum_generator->GetChecksum();