@nxtedition/rocksdb 10.1.5 → 10.1.6

package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h

@@ -32,22 +32,25 @@ namespace ROCKSDB_NAMESPACE {
 class TestFSWritableFile;
 class FaultInjectionTestFS;
 
+enum class FaultInjectionIOType {
+  kRead = 0,
+  kWrite,
+  kMetadataRead,
+  kMetadataWrite,
+};
+
 struct FSFileState {
   std::string filename_;
-  ssize_t pos_;
-  ssize_t pos_at_last_sync_;
-  ssize_t pos_at_last_flush_;
+  uint64_t pos_at_last_append_ = 0;
+  uint64_t pos_at_last_sync_ = 0;
   std::string buffer_;
 
-  explicit FSFileState(const std::string& filename)
-      : filename_(filename),
-        pos_(-1),
-        pos_at_last_sync_(-1),
-        pos_at_last_flush_(-1) {}
-
-  FSFileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {}
+  explicit FSFileState(const std::string& filename = {})
+      : filename_(filename) {}
 
-  bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; }
+  bool IsFullySynced() const {
+    return pos_at_last_append_ == pos_at_last_sync_;
+  }
 
   IOStatus DropUnsyncedData();
 
@@ -69,9 +72,7 @@ class TestFSWritableFile : public FSWritableFile {
                   const DataVerificationInfo& verification_info,
                   IODebugContext* dbg) override;
   IOStatus Truncate(uint64_t size, const IOOptions& options,
-                    IODebugContext* dbg) override {
-    return target_->Truncate(size, options, dbg);
-  }
+                    IODebugContext* dbg) override;
   IOStatus Close(const IOOptions& options, IODebugContext* dbg) override;
   IOStatus Flush(const IOOptions&, IODebugContext*) override;
   IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override;
@@ -80,9 +81,7 @@ class TestFSWritableFile : public FSWritableFile {
   bool IsSyncThreadSafe() const override { return true; }
   IOStatus PositionedAppend(const Slice& data, uint64_t offset,
                             const IOOptions& options,
-                            IODebugContext* dbg) override {
-    return target_->PositionedAppend(data, offset, options, dbg);
-  }
+                            IODebugContext* dbg) override;
   IOStatus PositionedAppend(const Slice& data, uint64_t offset,
                             const IOOptions& options,
                             const DataVerificationInfo& verification_info,
@@ -104,6 +103,7 @@ class TestFSWritableFile : public FSWritableFile {
   bool writable_file_opened_;
   FaultInjectionTestFS* fs_;
   port::Mutex mutex_;
+  const bool unsync_data_loss_;
 };
 
 // A wrapper around WritableFileWriter* file
@@ -163,8 +163,10 @@ class TestFSRandomAccessFile : public FSRandomAccessFile {
 class TestFSSequentialFile : public FSSequentialFileOwnerWrapper {
  public:
   explicit TestFSSequentialFile(std::unique_ptr<FSSequentialFile>&& f,
-                                FaultInjectionTestFS* fs)
-      : FSSequentialFileOwnerWrapper(std::move(f)), fs_(fs) {}
+                                FaultInjectionTestFS* fs, std::string fname)
+      : FSSequentialFileOwnerWrapper(std::move(f)),
+        fs_(fs),
+        fname_(std::move(fname)) {}
   IOStatus Read(size_t n, const IOOptions& options, Slice* result,
                 char* scratch, IODebugContext* dbg) override;
   IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options,
@@ -173,13 +175,16 @@ class TestFSSequentialFile : public FSSequentialFileOwnerWrapper {
 
  private:
   FaultInjectionTestFS* fs_;
+  std::string fname_;
+  uint64_t read_pos_ = 0;
+  uint64_t target_read_pos_ = 0;
 };
 
 class TestFSDirectory : public FSDirectory {
  public:
   explicit TestFSDirectory(FaultInjectionTestFS* fs, std::string dirname,
                            FSDirectory* dir)
-      : fs_(fs), dirname_(dirname), dir_(dir) {}
+      : fs_(fs), dirname_(std::move(dirname)), dir_(dir) {}
   ~TestFSDirectory() {}
 
   IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override;
@@ -202,17 +207,19 @@ class FaultInjectionTestFS : public FileSystemWrapper {
       : FileSystemWrapper(base),
         filesystem_active_(true),
         filesystem_writable_(false),
-        thread_local_error_(new ThreadLocalPtr(DeleteThreadLocalErrorContext)),
-        enable_write_error_injection_(false),
-        enable_metadata_write_error_injection_(false),
-        write_error_rand_(0),
-        write_error_one_in_(0),
-        metadata_write_error_one_in_(0),
-        read_error_one_in_(0),
+        inject_unsynced_data_loss_(false),
+        read_unsynced_data_(true),
+        allow_link_open_file_(false),
+        injected_thread_local_read_error_(DeleteThreadLocalErrorContext),
+        injected_thread_local_write_error_(DeleteThreadLocalErrorContext),
+        injected_thread_local_metadata_read_error_(
+            DeleteThreadLocalErrorContext),
+        injected_thread_local_metadata_write_error_(
+            DeleteThreadLocalErrorContext),
         ingest_data_corruption_before_write_(false),
         checksum_handoff_func_type_(kCRC32c),
         fail_get_file_unique_id_(false) {}
-  virtual ~FaultInjectionTestFS() { error_.PermitUncheckedError(); }
+  virtual ~FaultInjectionTestFS() override { fs_error_.PermitUncheckedError(); }
 
   static const char* kClassName() { return "FaultInjectionTestFS"; }
   const char* Name() const override { return kClassName(); }
@@ -221,6 +228,18 @@ class FaultInjectionTestFS : public FileSystemWrapper {
                         std::unique_ptr<FSDirectory>* result,
                         IODebugContext* dbg) override;
 
+  IOStatus FileExists(const std::string& fname, const IOOptions& options,
+                      IODebugContext* dbg) override;
+
+  IOStatus GetChildren(const std::string& dir, const IOOptions& options,
+                       std::vector<std::string>* result,
+                       IODebugContext* dbg) override;
+
+  IOStatus GetChildrenFileAttributes(const std::string& dir,
+                                     const IOOptions& options,
+                                     std::vector<FileAttributes>* result,
+                                     IODebugContext* dbg) override;
+
   IOStatus NewWritableFile(const std::string& fname,
                            const FileOptions& file_opts,
                            std::unique_ptr<FSWritableFile>* result,
@@ -253,26 +272,51 @@ class FaultInjectionTestFS : public FileSystemWrapper {
   IOStatus DeleteFile(const std::string& f, const IOOptions& options,
                       IODebugContext* dbg) override;
 
+  IOStatus GetFileSize(const std::string& f, const IOOptions& options,
+                       uint64_t* file_size, IODebugContext* dbg) override;
+
+  IOStatus GetFileModificationTime(const std::string& fname,
+                                   const IOOptions& options,
+                                   uint64_t* file_mtime,
+                                   IODebugContext* dbg) override;
+
   IOStatus RenameFile(const std::string& s, const std::string& t,
                       const IOOptions& options, IODebugContext* dbg) override;
 
   IOStatus LinkFile(const std::string& src, const std::string& target,
                     const IOOptions& options, IODebugContext* dbg) override;
 
+  IOStatus NumFileLinks(const std::string& fname, const IOOptions& options,
+                        uint64_t* count, IODebugContext* dbg) override;
+
+  IOStatus AreFilesSame(const std::string& first, const std::string& second,
+                        const IOOptions& options, bool* res,
+                        IODebugContext* dbg) override;
+  IOStatus GetAbsolutePath(const std::string& db_path, const IOOptions& options,
+                           std::string* output_path,
+                           IODebugContext* dbg) override;
+
 // Undef to eliminate clash on Windows
 #undef GetFreeSpace
   IOStatus GetFreeSpace(const std::string& path, const IOOptions& options,
                         uint64_t* disk_free, IODebugContext* dbg) override {
     IOStatus io_s;
     if (!IsFilesystemActive() &&
-        error_.subcode() == IOStatus::SubCode::kNoSpace) {
+        fs_error_.subcode() == IOStatus::SubCode::kNoSpace) {
       *disk_free = 0;
     } else {
-      io_s = target()->GetFreeSpace(path, options, disk_free, dbg);
+      io_s = MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead,
+                                         options);
+      if (io_s.ok()) {
+        io_s = target()->GetFreeSpace(path, options, disk_free, dbg);
+      }
     }
     return io_s;
   }
 
+  IOStatus IsDirectory(const std::string& path, const IOOptions& options,
+                       bool* is_dir, IODebugContext* dgb) override;
+
   IOStatus Poll(std::vector<void*>& io_handles,
                 size_t min_completions) override;
 
@@ -316,25 +360,12 @@ class FaultInjectionTestFS : public FileSystemWrapper {
     MutexLock l(&mutex_);
     return filesystem_writable_;
   }
-  bool ShouldUseDiretWritable(const std::string& file_name) {
-    MutexLock l(&mutex_);
-    if (filesystem_writable_) {
-      return true;
-    }
-    FileType file_type = kTempFile;
-    uint64_t file_number = 0;
-    if (!TryParseFileName(file_name, &file_number, &file_type)) {
-      return false;
-    }
-    return direct_writable_types_.find(file_type) !=
-           direct_writable_types_.end();
-  }
   void SetFilesystemActiveNoLock(
       bool active, IOStatus error = IOStatus::Corruption("Not active")) {
     error.PermitUncheckedError();
     filesystem_active_ = active;
     if (!active) {
-      error_ = error;
+      fs_error_ = error;
     }
   }
   void SetFilesystemActive(
@@ -347,14 +378,56 @@ class FaultInjectionTestFS : public FileSystemWrapper {
     MutexLock l(&mutex_);
     filesystem_writable_ = writable;
   }
+
+  // If true, we buffer write data in memory to simulate data loss upon system
+  // crash by only having process crashes
+  void SetInjectUnsyncedDataLoss(bool inject) {
+    MutexLock l(&mutex_);
+    inject_unsynced_data_loss_ = inject;
+  }
+
+  bool InjectUnsyncedDataLoss() {
+    MutexLock l(&mutex_);
+    return inject_unsynced_data_loss_;
+  }
+
+  // In places (e.g. GetSortedWals()) RocksDB relies on querying the file size
+  // or even reading the contents of files currently open for writing, and
+  // as in POSIX semantics, expects to see the flushed size and contents
+  // regardless of what has been synced. FaultInjectionTestFS historically
+  // did not emulate this behavior, only showing synced data from such read
+  // operations. (Different from FaultInjectionTestEnv--sigh.) Calling this
+  // function with false restores this historical behavior for testing
+  // stability, but use of this semantics must be phased out as it is
+  // inconsistent with expected FileSystem semantics. In other words, this
+  // functionality is DEPRECATED. Intended to be set after construction and
+  // unchanged (not thread safe).
+  void SetReadUnsyncedData(bool read_unsynced_data) {
+    read_unsynced_data_ = read_unsynced_data;
+  }
+  bool ReadUnsyncedData() const { return read_unsynced_data_; }
+
+  // FaultInjectionTestFS normally includes a hygiene check for FileSystem
+  // implementations that only support LinkFile() on closed files (not open
+  // for write). Setting this to true bypasses the check.
+  void SetAllowLinkOpenFile(bool allow_link_open_file = true) {
+    allow_link_open_file_ = allow_link_open_file;
+  }
+
+  bool ShouldIOActivtiesExcludedFromFaultInjection(Env::IOActivity io_activty) {
+    MutexLock l(&mutex_);
+    return io_activties_excluded_from_fault_injection.find(io_activty) !=
+           io_activties_excluded_from_fault_injection.end();
+  }
+
   void AssertNoOpenFile() { assert(open_managed_files_.empty()); }
 
-  IOStatus GetError() { return error_; }
+  IOStatus GetError() { return fs_error_; }
 
   void SetFileSystemIOError(IOStatus io_error) {
     MutexLock l(&mutex_);
     io_error.PermitUncheckedError();
-    error_ = io_error;
+    fs_error_ = io_error;
   }
 
   // To simulate the data corruption before data is written in FS
@@ -399,23 +472,19 @@ class FaultInjectionTestFS : public FileSystemWrapper {
     kMultiReadSingleReq = 1,
     kMultiRead = 2,
     kOpen,
+    kUnknown,
   };
 
-  // Set thread-local parameters for error injection. The first argument,
-  // seed is the seed for the random number generator, and one_in determines
-  // the probability of injecting error (i.e an error is injected with
-  // 1/one_in probability)
-  void SetThreadLocalReadErrorContext(uint32_t seed, int one_in,
-                                      bool retryable) {
-    struct ErrorContext* ctx =
-        static_cast<struct ErrorContext*>(thread_local_error_->Get());
-    if (ctx == nullptr) {
-      ctx = new ErrorContext(seed);
-      thread_local_error_->Reset(ctx);
-    }
-    ctx->one_in = one_in;
-    ctx->count = 0;
-    ctx->retryable = retryable;
+  void SetThreadLocalErrorContext(FaultInjectionIOType type, uint32_t seed,
+                                  int one_in, bool retryable,
+                                  bool has_data_loss) {
+    struct ErrorContext* new_ctx = new ErrorContext(seed);
+    new_ctx->one_in = one_in;
+    new_ctx->count = 0;
+    new_ctx->retryable = retryable;
+    new_ctx->has_data_loss = has_data_loss;
+
+    SetErrorContextOfFaultInjectionIOType(type, new_ctx);
   }
 
   static void DeleteThreadLocalErrorContext(void* p) {
@@ -423,112 +492,69 @@ class FaultInjectionTestFS : public FileSystemWrapper {
     delete ctx;
   }
 
-  // This is to set the parameters for the write error injection.
-  // seed is the seed for the random number generator, and one_in determines
-  // the probability of injecting error (i.e an error is injected with
-  // 1/one_in probability). For write error, we can specify the error we
-  // want to inject. Types decides the file types we want to inject the
-  // error (e.g., Wal files, SST files), which is empty by default.
-  void SetRandomWriteError(uint32_t seed, int one_in, IOStatus error,
-                           bool inject_for_all_file_types,
-                           const std::vector<FileType>& types) {
-    MutexLock l(&mutex_);
-    Random tmp_rand(seed);
-    error.PermitUncheckedError();
-    error_ = error;
-    write_error_rand_ = tmp_rand;
-    write_error_one_in_ = one_in;
-    inject_for_all_file_types_ = inject_for_all_file_types;
-    write_error_allowed_types_ = types;
-  }
-
-  void SetDirectWritableTypes(const std::set<FileType>& types) {
-    MutexLock l(&mutex_);
-    direct_writable_types_ = types;
-  }
-
-  void SetRandomMetadataWriteError(int one_in) {
-    MutexLock l(&mutex_);
-    metadata_write_error_one_in_ = one_in;
-  }
-  // If the value is not 0, it is enabled. Otherwise, it is disabled.
-  void SetRandomReadError(int one_in) { read_error_one_in_ = one_in; }
-
-  bool ShouldInjectRandomReadError() {
-    auto one_in = read_error_one_in();
-    return one_in > 0 && Random::GetTLSInstance()->OneIn(one_in);
-  }
-
-  // Inject an write error with randomlized parameter and the predefined
-  // error type. Only the allowed file types will inject the write error
-  IOStatus InjectWriteError(const std::string& file_name);
-
-  // Ingest error to metadata operations.
-  IOStatus InjectMetadataWriteError();
+  IOStatus MaybeInjectThreadLocalError(
+      FaultInjectionIOType type, const IOOptions& io_options,
+      const std::string& file_name = "", ErrorOperation op = kUnknown,
+      Slice* slice = nullptr, bool direct_io = false, char* scratch = nullptr,
+      bool need_count_increase = false, bool* fault_injected = nullptr);
 
-  // Inject an error. For a READ operation, a status of IOError(), a
-  // corruption in the contents of scratch, or truncation of slice
-  // are the types of error with equal probability. For OPEN,
-  // its always an IOError.
-  // fault_injected returns whether a fault is injected. It is needed
-  // because some fault is inected with IOStatus to be OK.
-  IOStatus InjectThreadSpecificReadError(ErrorOperation op, Slice* slice,
-                                         bool direct_io, char* scratch,
-                                         bool need_count_increase,
-                                         bool* fault_injected);
-
-  // Get the count of how many times we injected since the previous call
-  int GetAndResetErrorCount() {
-    ErrorContext* ctx = static_cast<ErrorContext*>(thread_local_error_->Get());
+  int GetAndResetInjectedThreadLocalErrorCount(FaultInjectionIOType type) {
+    ErrorContext* ctx = GetErrorContextFromFaultInjectionIOType(type);
     int count = 0;
-    if (ctx != nullptr) {
+    if (ctx) {
       count = ctx->count;
       ctx->count = 0;
     }
     return count;
   }
 
-  void EnableErrorInjection() {
-    ErrorContext* ctx = static_cast<ErrorContext*>(thread_local_error_->Get());
-    if (ctx) {
-      ctx->enable_error_injection = true;
-    }
-  }
-
-  void EnableWriteErrorInjection() {
+  void SetIOActivtiesExcludedFromFaultInjection(
+      const std::set<Env::IOActivity>& io_activties) {
     MutexLock l(&mutex_);
-    enable_write_error_injection_ = true;
+    io_activties_excluded_from_fault_injection = io_activties;
   }
-  void EnableMetadataWriteErrorInjection() {
+
+  void SetFileTypesExcludedFromWriteFaultInjection(
+      const std::set<FileType>& types) {
     MutexLock l(&mutex_);
-    enable_metadata_write_error_injection_ = true;
+    file_types_excluded_from_write_fault_injection_ = types;
   }
 
-  void DisableWriteErrorInjection() {
+  bool ShouldExcludeFromWriteFaultInjection(const std::string& file_name) {
     MutexLock l(&mutex_);
-    enable_write_error_injection_ = false;
+    FileType file_type = kTempFile;
+    uint64_t file_number = 0;
+    if (!TryParseFileName(file_name, &file_number, &file_type)) {
+      return false;
+    }
+    return file_types_excluded_from_write_fault_injection_.find(file_type) !=
+           file_types_excluded_from_write_fault_injection_.end();
   }
 
-  void DisableErrorInjection() {
-    ErrorContext* ctx = static_cast<ErrorContext*>(thread_local_error_->Get());
+  void EnableThreadLocalErrorInjection(FaultInjectionIOType type) {
+    ErrorContext* ctx = GetErrorContextFromFaultInjectionIOType(type);
     if (ctx) {
-      ctx->enable_error_injection = false;
+      ctx->enable_error_injection = true;
     }
   }
 
-  void DisableMetadataWriteErrorInjection() {
-    MutexLock l(&mutex_);
-    enable_metadata_write_error_injection_ = false;
+  void DisableThreadLocalErrorInjection(FaultInjectionIOType type) {
+    ErrorContext* ctx = GetErrorContextFromFaultInjectionIOType(type);
+    if (ctx) {
+      ctx->enable_error_injection = false;
+    }
   }
 
-  int read_error_one_in() const { return read_error_one_in_.load(); }
-
-  int write_error_one_in() const { return write_error_one_in_; }
+  void PrintInjectedThreadLocalErrorBacktrace(FaultInjectionIOType type);
 
-  // We capture a backtrace every time a fault is injected, for debugging
-  // purposes. This call prints the backtrace to stderr and frees the
-  // saved callstack
-  void PrintFaultBacktrace();
+  // If there is unsynced data in the specified file within the specified
+  // range [offset, offset + n), return the unsynced data overlapping with
+  // that range, in a corresponding range of scratch. When known, also return
+  // the position of the last sync, so that the caller can determine whether
+  // more data is available from the target file when not available from
+  // unsynced.
+  void ReadUnsynced(const std::string& fname, uint64_t offset, size_t n,
+                    Slice* result, char* scratch, int64_t* pos_at_last_sync);
 
  private:
   port::Mutex mutex_;
@@ -543,7 +569,10 @@ class FaultInjectionTestFS : public FileSystemWrapper {
   bool filesystem_active_;    // Record flushes, syncs, writes
   bool filesystem_writable_;  // Bypass FaultInjectionTestFS and go directly
                               // to underlying FS for writable files
-  IOStatus error_;
+  bool inject_unsynced_data_loss_;  // See InjectUnsyncedDataLoss()
+  bool read_unsynced_data_;   // See SetReadUnsyncedData()
+  bool allow_link_open_file_;  // See SetAllowLinkOpenFile()
+  IOStatus fs_error_;
 
   enum ErrorType : int {
     kErrorTypeStatus = 0,
@@ -562,13 +591,15 @@ class FaultInjectionTestFS : public FileSystemWrapper {
     int frames;
     ErrorType type;
     bool retryable;
+    bool has_data_loss;
 
     explicit ErrorContext(uint32_t seed)
         : rand(seed),
           enable_error_injection(false),
           callstack(nullptr),
           frames(0),
-          retryable(false) {}
+          retryable(false),
+          has_data_loss(false) {}
     ~ErrorContext() {
       if (callstack) {
         free(callstack);
@@ -576,25 +607,111 @@ class FaultInjectionTestFS : public FileSystemWrapper {
     }
   };
 
-  std::unique_ptr<ThreadLocalPtr> thread_local_error_;
-  bool enable_write_error_injection_;
-  bool enable_metadata_write_error_injection_;
-  Random write_error_rand_;
-  int write_error_one_in_;
-  int metadata_write_error_one_in_;
-  std::atomic<int> read_error_one_in_;
-  bool inject_for_all_file_types_;
-  std::vector<FileType> write_error_allowed_types_;
-  // File types where direct writable is skipped.
-  std::set<FileType> direct_writable_types_;
+  std::set<FileType> file_types_excluded_from_write_fault_injection_;
+  std::set<Env::IOActivity> io_activties_excluded_from_fault_injection;
+  ThreadLocalPtr injected_thread_local_read_error_;
+  ThreadLocalPtr injected_thread_local_write_error_;
+  ThreadLocalPtr injected_thread_local_metadata_read_error_;
+  ThreadLocalPtr injected_thread_local_metadata_write_error_;
   bool ingest_data_corruption_before_write_;
   ChecksumType checksum_handoff_func_type_;
   bool fail_get_file_unique_id_;
 
+  // Inject an error. For a READ operation, a status of IOError(), a
+  // corruption in the contents of scratch, or truncation of slice
+  // are the types of error with equal probability. For OPEN,
+  // its always an IOError.
+  // fault_injected returns whether a fault is injected. It is needed
+  // because some fault is inected with IOStatus to be OK.
+  IOStatus MaybeInjectThreadLocalReadError(const IOOptions& io_options,
+                                           ErrorOperation op, Slice* slice,
+                                           bool direct_io, char* scratch,
+                                           bool need_count_increase,
+                                           bool* fault_injected);
   // Extract number of type from file name. Return false if failing to fine
   // them.
   bool TryParseFileName(const std::string& file_name, uint64_t* number,
                         FileType* type);
+
+  ErrorContext* GetErrorContextFromFaultInjectionIOType(
+      FaultInjectionIOType type) {
+    ErrorContext* ctx = nullptr;
+    switch (type) {
+      case FaultInjectionIOType::kRead:
+        ctx = static_cast<struct ErrorContext*>(
+            injected_thread_local_read_error_.Get());
+        break;
+      case FaultInjectionIOType::kWrite:
+        ctx = static_cast<struct ErrorContext*>(
+            injected_thread_local_write_error_.Get());
+        break;
+      case FaultInjectionIOType::kMetadataRead:
+        ctx = static_cast<struct ErrorContext*>(
+            injected_thread_local_metadata_read_error_.Get());
+        break;
+      case FaultInjectionIOType::kMetadataWrite:
+        ctx = static_cast<struct ErrorContext*>(
+            injected_thread_local_metadata_write_error_.Get());
+        break;
+      default:
+        assert(false);
+        break;
+    }
+    return ctx;
+  }
+
+  void SetErrorContextOfFaultInjectionIOType(FaultInjectionIOType type,
+                                             ErrorContext* new_ctx) {
+    ErrorContext* old_ctx = nullptr;
+    switch (type) {
+      case FaultInjectionIOType::kRead:
+        old_ctx = static_cast<struct ErrorContext*>(
+            injected_thread_local_read_error_.Swap(new_ctx));
+        break;
+      case FaultInjectionIOType::kWrite:
+        old_ctx = static_cast<struct ErrorContext*>(
+            injected_thread_local_write_error_.Swap(new_ctx));
+        break;
+      case FaultInjectionIOType::kMetadataRead:
+        old_ctx = static_cast<struct ErrorContext*>(
+            injected_thread_local_metadata_read_error_.Swap(new_ctx));
+        break;
+      case FaultInjectionIOType::kMetadataWrite:
+        old_ctx = static_cast<struct ErrorContext*>(
+            injected_thread_local_metadata_write_error_.Swap(new_ctx));
+        break;
+      default:
+        assert(false);
+        break;
+    }
+
+    if (old_ctx) {
+      DeleteThreadLocalErrorContext(old_ctx);
+    }
+  }
+
+  std::string GetErrorMessageFromFaultInjectionIOType(
+      FaultInjectionIOType type) {
+    std::string msg = "";
+    switch (type) {
+      case FaultInjectionIOType::kRead:
+        msg = "injected read error";
+        break;
+      case FaultInjectionIOType::kWrite:
+        msg = "injected write error";
+        break;
+      case FaultInjectionIOType::kMetadataRead:
+        msg = "injected metadata read error";
+        break;
+      case FaultInjectionIOType::kMetadataWrite:
+        msg = "injected metadata write error";
+        break;
+      default:
+        assert(false);
+        break;
+    }
+    return msg;
+  }
 };
 
 }  // namespace ROCKSDB_NAMESPACE