@nxtedition/rocksdb 13.1.5 → 13.3.0

package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_plot.py

@@ -43,9 +43,9 @@ random.shuffle(colors)
 def num_to_gb(n):
     one_gb = 1024 * 1024 * 1024
     if float(n) % one_gb == 0:
-        return "{}".format(n / one_gb)
+        return f"{n / one_gb}"
     # Keep two decimal points.
-    return "{0:.2f}".format(float(n) / one_gb)
+    return f"{float(n) / one_gb:.2f}"
 
 
 def plot_miss_stats_graphs(
@@ -57,9 +57,9 @@ def plot_miss_stats_graphs(
             continue
         if not file.endswith(file_suffix):
             continue
-        print("Processing file {}/{}".format(csv_result_dir, file))
+        print(f"Processing file {csv_result_dir}/{file}")
         mrc_file_path = csv_result_dir + "/" + file
-        with open(mrc_file_path, "r") as csvfile:
+        with open(mrc_file_path) as csvfile:
             rows = csv.reader(csvfile, delimiter=",")
             for row in rows:
                 cache_name = row[0]
@@ -67,7 +67,7 @@ def plot_miss_stats_graphs(
                 ghost_capacity = int(row[2])
                 capacity = int(row[3])
                 miss_ratio = float(row[4])
-                config = "{}-{}-{}".format(cache_name, num_shard_bits, ghost_capacity)
+                config = f"{cache_name}-{num_shard_bits}-{ghost_capacity}"
                 if config not in miss_ratios:
                     miss_ratios[config] = {}
                     miss_ratios[config]["x"] = []
@@ -83,10 +83,10 @@ def plot_miss_stats_graphs(
             plt.ylabel(ylabel)
             plt.xscale("log", basex=2)
             plt.ylim(ymin=0)
-            plt.title("{}".format(file))
+            plt.title(f"{file}")
             plt.legend()
             fig.savefig(
-                output_result_dir + "/{}.pdf".format(pdf_file_name), bbox_inches="tight"
+                output_result_dir + f"/{pdf_file_name}.pdf", bbox_inches="tight"
             )
 
 
@@ -99,9 +99,9 @@ def plot_miss_stats_diff_lru_graphs(
             continue
         if not file.endswith(file_suffix):
             continue
-        print("Processing file {}/{}".format(csv_result_dir, file))
+        print(f"Processing file {csv_result_dir}/{file}")
         mrc_file_path = csv_result_dir + "/" + file
-        with open(mrc_file_path, "r") as csvfile:
+        with open(mrc_file_path) as csvfile:
             rows = csv.reader(csvfile, delimiter=",")
             for row in rows:
                 cache_name = row[0]
@@ -109,7 +109,7 @@ def plot_miss_stats_diff_lru_graphs(
                 ghost_capacity = int(row[2])
                 capacity = int(row[3])
                 miss_ratio = float(row[4])
-                config = "{}-{}-{}".format(cache_name, num_shard_bits, ghost_capacity)
+                config = f"{cache_name}-{num_shard_bits}-{ghost_capacity}"
                 if config not in miss_ratios:
                     miss_ratios[config] = {}
                     miss_ratios[config]["x"] = []
@@ -132,10 +132,10 @@ def plot_miss_stats_diff_lru_graphs(
     plt.xlabel("Cache capacity")
     plt.ylabel(ylabel)
     plt.xscale("log", basex=2)
-    plt.title("{}".format(file))
+    plt.title(f"{file}")
     plt.legend()
     fig.savefig(
-        output_result_dir + "/{}.pdf".format(pdf_file_name), bbox_inches="tight"
+        output_result_dir + f"/{pdf_file_name}.pdf", bbox_inches="tight"
     )
 
 
@@ -226,8 +226,8 @@ def plot_line_charts(
             continue
         if not file.startswith(filename_prefix):
             continue
-        print("Processing file {}/{}".format(csv_result_dir, file))
-        with open(csv_result_dir + "/" + file, "r") as csvfile:
+        print(f"Processing file {csv_result_dir}/{file}")
+        with open(csv_result_dir + "/" + file) as csvfile:
             x, labels, label_stats = read_data_for_plot(csvfile, vertical)
             if len(x) == 0 or len(labels) == 0:
                 continue
@@ -247,11 +247,11 @@ def plot_line_charts(
 
             # Translate time unit into x labels.
             if "_60" in file:
-                plt.xlabel("{} (Minute)".format(xlabel))
+                plt.xlabel(f"{xlabel} (Minute)")
             if "_3600" in file:
-                plt.xlabel("{} (Hour)".format(xlabel))
+                plt.xlabel(f"{xlabel} (Hour)")
             plt.ylabel(ylabel)
-            plt.title("{} {}".format(title, file))
+            plt.title(f"{title} {file}")
             if legend:
                 plt.legend()
             pdf.savefig(fig)
@@ -271,13 +271,13 @@ def plot_stacked_bar_charts(
 ):
     global color_index, bar_color_maps, colors
     pdf = matplotlib.backends.backend_pdf.PdfPages(
-        "{}/{}".format(output_result_dir, pdf_name)
+        f"{output_result_dir}/{pdf_name}"
     )
     for file in os.listdir(csv_result_dir):
         if not file.endswith(filename_suffix):
             continue
-        with open(csv_result_dir + "/" + file, "r") as csvfile:
-            print("Processing file {}/{}".format(csv_result_dir, file))
+        with open(csv_result_dir + "/" + file) as csvfile:
+            print(f"Processing file {csv_result_dir}/{file}")
             x, labels, label_stats = read_data_for_plot(csvfile, vertical)
             if len(x) == 0 or len(label_stats) == 0:
                 continue
@@ -310,25 +310,25 @@ def plot_stacked_bar_charts(
                 ind, [x_prefix + x[i] for i in range(len(x))], rotation=20, fontsize=8
             )
             plt.legend(bars, labels)
-            plt.title("{} filename:{}".format(title, file))
+            plt.title(f"{title} filename:{file}")
             pdf.savefig(fig)
     pdf.close()
 
 
 def plot_heatmap(csv_result_dir, output_result_dir, filename_suffix, pdf_name, title):
     pdf = matplotlib.backends.backend_pdf.PdfPages(
-        "{}/{}".format(output_result_dir, pdf_name)
+        f"{output_result_dir}/{pdf_name}"
     )
     for file in os.listdir(csv_result_dir):
         if not file.endswith(filename_suffix):
             continue
-        csv_file_name = "{}/{}".format(csv_result_dir, file)
-        print("Processing file {}/{}".format(csv_result_dir, file))
+        csv_file_name = f"{csv_result_dir}/{file}"
+        print(f"Processing file {csv_result_dir}/{file}")
         corr_table = pd.read_csv(csv_file_name)
         corr_table = corr_table.pivot("label", "corr", "value")
         fig = plt.figure()
         sns.heatmap(corr_table, annot=True, linewidths=0.5, fmt=".2")
-        plt.title("{} filename:{}".format(title, file))
+        plt.title(f"{title} filename:{file}")
         pdf.savefig(fig)
     pdf.close()
 
@@ -360,16 +360,16 @@ def plot_correlation(csv_result_dir, output_result_dir):
     for file in os.listdir(csv_result_dir):
         if not file.endswith("correlation_input"):
             continue
-        csv_file_name = "{}/{}".format(csv_result_dir, file)
-        print("Processing file {}/{}".format(csv_result_dir, file))
+        csv_file_name = f"{csv_result_dir}/{file}"
+        print(f"Processing file {csv_result_dir}/{file}")
         corr_table = pd.read_csv(csv_file_name)
         label_str = file.split("_")[0]
         label = file[len(label_str) + 1 :]
         label = label[: len(label) - len("_correlation_input")]
 
-        output_file = "{}/{}_correlation_output".format(csv_result_dir, label_str)
+        output_file = f"{csv_result_dir}/{label_str}_correlation_output"
         if output_file not in label_str_file:
-            f = open("{}/{}_correlation_output".format(csv_result_dir, label_str), "w+")
+            f = open(f"{csv_result_dir}/{label_str}_correlation_output", "w+")
             label_str_file[output_file] = f
             f.write("label,corr,value\n")
         f = label_str_file[output_file]
@@ -666,9 +666,9 @@ if __name__ == "__main__":
         csv_abs_dir = csv_result_dir + "/" + csv_relative_dir
         result_dir = output_result_dir + "/" + csv_relative_dir
         if not os.path.isdir(csv_abs_dir):
-            print("{} is not a directory".format(csv_abs_dir))
+            print(f"{csv_abs_dir} is not a directory")
             continue
-        print("Processing experiment dir: {}".format(csv_relative_dir))
+        print(f"Processing experiment dir: {csv_relative_dir}")
         if not os.path.exists(result_dir):
             os.makedirs(result_dir)
         plot_access_count_summary(csv_abs_dir, result_dir)
@@ -698,32 +698,32 @@ if __name__ == "__main__":
             plot_miss_stats_graphs(
                 csv_abs_dir,
                 result_dir,
-                file_prefix="ml_{}_".format(time_unit),
+                file_prefix=f"ml_{time_unit}_",
                 file_suffix="p95mb",
-                ylabel="p95 number of byte miss per {} seconds".format(time_unit),
-                pdf_file_name="p95mb_per{}_seconds".format(time_unit),
+                ylabel=f"p95 number of byte miss per {time_unit} seconds",
+                pdf_file_name=f"p95mb_per{time_unit}_seconds",
             )
             plot_miss_stats_graphs(
                 csv_abs_dir,
                 result_dir,
-                file_prefix="ml_{}_".format(time_unit),
+                file_prefix=f"ml_{time_unit}_",
                 file_suffix="avgmb",
-                ylabel="Average number of byte miss per {} seconds".format(time_unit),
-                pdf_file_name="avgmb_per{}_seconds".format(time_unit),
+                ylabel=f"Average number of byte miss per {time_unit} seconds",
+                pdf_file_name=f"avgmb_per{time_unit}_seconds",
             )
             plot_miss_stats_diff_lru_graphs(
                 csv_abs_dir,
                 result_dir,
-                file_prefix="ml_{}_".format(time_unit),
+                file_prefix=f"ml_{time_unit}_",
                 file_suffix="p95mb",
-                ylabel="p95 number of byte miss per {} seconds".format(time_unit),
-                pdf_file_name="p95mb_per{}_seconds_diff_lru".format(time_unit),
+                ylabel=f"p95 number of byte miss per {time_unit} seconds",
+                pdf_file_name=f"p95mb_per{time_unit}_seconds_diff_lru",
             )
             plot_miss_stats_diff_lru_graphs(
                 csv_abs_dir,
                 result_dir,
-                file_prefix="ml_{}_".format(time_unit),
+                file_prefix=f"ml_{time_unit}_",
                 file_suffix="avgmb",
-                ylabel="Average number of byte miss per {} seconds".format(time_unit),
-                pdf_file_name="avgmb_per{}_seconds_diff_lru".format(time_unit),
+                ylabel=f"Average number of byte miss per {time_unit} seconds",
+                pdf_file_name=f"avgmb_per{time_unit}_seconds_diff_lru",
             )

package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc

@@ -153,10 +153,11 @@ DEFINE_string(
     "randomtransaction,"
     "randomreplacekeys,"
     "timeseries,"
-    "getmergeoperands,",
+    "getmergeoperands,"
     "readrandomoperands,"
     "backup,"
-    "restore"
+    "restore,"
+    "approximatememtablestats",
 
     "Comma-separated list of operations to run in the specified"
     " order. Available benchmarks:\n"
@@ -243,9 +244,14 @@ DEFINE_string(
     "operation includes a rare but possible retry in case it got "
     "`Status::Incomplete()`. This happens upon encountering more keys than "
     "have ever been seen by the thread (or eight initially)\n"
-    "\tbackup --  Create a backup of the current DB and verify that a new backup is corrected. "
+    "\tbackup --  Create a backup of the current DB and verify that a new "
+    "backup is corrected. "
     "Rate limit can be specified through --backup_rate_limit\n"
-    "\trestore -- Restore the DB from the latest backup available, rate limit can be specified through --restore_rate_limit\n");
+    "\trestore -- Restore the DB from the latest backup available, rate limit "
+    "can be specified through --restore_rate_limit\n"
+    "\tapproximatememtablestats -- Tests accuracy of "
+    "GetApproximateMemTableStats, ideally\n"
+    "after fillrandom, where actual answer is batch_size");
 
 DEFINE_int64(num, 1000000, "Number of key/values to place in database");
 
@@ -3621,6 +3627,8 @@ class Benchmark {
         fprintf(stderr, "entries_per_batch = %" PRIi64 "\n",
                 entries_per_batch_);
         method = &Benchmark::ApproximateSizeRandom;
+      } else if (name == "approximatememtablestats") {
+        method = &Benchmark::ApproximateMemtableStats;
       } else if (name == "mixgraph") {
         method = &Benchmark::MixGraph;
       } else if (name == "readmissing") {
@@ -6298,6 +6306,35 @@ class Benchmark {
     thread->stats.AddMessage(msg);
   }
 
+  void ApproximateMemtableStats(ThreadState* thread) {
+    const size_t batch_size = entries_per_batch_;
+    std::unique_ptr<const char[]> skey_guard;
+    Slice skey = AllocateKey(&skey_guard);
+    std::unique_ptr<const char[]> ekey_guard;
+    Slice ekey = AllocateKey(&ekey_guard);
+    Duration duration(FLAGS_duration, reads_);
+    if (FLAGS_num < static_cast<int64_t>(batch_size)) {
+      std::terminate();
+    }
+    uint64_t range = static_cast<uint64_t>(FLAGS_num) - batch_size;
+    auto count_hist = std::make_shared<HistogramImpl>();
+    while (!duration.Done(1)) {
+      DB* db = SelectDB(thread);
+      uint64_t start_key = thread->rand.Uniform(range);
+      GenerateKeyFromInt(start_key, FLAGS_num, &skey);
+      uint64_t end_key = start_key + batch_size;
+      GenerateKeyFromInt(end_key, FLAGS_num, &ekey);
+      uint64_t count = UINT64_MAX;
+      uint64_t size = UINT64_MAX;
+      db->GetApproximateMemTableStats({skey, ekey}, &count, &size);
+      count_hist->Add(count);
+      thread->stats.FinishedOps(nullptr, db, 1, kOthers);
+    }
+    thread->stats.AddMessage("\nReported entry count stats (expected " +
+                             std::to_string(batch_size) + "):");
+    thread->stats.AddMessage("\n" + count_hist->ToString());
+  }
+
   // Calls ApproximateSize over random key ranges.
   void ApproximateSizeRandom(ThreadState* thread) {
     int64_t size_sum = 0;

package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc

@@ -3053,6 +3053,7 @@ void DumpWalFile(Options options, const std::string& wal_file,
           status = HandleWriteBatchTimestampSizeDifference(
               &batch, running_ts_sz, recorded_ts_sz,
               TimestampSizeConsistencyMode::kVerifyConsistency,
+              /* seq_per_batch */ false, /* batch_per_txn */ true,
               /*new_batch=*/nullptr);
           if (!status.ok()) {
             std::stringstream oss;

package/deps/rocksdb/rocksdb/tools/sst_dump_test.cc

@@ -132,7 +132,7 @@ class SSTDumpToolTest : public testing::Test {
             &internal_tbl_prop_coll_factories, CompressionType::kNoCompression,
             CompressionOptions(),
             TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
-            column_family_name, unknown_level),
+            column_family_name, unknown_level, kUnknownNewestKeyTime),
         file_writer.get()));
 
     // Populate slightly more than 1K keys

package/deps/rocksdb/rocksdb/unreleased_history/add.sh

@@ -4,7 +4,20 @@
 set -e
 set -o pipefail
 
+cat << EOF
+Release note advice:
+* Speak to the RocksDB user and what is impacted in the public API
+
+Formatting with markdown:
+* This is an example list item referring to \`BlockBasedTableOptions\`.
+If not starting with "* " it will be inserted at release time. This is OK.
+*This asterisk begins italics, NOT a list item. PROBABLY WRONG
+
+EOF
+
 if [ "$1" ]; then
+  echo "Press return to continue"
+  read
   # Target file specified on command line
   TARGET="$1"
 else

package/deps/rocksdb/rocksdb/util/aligned_buffer.h

@@ -12,7 +12,7 @@
 #include <cassert>
 
 #include "port/port.h"
-
+#include "rocksdb/file_system.h"
 namespace ROCKSDB_NAMESPACE {
 
 // This file contains utilities to handle the alignment of pages and buffers.
@@ -56,7 +56,7 @@ inline size_t Rounddown(size_t x, size_t y) { return (x / y) * y; }
 //                         copy_offset, copy_len);
 class AlignedBuffer {
   size_t alignment_;
-  std::unique_ptr<char[]> buf_;
+  FSAllocationPtr buf_;
   size_t capacity_;
   size_t cursize_;
   char* bufstart_;
@@ -100,11 +100,11 @@ class AlignedBuffer {
 
   void Clear() { cursize_ = 0; }
 
-  char* Release() {
+  FSAllocationPtr Release() {
     cursize_ = 0;
     capacity_ = 0;
     bufstart_ = nullptr;
-    return buf_.release();
+    return std::move(buf_);
   }
 
   void Alignment(size_t alignment) {
@@ -113,6 +113,21 @@ class AlignedBuffer {
     alignment_ = alignment;
   }
 
+  // Points the buffer to the result without allocating extra
+  // memory or performing any data copies. Takes ownership of the
+  // FSAllocationPtr. This method is called when we want to reuse the buffer
+  // provided by the file system
+  void SetBuffer(Slice& result, FSAllocationPtr new_buf) {
+    alignment_ = 1;
+    capacity_ = result.size();
+    cursize_ = result.size();
+    buf_ = std::move(new_buf);
+    assert(buf_.get() != nullptr);
+    // Note: bufstart_ must point to result.data() and not new_buf, which can
+    // point to any arbitrary object
+    bufstart_ = const_cast<char*>(result.data());
+  }
+
   // Allocates a new buffer and sets the start position to the first aligned
   // byte.
   //
@@ -156,7 +171,11 @@ class AlignedBuffer {
 
     bufstart_ = new_bufstart;
     capacity_ = new_capacity;
-    buf_.reset(new_buf);
+    // buf_ is a FSAllocationPtr which takes in a deleter
+    // we can just wrap the regular default delete that would have been called
+    buf_ = std::unique_ptr<void, std::function<void(void*)>>(
+        static_cast<void*>(new_buf),
+        [](void* p) { delete[] static_cast<char*>(p); });
   }
 
   // Append to the buffer.

package/deps/rocksdb/rocksdb/util/compaction_job_stats_impl.cc

@@ -17,6 +17,8 @@ void CompactionJobStats::Reset() {
   num_blobs_read = 0;
   num_input_files = 0;
   num_input_files_at_output_level = 0;
+  num_filtered_input_files = 0;
+  num_filtered_input_files_at_output_level = 0;
 
   num_output_records = 0;
   num_output_files = 0;
@@ -30,6 +32,7 @@ void CompactionJobStats::Reset() {
   total_blob_bytes_read = 0;
   total_output_bytes = 0;
   total_output_bytes_blob = 0;
+  total_skipped_input_bytes = 0;
 
   num_records_replaced = 0;
 
@@ -62,6 +65,9 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) {
   num_blobs_read += stats.num_blobs_read;
   num_input_files += stats.num_input_files;
   num_input_files_at_output_level += stats.num_input_files_at_output_level;
+  num_filtered_input_files += stats.num_filtered_input_files;
+  num_filtered_input_files_at_output_level +=
+      stats.num_filtered_input_files_at_output_level;
 
   num_output_records += stats.num_output_records;
   num_output_files += stats.num_output_files;
@@ -71,6 +77,7 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) {
   total_blob_bytes_read += stats.total_blob_bytes_read;
   total_output_bytes += stats.total_output_bytes;
   total_output_bytes_blob += stats.total_output_bytes_blob;
+  total_skipped_input_bytes += stats.total_skipped_input_bytes;
 
   num_records_replaced += stats.num_records_replaced;
 

package/deps/rocksdb/rocksdb/util/file_checksum_helper.cc

@@ -9,12 +9,6 @@
 
 #include "util/file_checksum_helper.h"
 
-#include <unordered_set>
-
-#include "db/log_reader.h"
-#include "db/version_edit.h"
-#include "db/version_edit_handler.h"
-#include "file/sequence_file_reader.h"
 #include "rocksdb/utilities/customizable_util.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -91,52 +85,6 @@ std::shared_ptr<FileChecksumGenFactory> GetFileChecksumGenCrc32cFactory() {
   return default_crc32c_gen_factory;
 }
 
-Status GetFileChecksumsFromManifest(Env* src_env, const std::string& abs_path,
-                                    uint64_t manifest_file_size,
-                                    FileChecksumList* checksum_list) {
-  if (checksum_list == nullptr) {
-    return Status::InvalidArgument("checksum_list is nullptr");
-  }
-  assert(checksum_list);
-  // TODO: plumb Env::IOActivity, Env::IOPriority
-  const ReadOptions read_options;
-  checksum_list->reset();
-  Status s;
-
-  std::unique_ptr<SequentialFileReader> file_reader;
-  {
-    std::unique_ptr<FSSequentialFile> file;
-    const std::shared_ptr<FileSystem>& fs = src_env->GetFileSystem();
-    s = fs->NewSequentialFile(abs_path,
-                              fs->OptimizeForManifestRead(FileOptions()), &file,
-                              nullptr /* dbg */);
-    if (!s.ok()) {
-      return s;
-    }
-    file_reader.reset(new SequentialFileReader(std::move(file), abs_path));
-  }
-
-  struct LogReporter : public log::Reader::Reporter {
-    Status* status_ptr;
-    void Corruption(size_t /*bytes*/, const Status& st) override {
-      if (status_ptr->ok()) {
-        *status_ptr = st;
-      }
-    }
-  } reporter;
-  reporter.status_ptr = &s;
-  log::Reader reader(nullptr, std::move(file_reader), &reporter,
-                     true /* checksum */, 0 /* log_number */);
-  FileChecksumRetriever retriever(read_options, manifest_file_size,
-                                  *checksum_list);
-  retriever.Iterate(reader, &s);
-  assert(!retriever.status().ok() ||
-         manifest_file_size == std::numeric_limits<uint64_t>::max() ||
-         reader.LastRecordEnd() == manifest_file_size);
-
-  return retriever.status();
-}
-
 namespace {
 static int RegisterFileChecksumGenFactories(ObjectLibrary& library,
                                             const std::string& /*arg*/) {

package/deps/rocksdb/rocksdb/util/file_checksum_helper.h

@@ -4,13 +4,10 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #pragma once
-#include <cassert>
+
 #include <unordered_map>
 
-#include "port/port.h"
-#include "rocksdb/env.h"
 #include "rocksdb/file_checksum.h"
-#include "rocksdb/status.h"
 #include "util/coding.h"
 #include "util/crc32c.h"
 #include "util/math.h"
@@ -92,10 +89,4 @@ class FileChecksumListImpl : public FileChecksumList {
       checksum_map_;
 };
 
-// If manifest_file_size < std::numeric_limits<uint64_t>::max(), only use
-// that length prefix of the manifest file.
-Status GetFileChecksumsFromManifest(Env* src_env, const std::string& abs_path,
-                                    uint64_t manifest_file_size,
-                                    FileChecksumList* checksum_list);
-
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/util/file_reader_writer_test.cc

@@ -230,6 +230,98 @@ TEST_F(WritableFileWriterTest, IncrementalBuffer) {
   }
 }
 
+TEST_F(WritableFileWriterTest, AlignedBufferedWrites) {
+  class FakeWF : public FSWritableFile {
+   public:
+    explicit FakeWF(std::string* _file_data) : file_data_(_file_data) {}
+    ~FakeWF() override = default;
+
+    using FSWritableFile::Append;
+    IOStatus Append(const Slice& data, const IOOptions& /*options*/,
+                    IODebugContext* /*dbg*/) override {
+      EXPECT_EQ(data.size() & (data.size() - 1), 0);
+      file_data_->append(data.data(), data.size());
+      size_ += data.size();
+      return IOStatus::OK();
+    }
+    using FSWritableFile::PositionedAppend;
+    IOStatus PositionedAppend(const Slice& data, uint64_t pos,
+                              const IOOptions& /*options*/,
+                              IODebugContext* /*dbg*/) override {
+      EXPECT_TRUE(pos % 512 == 0);
+      EXPECT_TRUE(data.size() % 512 == 0);
+      file_data_->resize(pos);
+      file_data_->append(data.data(), data.size());
+      size_ += data.size();
+      return IOStatus::OK();
+    }
+
+    IOStatus Truncate(uint64_t size, const IOOptions& /*options*/,
+                      IODebugContext* /*dbg*/) override {
+      file_data_->resize(size);
+      return IOStatus::OK();
+    }
+    IOStatus Close(const IOOptions& /*options*/,
+                   IODebugContext* /*dbg*/) override {
+      return IOStatus::OK();
+    }
+    IOStatus Flush(const IOOptions& /*options*/,
+                   IODebugContext* /*dbg*/) override {
+      return IOStatus::OK();
+    }
+    IOStatus Sync(const IOOptions& /*options*/,
+                  IODebugContext* /*dbg*/) override {
+      return IOStatus::OK();
+    }
+    IOStatus Fsync(const IOOptions& /*options*/,
+                   IODebugContext* /*dbg*/) override {
+      return IOStatus::OK();
+    }
+    void SetIOPriority(Env::IOPriority /*pri*/) override {}
+    uint64_t GetFileSize(const IOOptions& /*options*/,
+                         IODebugContext* /*dbg*/) override {
+      return size_;
+    }
+    void GetPreallocationStatus(size_t* /*block_size*/,
+                                size_t* /*last_allocated_block*/) override {}
+    size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const override {
+      return 0;
+    }
+    IOStatus InvalidateCache(size_t /*offset*/, size_t /*length*/) override {
+      return IOStatus::OK();
+    }
+    bool use_direct_io() const override { return false; }
+
+    std::string* file_data_;
+    size_t size_ = 0;
+  };
+
+  Random r(301);
+  EnvOptions env_options;
+  env_options.writable_file_max_buffer_size = 64 * 1024 * 1024;
+  std::string actual;
+  std::unique_ptr<FakeWF> wf(new FakeWF(&actual));
+  std::unique_ptr<WritableFileWriter> writer(
+      new WritableFileWriter(std::move(wf), "" /* don't care */, env_options));
+
+  std::string target;
+  uint32_t left =
+      static_cast<uint32_t>(2 * env_options.writable_file_max_buffer_size);
+  ;
+  while (left > 0) {
+    uint32_t num = 4096 + r.Uniform(8192);
+    num = std::min<uint32_t>(num, left);
+    std::string random_string = r.RandomString(num);
+    ASSERT_OK(writer->Append(IOOptions(), Slice(random_string.c_str(), num)));
+    target.append(random_string.c_str(), num);
+    left -= num;
+  }
+  ASSERT_OK(writer->Flush(IOOptions()));
+  ASSERT_OK(writer->Close(IOOptions()));
+  ASSERT_EQ(target.size(), actual.size());
+  ASSERT_EQ(target, actual);
+}
+
 TEST_F(WritableFileWriterTest, BufferWithZeroCapacityDirectIO) {
   EnvOptions env_opts;
   env_opts.use_direct_writes = true;

package/deps/rocksdb/rocksdb/util/thread_operation.h

@@ -47,6 +47,7 @@ static OperationInfo global_operation_table[] = {
     {ThreadStatus::OP_VERIFY_FILE_CHECKSUMS, "VerifyFileChecksums"},
     {ThreadStatus::OP_GETENTITY, "GetEntity"},
     {ThreadStatus::OP_MULTIGETENTITY, "MultiGetEntity"},
+    {ThreadStatus::OP_READ_MANIFEST, "ReadManifest"},
 
 };