@nxtedition/rocksdb 10.0.13 → 10.0.15

package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc

@@ -92,7 +92,7 @@ class ExternalSSTFileTest
     : public ExternalSSTFileTestBase,
       public ::testing::WithParamInterface<std::tuple<bool, bool>> {
  public:
-  ExternalSSTFileTest() {}
+  ExternalSSTFileTest() = default;
 
   Status GenerateOneExternalFile(
       const Options& options, ColumnFamilyHandle* cfh,
@@ -295,6 +295,25 @@ class ExternalSSTFileTest
   int last_file_id_ = 0;
 };
 
+TEST_F(ExternalSSTFileTest, ComparatorMismatch) {
+  Options options = CurrentOptions();
+  Options options_diff_ucmp = options;
+
+  options.comparator = BytewiseComparator();
+  options_diff_ucmp.comparator = ReverseBytewiseComparator();
+
+  SstFileWriter sst_file_writer(EnvOptions(), options_diff_ucmp);
+
+  std::string file = sst_files_dir_ + "file.sst";
+  ASSERT_OK(sst_file_writer.Open(file));
+  ASSERT_OK(sst_file_writer.Put("foo", "val"));
+  ASSERT_OK(sst_file_writer.Put("bar", "val1"));
+  ASSERT_OK(sst_file_writer.Finish());
+
+  DestroyAndReopen(options);
+  ASSERT_NOK(DeprecatedAddFile({file}));
+}
+
 TEST_F(ExternalSSTFileTest, Basic) {
   do {
     Options options = CurrentOptions();
@@ -832,7 +851,7 @@ TEST_F(ExternalSSTFileTest, AddList) {
     TablePropertiesCollection props;
     ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
     ASSERT_EQ(props.size(), 2);
-    for (auto file_props : props) {
+    for (const auto& file_props : props) {
       auto user_props = file_props.second->user_collected_properties;
       ASSERT_EQ(user_props["abc_SstFileWriterCollector"], "YES");
       ASSERT_EQ(user_props["xyz_SstFileWriterCollector"], "YES");
@@ -855,7 +874,7 @@ TEST_F(ExternalSSTFileTest, AddList) {
 
     ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
     ASSERT_EQ(props.size(), 3);
-    for (auto file_props : props) {
+    for (const auto& file_props : props) {
       auto user_props = file_props.second->user_collected_properties;
       ASSERT_EQ(user_props["abc_SstFileWriterCollector"], "YES");
       ASSERT_EQ(user_props["xyz_SstFileWriterCollector"], "YES");
@@ -1714,9 +1733,8 @@ TEST_F(ExternalSSTFileTest, WithUnorderedWrite) {
        {"DBImpl::WaitForPendingWrites:BeforeBlock",
         "DBImpl::WriteImpl:BeforeUnorderedWriteMemtable"}});
   SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::IngestExternalFile:NeedFlush", [&](void* need_flush) {
-        ASSERT_TRUE(*reinterpret_cast<bool*>(need_flush));
-      });
+      "DBImpl::IngestExternalFile:NeedFlush",
+      [&](void* need_flush) { ASSERT_TRUE(*static_cast<bool*>(need_flush)); });
 
   Options options = CurrentOptions();
   options.unordered_write = true;
@@ -1848,6 +1866,92 @@ TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
   VerifyDBFromMap(true_data, &kcnt, false);
 }
 
+TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedUniversal) {
+  bool write_global_seqno = std::get<0>(GetParam());
+  bool verify_checksums_before_ingest = std::get<1>(GetParam());
+  Options options = CurrentOptions();
+  options.num_levels = 5;
+  options.compaction_style = kCompactionStyleUniversal;
+  options.disable_auto_compactions = true;
+  DestroyAndReopen(options);
+  std::vector<std::pair<std::string, std::string>> file_data;
+  std::map<std::string, std::string> true_data;
+
+  // Write 200 -> 250 into the bottommost level
+  for (int i = 200; i <= 250; i++) {
+    ASSERT_OK(Put(Key(i), "bottommost"));
+    true_data[Key(i)] = "bottommost";
+  }
+  CompactRangeOptions cro;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_EQ("0,0,0,0,1", FilesPerLevel());
+
+  // Take a snapshot to enforce global sequence number.
+  const Snapshot* snap = db_->GetSnapshot();
+
+  // Insert 100 -> 200 into the memtable
+  for (int i = 100; i <= 200; i++) {
+    ASSERT_OK(Put(Key(i), "memtable"));
+    true_data[Key(i)] = "memtable";
+  }
+
+  // Insert 0 -> 20 using AddFile
+  file_data.clear();
+  for (int i = 0; i <= 20; i++) {
+    file_data.emplace_back(Key(i), "L4");
+  }
+
+  ASSERT_OK(GenerateAndAddExternalFile(
+      options, file_data, -1, true, write_global_seqno,
+      verify_checksums_before_ingest, false, false, &true_data));
+
+  // This file don't overlap with anything in the DB, will go to L4
+  ASSERT_EQ("0,0,0,0,2", FilesPerLevel());
+
+  // Insert 80 -> 130 using AddFile
+  file_data.clear();
+  for (int i = 80; i <= 130; i++) {
+    file_data.emplace_back(Key(i), "L0");
+  }
+  ASSERT_OK(GenerateAndAddExternalFile(
+      options, file_data, -1, true, write_global_seqno,
+      verify_checksums_before_ingest, false, false, &true_data));
+
+  // This file overlap with the memtable, so it will flush it and add
+  // it self to L0
+  ASSERT_EQ("2,0,0,0,2", FilesPerLevel());
+
+  // Insert 30 -> 50 using AddFile
+  file_data.clear();
+  for (int i = 30; i <= 50; i++) {
+    file_data.emplace_back(Key(i), "L4");
+  }
+  ASSERT_OK(GenerateAndAddExternalFile(
+      options, file_data, -1, true, write_global_seqno,
+      verify_checksums_before_ingest, false, false, &true_data));
+
+  // This file don't overlap with anything in the DB and fit in L4 as well
+  ASSERT_EQ("2,0,0,0,3", FilesPerLevel());
+
+  // Insert 10 -> 40 using AddFile
+  file_data.clear();
+  for (int i = 10; i <= 40; i++) {
+    file_data.emplace_back(Key(i), "L3");
+  }
+  ASSERT_OK(GenerateAndAddExternalFile(
+      options, file_data, -1, true, write_global_seqno,
+      verify_checksums_before_ingest, false, false, &true_data));
+
+  // This file overlap with files in L4, we will ingest it into the last
+  // non-overlapping and non-empty level, in this case, it's L0.
+  ASSERT_EQ("3,0,0,0,3", FilesPerLevel());
+
+  size_t kcnt = 0;
+  VerifyDBFromMap(true_data, &kcnt, false);
+  db_->ReleaseSnapshot(snap);
+}
+
 TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoMemtableFlush) {
   Options options = CurrentOptions();
   DestroyAndReopen(options);
@@ -2878,7 +2982,7 @@ TEST_P(ExternalSSTFileTest, IngestFilesTriggerFlushingWithTwoWriteQueue) {
   // currently at the front of the 2nd writer queue. We must make
   // sure that it won't enter the 2nd writer queue for the second time.
   std::vector<std::pair<std::string, std::string>> data;
-  data.push_back(std::make_pair("1001", "v2"));
+  data.emplace_back("1001", "v2");
   ASSERT_OK(GenerateAndAddExternalFile(options, data, -1, true));
 }
 
@@ -2978,6 +3082,593 @@ TEST_P(ExternalSSTFileTest,
   delete iter;
 }
 
+TEST_F(ExternalSSTFileTest, FIFOCompaction) {
+  // FIFO always ingests SST files to L0 and assign latest sequence number.
+  Options options = CurrentOptions();
+  options.num_levels = 1;
+  options.compaction_style = kCompactionStyleFIFO;
+  options.max_open_files = -1;
+  DestroyAndReopen(options);
+  std::map<std::string, std::string> true_data;
+
+  for (int i = 0; i < 100; ++i) {
+    ASSERT_OK(Put(Key(i), Key(i) + "_val"));
+    true_data[Key(i)] = Key(i) + "_val";
+  }
+  ASSERT_OK(Flush());
+  ASSERT_EQ("1", FilesPerLevel());
+  std::vector<std::pair<std::string, std::string>> file_data;
+  for (int i = 0; i <= 20; i++) {
+    file_data.emplace_back(Key(i), Key(i) + "_ingest");
+  }
+  // Overlaps with memtable, will trigger flush
+  ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1,
+                                       /*allow_global_seqno=*/true, true, false,
+                                       false, false, &true_data));
+  ASSERT_EQ("2", FilesPerLevel());
+
+  file_data.clear();
+  for (int i = 100; i <= 120; i++) {
+    file_data.emplace_back(Key(i), Key(i) + "_ingest");
+  }
+  // global sequence number is always assigned, so this will fail
+  ASSERT_NOK(GenerateAndAddExternalFile(options, file_data, -1,
+                                        /*allow_global_seqno=*/false, true,
+                                        false, false, false, &true_data));
+  ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1,
+                                       /*allow_global_seqno=*/true, true, false,
+                                       false, false, &true_data));
+
+  // Compact to data to lower level to test multi-level FIFO later
+  options.num_levels = 7;
+  options.compaction_style = kCompactionStyleUniversal;
+  ASSERT_OK(TryReopen(options));
+  CompactRangeOptions cro;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized;
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
+
+  options.num_levels = 7;
+  options.compaction_style = kCompactionStyleFIFO;
+  ASSERT_OK(TryReopen(options));
+  file_data.clear();
+  for (int i = 200; i <= 220; i++) {
+    file_data.emplace_back(Key(i), Key(i) + "_ingest");
+  }
+  // Files are ingested into L0 for multi-level FIFO
+  ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1,
+                                       /*allow_global_seqno=*/true, true, false,
+                                       false, false, &true_data));
+
+  ASSERT_EQ("1,0,0,0,0,0,1", FilesPerLevel());
+  VerifyDBFromMap(true_data);
+}
+
+class ExternalSSTFileWithTimestampTest : public ExternalSSTFileTest {
+ public:
+  ExternalSSTFileWithTimestampTest() = default;
+
+  static const std::string kValueNotFound;
+  static const std::string kTsNotFound;
+
+  std::string EncodeAsUint64(uint64_t v) {
+    std::string dst;
+    PutFixed64(&dst, v);
+    return dst;
+  }
+
+  Status IngestExternalUDTFile(const std::vector<std::string>& files,
+                               bool allow_global_seqno = true) {
+    IngestExternalFileOptions opts;
+    opts.snapshot_consistency = true;
+    opts.allow_global_seqno = allow_global_seqno;
+    return db_->IngestExternalFile(files, opts);
+  }
+
+  void VerifyValueAndTs(const std::string& key,
+                        const std::string& read_timestamp,
+                        const std::string& expected_value,
+                        const std::string& expected_timestamp) {
+    Slice read_ts = read_timestamp;
+    ReadOptions read_options;
+    read_options.timestamp = &read_ts;
+    std::string value;
+    std::string timestamp;
+    Status s = db_->Get(read_options, key, &value, &timestamp);
+    if (s.ok()) {
+      ASSERT_EQ(value, expected_value);
+      ASSERT_EQ(timestamp, expected_timestamp);
+    } else if (s.IsNotFound()) {
+      ASSERT_EQ(kValueNotFound, expected_value);
+      ASSERT_EQ(kTsNotFound, expected_timestamp);
+    } else {
+      assert(false);
+    }
+  }
+};
+
+const std::string ExternalSSTFileWithTimestampTest::kValueNotFound =
+    "NOT_FOUND";
+const std::string ExternalSSTFileWithTimestampTest::kTsNotFound =
+    "NOT_FOUND_TS";
+
+TEST_F(ExternalSSTFileWithTimestampTest, Basic) {
+  do {
+    Options options = CurrentOptions();
+    options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+    options.persist_user_defined_timestamps = true;
+
+    DestroyAndReopen(options);
+
+    SstFileWriter sst_file_writer(EnvOptions(), options);
+
+    // Current file size should be 0 after sst_file_writer init and before open
+    // a file.
+    ASSERT_EQ(sst_file_writer.FileSize(), 0);
+
+    // file1.sst [0, 50)
+    std::string file1 = sst_files_dir_ + "file1.sst";
+    ASSERT_OK(sst_file_writer.Open(file1));
+    for (int k = 0; k < 50; k++) {
+      // write 3 versions of values for each key, write newer version first
+      // they are treated as logically smaller by the comparator.
+      for (int version = 3; version > 0; version--) {
+        ASSERT_OK(
+            sst_file_writer.Put(Key(k), EncodeAsUint64(k + version),
+                                Key(k) + "_val" + std::to_string(version)));
+      }
+    }
+
+    ExternalSstFileInfo file1_info;
+    ASSERT_OK(sst_file_writer.Finish(&file1_info));
+    // sst_file_writer already finished, cannot add this value
+    ASSERT_NOK(sst_file_writer.Put(Key(100), EncodeAsUint64(1), "bad_val"));
+
+    ASSERT_EQ(file1_info.file_path, file1);
+    ASSERT_EQ(file1_info.num_entries, 150);
+    ASSERT_EQ(file1_info.smallest_key, Key(0) + EncodeAsUint64(0 + 3));
+    ASSERT_EQ(file1_info.largest_key, Key(49) + EncodeAsUint64(49 + 1));
+    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
+    // Add file using file path
+    ASSERT_OK(IngestExternalUDTFile({file1}));
+    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
+
+    for (int k = 0; k < 50; k++) {
+      for (int version = 3; version > 0; version--) {
+        VerifyValueAndTs(Key(k), EncodeAsUint64(k + version),
+                         Key(k) + "_val" + std::to_string(version),
+                         EncodeAsUint64(k + version));
+      }
+    }
+
+    // file2.sst [50, 200)
+    // Put [key=k, ts=k, value=k_val] for k in [50, 200)
+    // RangeDelete[start_key=75, end_key=125, ts=100]
+    std::string file2 = sst_files_dir_ + "file2.sst";
+    int range_del_begin = 75, range_del_end = 125, range_del_ts = 100;
+    ASSERT_OK(sst_file_writer.Open(file2));
+    for (int k = 50; k < 200; k++) {
+      ASSERT_OK(
+          sst_file_writer.Put(Key(k), EncodeAsUint64(k), Key(k) + "_val"));
+      if (k == range_del_ts) {
+        ASSERT_OK(sst_file_writer.DeleteRange(
+            Key(range_del_begin), Key(range_del_end), EncodeAsUint64(k)));
+      }
+    }
+
+    ExternalSstFileInfo file2_info;
+    ASSERT_OK(sst_file_writer.Finish(&file2_info));
+
+    // Current file size should be non-zero after success write.
+    ASSERT_GT(sst_file_writer.FileSize(), 0);
+
+    ASSERT_EQ(file2_info.file_path, file2);
+    ASSERT_EQ(file2_info.num_entries, 150);
+    ASSERT_EQ(file2_info.smallest_key, Key(50) + EncodeAsUint64(50));
+    ASSERT_EQ(file2_info.largest_key, Key(199) + EncodeAsUint64(199));
+    ASSERT_EQ(file2_info.num_range_del_entries, 1);
+    ASSERT_EQ(file2_info.smallest_range_del_key,
+              Key(range_del_begin) + EncodeAsUint64(range_del_ts));
+    ASSERT_EQ(file2_info.largest_range_del_key,
+              Key(range_del_end) + EncodeAsUint64(range_del_ts));
+    // Add file using file path
+    ASSERT_OK(IngestExternalUDTFile({file2}));
+    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
+
+    for (int k = 50; k < 200; k++) {
+      if (k < range_del_begin || k >= range_del_end) {
+        VerifyValueAndTs(Key(k), EncodeAsUint64(k), Key(k) + "_val",
+                         EncodeAsUint64(k));
+      }
+      //      else {
+      //        // FIXME(yuzhangyu): when range tombstone and point data has the
+      //        // same seq, on read path, make range tombstone overrides point
+      //        // data if it has a newer user-defined timestamp. This is how
+      //        // we determine point data's overriding relationship, so we
+      //        //  should keep it consistent.
+      //        VerifyValueAndTs(Key(k), EncodeAsUint64(k), Key(k) + "_val",
+      //                         EncodeAsUint64(k));
+      //        VerifyValueAndTs(Key(k), EncodeAsUint64(range_del_ts),
+      //        kValueNotFound,
+      //                         kTsNotFound);
+      //      }
+    }
+
+    // file3.sst [100, 200), key range overlap with db
+    std::string file3 = sst_files_dir_ + "file3.sst";
+    ASSERT_OK(sst_file_writer.Open(file3));
+    for (int k = 100; k < 200; k++) {
+      ASSERT_OK(
+          sst_file_writer.Put(Key(k), EncodeAsUint64(k + 1), Key(k) + "_val1"));
+    }
+    ExternalSstFileInfo file3_info;
+    ASSERT_OK(sst_file_writer.Finish(&file3_info));
+    ASSERT_EQ(file3_info.file_path, file3);
+    ASSERT_EQ(file3_info.num_entries, 100);
+    ASSERT_EQ(file3_info.smallest_key, Key(100) + EncodeAsUint64(101));
+    ASSERT_EQ(file3_info.largest_key, Key(199) + EncodeAsUint64(200));
+
+    // Allowing ingesting a file containing overlap key range with the db is
+    // not safe without verifying the overlapped key has a higher timestamp
+    // than what the db contains, so we do not allow this regardless of
+    // whether global sequence number is allowed.
+    ASSERT_NOK(IngestExternalUDTFile({file2}));
+    ASSERT_NOK(IngestExternalUDTFile({file2}, /*allow_global_seqno*/ false));
+
+    // Write [0, 50)
+    // Write to DB newer versions to cover ingested data and move sequence
+    // number forward.
+    for (int k = 0; k < 50; k++) {
+      ASSERT_OK(dbfull()->Put(WriteOptions(), Key(k), EncodeAsUint64(k + 4),
+                              Key(k) + "_val" + std::to_string(4)));
+    }
+
+    // Read all 4 versions (3 from ingested, 1 from live writes).
+    for (int k = 0; k < 50; k++) {
+      for (int version = 4; version > 0; version--) {
+        VerifyValueAndTs(Key(k), EncodeAsUint64(k + version),
+                         Key(k) + "_val" + std::to_string(version),
+                         EncodeAsUint64(k + version));
+      }
+    }
+    SequenceNumber seq_num_before_ingestion = db_->GetLatestSequenceNumber();
+    ASSERT_GT(seq_num_before_ingestion, 0U);
+
+    // file4.sst [200, 250)
+    std::string file4 = sst_files_dir_ + "file4.sst";
+    ASSERT_OK(sst_file_writer.Open(file4));
+    for (int k = 200; k < 250; k++) {
+      ASSERT_OK(
+          sst_file_writer.Put(Key(k), EncodeAsUint64(k), Key(k) + "_val"));
+    }
+
+    ExternalSstFileInfo file4_info;
+    ASSERT_OK(sst_file_writer.Finish(&file4_info));
+
+    // Current file size should be non-zero after success write.
+    ASSERT_GT(sst_file_writer.FileSize(), 0);
+
+    ASSERT_EQ(file4_info.file_path, file4);
+    ASSERT_EQ(file4_info.num_entries, 50);
+    ASSERT_EQ(file4_info.smallest_key, Key(200) + EncodeAsUint64(200));
+    ASSERT_EQ(file4_info.largest_key, Key(249) + EncodeAsUint64(249));
+    ASSERT_EQ(file4_info.num_range_del_entries, 0);
+    ASSERT_EQ(file4_info.smallest_range_del_key, "");
+    ASSERT_EQ(file4_info.largest_range_del_key, "");
+
+    ASSERT_OK(IngestExternalUDTFile({file4}));
+
+    // In UDT mode, any external file that can be successfully ingested also
+    // should not overlap with the db. As a result, they can always get the
+    // seq 0 assigned.
+    ASSERT_EQ(db_->GetLatestSequenceNumber(), seq_num_before_ingestion);
+
+    DestroyAndRecreateExternalSSTFilesDir();
+  } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction |
+                         kRangeDelSkipConfigs));
+}
+
+TEST_F(ExternalSSTFileWithTimestampTest, SanityCheck) {
+  Options options = CurrentOptions();
+  options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  options.persist_user_defined_timestamps = true;
+  DestroyAndReopen(options);
+
+  SstFileWriter sst_file_writer(EnvOptions(), options);
+
+  // file1.sst [0, 100)
+  std::string file1 = sst_files_dir_ + "file1.sst";
+  ASSERT_OK(sst_file_writer.Open(file1));
+  for (int k = 0; k < 100; k++) {
+    ASSERT_OK(sst_file_writer.Put(Key(k), EncodeAsUint64(k), Key(k) + "_val"));
+  }
+
+  ExternalSstFileInfo file1_info;
+  ASSERT_OK(sst_file_writer.Finish(&file1_info));
+
+  // file2.sst [50, 75)
+  std::string file2 = sst_files_dir_ + "file2.sst";
+  ASSERT_OK(sst_file_writer.Open(file2));
+  for (int k = 50; k < 75; k++) {
+    ASSERT_OK(
+        sst_file_writer.Put(Key(k), EncodeAsUint64(k + 2), Key(k) + "_val"));
+  }
+  ExternalSstFileInfo file2_info;
+  ASSERT_OK(sst_file_writer.Finish(&file2_info));
+
+  // Cannot ingest when files' user key range overlaps. There is no
+  // straightforward way to assign sequence number to the files so that they
+  // meet the user-defined timestamps invariant: for the same user provided key,
+  // the entry with a higher sequence number should not have a smaller
+  // timestamp. In this case: file1 has (key=k, ts=k) for k in [50, 75),
+  //               file2 has (key=k, ts=k+2) for k in [50, 75).
+  // The invariant is only met if file2 is ingested after file1. In other cases
+  // when user key ranges are interleaved in files, no order of ingestion can
+  // guarantee this invariant. So we do not allow ingesting files with
+  // overlapping key ranges.
+  ASSERT_TRUE(IngestExternalUDTFile({file1, file2}).IsNotSupported());
+
+  options.allow_ingest_behind = true;
+  DestroyAndReopen(options);
+  IngestExternalFileOptions opts;
+
+  // TODO(yuzhangyu): support ingestion behind for user-defined timestamps?
+  // Ingesting external files with user-defined timestamps requires searching
+  // through the whole lsm tree to make sure there is no key range overlap with
+  // the db. Ingestion behind currently is doing a simply placing it at the
+  // bottom level step without a search, so we don't allow it either.
+  opts.ingest_behind = true;
+  ASSERT_TRUE(db_->IngestExternalFile({file1}, opts).IsNotSupported());
+
+  DestroyAndRecreateExternalSSTFilesDir();
+}
+
+TEST_F(ExternalSSTFileWithTimestampTest, UDTSettingsCompatibilityCheck) {
+  Options options = CurrentOptions();
+  Options disable_udt_options = options;
+  Options not_persist_udt_options = options;
+  Options persist_udt_options = options;
+  disable_udt_options.comparator = BytewiseComparator();
+  not_persist_udt_options.comparator =
+      test::BytewiseComparatorWithU64TsWrapper();
+  not_persist_udt_options.persist_user_defined_timestamps = false;
+  not_persist_udt_options.allow_concurrent_memtable_write = false;
+  persist_udt_options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  persist_udt_options.persist_user_defined_timestamps = true;
+
+  EnvOptions env_options = EnvOptions();
+
+  SstFileWriter disable_udt_sst_writer(env_options, disable_udt_options);
+  SstFileWriter not_persist_udt_sst_writer(env_options,
+                                           not_persist_udt_options);
+  SstFileWriter persist_udt_sst_writer(env_options, persist_udt_options);
+
+  // File1: [0, 50), contains no timestamps
+  // comparator name: leveldb.BytewiseComparator
+  // user_defined_timestamps_persisted: true
+  std::string disable_udt_sst_file = sst_files_dir_ + "file1.sst";
+  ASSERT_OK(disable_udt_sst_writer.Open(disable_udt_sst_file));
+  for (int k = 0; k < 50; k++) {
+    ASSERT_NOK(
+        disable_udt_sst_writer.Put(Key(k), EncodeAsUint64(1), Key(k) + "_val"));
+    ASSERT_OK(disable_udt_sst_writer.Put(Key(k), Key(k) + "_val"));
+  }
+  ASSERT_OK(disable_udt_sst_writer.Finish());
+
+  // File2: [50, 100), contains no timestamps
+  // comparator name: leveldb.BytewiseComparator.u64ts
+  // user_defined_timestamps_persisted: false
+  std::string not_persist_udt_sst_file = sst_files_dir_ + "file2.sst";
+  ASSERT_OK(not_persist_udt_sst_writer.Open(not_persist_udt_sst_file));
+  for (int k = 50; k < 100; k++) {
+    ASSERT_NOK(not_persist_udt_sst_writer.Put(Key(k), Key(k) + "_val"));
+    ASSERT_NOK(not_persist_udt_sst_writer.Put(Key(k), EncodeAsUint64(k),
+                                              Key(k) + "_val"));
+    ASSERT_OK(not_persist_udt_sst_writer.Put(Key(k), EncodeAsUint64(0),
+                                             Key(k) + "_val"));
+  }
+  ASSERT_OK(not_persist_udt_sst_writer.Finish());
+
+  // File3: [100, 150), contains timestamp
+  // comparator name: leveldb.BytewiseComparator.u64ts
+  // user_defined_timestamps_persisted: true
+  std::string persist_udt_sst_file = sst_files_dir_ + "file3.sst";
+  ASSERT_OK(persist_udt_sst_writer.Open(persist_udt_sst_file));
+  for (int k = 100; k < 150; k++) {
+    ASSERT_NOK(persist_udt_sst_writer.Put(Key(k), Key(k) + "_val"));
+    ASSERT_OK(
+        persist_udt_sst_writer.Put(Key(k), EncodeAsUint64(k), Key(k) + "_val"));
+  }
+  ASSERT_OK(persist_udt_sst_writer.Finish());
+
+  DestroyAndReopen(disable_udt_options);
+  ASSERT_OK(
+      IngestExternalUDTFile({disable_udt_sst_file, not_persist_udt_sst_file}));
+  ASSERT_NOK(IngestExternalUDTFile({persist_udt_sst_file}));
+  for (int k = 0; k < 100; k++) {
+    ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
+  }
+
+  DestroyAndReopen(not_persist_udt_options);
+  ASSERT_OK(
+      IngestExternalUDTFile({disable_udt_sst_file, not_persist_udt_sst_file}));
+  ASSERT_NOK(IngestExternalUDTFile({persist_udt_sst_file}));
+  for (int k = 0; k < 100; k++) {
+    VerifyValueAndTs(Key(k), EncodeAsUint64(0), Key(k) + "_val",
+                     EncodeAsUint64(0));
+  }
+
+  DestroyAndReopen(persist_udt_options);
+  ASSERT_NOK(
+      IngestExternalUDTFile({disable_udt_sst_file, not_persist_udt_sst_file}));
+  ASSERT_OK(IngestExternalUDTFile({persist_udt_sst_file}));
+  for (int k = 100; k < 150; k++) {
+    VerifyValueAndTs(Key(k), EncodeAsUint64(k), Key(k) + "_val",
+                     EncodeAsUint64(k));
+  }
+
+  DestroyAndRecreateExternalSSTFilesDir();
+}
+
+TEST_F(ExternalSSTFileWithTimestampTest, TimestampsNotPersistedBasic) {
+  do {
+    Options options = CurrentOptions();
+    options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+    options.persist_user_defined_timestamps = false;
+    options.allow_concurrent_memtable_write = false;
+
+    DestroyAndReopen(options);
+
+    SstFileWriter sst_file_writer(EnvOptions(), options);
+
+    // file1.sst [0, 50)
+    std::string file1 = sst_files_dir_ + "file1.sst";
+    ASSERT_OK(sst_file_writer.Open(file1));
+    for (int k = 0; k < 50; k++) {
+      // Attempting to write 2 versions of values for each key, only the version
+      // with timestamp 0 goes through.
+      for (int version = 1; version >= 0; version--) {
+        if (version == 1) {
+          ASSERT_NOK(
+              sst_file_writer.Put(Key(k), EncodeAsUint64(version),
+                                  Key(k) + "_val" + std::to_string(version)));
+        } else {
+          ASSERT_OK(
+              sst_file_writer.Put(Key(k), EncodeAsUint64(version),
+                                  Key(k) + "_val" + std::to_string(version)));
+        }
+      }
+    }
+
+    ExternalSstFileInfo file1_info;
+    ASSERT_OK(sst_file_writer.Finish(&file1_info));
+    // sst_file_writer already finished, cannot add this value
+    ASSERT_NOK(sst_file_writer.Put(Key(100), EncodeAsUint64(0), "bad_val"));
+
+    ASSERT_EQ(file1_info.file_path, file1);
+    ASSERT_EQ(file1_info.num_entries, 50);
+    ASSERT_EQ(file1_info.smallest_key, Key(0));
+    ASSERT_EQ(file1_info.largest_key, Key(49));
+    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
+    // Add file using file path
+    ASSERT_OK(IngestExternalUDTFile({file1}));
+    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
+
+    // Read ingested file: all data contain minimum timestamps.
+    for (int k = 0; k < 50; k++) {
+      VerifyValueAndTs(Key(k), EncodeAsUint64(0),
+                       Key(k) + "_val" + std::to_string(0), EncodeAsUint64(0));
+    }
+
+    // file2.sst [50, 200)
+    // Put [key=k, ts=0, value=k_val0] for k in [50, 200)
+    // RangeDelete[start_key=75, end_key=125, ts=0]
+    std::string file2 = sst_files_dir_ + "file2.sst";
+    int range_del_begin = 75, range_del_end = 125;
+    ASSERT_OK(sst_file_writer.Open(file2));
+    for (int k = 50; k < 200; k++) {
+      // All these timestamps will later be effectively 0
+      ASSERT_OK(
+          sst_file_writer.Put(Key(k), EncodeAsUint64(0), Key(k) + "_val0"));
+    }
+    ASSERT_OK(sst_file_writer.DeleteRange(
+        Key(range_del_begin), Key(range_del_end), EncodeAsUint64(0)));
+
+    ExternalSstFileInfo file2_info;
+    ASSERT_OK(sst_file_writer.Finish(&file2_info));
+
+    // Current file size should be non-zero after success write.
+    ASSERT_GT(sst_file_writer.FileSize(), 0);
+
+    ASSERT_EQ(file2_info.file_path, file2);
+    ASSERT_EQ(file2_info.num_entries, 150);
+    ASSERT_EQ(file2_info.smallest_key, Key(50));
+    ASSERT_EQ(file2_info.largest_key, Key(199));
+    ASSERT_EQ(file2_info.num_range_del_entries, 1);
+    ASSERT_EQ(file2_info.smallest_range_del_key, Key(range_del_begin));
+    ASSERT_EQ(file2_info.largest_range_del_key, Key(range_del_end));
+    // Add file using file path
+    ASSERT_OK(IngestExternalUDTFile({file2}));
+    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
+
+    // Range deletion covering point data in the same file is over-written.
+    for (int k = 50; k < 200; k++) {
+      VerifyValueAndTs(Key(k), EncodeAsUint64(0), Key(k) + "_val0",
+                       EncodeAsUint64(0));
+    }
+
+    // file3.sst [100, 200), key range overlap with db
+    std::string file3 = sst_files_dir_ + "file3.sst";
+    ASSERT_OK(sst_file_writer.Open(file3));
+    for (int k = 100; k < 200; k++) {
+      ASSERT_OK(
+          sst_file_writer.Put(Key(k), EncodeAsUint64(0), Key(k) + "_val0"));
+    }
+    ExternalSstFileInfo file3_info;
+    ASSERT_OK(sst_file_writer.Finish(&file3_info));
+    ASSERT_EQ(file3_info.file_path, file3);
+    ASSERT_EQ(file3_info.num_entries, 100);
+    ASSERT_EQ(file3_info.smallest_key, Key(100));
+    ASSERT_EQ(file3_info.largest_key, Key(199));
+
+    // In UDT mode, file with overlapping key range cannot be ingested.
+    ASSERT_NOK(IngestExternalUDTFile({file3}));
+    ASSERT_NOK(IngestExternalUDTFile({file3}, /*allow_global_seqno*/ false));
+
+    // Write [0, 50)
+    // Write to DB newer versions to cover ingested data and move sequence
+    // number forward.
+    for (int k = 0; k < 50; k++) {
+      for (int version = 1; version < 3; version++) {
+        ASSERT_OK(dbfull()->Put(WriteOptions(), Key(k), EncodeAsUint64(version),
+                                Key(k) + "_val" + std::to_string(version)));
+      }
+    }
+
+    // Read three versions (1 from ingested, 2 from live writes)
+    for (int k = 0; k < 50; k++) {
+      for (int version = 0; version < 3; version++) {
+        VerifyValueAndTs(Key(k), EncodeAsUint64(version),
+                         Key(k) + "_val" + std::to_string(version),
+                         EncodeAsUint64(version));
+      }
+    }
+    SequenceNumber seq_num_before_ingestion = db_->GetLatestSequenceNumber();
+    ASSERT_GT(seq_num_before_ingestion, 0U);
+
+    // file4.sst [200, 250)
+    std::string file4 = sst_files_dir_ + "file4.sst";
+    ASSERT_OK(sst_file_writer.Open(file4));
+    for (int k = 200; k < 250; k++) {
+      ASSERT_OK(
+          sst_file_writer.Put(Key(k), EncodeAsUint64(0), Key(k) + "_val"));
+    }
+
+    ExternalSstFileInfo file4_info;
+    ASSERT_OK(sst_file_writer.Finish(&file4_info));
+
+    // Current file size should be non-zero after success write.
+    ASSERT_GT(sst_file_writer.FileSize(), 0);
+
+    ASSERT_EQ(file4_info.file_path, file4);
+    ASSERT_EQ(file4_info.num_entries, 50);
+    ASSERT_EQ(file4_info.smallest_key, Key(200));
+    ASSERT_EQ(file4_info.largest_key, Key(249));
+    ASSERT_EQ(file4_info.num_range_del_entries, 0);
+    ASSERT_EQ(file4_info.smallest_range_del_key, "");
+    ASSERT_EQ(file4_info.largest_range_del_key, "");
+
+    ASSERT_OK(IngestExternalUDTFile({file4}));
+
+    // Ingested files do not overlap with db, they can always have global seqno
+    // 0 assigned.
+    ASSERT_EQ(db_->GetLatestSequenceNumber(), seq_num_before_ingestion);
+
+    DestroyAndRecreateExternalSSTFilesDir();
+  } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction |
+                         kRangeDelSkipConfigs));
+}
+
 INSTANTIATE_TEST_CASE_P(ExternalSSTFileTest, ExternalSSTFileTest,
                         testing::Values(std::make_tuple(false, false),
                                         std::make_tuple(false, true),