@nxtedition/rocksdb 13.1.5 → 13.2.0

package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc

@@ -301,7 +301,7 @@ Status DBImpl::NewDB(std::vector<std::string>* new_filenames) {
   VersionEdit new_db_edit;
   const WriteOptions write_options(Env::IOActivity::kDBOpen);
   Status s = SetupDBId(write_options, /*read_only=*/false, /*is_new_db=*/true,
-                       &new_db_edit);
+                       /*is_retry=*/false, &new_db_edit);
   if (!s.ok()) {
     return s;
   }
@@ -575,6 +575,7 @@ Status DBImpl::Recover(
   }
   if (s.ok() && !read_only) {
     for (auto cfd : *versions_->GetColumnFamilySet()) {
+      auto& moptions = *cfd->GetLatestMutableCFOptions();
       // Try to trivially move files down the LSM tree to start from bottommost
       // level when level_compaction_dynamic_level_bytes is enabled. This should
       // only be useful when user is migrating to turning on this option.
@@ -592,14 +593,14 @@ Status DBImpl::Recover(
       if (cfd->ioptions()->compaction_style ==
               CompactionStyle::kCompactionStyleLevel &&
           cfd->ioptions()->level_compaction_dynamic_level_bytes &&
-          !cfd->GetLatestMutableCFOptions()->disable_auto_compactions) {
+          !moptions.disable_auto_compactions) {
         int to_level = cfd->ioptions()->num_levels - 1;
         // last level is reserved
         // allow_ingest_behind does not support Level Compaction,
         // and per_key_placement can have infinite compaction loop for Level
         // Compaction. Adjust to_level here just to be safe.
         if (cfd->ioptions()->allow_ingest_behind ||
-            cfd->ioptions()->preclude_last_level_data_seconds > 0) {
+            moptions.preclude_last_level_data_seconds > 0) {
           to_level -= 1;
         }
         // Whether this column family has a level trivially moved
@@ -675,11 +676,11 @@ Status DBImpl::Recover(
     // Already set up DB ID in NewDB
   } else if (immutable_db_options_.write_dbid_to_manifest && recovery_ctx) {
     VersionEdit edit;
-    s = SetupDBId(write_options, read_only, is_new_db, &edit);
+    s = SetupDBId(write_options, read_only, is_new_db, is_retry, &edit);
     recovery_ctx->UpdateVersionEdits(
         versions_->GetColumnFamilySet()->GetDefault(), edit);
   } else {
-    s = SetupDBId(write_options, read_only, is_new_db, nullptr);
+    s = SetupDBId(write_options, read_only, is_new_db, is_retry, nullptr);
   }
   assert(!s.ok() || !db_id_.empty());
   ROCKS_LOG_INFO(immutable_db_options_.info_log, "DB ID: %s\n", db_id_.c_str());
@@ -1274,7 +1275,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
           reader.GetRecordedTimestampSize();
       status = HandleWriteBatchTimestampSizeDifference(
           &batch, running_ts_sz, record_ts_sz,
-          TimestampSizeConsistencyMode::kReconcileInconsistency, &new_batch);
+          TimestampSizeConsistencyMode::kReconcileInconsistency, seq_per_batch_,
+          batch_per_txn_, &new_batch);
       if (!status.ok()) {
         return status;
       }
@@ -1371,6 +1373,9 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
         }
       }
     }
+    ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                   "Recovered to log #%" PRIu64 " seq #%" PRIu64, wal_number,
+                   *next_sequence);
 
     if (!status.ok() || old_log_record) {
       if (status.IsNotSupported()) {
@@ -1403,10 +1408,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
         if (corrupted_wal_found != nullptr) {
           *corrupted_wal_found = true;
         }
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "Point in time recovered to log #%" PRIu64
-                       " seq #%" PRIu64,
-                       wal_number, *next_sequence);
       } else {
         assert(immutable_db_options_.wal_recovery_mode ==
                    WALRecoveryMode::kTolerateCorruptedTailRecords ||
@@ -1667,10 +1668,20 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
   Arena arena;
   Status s;
   TableProperties table_properties;
+  const auto* ucmp = cfd->internal_comparator().user_comparator();
+  assert(ucmp);
+  const size_t ts_sz = ucmp->timestamp_size();
+  const bool logical_strip_timestamp =
+      ts_sz > 0 && !cfd->ioptions()->persist_user_defined_timestamps;
   {
     ScopedArenaPtr<InternalIterator> iter(
-        mem->NewIterator(ro, /*seqno_to_time_mapping=*/nullptr, &arena,
-                         /*prefix_extractor=*/nullptr));
+        logical_strip_timestamp
+            ? mem->NewTimestampStrippingIterator(
+                  ro, /*seqno_to_time_mapping=*/nullptr, &arena,
+                  /*prefix_extractor=*/nullptr, ts_sz)
+            : mem->NewIterator(ro, /*seqno_to_time_mapping=*/nullptr, &arena,
+                               /*prefix_extractor=*/nullptr,
+                               /*for_flush=*/true));
     ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                     "[%s] [WriteLevel0TableForRecovery]"
                     " Level-0 table #%" PRIu64 ": started",
@@ -1705,11 +1716,14 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
       std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
           range_del_iters;
       auto range_del_iter =
-          // This is called during recovery, where a live memtable is flushed
-          // directly. In this case, no fragmented tombstone list is cached in
-          // this memtable yet.
-          mem->NewRangeTombstoneIterator(ro, kMaxSequenceNumber,
-                                         false /* immutable_memtable */);
+          logical_strip_timestamp
+              ? mem->NewTimestampStrippingRangeTombstoneIterator(
+                    ro, kMaxSequenceNumber, ts_sz)
+              // This is called during recovery, where a live memtable is
+              // flushed directly. In this case, no fragmented tombstone list is
+              // cached in this memtable yet.
+              : mem->NewRangeTombstoneIterator(ro, kMaxSequenceNumber,
+                                               false /* immutable_memtable */);
       if (range_del_iter != nullptr) {
         range_del_iters.emplace_back(range_del_iter);
       }
@@ -1723,10 +1737,11 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
           cfd->internal_comparator(), cfd->internal_tbl_prop_coll_factories(),
           GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
           mutable_cf_options.compression_opts, cfd->GetID(), cfd->GetName(),
-          0 /* level */, false /* is_bottommost */,
-          TableFileCreationReason::kRecovery, 0 /* oldest_key_time */,
-          0 /* file_creation_time */, db_id_, db_session_id_,
-          0 /* target_file_size */, meta.fd.GetNumber(), kMaxSequenceNumber);
+          0 /* level */, current_time /* newest_key_time */,
+          false /* is_bottommost */, TableFileCreationReason::kRecovery,
+          0 /* oldest_key_time */, 0 /* file_creation_time */, db_id_,
+          db_session_id_, 0 /* target_file_size */, meta.fd.GetNumber(),
+          kMaxSequenceNumber);
       Version* version = cfd->current();
       version->Ref();
       uint64_t num_input_entries = 0;
@@ -1756,7 +1771,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
         s = io_s;
       }
 
-      uint64_t total_num_entries = mem->num_entries();
+      uint64_t total_num_entries = mem->NumEntries();
       if (s.ok() && total_num_entries != num_input_entries) {
         std::string msg = "Expected " + std::to_string(total_num_entries) +
                           " entries in memtable, but read " +
@@ -1795,9 +1810,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
 
     // For UDT in memtable only feature, move up the cutoff timestamp whenever
     // a flush happens.
-    const Comparator* ucmp = cfd->user_comparator();
-    size_t ts_sz = ucmp->timestamp_size();
-    if (ts_sz > 0 && !cfd->ioptions()->persist_user_defined_timestamps) {
+    if (logical_strip_timestamp) {
       Slice mem_newest_udt = mem->GetNewestUDT();
       std::string full_history_ts_low = cfd->GetFullHistoryTsLow();
       if (full_history_ts_low.empty() ||

package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc

@@ -8,7 +8,7 @@
 #include "db/arena_wrapped_db_iter.h"
 #include "db/db_impl/compacted_db_impl.h"
 #include "db/db_impl/db_impl.h"
-#include "db/db_iter.h"
+#include "db/manifest_ops.h"
 #include "db/merge_context.h"
 #include "logging/logging.h"
 #include "monitoring/perf_context_imp.h"
@@ -265,8 +265,8 @@ Status OpenForReadOnlyCheckExistence(const DBOptions& db_options,
     const std::shared_ptr<FileSystem>& fs = db_options.env->GetFileSystem();
     std::string manifest_path;
     uint64_t manifest_file_number;
-    s = VersionSet::GetCurrentManifestPath(dbname, fs.get(), &manifest_path,
-                                           &manifest_file_number);
+    s = GetCurrentManifestPath(dbname, fs.get(), /*is_retry=*/false,
+                               &manifest_path, &manifest_file_number);
   } else {
     // Historic behavior that doesn't necessarily make sense
     s = db_options.env->CreateDirIfMissing(dbname);

package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc

@@ -233,7 +233,8 @@ Status DBImplSecondary::RecoverLogFiles(
           reader->GetRecordedTimestampSize();
       status = HandleWriteBatchTimestampSizeDifference(
           &batch, running_ts_sz, record_ts_sz,
-          TimestampSizeConsistencyMode::kVerifyConsistency);
+          TimestampSizeConsistencyMode::kVerifyConsistency, seq_per_batch_,
+          batch_per_txn_);
       if (!status.ok()) {
         break;
       }
@@ -247,9 +248,7 @@ Status DBImplSecondary::RecoverLogFiles(
           if (cfd == nullptr) {
             continue;
           }
-          if (cfds_changed->count(cfd) == 0) {
-            cfds_changed->insert(cfd);
-          }
+          cfds_changed->insert(cfd);
           const std::vector<FileMetaData*>& l0_files =
               cfd->current()->storage_info()->LevelFiles(0);
           SequenceNumber seq =
@@ -957,6 +956,10 @@ Status DB::OpenAndCompact(
   config_options.env = override_options.env;
   std::vector<ColumnFamilyDescriptor> all_column_families;
 
+  TEST_SYNC_POINT_CALLBACK(
+      "DBImplSecondary::OpenAndCompact::BeforeLoadingOptions:0",
+      &compaction_input.options_file_number);
+  TEST_SYNC_POINT("DBImplSecondary::OpenAndCompact::BeforeLoadingOptions:1");
   std::string options_file_name =
       OptionsFileName(name, compaction_input.options_file_number);
 

package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc

@@ -12,6 +12,7 @@
 #include "db/error_handler.h"
 #include "db/event_helpers.h"
 #include "logging/logging.h"
+#include "memtable/wbwi_memtable.h"
 #include "monitoring/perf_context_imp.h"
 #include "options/options_helper.h"
 #include "test_util/sync_point.h"
@@ -189,16 +190,137 @@ Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
   return s;
 }
 
-// The main write queue. This is the only write queue that updates LastSequence.
-// When using one write queue, the same sequence also indicates the last
-// published sequence.
+Status DBImpl::IngestWBWI(std::shared_ptr<WriteBatchWithIndex> wbwi,
+                          const WBWIMemTable::SeqnoRange& assigned_seqno,
+                          uint64_t prep_log,
+                          SequenceNumber last_seqno_after_ingest,
+                          bool memtable_updated, bool ignore_missing_cf) {
+  // Keys in new memtable have seqno > last_seqno_after_ingest >= keys in wbwi.
+  assert(assigned_seqno.upper_bound <= last_seqno_after_ingest);
+  // Keys in the current memtable have seqno <= LastSequence() < keys in wbwi.
+  assert(assigned_seqno.lower_bound > versions_->LastSequence());
+  autovector<ReadOnlyMemTable*> memtables;
+  autovector<ColumnFamilyData*> cfds;
+  InstrumentedMutexLock lock(&mutex_);
+  ColumnFamilySet* cf_set = versions_->GetColumnFamilySet();
+
+  // Create WBWIMemTables
+  for (const auto [cf_id, stat] : wbwi->GetCFStats()) {
+    ColumnFamilyData* cfd = cf_set->GetColumnFamily(cf_id);
+    if (!cfd) {
+      if (ignore_missing_cf) {
+        continue;
+      }
+      for (auto mem : memtables) {
+        mem->Unref();
+        delete mem;
+      }
+      for (auto cfd_ptr : cfds) {
+        cfd_ptr->UnrefAndTryDelete();
+      }
+      Status s = Status::InvalidArgument(
+          "Invalid column family id from WriteBatchWithIndex: " +
+          std::to_string(cf_id));
+      if (memtable_updated) {
+        s = Status::Corruption(
+            "Part of the write batch is applied. Memtable is in a inconsistent "
+            "state. " +
+            s.ToString());
+        error_handler_.SetBGError(s, BackgroundErrorReason::kMemTable);
+      }
+
+      return s;
+    }
+    WBWIMemTable* wbwi_memtable =
+        new WBWIMemTable(wbwi, cfd->user_comparator(), cf_id, cfd->ioptions(),
+                         cfd->GetLatestMutableCFOptions(), stat);
+    wbwi_memtable->Ref();
+    wbwi_memtable->AssignSequenceNumbers(assigned_seqno);
+    // This is needed to keep the WAL that contains Prepare alive until
+    // committed data in this memtable is persisted.
+    wbwi_memtable->SetMinPrepLog(prep_log);
+    memtables.push_back(wbwi_memtable);
+    cfd->Ref();
+    cfds.push_back(cfd);
+  }
+
+  // Stop writes to the DB by entering both write threads
+  WriteThread::Writer nonmem_w;
+  if (two_write_queues_) {
+    nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
+  }
+  WaitForPendingWrites();
+
+  // Switch memtable and add WBWIMemTables
+  Status s;
+  for (size_t i = 0; i < memtables.size(); ++i) {
+    assert(!immutable_db_options_.atomic_flush);
+    // NOTE: to support atomic flush, need to call
+    // SelectColumnFamiliesForAtomicFlush()
+    WriteContext write_context;
+    // TODO: not switch on empty memtable, may need to update metadata
+    //   like NextLogNumber(), earliest_seqno and memtable id.
+    s = SwitchMemtable(cfds[i], &write_context, memtables[i],
+                       last_seqno_after_ingest);
+    if (!s.ok()) {
+      // SwitchMemtable() can only fail if a new WAL is to be created, this
+      // should only happen for the first call to SwitchMemtable(). log will
+      // be empty and no new WAL is created for the rest of the calls.
+      assert(i == 0);
+      if (i != 0 || memtable_updated) {
+        // escalate error to non-recoverable
+        s = Status::Corruption(
+            "Part of the write batch is applied. Memtable is in a inconsistent "
+            "state. " +
+            s.ToString());
+        error_handler_.SetBGError(s, BackgroundErrorReason::kMemTable);
+      } else {
+        // SwitchMemtable() already sets appropriate bg error
+      }
+      for (size_t j = i; j < memtables.size(); j++) {
+        memtables[j]->Unref();
+        delete memtables[j];
+      }
+      break;
+    }
+  }
+  for (size_t i = 0; i < cfds.size(); ++i) {
+    if (cfds[i]->UnrefAndTryDelete()) {
+      cfds[i] = nullptr;
+    }
+  }
+
+  // exit the second queue before returning
+  if (two_write_queues_) {
+    nonmem_write_thread_.ExitUnbatched(&nonmem_w);
+  }
+  if (s.ok()) {
+    // Trigger flushes for the new immutable memtables.
+    for (const auto cfd : cfds) {
+      if (cfd == nullptr) {
+        continue;
+      }
+      cfd->imm()->FlushRequested();
+      FlushRequest flush_req;
+      // TODO: a new flush reason for ingesting memtable
+      GenerateFlushRequest({cfd}, FlushReason::kExternalFileIngestion,
+                           &flush_req);
+      EnqueuePendingFlush(flush_req);
+    }
+    MaybeScheduleFlushOrCompaction();
+  }
+  return s;
+}
+
 Status DBImpl::WriteImpl(const WriteOptions& write_options,
                          WriteBatch* my_batch, WriteCallback* callback,
                          UserWriteCallback* user_write_cb, uint64_t* log_used,
                          uint64_t log_ref, bool disable_memtable,
                          uint64_t* seq_used, size_t batch_cnt,
                          PreReleaseCallback* pre_release_callback,
-                         PostMemTableCallback* post_memtable_callback) {
+                         PostMemTableCallback* post_memtable_callback,
+                         std::shared_ptr<WriteBatchWithIndex> wbwi,
+                         uint64_t prep_log) {
   assert(!seq_per_batch_ || batch_cnt != 0);
   assert(my_batch == nullptr || my_batch->Count() == 0 ||
          write_options.protection_bytes_per_key == 0 ||
@@ -287,6 +409,23 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     return Status::NotSupported(
         "DeleteRange is not compatible with row cache.");
   }
+  if (wbwi) {
+    assert(prep_log > 0);
+    // Used only in WriteCommittedTxn::CommitInternal() with no `callback`.
+    assert(!callback);
+    if (immutable_db_options_.unordered_write) {
+      return Status::NotSupported(
+          "Ingesting WriteBatch does not support unordered_write");
+    }
+    if (immutable_db_options_.enable_pipelined_write) {
+      return Status::NotSupported(
+          "Ingesting WriteBatch does not support pipelined_write");
+    }
+    if (immutable_db_options_.atomic_flush) {
+      return Status::NotSupported(
+          "Ingesting WriteBatch does not support atomic_flush");
+    }
+  }
   // Otherwise IsLatestPersistentState optimization does not make sense
   assert(!WriteBatchInternal::IsLatestPersistentState(my_batch) ||
          disable_memtable);
@@ -344,7 +483,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
   WriteThread::Writer w(write_options, my_batch, callback, user_write_cb,
                         log_ref, disable_memtable, batch_cnt,
-                        pre_release_callback, post_memtable_callback);
+                        pre_release_callback, post_memtable_callback,
+                        /*_ingest_wbwi=*/wbwi != nullptr);
   StopWatch write_sw(immutable_db_options_.clock, stats_, DB_WRITE);
 
   write_thread_.JoinBatchGroup(&w);
@@ -441,6 +581,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   TEST_SYNC_POINT("DBImpl::WriteImpl:BeforeLeaderEnters");
   last_batch_group_size_ =
       write_thread_.EnterAsBatchGroupLeader(&w, &write_group);
+  if (wbwi) {
+    assert(write_group.size == 1);
+  }
 
   IOStatus io_s;
   Status pre_release_cb_status;
@@ -494,10 +637,25 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     // Note about seq_per_batch_: either disableWAL is set for the entire write
     // group or not. In either case we inc seq for each write batch with no
     // failed callback. This means that there could be a batch with
-    // disalbe_memtable in between; although we do not write this batch to
+    // disable_memtable in between; although we do not write this batch to
     // memtable it still consumes a seq. Otherwise, if !seq_per_batch_, we inc
     // the seq per valid written key to mem.
     size_t seq_inc = seq_per_batch_ ? valid_batches : total_count;
+    if (wbwi) {
+      // Reserve sequence numbers for the ingested memtable. We need to reserve
+      // at lease this amount for recovery. During recovery,
+      // transactions do not commit by ingesting WBWI. The sequence number
+      // associated with the commit entry in WAL is used as the starting
+      // sequence number for inserting into memtable. We need to reserve
+      // enough sequence numbers here (at least the number of operations
+      // in write batch) to assign to memtable entries for this transaction.
+      // This prevents updates in different transactions from using out-of-order
+      // sequence numbers or the same key+seqno.
+      //
+      // WBWI ingestion requires not grouping writes, so we don't need to
+      // consider incrementing sequence number for WBWI from other writers.
+      seq_inc += wbwi->GetWriteBatch()->Count();
+    }
 
     const bool concurrent_update = two_write_queues_;
     // Update stats while we are an exclusive group leader, so we know
@@ -674,6 +832,27 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     // handle exit, false means somebody else did
     should_exit_batch_group = write_thread_.CompleteParallelMemTableWriter(&w);
   }
+  if (wbwi) {
+    if (status.ok() && w.status.ok()) {
+      // w.batch contains (potentially empty) commit time batch updates,
+      // only ingest wbwi if w.batch is applied to memtable successfully
+      assert(wbwi->GetWriteBatch()->Count() > 0);
+
+      uint32_t memtable_update_count = w.batch->Count();
+      SequenceNumber lb = versions_->LastSequence() + memtable_update_count + 1;
+      SequenceNumber ub = versions_->LastSequence() + memtable_update_count +
+                          wbwi->GetWriteBatch()->Count();
+      assert(ub == last_sequence);
+      if (two_write_queues_) {
+        assert(ub <= versions_->LastAllocatedSequence());
+      }
+      status = IngestWBWI(wbwi, {/*lower_bound=*/lb, /*upper_bound=*/ub},
+                          prep_log, last_sequence,
+                          /*memtable_updated=*/memtable_update_count > 0,
+                          write_options.ignore_missing_column_families);
+    }
+  }
+
   if (should_exit_batch_group) {
     if (status.ok()) {
       for (auto* tmp_w : write_group) {
@@ -687,7 +866,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
         }
       }
       // Note: if we are to resume after non-OK statuses we need to revisit how
-      // we reacts to non-OK statuses here.
+      // we react to non-OK statuses here.
       versions_->SetLastSequence(last_sequence);
     }
     MemTableInsertStatusCheck(w.status);
@@ -735,17 +914,6 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
     size_t total_byte_size = 0;
 
     if (w.status.ok()) {
-      // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock
-      // grabs but does not seem thread-safe.
-      if (tracer_) {
-        InstrumentedMutexLock lock(&trace_mutex_);
-        if (tracer_ != nullptr && tracer_->IsWriteOrderPreserved()) {
-          for (auto* writer : wal_write_group) {
-            // TODO: maybe handle the tracing status?
-            tracer_->Write(writer->batch).PermitUncheckedError();
-          }
-        }
-      }
       SequenceNumber next_sequence = current_sequence;
       for (auto* writer : wal_write_group) {
         assert(writer);
@@ -760,6 +928,22 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
           }
         }
       }
+      // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock
+      // grabs but does not seem thread-safe.
+      if (tracer_) {
+        InstrumentedMutexLock lock(&trace_mutex_);
+        if (tracer_ != nullptr && tracer_->IsWriteOrderPreserved()) {
+          for (auto* writer : wal_write_group) {
+            if (writer->CallbackFailed()) {
+              // When optimisitc txn conflict checking fails, we should
+              // not record to trace.
+              continue;
+            }
+            // TODO: maybe handle the tracing status?
+            tracer_->Write(writer->batch).PermitUncheckedError();
+          }
+        }
+      }
       if (w.disable_wal) {
         has_unpersisted_data_.store(true, std::memory_order_relaxed);
       }
@@ -1005,19 +1189,6 @@ Status DBImpl::WriteImplWALOnly(
   WriteThread::WriteGroup write_group;
   uint64_t last_sequence;
   write_thread->EnterAsBatchGroupLeader(&w, &write_group);
-  // Note: no need to update last_batch_group_size_ here since the batch writes
-  // to WAL only
-  // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock
-  // grabs but does not seem thread-safe.
-  if (tracer_) {
-    InstrumentedMutexLock lock(&trace_mutex_);
-    if (tracer_ != nullptr && tracer_->IsWriteOrderPreserved()) {
-      for (auto* writer : write_group) {
-        // TODO: maybe handle the tracing status?
-        tracer_->Write(writer->batch).PermitUncheckedError();
-      }
-    }
-  }
 
   size_t pre_release_callback_cnt = 0;
   size_t total_byte_size = 0;
@@ -1032,6 +1203,23 @@ Status DBImpl::WriteImplWALOnly(
     }
   }
 
+  // Note: no need to update last_batch_group_size_ here since the batch writes
+  // to WAL only
+  // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock
+  // grabs but does not seem thread-safe.
+  if (tracer_) {
+    InstrumentedMutexLock lock(&trace_mutex_);
+    if (tracer_ != nullptr && tracer_->IsWriteOrderPreserved()) {
+      for (auto* writer : write_group) {
+        if (writer->CallbackFailed()) {
+          continue;
+        }
+        // TODO: maybe handle the tracing status?
+        tracer_->Write(writer->batch).PermitUncheckedError();
+      }
+    }
+  }
+
   const bool concurrent_update = true;
   // Update stats while we are an exclusive group leader, so we know
   // that nobody else can be writing to these particular stats.
@@ -1201,7 +1389,6 @@ void DBImpl::MemTableInsertStatusCheck(const Status& status) {
   if (!status.ok()) {
     mutex_.Lock();
     assert(!error_handler_.IsBGWorkStopped());
-    // Maybe change the return status to void?
     error_handler_.SetBGError(status, BackgroundErrorReason::kMemTable);
     mutex_.Unlock();
   }
@@ -1601,6 +1788,8 @@ IOStatus DBImpl::ConcurrentWriteToWAL(
 Status DBImpl::WriteRecoverableState() {
   mutex_.AssertHeld();
   if (!cached_recoverable_state_empty_) {
+    // Only for write-prepared and write-unprepared.
+    assert(seq_per_batch_);
     bool dont_care_bool;
     SequenceNumber next_seq;
     if (two_write_queues_) {
@@ -2193,16 +2382,13 @@ void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* /*cfd*/,
   mutex_.Lock();
 }
 
-// REQUIRES: mutex_ is held
-// REQUIRES: this thread is currently at the front of the writer queue
-// REQUIRES: this thread is currently at the front of the 2nd writer queue if
-// two_write_queues_ is true (This is to simplify the reasoning.)
-Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
+Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
+                              ReadOnlyMemTable* new_imm,
+                              SequenceNumber last_seqno) {
   mutex_.AssertHeld();
   assert(lock_wal_count_ == 0);
 
   // TODO: plumb Env::IOActivity, Env::IOPriority
-  const ReadOptions read_options;
   const WriteOptions write_options;
 
   log::Writer* new_log = nullptr;
@@ -2238,12 +2424,13 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
 
   // Set memtable_info for memtable sealed callback
+  // TODO: memtable_info for `new_imm`
   MemTableInfo memtable_info;
   memtable_info.cf_name = cfd->GetName();
   memtable_info.first_seqno = cfd->mem()->GetFirstSequenceNumber();
   memtable_info.earliest_seqno = cfd->mem()->GetEarliestSequenceNumber();
-  memtable_info.num_entries = cfd->mem()->num_entries();
-  memtable_info.num_deletes = cfd->mem()->num_deletes();
+  memtable_info.num_entries = cfd->mem()->NumEntries();
+  memtable_info.num_deletes = cfd->mem()->NumDeletion();
   if (!cfd->ioptions()->persist_user_defined_timestamps &&
       cfd->user_comparator()->timestamp_size() > 0) {
     const Slice& newest_udt = cfd->mem()->GetNewestUDT();
@@ -2265,8 +2452,20 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
     }
   }
   if (s.ok()) {
-    SequenceNumber seq = versions_->LastSequence();
-    new_mem = cfd->ConstructNewMemtable(mutable_cf_options, seq);
+    // FIXME: from the comment for GetEarliestSequenceNumber(), any key with
+    //  seqno >= earliest_seqno should be in this or later memtable. This means
+    //  we should use LastSequence() + 1 or last_seqno + 1 here. And it needs to
+    //  be incremented with file ingestion and other operations that consumes
+    //  sequence number.
+    SequenceNumber seq;
+    if (new_imm) {
+      assert(last_seqno > versions_->LastSequence());
+      seq = last_seqno;
+    } else {
+      seq = versions_->LastSequence();
+    }
+    new_mem =
+        cfd->ConstructNewMemtable(mutable_cf_options, /*earliest_seq=*/seq);
     context->superversion_context.NewSuperVersion();
 
     ROCKS_LOG_INFO(immutable_db_options_.info_log,
@@ -2348,6 +2547,8 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
         versions_->PreComputeMinLogNumberWithUnflushedData(logfile_number_);
     if (min_wal_number_to_keep >
         versions_->GetWalSet().GetMinWalNumberToKeep()) {
+      // TODO: plumb Env::IOActivity, Env::IOPriority
+      const ReadOptions read_options;
       // Get a snapshot of the empty column families.
       // LogAndApply may release and reacquire db
       // mutex, during that period, column family may become empty (e.g. its
@@ -2405,6 +2606,18 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   cfd->mem()->SetNextLogNumber(logfile_number_);
   assert(new_mem != nullptr);
   cfd->imm()->Add(cfd->mem(), &context->memtables_to_free_);
+  if (new_imm) {
+    // Need to assign memtable id here before SetMemtable() below assigns id to
+    // the new live memtable
+    cfd->AssignMemtableID(new_imm);
+    // NOTE: new_imm and cfd->mem() references the same WAL and has the same
+    // NextLogNumber(). They should be flushed together. For non-atomic-flush,
+    // we always try to flush all immutable memtable. For atomic flush, these
+    // two memtables will be marked eligible for flush in the same call to
+    // AssignAtomicFlushSeq().
+    new_imm->SetNextLogNumber(logfile_number_);
+    cfd->imm()->Add(new_imm, &context->memtables_to_free_);
+  }
   new_mem->Ref();
   cfd->SetMemtable(new_mem);
   InstallSuperVersionAndScheduleWork(cfd, &context->superversion_context,
@@ -2417,6 +2630,9 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   // that is okay.  If we did, it most likely means that s was already an error.
   // In any case, ignore any unchecked error for i_os here.
   io_s.PermitUncheckedError();
+  // We guarantee that if a non-ok status is returned, `new_imm` was not added
+  // to the db.
+  assert(s.ok());
   return s;
 }