@nxtedition/rocksdb 14.0.0 → 15.1.0

package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h

@@ -176,9 +176,20 @@ class CompactionJob {
   // and organizing seqno <-> time info. `known_single_subcompact` is non-null
   // if we already have a known single subcompaction, with optional key bounds
   // (currently for executing a remote compaction).
+  //
+  // @param compaction_progress Previously saved compaction progress
+  //   to resume from. If empty, compaction starts fresh from the
+  //   beginning.
+  //
+  // @param compaction_progress_writer Writer for persisting
+  //   subcompaction progress periodically during compaction
+  //   execution. If nullptr, progress tracking is disabled and compaction
+  //   cannot be resumed later.
   void Prepare(
       std::optional<std::pair<std::optional<Slice>, std::optional<Slice>>>
-          known_single_subcompact);
+          known_single_subcompact,
+      const CompactionProgress& compaction_progress = CompactionProgress{},
+      log::Writer* compaction_progress_writer = nullptr);
 
   // REQUIRED mutex not held
   // Launch threads for each subcompaction and wait for them to finish. After
@@ -197,6 +208,7 @@ class CompactionJob {
 
  protected:
   void UpdateCompactionJobOutputStatsFromInternalStats(
+      const Status& status,
       const InternalStats::CompactionStatsFull& internal_stats) const;
 
   void LogCompaction();
@@ -259,6 +271,10 @@ class CompactionJob {
   // consecutive groups such that each group has a similar size.
   void GenSubcompactionBoundaries();
 
+  void MaybeAssignCompactionProgressAndWriter(
+      const CompactionProgress& compaction_progress,
+      log::Writer* compaction_progress_writer);
+
   // Get the number of planned subcompactions based on max_subcompactions and
   // extra reserved resources
   uint64_t GetSubcompactionsLimit();
@@ -359,8 +375,9 @@ class CompactionJob {
       const CompactionFilter* configured_compaction_filter,
       const CompactionFilter*& compaction_filter,
       std::unique_ptr<CompactionFilter>& compaction_filter_from_factory);
-  void InitializeReadOptions(ColumnFamilyData* cfd, ReadOptions& read_options,
-                             SubcompactionKeyBoundaries& boundaries);
+  void InitializeReadOptionsAndBoundaries(
+      size_t ts_sz, ReadOptions& read_options,
+      SubcompactionKeyBoundaries& boundaries);
   InternalIterator* CreateInputIterator(
       SubcompactionState* sub_compact, ColumnFamilyData* cfd,
       SubcompactionInternalIterators& iterators,
@@ -411,12 +428,12 @@ class CompactionJob {
   // update the thread status for starting a compaction.
   void ReportStartedCompaction(Compaction* compaction);
 
-  Status FinishCompactionOutputFile(const Status& input_status,
-                                    SubcompactionState* sub_compact,
-                                    CompactionOutputs& outputs,
-                                    const Slice& next_table_min_key,
-                                    const Slice* comp_start_user_key,
-                                    const Slice* comp_end_user_key);
+  Status FinishCompactionOutputFile(
+      const Status& input_status,
+      const ParsedInternalKey& prev_table_last_internal_key,
+      const Slice& next_table_min_key, const Slice* comp_start_user_key,
+      const Slice* comp_end_user_key, const CompactionIterator* c_iter,
+      SubcompactionState* sub_compact, CompactionOutputs& outputs);
   Status InstallCompactionResults(bool* compaction_released);
   Status OpenCompactionOutputFile(SubcompactionState* sub_compact,
                                   CompactionOutputs& outputs);
@@ -493,6 +510,9 @@ class CompactionJob {
   // Setting this requires DBMutex.
   uint64_t options_file_number_ = 0;
 
+  // Writer for persisting compaction progress during compaction
+  log::Writer* compaction_progress_writer_ = nullptr;
+
   // Get table file name in where it's outputting to, which should also be in
   // `output_directory_`.
   virtual std::string GetTableFileName(uint64_t file_number);
@@ -500,6 +520,43 @@ class CompactionJob {
   // The Compaction Read and Write priorities are the same for different
   // scenarios, such as write stalled.
   Env::IOPriority GetRateLimiterPriority();
+
+  Status MaybeResumeSubcompactionProgressOnInputIterator(
+      SubcompactionState* sub_compact, InternalIterator* input_iter);
+
+  Status ReadOutputFilesTableProperties(
+      const autovector<FileMetaData>& temporary_output_file_allocation,
+      const ReadOptions& read_options,
+      std::vector<std::shared_ptr<const TableProperties>>&
+          output_files_table_properties,
+      bool is_proximal_level = false);
+
+  Status ReadTablePropertiesDirectly(
+      const ImmutableOptions& ioptions, const MutableCFOptions& moptions,
+      const FileMetaData* file_meta, const ReadOptions& read_options,
+      std::shared_ptr<const TableProperties>* tp);
+
+  void RestoreCompactionOutputs(
+      const ColumnFamilyData* cfd,
+      const std::vector<std::shared_ptr<const TableProperties>>&
+          output_files_table_properties,
+      SubcompactionProgressPerLevel& subcompaction_progress_per_level,
+      CompactionOutputs* outputs_to_restore);
+
+  bool ShouldUpdateSubcompactionProgress(
+      const SubcompactionState* sub_compact, const CompactionIterator* c_iter,
+      const ParsedInternalKey& prev_table_last_internal_key,
+      const Slice& next_table_min_internal_key, const FileMetaData* meta) const;
+
+  void UpdateSubcompactionProgress(const CompactionIterator* c_iter,
+                                   const Slice next_table_min_key,
+                                   SubcompactionState* sub_compact);
+
+  Status PersistSubcompactionProgress(SubcompactionState* sub_compact);
+
+  void UpdateSubcompactionProgressPerLevel(
+      SubcompactionState* sub_compact, bool is_proximal_level,
+      SubcompactionProgress& subcompaction_progress);
 };
 
 // CompactionServiceInput is used the pass compaction information between two
@@ -649,7 +706,9 @@ class CompactionServiceCompactionJob : private CompactionJob {
 
   // REQUIRED: mutex held
   // Like CompactionJob::Prepare()
-  void Prepare();
+  void Prepare(
+      const CompactionProgress& compaction_progress = CompactionProgress{},
+      log::Writer* compaction_progress_writer = nullptr);
 
   // Run the compaction in current thread and return the result
   Status Run();

package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc

@@ -17,6 +17,7 @@
 #include "db/db_impl/db_impl.h"
 #include "db/error_handler.h"
 #include "db/version_set.h"
+#include "file/filename.h"
 #include "file/random_access_file_reader.h"
 #include "file/writable_file_writer.h"
 #include "options/options_helper.h"
@@ -2409,6 +2410,448 @@ TEST_F(CompactionJobIOPriorityTest, GetRateLimiterPriority) {
                 Env::IO_LOW, Env::IO_LOW);
 }
 
+class ResumableCompactionJobTest : public CompactionJobTestBase {
+ public:
+  ResumableCompactionJobTest()
+      : CompactionJobTestBase(
+            test::PerThreadDBPath("allow_resumption_job_test"),
+            BytewiseComparator(), [](uint64_t /*ts*/) { return ""; },
+            /*test_io_priority=*/false, TableTypeForTest::kBlockBasedTable) {}
+
+ protected:
+  static constexpr const char* kCancelBeforeThisKey = "cancel_before_this_key";
+  std::string progress_dir_ = "";
+  bool enable_cancel_ = false;
+  std::atomic<int> stop_count_{0};
+  std::atomic<bool> cancel_{false};
+
+  void SetUp() override {
+    CompactionJobTestBase::SetUp();
+    SyncPoint::GetInstance()->SetCallBack(
+        "CompactionOutputs::ShouldStopBefore::manual_decision",
+        [this](void* p) {
+          auto* pair = static_cast<std::pair<bool*, const Slice>*>(p);
+          *(pair->first) = true;
+
+          // Cancel after outputting a specific key
+          if (enable_cancel_) {
+            ParsedInternalKey parsed_key;
+            if (ParseInternalKey(pair->second, &parsed_key, true).ok()) {
+              if (parsed_key.user_key == kCancelBeforeThisKey) {
+                cancel_.store(true);
+              }
+            }
+          }
+        });
+    SyncPoint::GetInstance()->EnableProcessing();
+  }
+
+  void TearDown() override {
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+
+    if (env_->FileExists(progress_dir_).ok()) {
+      std::vector<std::string> files;
+      EXPECT_OK(env_->GetChildren(progress_dir_, &files));
+      for (const auto& file : files) {
+        if (file != "." && file != "..") {
+          EXPECT_OK(env_->DeleteFile(progress_dir_ + "/" + file));
+        }
+      }
+      EXPECT_OK(env_->DeleteDir(progress_dir_));
+    }
+
+    CompactionJobTestBase::TearDown();
+  }
+
+  void NewDB() {
+    if (env_->FileExists(progress_dir_).ok()) {
+      std::vector<std::string> files;
+      EXPECT_OK(env_->GetChildren(progress_dir_, &files));
+      for (const auto& file : files) {
+        if (file != "." && file != "..") {
+          EXPECT_OK(env_->DeleteFile(progress_dir_ + "/" + file));
+        }
+      }
+      EXPECT_OK(env_->DeleteDir(progress_dir_));
+    }
+
+    CompactionJobTestBase::NewDB();
+
+    progress_dir_ = test::PerThreadDBPath("compaction_progress");
+    ASSERT_OK(env_->CreateDirIfMissing(progress_dir_));
+  }
+
+  void EnableCompactionCancel() { enable_cancel_ = true; }
+
+  void DisableCompactionCancel() {
+    enable_cancel_ = false;
+    cancel_.store(false);
+  }
+
+  std::unique_ptr<log::Writer> CreateCompactionProgressWriter(
+      const std::string& compaction_progress_file) {
+    std::unique_ptr<FSWritableFile> file;
+    EXPECT_OK(fs_->NewWritableFile(compaction_progress_file, FileOptions(),
+                                   &file, nullptr));
+    auto file_writer = std::make_unique<WritableFileWriter>(
+        std::move(file), compaction_progress_file, FileOptions());
+    auto compaction_progress_writer =
+        std::make_unique<log::Writer>(std::move(file_writer), 0, false);
+    return compaction_progress_writer;
+  }
+
+  Status RunCompactionWithProgressTracking(
+      const CompactionProgress& compaction_progress,
+      log::Writer* compaction_progress_writer,
+      std::vector<SequenceNumber> snapshots = {},
+      std::shared_ptr<Statistics> stats = nullptr) {
+    mutex_.Lock();
+
+    auto cfd = versions_->GetColumnFamilySet()->GetDefault();
+    auto files = cfd->current()->storage_info()->LevelFiles(0);
+
+    db_options_.statistics = stats;
+    db_options_.stats = db_options_.statistics.get();
+
+    std::vector<CompactionInputFiles> compaction_input_files;
+    CompactionInputFiles level;
+    level.level = 0;
+    level.files = files;
+    compaction_input_files.push_back(level);
+
+    Compaction compaction(
+        cfd->current()->storage_info(), cfd->ioptions(),
+        cfd->GetLatestMutableCFOptions(), mutable_db_options_,
+        compaction_input_files, 1, mutable_cf_options_.target_file_size_base,
+        mutable_cf_options_.max_compaction_bytes, 0, kNoCompression,
+        cfd->GetLatestMutableCFOptions().compression_opts,
+        Temperature::kUnknown, 0, {}, std::nullopt, nullptr,
+        CompactionReason::kManualCompaction);
+    compaction.FinalizeInputInfo(cfd->current());
+
+    LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
+    EventLogger event_logger(db_options_.info_log.get());
+    JobContext job_context(1, false);
+    job_context.InitSnapshotContext(nullptr, nullptr, kMaxSequenceNumber,
+                                    std::move(snapshots));
+    CompactionJobStats job_stats;
+
+    CompactionJob compaction_job(
+        0, &compaction, db_options_, mutable_db_options_, env_options_,
+        versions_.get(), &shutting_down_, &log_buffer, nullptr, nullptr,
+        nullptr, stats.get(), &mutex_, &error_handler_, &job_context,
+        table_cache_, &event_logger, false, false, dbname_, &job_stats,
+        Env::Priority::USER, nullptr, cancel_, env_->GenerateUniqueId(),
+        DBImpl::GenerateDbSessionId(nullptr), "");
+
+    compaction_job.Prepare(std::nullopt, compaction_progress,
+                           compaction_progress_writer);
+    mutex_.Unlock();
+
+    compaction_job.Run().PermitUncheckedError();
+    EXPECT_OK(compaction_job.io_status());
+
+    mutex_.Lock();
+
+    bool compaction_released = false;
+    Status s = compaction_job.Install(&compaction_released);
+
+    mutex_.Unlock();
+    if (!compaction_released) {
+      compaction.ReleaseCompactionFiles(s);
+    }
+
+    return s;
+  }
+
+  SubcompactionProgress ReadAndParseProgress(
+      const std::string& compaction_progress_file) {
+    std::unique_ptr<FSSequentialFile> seq_file;
+    EXPECT_OK(fs_->NewSequentialFile(compaction_progress_file, FileOptions(),
+                                     &seq_file, nullptr));
+    auto file_reader = std::make_unique<SequentialFileReader>(
+        std::move(seq_file), compaction_progress_file, 0, nullptr);
+    log::Reader reader(nullptr, std::move(file_reader), nullptr, true, 0);
+
+    SubcompactionProgressBuilder builder;
+    std::string record;
+    Slice slice;
+
+    while (reader.ReadRecord(&slice, &record)) {
+      VersionEdit edit;
+      if (!edit.DecodeFrom(slice).ok()) continue;
+      builder.ProcessVersionEdit(edit);
+    }
+
+    EXPECT_TRUE(builder.HasAccumulatedSubcompactionProgress());
+
+    return builder.GetAccumulatedSubcompactionProgress();
+  }
+
+  // Test utility function to verify that compaction progress was correctly
+  // persisted to the progress file after compaction interruption.
+  //
+  // VERIFIES:
+  // - Progress file exists and has expected size (empty if no progress
+  // expected)
+  // - Next internal key to compact matches expected user key with proper format
+  // - Number of processed input records matches position in ordered input keys
+  // - Number of processed output records equals number of processed input
+  // records (by test design to simplify verification)
+  // - Each output file contains exactly one user key (by test design to
+  // simplify verification)
+  void VerifyCompactionProgressPersisted(
+      const std::string& compaction_progress_file,
+      const std::string& next_user_key_to_compact,
+      const std::vector<std::string>& ordered_intput_keys) {
+    ASSERT_OK(env_->FileExists(compaction_progress_file));
+
+    uint64_t file_size;
+    ASSERT_OK(env_->GetFileSize(compaction_progress_file, &file_size));
+
+    if (next_user_key_to_compact.empty()) {
+      ASSERT_EQ(file_size, 0);
+      return;
+    }
+
+    const auto& subcompaction_progress =
+        ReadAndParseProgress(compaction_progress_file);
+
+    ASSERT_FALSE(subcompaction_progress.next_internal_key_to_compact.empty());
+    ParsedInternalKey parsed_next_key;
+    ASSERT_OK(
+        ParseInternalKey(subcompaction_progress.next_internal_key_to_compact,
+                         &parsed_next_key, true /* log_err_key */));
+    ASSERT_EQ(parsed_next_key.user_key, next_user_key_to_compact);
+    ASSERT_EQ(parsed_next_key.sequence, kMaxSequenceNumber);
+    ASSERT_EQ(parsed_next_key.type, kValueTypeForSeek);
+
+    auto it = std::find(ordered_intput_keys.begin(), ordered_intput_keys.end(),
+                        next_user_key_to_compact);
+    ASSERT_TRUE(it != ordered_intput_keys.end());
+
+    auto next_key_index = std::distance(ordered_intput_keys.begin(), it);
+
+    ASSERT_EQ(subcompaction_progress.num_processed_input_records,
+              next_key_index);
+
+    ASSERT_EQ(subcompaction_progress.output_level_progress
+                  .GetNumProcessedOutputRecords(),
+              next_key_index);
+
+    ASSERT_EQ(
+        subcompaction_progress.output_level_progress.GetOutputFiles().size(),
+
+        next_key_index);
+
+    for (size_t i = 0;
+         i <
+         subcompaction_progress.output_level_progress.GetOutputFiles().size();
+         ++i) {
+      const auto& output_file =
+          subcompaction_progress.output_level_progress.GetOutputFiles()[i];
+      ASSERT_EQ(output_file.smallest.user_key().ToString(),
+                output_file.largest.user_key().ToString());
+      ASSERT_EQ(output_file.largest.user_key().ToString(),
+                ordered_intput_keys[i]);
+    }
+  }
+
+  void RunCancelAndResumeTest(
+      const std::initializer_list<mock::KVPair>& input_file_1,
+      const std::initializer_list<mock::KVPair>& input_file_2,
+      uint64_t last_sequence, const std::vector<uint64_t>& snapshots,
+      const std::string& expected_next_key_to_compact,
+      const std::vector<std::string>& expected_input_keys, bool exists_progress,
+      bool cancelled_past_mid_point = false) {
+    std::shared_ptr<Statistics> stats = ROCKSDB_NAMESPACE::CreateDBStatistics();
+
+    auto file1 = mock::MakeMockFile(input_file_1);
+    AddMockFile(file1);
+    auto file2 = mock::MakeMockFile(input_file_2);
+    AddMockFile(file2);
+    SetLastSequence(last_sequence);
+
+    // First compaction (will be cancelled)
+    std::string compaction_progress_file =
+        CompactionProgressFileName(progress_dir_, 123);
+    std::unique_ptr<log::Writer> compaction_progress_writer =
+        CreateCompactionProgressWriter(compaction_progress_file);
+
+    ASSERT_OK(stats->Reset());
+    EnableCompactionCancel();
+
+    Status status = RunCompactionWithProgressTracking(
+        CompactionProgress{}, compaction_progress_writer.get(), snapshots,
+        stats);
+
+    ASSERT_TRUE(status.IsManualCompactionPaused());
+    DisableCompactionCancel();
+
+    HistogramData cancelled_compaction_stats;
+    stats->histogramData(FILE_WRITE_COMPACTION_MICROS,
+                         &cancelled_compaction_stats);
+
+    VerifyCompactionProgressPersisted(compaction_progress_file,
+                                      expected_next_key_to_compact,
+                                      expected_input_keys);
+
+    // Resume compaction
+    CompactionProgress compaction_progress;
+    if (exists_progress) {
+      compaction_progress.push_back(
+          ReadAndParseProgress(compaction_progress_file));
+    }
+
+    std::string compaction_progress_file_2 =
+        CompactionProgressFileName(progress_dir_, 234);
+    std::unique_ptr<log::Writer> compaction_progress_writer_2 =
+        CreateCompactionProgressWriter(compaction_progress_file_2);
+
+    ASSERT_OK(stats->Reset());
+
+    status = RunCompactionWithProgressTracking(
+        compaction_progress, compaction_progress_writer_2.get(),
+        {} /* snapshots */, stats);
+
+    ASSERT_OK(status);
+
+    if (cancelled_past_mid_point) {
+      HistogramData resumed_compaction_stats;
+      stats->histogramData(FILE_WRITE_COMPACTION_MICROS,
+                           &resumed_compaction_stats);
+      ASSERT_GT(cancelled_compaction_stats.count,
+                resumed_compaction_stats.count);
+    }
+  }
+};
+
+TEST_F(ResumableCompactionJobTest, BasicProgressPersistence) {
+  NewDB();
+
+  auto file1 = mock::MakeMockFile({
+      {KeyStr("a", 1U, kTypeValue), "val1"},
+      {KeyStr("b", 2U, kTypeValue), "val2"},
+  });
+  AddMockFile(file1);
+
+  auto file2 = mock::MakeMockFile({
+      {KeyStr("c", 3U, kTypeValue), "val3"},
+      {KeyStr("d", 4U, kTypeValue), "val4"},
+  });
+  AddMockFile(file2);
+
+  SetLastSequence(4U);
+
+  std::string compaction_progress_file =
+      CompactionProgressFileName(progress_dir_, 123);
+
+  std::unique_ptr<log::Writer> compaction_progress_writer =
+      CreateCompactionProgressWriter(compaction_progress_file);
+
+  Status status = RunCompactionWithProgressTracking(
+      CompactionProgress(), compaction_progress_writer.get());
+
+  ASSERT_OK(status);
+
+  VerifyCompactionProgressPersisted(
+      compaction_progress_file, "d" /* next_user_key_to_compact */,
+      {"a", "b", "c", "d"} /* ordered_intput_keys */);
+}
+
+TEST_F(ResumableCompactionJobTest, BasicProgressResume) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr("a", 1U, kTypeValue), "val1"},
+       {KeyStr("b", 2U, kTypeValue), "val2"}} /* input_file_1 */,
+      {{KeyStr("bb", 3U, kTypeValue), "val3"},
+       {KeyStr(kCancelBeforeThisKey, 4U, kTypeValue),
+        "val4"}} /* input_file_2 */,
+      4U /* last_sequence */, {} /* snapshots */,
+      kCancelBeforeThisKey /* expected_next_key_to_compact */,
+      {"a", "b", "bb", kCancelBeforeThisKey} /* expected_input_keys */,
+      true /* exists_progress */, true /* cancelled_past_mid_point*/);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnSameKey) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr(kCancelBeforeThisKey, 1U, kTypeValue),
+        "val1"}} /* input_file_1 */,
+      {{KeyStr(kCancelBeforeThisKey, 2U, kTypeValue),
+        "val2"}} /* input_file_2 */,
+      2U /* last_sequence */, {1U} /* snapshots */,
+      "" /* expected_next_key_to_compact */,
+      {kCancelBeforeThisKey, kCancelBeforeThisKey} /* expected_input_keys */,
+      false /* exists_progress */);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnDeleteRange) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr(kCancelBeforeThisKey, 1U, kTypeValue),
+        "val1"}} /* input_file_1 */,
+      {{KeyStr(kCancelBeforeThisKey, 2U, kTypeRangeDeletion),
+        "val2"}} /* input_file_2 */,
+      2U /* last_sequence */, {1U} /* snapshots */,
+      "" /* expected_next_key_to_compact */,
+      {kCancelBeforeThisKey, kCancelBeforeThisKey} /* expected_input_keys */,
+      false /* exists_progress */);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnMerge) {
+  merge_op_ = MergeOperators::CreateStringAppendOperator();
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr("a", 1U, kTypeValue), "val1"},
+       {KeyStr("b", 2U, kTypeValue), "val2"}} /* input_file_1 */,
+      {{KeyStr("bb", 3U, kTypeValue), "val3"},
+       {KeyStr(kCancelBeforeThisKey, 4U, kTypeMerge),
+        "val4"}} /* input_file_2 */,
+      4U /* last_sequence */, {} /* snapshots */,
+      "bb" /* expected_next_key_to_compact */,
+      {"a", "b", "bb", kCancelBeforeThisKey} /* expected_input_keys */,
+      true /* exists_progress */);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnSingleDelete) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr("a", 1U, kTypeValue), "val1"},
+       {KeyStr("b", 2U, kTypeValue), "val2"},
+       {KeyStr(kCancelBeforeThisKey, 3U, kTypeValue),
+        "val3"}} /* input_file_1 */,
+      {{KeyStr(kCancelBeforeThisKey, 4U, kTypeSingleDeletion), ""},
+       {KeyStr("d", 5U, kTypeValue), "val4"}} /* input_file_2 */,
+      5U /* last_sequence */, {3U} /* snapshots */,
+      "b" /* expected_next_key_to_compact */,
+      {"a", "b", kCancelBeforeThisKey, kCancelBeforeThisKey,
+       "d"} /* expected_input_keys */,
+      true /* exists_progress */);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnDeletionAtBottom) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr("a", 1U, kTypeValue), "val1"},
+       {KeyStr("b", 2U, kTypeValue), "val2"},
+       {KeyStr(kCancelBeforeThisKey, 3U, kTypeValue),
+        "val3"}} /* input_file_1 */,
+      {{KeyStr(kCancelBeforeThisKey, 4U, kTypeDeletion), ""},
+       {KeyStr("d", 5U, kTypeValue), "val4"}} /* input_file_2 */,
+      5U /* last_sequence */, {3U} /* snapshots */,
+      "b" /* expected_next_key_to_compact */,
+      {"a", "b", kCancelBeforeThisKey, kCancelBeforeThisKey,
+       "d"} /* expected_input_keys */,
+      true /* exists_progress */);
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc

@@ -359,7 +359,8 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
 Status CompactionOutputs::AddToOutput(
     const CompactionIterator& c_iter,
     const CompactionFileOpenFunc& open_file_func,
-    const CompactionFileCloseFunc& close_file_func) {
+    const CompactionFileCloseFunc& close_file_func,
+    const ParsedInternalKey& prev_table_last_internal_key) {
   Status s;
   bool is_range_del = c_iter.IsDeleteRangeSentinelKey();
   if (is_range_del && compaction_->bottommost_level()) {
@@ -370,7 +371,8 @@ Status CompactionOutputs::AddToOutput(
   }
   const Slice& key = c_iter.key();
   if (ShouldStopBefore(c_iter) && HasBuilder()) {
-    s = close_file_func(*this, c_iter.InputStatus(), key);
+    s = close_file_func(c_iter.InputStatus(), prev_table_last_internal_key, key,
+                        &c_iter, *this);
     if (!s.ok()) {
       return s;
     }

package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h

@@ -21,7 +21,8 @@ namespace ROCKSDB_NAMESPACE {
 class CompactionOutputs;
 using CompactionFileOpenFunc = std::function<Status(CompactionOutputs&)>;
 using CompactionFileCloseFunc =
-    std::function<Status(CompactionOutputs&, const Status&, const Slice&)>;
+    std::function<Status(const Status&, const ParsedInternalKey&, const Slice&,
+                         const CompactionIterator*, CompactionOutputs&)>;
 
 // Files produced by subcompaction, most of the functions are used by
 // compaction_job Open/Close compaction file functions.
@@ -58,6 +59,8 @@ class CompactionOutputs {
                           precalculated_hash, is_proximal_level_);
   }
 
+  const std::vector<Output>& GetOutputs() const { return outputs_; }
+
   // Set new table builder for the current output
   void NewBuilder(const TableBuilderOptions& tboptions);
 
@@ -195,6 +198,10 @@ class CompactionOutputs {
       std::pair<SequenceNumber, SequenceNumber> keep_seqno_range,
       const Slice& next_table_min_key, const std::string& full_history_ts_low);
 
+  void SetNumOutputRecords(uint64_t num_output_records) {
+    stats_.num_output_records = num_output_records;
+  }
+
  private:
   friend class SubcompactionState;
 
@@ -254,7 +261,8 @@ class CompactionOutputs {
   // close and open new compaction output with the functions provided.
   Status AddToOutput(const CompactionIterator& c_iter,
                      const CompactionFileOpenFunc& open_file_func,
-                     const CompactionFileCloseFunc& close_file_func);
+                     const CompactionFileCloseFunc& close_file_func,
+                     const ParsedInternalKey& prev_table_last_internal_key);
 
   // Close the current output. `open_file_func` is needed for creating new file
   // for range-dels only output file.
@@ -270,9 +278,12 @@ class CompactionOutputs {
         !range_del_agg->IsEmpty()) {
       status = open_file_func(*this);
     }
+
     if (HasBuilder()) {
+      const ParsedInternalKey empty_internal_key{};
       const Slice empty_key{};
-      Status s = close_file_func(*this, status, empty_key);
+      Status s = close_file_func(status, empty_internal_key, empty_key,
+                                 nullptr /* c_iter */, *this);
       if (!s.ok() && status.ok()) {
         status = s;
       }