@nxtedition/rocksdb 15.1.2 → 15.1.4

package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc

@@ -23,147 +23,339 @@ Options GetNoCompactionOptions(const Options& opts) {
   return ret_opts;
 }
 
-Status OpenDb(const Options& options, const std::string& dbname,
-              std::unique_ptr<DB>* db) {
-  db->reset();
-  DB* tmpdb;
-  Status s = DB::Open(options, dbname, &tmpdb);
-  if (s.ok()) {
-    db->reset(tmpdb);
-  }
-  return s;
-}
+// Compact a specific CF to a specific level
+//  cf_handle should not be null
+Status CompactToLevel(DB* db, ColumnFamilyHandle* cf_handle, int dest_level) {
+  assert(cf_handle != nullptr);
 
-// l0_file_size specifies size of file on L0. Files will be range partitioned
-// after a full compaction so they are likely qualified to put on L0. If
-// left as 0, the files are compacted in a single file and put to L0. Otherwise,
-// will try to compact the files as size l0_file_size.
-Status CompactToLevel(const Options& options, const std::string& dbname,
-                      int dest_level, uint64_t l0_file_size, bool need_reopen) {
-  std::unique_ptr<DB> db;
-  Options no_compact_opts = GetNoCompactionOptions(options);
-  if (dest_level == 0) {
-    if (l0_file_size == 0) {
-      // Single file.
-      l0_file_size = 999999999999999;
-    }
-    // L0 has strict sequenceID requirements to files to it. It's safer
-    // to only put one compacted file to there.
-    // This is only used for converting to universal compaction with
-    // only one level. In this case, compacting to one file is also
-    // optimal.
-    no_compact_opts.target_file_size_base = l0_file_size;
-    no_compact_opts.max_compaction_bytes = l0_file_size;
-  }
-  Status s = OpenDb(no_compact_opts, dbname, &db);
-  if (!s.ok()) {
-    return s;
-  }
   CompactRangeOptions cro;
   cro.change_level = true;
   cro.target_level = dest_level;
+
   if (dest_level == 0) {
     // cannot use kForceOptimized because the compaction is expected to
-    // generate one output file
+    // generate one output file so to force the full compaction to skip trivial
+    // move to L0
     cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
   }
-  s = db->CompactRange(cro, nullptr, nullptr);
 
-  if (s.ok() && need_reopen) {
-    // Need to restart DB to rewrite the manifest file.
-    // In order to open a DB with specific num_levels, the manifest file should
-    // contain no record that mentiones any level beyond num_levels. Issuing a
-    // full compaction will move all the data to a level not exceeding
-    // num_levels, but the manifest may still contain previous record mentioning
-    // a higher level. Reopening the DB will force the manifest to be rewritten
-    // so that those records will be cleared.
-    db.reset();
-    s = OpenDb(no_compact_opts, dbname, &db);
-  }
-  return s;
+  return db->CompactRange(cro, cf_handle, nullptr, nullptr);
 }
 
-Status MigrateToUniversal(std::string dbname, const Options& old_opts,
-                          const Options& new_opts) {
-  if (old_opts.num_levels <= new_opts.num_levels ||
-      old_opts.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
-    return Status::OK();
-  } else {
-    bool need_compact = false;
-    {
-      std::unique_ptr<DB> db;
-      Options opts = GetNoCompactionOptions(old_opts);
-      Status s = OpenDb(opts, dbname, &db);
-      if (!s.ok()) {
-        return s;
-      }
-      ColumnFamilyMetaData metadata;
-      db->GetColumnFamilyMetaData(&metadata);
-      if (!metadata.levels.empty() &&
-          metadata.levels.back().level >= new_opts.num_levels) {
-        need_compact = true;
-      }
-    }
-    if (need_compact) {
-      return CompactToLevel(old_opts, dbname, new_opts.num_levels - 1,
-                            /*l0_file_size=*/0, true);
-    }
+Status MigrateToUniversal(DB* db, ColumnFamilyHandle* cf_handle,
+                          int old_num_levels, int new_num_levels) {
+  assert(cf_handle != nullptr);
+
+  if (old_num_levels <= new_num_levels) {
     return Status::OK();
   }
+
+  // Check if compaction is needed
+  ColumnFamilyMetaData metadata;
+  db->GetColumnFamilyMetaData(cf_handle, &metadata);
+
+  if (!metadata.levels.empty() &&
+      metadata.levels.back().level >= new_num_levels) {
+    // Need to compact to fit new num_levels
+    return CompactToLevel(db, cf_handle, new_num_levels - 1);
+  }
+
+  return Status::OK();
 }
 
-Status MigrateToLevelBase(std::string dbname, const Options& old_opts,
-                          const Options& new_opts) {
-  if (!new_opts.level_compaction_dynamic_level_bytes) {
-    if (old_opts.num_levels == 1) {
+Status MigrateToLevelBase(DB* db, ColumnFamilyHandle* cf_handle,
+                          int old_num_levels, int new_num_levels,
+                          bool dynamic_level_bytes) {
+  assert(cf_handle != nullptr);
+
+  if (!dynamic_level_bytes) {
+    // Non-dynamic level mode
+    if (old_num_levels == 1) {
       return Status::OK();
     }
-    // Compact everything to level 1 to guarantee it can be safely opened.
-    Options opts = old_opts;
-    opts.target_file_size_base = new_opts.target_file_size_base;
-    // Although sometimes we can open the DB with the new option without error,
-    // We still want to compact the files to avoid the LSM tree to stuck
-    // in bad shape. For example, if the user changed the level size
-    // multiplier from 4 to 8, with the same data, we will have fewer
-    // levels. Unless we issue a full comaction, the LSM tree may stuck
-    // with more levels than needed and it won't recover automatically.
-    return CompactToLevel(opts, dbname, 1, /*l0_file_size=*/0, true);
+    // Compact to L1
+    return CompactToLevel(db, cf_handle, 1);
+
   } else {
-    // Compact everything to the last level to guarantee it can be safely
-    // opened.
-    if (old_opts.num_levels == 1) {
+    // Dynamic level mode
+    if (old_num_levels == 1) {
       return Status::OK();
-    } else if (new_opts.num_levels > old_opts.num_levels) {
-      // Dynamic level mode requires data to be put in the last level first.
-      return CompactToLevel(new_opts, dbname, new_opts.num_levels - 1,
-                            /*l0_file_size=*/0, false);
-    } else {
-      Options opts = old_opts;
-      opts.target_file_size_base = new_opts.target_file_size_base;
-      return CompactToLevel(opts, dbname, new_opts.num_levels - 1,
-                            /*l0_file_size=*/0, true);
     }
+    // Compact to last level
+    return CompactToLevel(db, cf_handle, new_num_levels - 1);
   }
 }
-}  // namespace
 
-Status OptionChangeMigration(std::string dbname, const Options& old_opts,
-                             const Options& new_opts) {
+Status MigrateToFIFO(DB* db, ColumnFamilyHandle* cf_handle) {
+  assert(cf_handle != nullptr);
+  return CompactToLevel(db, cf_handle, 0);
+}
+
+Status MigrateSingleColumnFamily(DB* db, ColumnFamilyHandle* cf_handle,
+                                 const Options& old_opts,
+                                 const Options& new_opts) {
+  assert(cf_handle != nullptr);
+
   if (old_opts.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
-    // LSM generated by FIFO compaction can be opened by any compaction.
     return Status::OK();
-  } else if (new_opts.compaction_style ==
-             CompactionStyle::kCompactionStyleUniversal) {
-    return MigrateToUniversal(dbname, old_opts, new_opts);
+  }
+
+  if (new_opts.compaction_style == CompactionStyle::kCompactionStyleUniversal) {
+    return MigrateToUniversal(db, cf_handle, old_opts.num_levels,
+                              new_opts.num_levels);
   } else if (new_opts.compaction_style ==
              CompactionStyle::kCompactionStyleLevel) {
-    return MigrateToLevelBase(dbname, old_opts, new_opts);
+    return MigrateToLevelBase(db, cf_handle, old_opts.num_levels,
+                              new_opts.num_levels,
+                              new_opts.level_compaction_dynamic_level_bytes);
   } else if (new_opts.compaction_style ==
              CompactionStyle::kCompactionStyleFIFO) {
-    return CompactToLevel(old_opts, dbname, 0, 0 /* l0_file_size */, true);
+    return MigrateToFIFO(db, cf_handle);
+  }
+
+  return Status::NotSupported(
+      "Do not know how to migrate to this compaction style");
+}
+
+Status ValidateCFDescriptors(
+    const std::vector<ColumnFamilyDescriptor>& old_cf_descs,
+    const std::vector<ColumnFamilyDescriptor>& new_cf_descs) {
+  if (old_cf_descs.size() != new_cf_descs.size()) {
+    return Status::InvalidArgument(
+        "old_cf_descs and new_cf_descs must have the same number of column "
+        "families. Got " +
+        std::to_string(old_cf_descs.size()) + " old CFs and " +
+        std::to_string(new_cf_descs.size()) +
+        " new CFs. Adding or dropping CFs is not supported.");
+  }
+
+  for (size_t i = 0; i < old_cf_descs.size(); ++i) {
+    if (old_cf_descs[i].name != new_cf_descs[i].name) {
+      return Status::InvalidArgument(
+          "Column family mismatch at index " + std::to_string(i) + ": " +
+          "old has '" + old_cf_descs[i].name + "', " + "new has '" +
+          new_cf_descs[i].name + "'. CF names and order must match exactly.");
+    }
+  }
+
+  return Status::OK();
+}
+
+struct BaseOptionsResult {
+  ColumnFamilyOptions base_opts;
+  bool need_reopen = true;
+};
+
+BaseOptionsResult DetermineBaseOptions(const ColumnFamilyOptions& old_opts,
+                                       const ColumnFamilyOptions& new_opts) {
+  BaseOptionsResult result;
+
+  if (new_opts.compaction_style == CompactionStyle::kCompactionStyleLevel) {
+    if (!new_opts.level_compaction_dynamic_level_bytes) {
+      result.base_opts = old_opts;
+      result.base_opts.target_file_size_base = new_opts.target_file_size_base;
+    } else {
+      if (new_opts.num_levels > old_opts.num_levels) {
+        result.base_opts = new_opts;
+        result.need_reopen = false;
+      } else {
+        result.base_opts = old_opts;
+        result.base_opts.target_file_size_base = new_opts.target_file_size_base;
+      }
+    }
+  } else {
+    result.base_opts = old_opts;
+  }
+
+  return result;
+}
+
+void ApplySpecialSingleLevelSettings(const ColumnFamilyOptions& new_opts,
+                                     ColumnFamilyOptions* base_opts) {
+  if (((new_opts.compaction_style ==
+            CompactionStyle::kCompactionStyleUniversal ||
+        new_opts.compaction_style == CompactionStyle::kCompactionStyleLevel) &&
+       new_opts.num_levels == 1) ||
+      new_opts.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
+    base_opts->target_file_size_base = 999999999999999;
+    base_opts->max_compaction_bytes = 999999999999999;
+  }
+}
+
+std::vector<ColumnFamilyDescriptor> PrepareNoCompactionCFDescriptors(
+    const DBOptions& old_db_opts,
+    const std::vector<ColumnFamilyDescriptor>& old_cf_descs,
+    const std::vector<ColumnFamilyDescriptor>& new_cf_descs,
+    bool* any_need_reopen) {
+  assert(old_cf_descs.size() == new_cf_descs.size());
+
+  std::vector<ColumnFamilyDescriptor> no_compact_cf_descs;
+  *any_need_reopen = false;
+
+  for (size_t i = 0; i < old_cf_descs.size(); ++i) {
+    const std::string& cf_name = old_cf_descs[i].name;
+    const ColumnFamilyOptions& old_opts = old_cf_descs[i].options;
+    const ColumnFamilyOptions& new_opts = new_cf_descs[i].options;
+
+    BaseOptionsResult result = DetermineBaseOptions(old_opts, new_opts);
+    ColumnFamilyOptions base_opts = result.base_opts;
+
+    if (result.need_reopen) {
+      *any_need_reopen = true;
+    }
+
+    ApplySpecialSingleLevelSettings(new_opts, &base_opts);
+
+    Options tmp_opts(old_db_opts, base_opts);
+    Options no_compact_opts = GetNoCompactionOptions(tmp_opts);
+
+    no_compact_cf_descs.emplace_back(cf_name,
+                                     ColumnFamilyOptions(no_compact_opts));
+  }
+
+  return no_compact_cf_descs;
+}
+
+Status OpenDBWithCFs(const DBOptions& db_opts, const std::string& dbname,
+                     const std::vector<ColumnFamilyDescriptor>& cf_descs,
+                     std::unique_ptr<DB>* db,
+                     std::vector<ColumnFamilyHandle*>* handles) {
+  handles->clear();
+  DB* tmpdb;
+  Status s = DB::Open(db_opts, dbname, cf_descs, handles, &tmpdb);
+
+  if (s.ok()) {
+    db->reset(tmpdb);
   } else {
-    return Status::NotSupported(
-        "Do not how to migrate to this compaction style");
+    for (auto* handle : *handles) {
+      delete handle;
+    }
+    handles->clear();
   }
+
+  return s;
+}
+
+Status CleanupCFHandles(DB* db, std::vector<ColumnFamilyHandle*>* handles) {
+  Status s;
+  for (auto* handle : *handles) {
+    if (handle != db->DefaultColumnFamily()) {
+      Status destroy_status = db->DestroyColumnFamilyHandle(handle);
+      if (!destroy_status.ok() && s.ok()) {
+        s = destroy_status;
+      }
+    }
+  }
+  handles->clear();
+  return s;
+}
+
+Status MigrateAllCFs(DB* db, const std::vector<ColumnFamilyHandle*>& handles,
+                     const DBOptions& old_db_opts, const DBOptions& new_db_opts,
+                     const std::vector<ColumnFamilyDescriptor>& old_cf_descs,
+                     const std::vector<ColumnFamilyDescriptor>& new_cf_descs) {
+  assert(handles.size() == old_cf_descs.size());
+  assert(old_cf_descs.size() == new_cf_descs.size());
+
+  for (size_t i = 0; i < handles.size(); ++i) {
+    const ColumnFamilyOptions& old_cf_opts = old_cf_descs[i].options;
+    const ColumnFamilyOptions& new_cf_opts = new_cf_descs[i].options;
+
+    Options old_opts(old_db_opts, old_cf_opts);
+    Options new_opts(new_db_opts, new_cf_opts);
+
+    Status s = MigrateSingleColumnFamily(db, handles[i], old_opts, new_opts);
+    if (!s.ok()) {
+      return s;
+    }
+  }
+
+  return Status::OK();
+}
+}  // namespace
+
+Status OptionChangeMigration(
+    const std::string& dbname, const DBOptions& old_db_opts,
+    const std::vector<ColumnFamilyDescriptor>& old_cf_descs,
+    const DBOptions& new_db_opts,
+    const std::vector<ColumnFamilyDescriptor>& new_cf_descs) {
+  // Step 1: Validate that old and new have same CFs in same order
+  Status s = ValidateCFDescriptors(old_cf_descs, new_cf_descs);
+  if (!s.ok()) {
+    return s;
+  }
+
+  // Step 2: Prepare no-compaction CF descriptors
+  bool any_need_reopen = false;
+  std::vector<ColumnFamilyDescriptor> no_compact_cf_descs =
+      PrepareNoCompactionCFDescriptors(old_db_opts, old_cf_descs, new_cf_descs,
+                                       &any_need_reopen);
+
+  // Step 3: Open DB with all CFs
+  std::unique_ptr<DB> db;
+  std::vector<ColumnFamilyHandle*> handles;
+  s = OpenDBWithCFs(old_db_opts, dbname, no_compact_cf_descs, &db, &handles);
+  if (!s.ok()) {
+    return s;
+  }
+  assert(db != nullptr);
+
+  // Step 4: Migrate all CFs
+  s = MigrateAllCFs(db.get(), handles, old_db_opts, new_db_opts, old_cf_descs,
+                    new_cf_descs);
+
+  // Step 5: Cleanup CF handles
+  Status cleanup_status = CleanupCFHandles(db.get(), &handles);
+  if (s.ok() && !cleanup_status.ok()) {
+    s = cleanup_status;
+  }
+
+  // Step 6: Close and reopen DB if needed to rewrite manifest
+  if (s.ok() && any_need_reopen) {
+    Status close_status = db->Close();
+    if (!close_status.ok()) {
+      return close_status;
+    }
+    db.reset();
+
+    s = OpenDBWithCFs(old_db_opts, dbname, no_compact_cf_descs, &db, &handles);
+    if (!s.ok()) {
+      return s;
+    }
+
+    // Cleanup CF handles before final close
+    cleanup_status = CleanupCFHandles(db.get(), &handles);
+    if (!cleanup_status.ok() && s.ok()) {
+      s = cleanup_status;
+    }
+  }
+
+  // Final step: Close DB (either after reopening or without reopening)
+  Status close_status = db->Close();
+  if (!close_status.ok() && s.ok()) {
+    s = close_status;
+  }
+
+  db.reset();
+
+  return s;
+}
+
+Status OptionChangeMigration(const std::string& dbname, const Options& old_opts,
+                             const Options& new_opts) {
+  DBOptions old_db_opts(old_opts);
+  DBOptions new_db_opts(new_opts);
+
+  ColumnFamilyOptions old_cf_opts(old_opts);
+  ColumnFamilyOptions new_cf_opts(new_opts);
+
+  std::vector<ColumnFamilyDescriptor> old_cf_descs = {
+      {kDefaultColumnFamilyName, old_cf_opts}};
+
+  std::vector<ColumnFamilyDescriptor> new_cf_descs = {
+      {kDefaultColumnFamilyName, new_cf_opts}};
+
+  return OptionChangeMigration(dbname, old_db_opts, old_cf_descs, new_db_opts,
+                               new_cf_descs);
 }
 }  // namespace ROCKSDB_NAMESPACE