@nxtedition/rocksdb 15.1.2 → 15.1.3

package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration_test.cc

@@ -556,6 +556,385 @@ TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) {
   }
 }
 
+class DBOptionChangeMigrationMultiCFTest : public DBTestBase {
+ public:
+  DBOptionChangeMigrationMultiCFTest()
+      : DBTestBase("db_option_change_migration_multi_cf_test",
+                   /*env_do_fsync=*/true) {}
+};
+
+TEST_F(DBOptionChangeMigrationMultiCFTest, BasicMultiCF) {
+  Options options = CurrentOptions();
+  options.compaction_style = CompactionStyle::kCompactionStyleLevel;
+  options.level_compaction_dynamic_level_bytes = false;
+  options.num_levels = 4;
+  options.write_buffer_size = 64 * 1024;
+  options.target_file_size_base = 128 * 1024;
+
+  // Create DB with default CF
+  Reopen(options);
+
+  // Create additional CF
+  ColumnFamilyHandle* cf_handle;
+  ASSERT_OK(db_->CreateColumnFamily(options, "cf1", &cf_handle));
+
+  // Write data to both CFs
+  Random rnd(301);
+  for (int i = 0; i < 100; i++) {
+    ASSERT_OK(Put("key" + std::to_string(i), rnd.RandomString(900)));
+    ASSERT_OK(db_->Put(WriteOptions(), cf_handle, "key" + std::to_string(i),
+                       rnd.RandomString(900)));
+  }
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->Flush(FlushOptions(), cf_handle));
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // Collect keys from both CFs
+  std::set<std::string> keys_default;
+  std::set<std::string> keys_cf1;
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+    for (it->SeekToFirst(); it->Valid(); it->Next()) {
+      keys_default.insert(it->key().ToString());
+    }
+    ASSERT_OK(it->status());
+  }
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions(), cf_handle));
+    for (it->SeekToFirst(); it->Valid(); it->Next()) {
+      keys_cf1.insert(it->key().ToString());
+    }
+    ASSERT_OK(it->status());
+  }
+
+  delete cf_handle;
+  Close();
+
+  // Prepare old and new options
+  DBOptions old_db_opts(options);
+  ColumnFamilyOptions old_cf_opts(options);
+
+  std::vector<ColumnFamilyDescriptor> old_cf_descs = {
+      {kDefaultColumnFamilyName, old_cf_opts}, {"cf1", old_cf_opts}};
+
+  // New options: migrate to Universal compaction
+  Options new_options = options;
+  new_options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  new_options.num_levels = 5;
+  new_options.target_file_size_base = 256 * 1024;
+
+  DBOptions new_db_opts(new_options);
+  ColumnFamilyOptions new_cf_opts(new_options);
+
+  std::vector<ColumnFamilyDescriptor> new_cf_descs = {
+      {kDefaultColumnFamilyName, new_cf_opts}, {"cf1", new_cf_opts}};
+
+  // Perform multi-CF migration
+  ASSERT_OK(OptionChangeMigration(dbname_, old_db_opts, old_cf_descs,
+                                  new_db_opts, new_cf_descs));
+
+  // Reopen with new options
+  std::vector<ColumnFamilyHandle*> handles;
+  ASSERT_OK(DB::Open(new_db_opts, dbname_, new_cf_descs, &handles, &db_));
+  ASSERT_EQ(handles.size(), 2);
+
+  // Verify data in default CF
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+    it->SeekToFirst();
+    for (const std::string& key : keys_default) {
+      ASSERT_TRUE(it->Valid());
+      ASSERT_EQ(key, it->key().ToString());
+      it->Next();
+    }
+    ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
+  }
+
+  // Verify data in cf1
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions(), handles[1]));
+    it->SeekToFirst();
+    for (const std::string& key : keys_cf1) {
+      ASSERT_TRUE(it->Valid());
+      ASSERT_EQ(key, it->key().ToString());
+      it->Next();
+    }
+    ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
+  }
+
+  // Cleanup
+  for (auto* handle : handles) {
+    if (handle != db_->DefaultColumnFamily()) {
+      ASSERT_OK(db_->DestroyColumnFamilyHandle(handle));
+    }
+  }
+}
+
+TEST_F(DBOptionChangeMigrationMultiCFTest, DifferentStylesPerCF) {
+  // Create DB with 2 CFs, both using Level compaction
+  Options options1 = CurrentOptions();
+  options1.compaction_style = CompactionStyle::kCompactionStyleLevel;
+  options1.num_levels = 4;
+  options1.write_buffer_size = 64 * 1024;
+
+  Reopen(options1);
+
+  ColumnFamilyHandle* cf_handle;
+  ASSERT_OK(db_->CreateColumnFamily(options1, "cf1", &cf_handle));
+
+  // Write data
+  Random rnd(301);
+  for (int i = 0; i < 50; i++) {
+    ASSERT_OK(Put("key" + std::to_string(i), rnd.RandomString(900)));
+    ASSERT_OK(db_->Put(WriteOptions(), cf_handle, "key" + std::to_string(i),
+                       rnd.RandomString(900)));
+  }
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->Flush(FlushOptions(), cf_handle));
+
+  // Collect keys from both CFs
+  std::set<std::string> keys_default;
+  std::set<std::string> keys_cf1;
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+    for (it->SeekToFirst(); it->Valid(); it->Next()) {
+      keys_default.insert(it->key().ToString());
+    }
+    ASSERT_OK(it->status());
+  }
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions(), cf_handle));
+    for (it->SeekToFirst(); it->Valid(); it->Next()) {
+      keys_cf1.insert(it->key().ToString());
+    }
+    ASSERT_OK(it->status());
+  }
+
+  delete cf_handle;
+  Close();
+
+  // Old descriptors
+  DBOptions old_db_opts(options1);
+  ColumnFamilyOptions old_cf_opts(options1);
+
+  std::vector<ColumnFamilyDescriptor> old_cf_descs = {
+      {kDefaultColumnFamilyName, old_cf_opts}, {"cf1", old_cf_opts}};
+
+  // New descriptors: default CF to Universal, cf1 to Level with dynamic
+  Options new_opts_default = options1;
+  new_opts_default.compaction_style =
+      CompactionStyle::kCompactionStyleUniversal;
+  new_opts_default.num_levels = 5;
+
+  Options new_opts_cf1 = options1;
+  new_opts_cf1.compaction_style = CompactionStyle::kCompactionStyleLevel;
+  new_opts_cf1.level_compaction_dynamic_level_bytes = true;
+  new_opts_cf1.num_levels = 5;
+
+  DBOptions new_db_opts(new_opts_default);
+
+  std::vector<ColumnFamilyDescriptor> new_cf_descs = {
+      {kDefaultColumnFamilyName, ColumnFamilyOptions(new_opts_default)},
+      {"cf1", ColumnFamilyOptions(new_opts_cf1)}};
+
+  // Perform migration
+  ASSERT_OK(OptionChangeMigration(dbname_, old_db_opts, old_cf_descs,
+                                  new_db_opts, new_cf_descs));
+
+  // Reopen and verify
+  std::vector<ColumnFamilyHandle*> handles;
+  ASSERT_OK(DB::Open(new_db_opts, dbname_, new_cf_descs, &handles, &db_));
+  ASSERT_EQ(handles.size(), 2);
+
+  // Verify data in default CF
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+    it->SeekToFirst();
+    for (const std::string& key : keys_default) {
+      ASSERT_TRUE(it->Valid());
+      ASSERT_EQ(key, it->key().ToString());
+      it->Next();
+    }
+    ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
+  }
+
+  // Verify data in cf1
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions(), handles[1]));
+    it->SeekToFirst();
+    for (const std::string& key : keys_cf1) {
+      ASSERT_TRUE(it->Valid());
+      ASSERT_EQ(key, it->key().ToString());
+      it->Next();
+    }
+    ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
+  }
+
+  // Cleanup
+  for (auto* handle : handles) {
+    if (handle != db_->DefaultColumnFamily()) {
+      ASSERT_OK(db_->DestroyColumnFamilyHandle(handle));
+    }
+  }
+}
+
+TEST_F(DBOptionChangeMigrationMultiCFTest, ValidationMismatched) {
+  Options options = CurrentOptions();
+  DBOptions db_opts(options);
+  ColumnFamilyOptions cf_opts(options);
+
+  // Test 1: Mismatched CF count (missing cf1)
+  {
+    std::vector<ColumnFamilyDescriptor> old_cf_descs = {
+        {kDefaultColumnFamilyName, cf_opts}, {"cf1", cf_opts}};
+
+    std::vector<ColumnFamilyDescriptor> new_cf_descs = {
+        {kDefaultColumnFamilyName, cf_opts}};  // Missing cf1
+
+    Status s = OptionChangeMigration(dbname_, db_opts, old_cf_descs, db_opts,
+                                     new_cf_descs);
+    ASSERT_TRUE(s.IsInvalidArgument());
+    ASSERT_TRUE(s.ToString().find("same number") != std::string::npos);
+  }
+
+  // Test 2: Mismatched CF names (cf2 instead of cf1)
+  {
+    std::vector<ColumnFamilyDescriptor> old_cf_descs = {
+        {kDefaultColumnFamilyName, cf_opts}, {"cf1", cf_opts}};
+
+    std::vector<ColumnFamilyDescriptor> new_cf_descs = {
+        {kDefaultColumnFamilyName, cf_opts},
+        {"cf2", cf_opts}};  // Different name
+
+    Status s = OptionChangeMigration(dbname_, db_opts, old_cf_descs, db_opts,
+                                     new_cf_descs);
+    ASSERT_TRUE(s.IsInvalidArgument());
+    ASSERT_TRUE(s.ToString().find("mismatch") != std::string::npos);
+  }
+
+  // Test 3: Mismatched CF order (swapped)
+  {
+    std::vector<ColumnFamilyDescriptor> old_cf_descs = {
+        {kDefaultColumnFamilyName, cf_opts}, {"cf1", cf_opts}};
+
+    std::vector<ColumnFamilyDescriptor> new_cf_descs = {
+        {"cf1", cf_opts},  // Swapped order
+        {kDefaultColumnFamilyName, cf_opts}};
+
+    Status s = OptionChangeMigration(dbname_, db_opts, old_cf_descs, db_opts,
+                                     new_cf_descs);
+    ASSERT_TRUE(s.IsInvalidArgument());
+    ASSERT_TRUE(s.ToString().find("mismatch") != std::string::npos);
+  }
+}
+
+TEST_F(DBOptionChangeMigrationMultiCFTest, FromFIFOMultiCF) {
+  Options options = CurrentOptions();
+  options.compaction_style = CompactionStyle::kCompactionStyleFIFO;
+  options.num_levels = 1;
+  options.max_open_files = -1;
+
+  Reopen(options);
+
+  ColumnFamilyHandle* cf_handle;
+  ASSERT_OK(db_->CreateColumnFamily(options, "cf1", &cf_handle));
+
+  // Write some data
+  Random rnd(301);
+  for (int i = 0; i < 50; i++) {
+    ASSERT_OK(Put("key" + std::to_string(i), rnd.RandomString(900)));
+    ASSERT_OK(db_->Put(WriteOptions(), cf_handle, "key" + std::to_string(i),
+                       rnd.RandomString(900)));
+  }
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->Flush(FlushOptions(), cf_handle));
+
+  // Collect keys from both CFs
+  std::set<std::string> keys_default;
+  std::set<std::string> keys_cf1;
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+    for (it->SeekToFirst(); it->Valid(); it->Next()) {
+      keys_default.insert(it->key().ToString());
+    }
+    ASSERT_OK(it->status());
+  }
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions(), cf_handle));
+    for (it->SeekToFirst(); it->Valid(); it->Next()) {
+      keys_cf1.insert(it->key().ToString());
+    }
+    ASSERT_OK(it->status());
+  }
+
+  delete cf_handle;
+  Close();
+
+  // Migrate from FIFO to Level
+  DBOptions old_db_opts(options);
+  ColumnFamilyOptions old_cf_opts(options);
+
+  std::vector<ColumnFamilyDescriptor> old_cf_descs = {
+      {kDefaultColumnFamilyName, old_cf_opts}, {"cf1", old_cf_opts}};
+
+  Options new_options = options;
+  new_options.compaction_style = CompactionStyle::kCompactionStyleLevel;
+  new_options.num_levels = 4;
+  new_options.max_open_files = 1000;
+
+  DBOptions new_db_opts(new_options);
+  ColumnFamilyOptions new_cf_opts(new_options);
+
+  std::vector<ColumnFamilyDescriptor> new_cf_descs = {
+      {kDefaultColumnFamilyName, new_cf_opts}, {"cf1", new_cf_opts}};
+
+  // Migration should succeed (FIFO is special case)
+  ASSERT_OK(OptionChangeMigration(dbname_, old_db_opts, old_cf_descs,
+                                  new_db_opts, new_cf_descs));
+
+  // Reopen and verify
+  std::vector<ColumnFamilyHandle*> handles;
+  ASSERT_OK(DB::Open(new_db_opts, dbname_, new_cf_descs, &handles, &db_));
+  ASSERT_EQ(handles.size(), 2);
+
+  // Verify data in default CF
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+    it->SeekToFirst();
+    for (const std::string& key : keys_default) {
+      ASSERT_TRUE(it->Valid());
+      ASSERT_EQ(key, it->key().ToString());
+      it->Next();
+    }
+    ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
+  }
+
+  // Verify data in cf1
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions(), handles[1]));
+    it->SeekToFirst();
+    for (const std::string& key : keys_cf1) {
+      ASSERT_TRUE(it->Valid());
+      ASSERT_EQ(key, it->key().ToString());
+      it->Next();
+    }
+    ASSERT_TRUE(!it->Valid());
+    ASSERT_OK(it->status());
+  }
+
+  // Cleanup
+  for (auto* handle : handles) {
+    if (handle != db_->DefaultColumnFamily()) {
+      ASSERT_OK(db_->DestroyColumnFamilyHandle(handle));
+    }
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb.gyp

@@ -9,6 +9,7 @@
         "include_dirs": ["rocksdb/include/"]
       },
       "defines": [
+        "NDEBUG=1",
         "ZSTD=1",
         "ZSTD_STATIC_LINKING_ONLY=1",
         "ROCKSDB_BACKTRACE=1",
@@ -257,6 +258,7 @@
         "rocksdb/db/write_controller.cc",
         "rocksdb/db/write_stall_stats.cc",
         "rocksdb/db/write_thread.cc",
+        # "rocksdb/db_stress_tool/db_stress_compression_manager.cc",
         "rocksdb/env/composite_env.cc",
         "rocksdb/env/env.cc",
         "rocksdb/env/env_chroot.cc",
@@ -367,10 +369,10 @@
         "rocksdb/table/table_properties.cc",
         "rocksdb/table/two_level_iterator.cc",
         "rocksdb/table/unique_id.cc",
-        "rocksdb/test_util/sync_point.cc",
-        "rocksdb/test_util/sync_point_impl.cc",
-        "rocksdb/test_util/testutil.cc",
-        "rocksdb/test_util/transaction_test_util.cc",
+        # "rocksdb/test_util/sync_point.cc",
+        # "rocksdb/test_util/sync_point_impl.cc",
+        # "rocksdb/test_util/testutil.cc",
+        # "rocksdb/test_util/transaction_test_util.cc",
         # "rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
         # "rocksdb/tools/dump/db_dump_tool.cc",
         # "rocksdb/tools/io_tracer_parser_tool.cc",

package/iterator.js

@@ -17,13 +17,12 @@ const kPosition = Symbol('position')
 const kBusy = Symbol('busy')
 
 const kEmpty = []
-const kForceSync = false
 
 class Iterator extends AbstractIterator {
   constructor (db, context, options) {
     super(db, options)
 
-    this[kContext] = binding.iterator_init(context, options)
+    this[kContext] = binding.iterator_init_sync(context, options)
 
     this[kFirst] = true
     this[kCache] = kEmpty
@@ -50,40 +49,9 @@ class Iterator extends AbstractIterator {
     return (this[kCache].length - this[kPosition]) / 2
   }
 
-  // nxt API
-
-  _seekSync (target) {
-    assert(this[kContext])
-
-    if (target.length === 0) {
-      throw new Error('cannot seek() to an empty target')
-    }
-
-    this[kFirst] = true
-    this[kCache] = kEmpty
-    this[kFinished] = false
-    this[kPosition] = 0
-
-    binding.iterator_seek(this[kContext], target)
-  }
-
-  _seekAsync (target, callback) {
-    assert(this[kContext])
-
-    callback = fromCallback(callback, kPromise)
-
-    try {
-      this._seekSync(target)
-      process.nextTick(callback)
-    } catch (err) {
-      process.nextTick(callback, err)
-    }
-
-    return callback[kPromise]
-  }
-
   _next (callback) {
     assert(this[kContext])
+    assert(!this[kBusy])
 
     if (this[kPosition] < this[kCache].length) {
       const key = this[kCache][this[kPosition]++]
@@ -112,43 +80,72 @@ class Iterator extends AbstractIterator {
 
   _nextv (size, options, callback) {
     assert(this[kContext])
+    assert(!this[kBusy])
 
     callback = fromCallback(callback, kPromise)
 
-    if (this[kFinished]) {
-      process.nextTick(callback, null, [])
-    } else {
-      this[kFirst] = false
+    this._nextvAsync(size, options, (err, val) => {
+      if (err) {
+        callback(err)
+      } else {
+        const { rows, finished } = val
 
-      try {
-        if (kForceSync) {
-          const { rows, finished } = this._nextvSync(size, options)
+        const entries = []
+        for (let n = 0; n < rows.length; n += 2) {
+          entries.push([rows[n + 0], rows[n + 1]])
+        }
 
-          const entries = []
-          for (let n = 0; n < rows.length; n += 2) {
-            entries.push([rows[n + 0], rows[n + 1]])
-          }
+        callback(null, entries, finished)
+      }
+    })
+
+    return callback[kPromise]
+  }
+
+  // nxt API
+
+  _seekSync (target) {
+    assert(this[kContext])
+    assert(!this[kBusy])
+
+    if (target.length === 0) {
+      throw new Error('cannot seek() to an empty target')
+    }
+
+    this[kFirst] = true
+    this[kCache] = kEmpty
+    this[kFinished] = false
+    this[kPosition] = 0
+
+    binding.iterator_seek_sync(this[kContext], target)
+  }
+
+  _seekAsync (target, callback) {
+    assert(this[kContext])
+    assert(!this[kBusy])
 
-          process.nextTick(callback, null, entries, finished)
+    callback = fromCallback(callback, kPromise)
+
+    this[kFirst] = true
+    this[kCache] = kEmpty
+    this[kFinished] = false
+    this[kPosition] = 0
+
+    try {
+      binding.iterator_seek(this[kContext], target, (err) => {
+        this[kBusy] = false
+        this[kDB][kUnref]()
+
+        if (err) {
+          callback(err)
         } else {
-          this._nextvAsync(size, options, (err, val) => {
-            if (err) {
-              process.nextTick(callback, err)
-            } else {
-              const { rows, finished } = val
-
-              const entries = []
-              for (let n = 0; n < rows.length; n += 2) {
-                entries.push([rows[n + 0], rows[n + 1]])
-              }
-
-              callback(null, entries, finished)
-            }
-          })
+          callback(null)
         }
-      } catch (err) {
-        process.nextTick(callback, err)
-      }
+      })
+      this[kDB][kRef]()
+      this[kBusy] = true
+    } catch (err) {
+      process.nextTick(callback, err)
     }
 
     return callback[kPromise]
@@ -156,12 +153,12 @@ class Iterator extends AbstractIterator {
 
   _nextvSync (size, options) {
     assert(this[kContext])
+    assert(!this[kBusy])
 
     if (this[kFinished]) {
       return { rows: [], finished: true }
     }
 
-    this[kFirst] = false
     const result = binding.iterator_nextv_sync(this[kContext], size, options)
     this[kFinished] = result.finished
 
@@ -170,6 +167,7 @@ class Iterator extends AbstractIterator {
 
   _nextvAsync (size, options, callback) {
     assert(this[kContext])
+    assert(!this[kBusy])
 
     callback = fromCallback(callback, kPromise)
 
@@ -177,14 +175,10 @@ class Iterator extends AbstractIterator {
       if (this[kFinished]) {
         process.nextTick(callback, null, { rows: [], finished: true })
       } else {
-        assert(!this[kBusy])
-
-        this[kFirst] = false
-        this[kBusy] = true
-        this[kDB][kRef]()
         binding.iterator_nextv(this[kContext], size, options, (err, result) => {
           this[kBusy] = false
           this[kDB][kUnref]()
+
           if (err) {
             callback(err)
           } else {
@@ -192,6 +186,8 @@ class Iterator extends AbstractIterator {
             callback(null, result)
           }
         })
+        this[kBusy] = true
+        this[kDB][kRef]()
       }
     } catch (err) {
       process.nextTick(callback, err)
@@ -204,7 +200,7 @@ class Iterator extends AbstractIterator {
     this[kCache] = kEmpty
 
     if (this[kContext]) {
-      binding.iterator_close(this[kContext])
+      binding.iterator_close_sync(this[kContext])
       this[kContext] = null
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "15.1.2",
+  "version": "15.1.3",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",
@@ -20,7 +20,7 @@
     "prebuildify": "^6.0.1"
   },
   "devDependencies": {
-    "@types/node": "^24.10.2",
+    "@types/node": "^25.2.1",
     "@voxpelli/tsconfig": "^16.1.0",
     "async-each": "^1.0.3",
     "cross-env": "^10.1.0",
@@ -28,17 +28,17 @@
     "dependency-check": "^4.1.0",
     "du": "^1.0.0",
     "faucet": "^0.0.4",
-    "glob": "^13.0.0",
-    "lru-cache": "^11.2.4",
+    "glob": "^13.0.1",
+    "lru-cache": "^11.2.5",
     "mitata": "^1.0.10",
     "mkfiletree": "^3.0.0",
-    "node-gyp": "^12.1.0",
+    "node-gyp": "^12.2.0",
     "nyc": "^17.1.0",
     "readfiletree": "^2.0.0",
     "rimraf": "^6.1.2",
     "standard": "^17.1.2",
     "tape": "^5.9.0",
-    "tempy": "^3.1.0"
+    "tempy": "^1.0.1"
   },
   "standard": {
     "ignore": [