@nxtedition/rocksdb 7.0.0-alpha.2 → 7.0.0-alpha.3

package/binding.cc

@@ -31,7 +31,6 @@ class NullLogger : public rocksdb::Logger {
 struct Database;
 struct Iterator;
 struct Updates;
-struct Column;
 
 #define NAPI_STATUS_RETURN(call) \
   {                              \
@@ -365,16 +364,6 @@ struct Database {
     DecrementPriorityWork(env);
   }
 
-  void AttachColumn(napi_env env, Column* column) {
-    columns_.insert(column);
-    IncrementPriorityWork(env);
-  }
-
-  void DetachColumn(napi_env env, Column* column) {
-    columns_.erase(column);
-    DecrementPriorityWork(env);
-  }
-
   void IncrementPriorityWork(napi_env env) { napi_reference_ref(env, priorityRef_, &priorityWork_); }
 
   void DecrementPriorityWork(napi_env env) {
@@ -392,7 +381,7 @@ struct Database {
   Worker* pendingCloseWorker_;
   std::set<Iterator*> iterators_;
   std::set<Updates*> updates_;
-  std::set<Column*> columns_;
+  std::vector<rocksdb::ColumnFamilyHandle*> columns_;
   napi_ref priorityRef_;
 
  private:
@@ -597,39 +586,9 @@ struct Updates {
   napi_ref ref_ = nullptr;
 };
 
-struct Column {
-  Column(Database* database, rocksdb::ColumnFamilyHandle* handle)
-      : database_(database), handle_(handle) {}
-
-  void Close() { 
-    if (handle_) {
-      database_->db_->DestroyColumnFamilyHandle(handle_); // TODO (fix): Handle error?
-      handle_ = nullptr;
-    }
-  }
-
-  void Attach(napi_env env, napi_value context) {
-    napi_create_reference(env, context, 1, &ref_);
-    database_->AttachColumn(env, this);
-  }
-
-  void Detach(napi_env env) {
-    database_->DetachColumn(env, this);
-    if (ref_) {
-      napi_delete_reference(env, ref_);
-    }
-  }
-
-  Database* database_;
-  rocksdb::ColumnFamilyHandle* handle_;
-
- private:
-  napi_ref ref_ = nullptr;
-};
-
 static rocksdb::ColumnFamilyHandle* GetColumnFamily(Database* database, napi_env env, napi_value options) {
-  auto column_opt = ExternalValueProperty<Column>(env, options, "column");
-  return column_opt ? (*column_opt)->handle_ : database->db_->DefaultColumnFamily();
+  auto column_opt = ExternalValueProperty<rocksdb::ColumnFamilyHandle>(env, options, "column");
+  return column_opt ? *column_opt : database->db_->DefaultColumnFamily();
 }
 
 /**
@@ -659,8 +618,7 @@ static void env_cleanup_hook(void* arg) {
     }
 
     for (auto it : database->columns_) {
-      // TODO: does not do `napi_delete_reference`. Problem?
-      it->Close();
+      database->db_->DestroyColumnFamilyHandle(it);
     }
 
     // Having closed the iterators (and released snapshots) we can safely close.
@@ -695,24 +653,42 @@ struct OpenWorker final : public Worker {
              Database* database,
              napi_value callback,
              const std::string& location,
-             rocksdb::Options options,
-             const bool readOnly)
+             const rocksdb::Options& options,
+             std::vector<rocksdb::ColumnFamilyDescriptor> column_families)
       : Worker(env, database, callback, "leveldown.db.open"),
         options_(options),
-        readOnly_(readOnly),
-        location_(location) {}
+        location_(location),
+        column_families_(std::move(column_families)) {}
 
   rocksdb::Status Execute(Database& database) override {
     rocksdb::DB* db = nullptr;
-    const auto status = readOnly_ ? rocksdb::DB::OpenForReadOnly(options_, location_, &db)
-                                  : rocksdb::DB::Open(options_, location_, &db);
+    const auto status = column_families_.empty()
+      ? rocksdb::DB::Open(options_, location_, &db)
+      : rocksdb::DB::Open(options_, location_, column_families_, &database.columns_, &db);
     database.db_.reset(db);
     return status;
   }
 
+  napi_status OnOk(napi_env env, napi_value callback) override {    
+    const auto size = database_->columns_.size();
+    napi_value result;
+    NAPI_STATUS_RETURN(napi_create_object(env, &result));
+  
+    for (size_t n = 0; n < size; ++n) {
+      napi_value column;
+      NAPI_STATUS_RETURN(napi_create_external(env, database_->columns_[n], nullptr, nullptr, &column));
+      NAPI_STATUS_RETURN(napi_set_named_property(env, result, column_families_[n].name.c_str(), column));
+    }
+
+    napi_value argv[2];
+    NAPI_STATUS_RETURN(napi_get_null(env, &argv[0]));
+    argv[1] = result;
+    return CallFunction(env, callback, 2, argv);
+  }
+
   rocksdb::Options options_;
-  const bool readOnly_;
   const std::string location_;
+  std::vector<rocksdb::ColumnFamilyDescriptor> column_families_;
 };
 
 napi_status InitOptions(napi_env env, auto& options, auto options2) {
@@ -805,26 +781,26 @@ NAPI_METHOD(db_open) {
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&database));
 
   const auto location = ToString(env, argv[1]);
+  const auto options = argv[2];
+  const auto callback = argv[3];
 
-  rocksdb::Options options;
-
-  options.IncreaseParallelism(
-      Uint32Property(env, argv[2], "parallelism").value_or(std::thread::hardware_concurrency() / 2));
+  rocksdb::Options dbOptions;
 
-  options.create_if_missing = BooleanProperty(env, argv[2], "createIfMissing").value_or(true);
-  options.error_if_exists = BooleanProperty(env, argv[2], "errorIfExists").value_or(false);
-  options.avoid_unnecessary_blocking_io = true;
-  options.use_adaptive_mutex = BooleanProperty(env, argv[2], "useAdaptiveMutex").value_or(true);
-  options.enable_pipelined_write = BooleanProperty(env, argv[2], "enablePipelinedWrite").value_or(true);
-  options.max_background_jobs =
-      Uint32Property(env, argv[2], "maxBackgroundJobs").value_or(std::thread::hardware_concurrency() / 4);
-  options.WAL_ttl_seconds = Uint32Property(env, argv[2], "walTTL").value_or(0) / 1e3;
-  options.WAL_size_limit_MB = Uint32Property(env, argv[2], "walSizeLimit").value_or(0) / 1e6;
+  dbOptions.IncreaseParallelism(
+      Uint32Property(env, options, "parallelism").value_or(std::thread::hardware_concurrency() / 2));
 
-  // TODO: Consider direct IO (https://github.com/facebook/rocksdb/wiki/Direct-IO) once
-  // secondary compressed cache is stable.
+  dbOptions.create_if_missing = BooleanProperty(env, options, "createIfMissing").value_or(true);
+  dbOptions.error_if_exists = BooleanProperty(env, options, "errorIfExists").value_or(false);
+  dbOptions.avoid_unnecessary_blocking_io = true;
+  dbOptions.use_adaptive_mutex = BooleanProperty(env, options, "useAdaptiveMutex").value_or(true);
+  dbOptions.enable_pipelined_write = BooleanProperty(env, options, "enablePipelinedWrite").value_or(true);
+  dbOptions.max_background_jobs =
+      Uint32Property(env, options, "maxBackgroundJobs").value_or(std::thread::hardware_concurrency() / 4);
+  dbOptions.WAL_ttl_seconds = Uint32Property(env, options, "walTTL").value_or(0) / 1e3;
+  dbOptions.WAL_size_limit_MB = Uint32Property(env, options, "walSizeLimit").value_or(0) / 1e6;
+  dbOptions.create_missing_column_families = true;
 
-  const auto infoLogLevel = StringProperty(env, argv[2], "infoLogLevel").value_or("");
+  const auto infoLogLevel = StringProperty(env, options, "infoLogLevel").value_or("");
   if (infoLogLevel.size() > 0) {
     rocksdb::InfoLogLevel lvl = {};
 
@@ -843,21 +819,43 @@ NAPI_METHOD(db_open) {
     else
       napi_throw_error(env, nullptr, "invalid log level");
 
-    options.info_log_level = lvl;
+    dbOptions.info_log_level = lvl;
   } else {
     // In some places RocksDB checks this option to see if it should prepare
     // debug information (ahead of logging), so set it to the highest level.
-    options.info_log_level = rocksdb::InfoLogLevel::HEADER_LEVEL;
-    options.info_log.reset(new NullLogger());
+    dbOptions.info_log_level = rocksdb::InfoLogLevel::HEADER_LEVEL;
+    dbOptions.info_log.reset(new NullLogger());
   }
 
-  const auto readOnly = BooleanProperty(env, argv[2], "readOnly").value_or(false);
+  NAPI_STATUS_THROWS(InitOptions(env, dbOptions, options));
 
-  NAPI_STATUS_THROWS(InitOptions(env, options, argv[2]));
+  std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
+  
+  if (HasProperty(env, options, "columns")) {
+    napi_value columns;
+    NAPI_STATUS_THROWS(napi_get_named_property(env, options, "columns", &columns));
 
-  const auto callback = argv[3];
+    napi_value keys;
+    NAPI_STATUS_THROWS(napi_get_property_names(env, columns, &keys));
+
+    uint32_t len;
+    NAPI_STATUS_THROWS(napi_get_array_length(env, keys, &len));
+
+    for (uint32_t n = 0; n < len; ++n) {
+      napi_value key;
+      NAPI_STATUS_THROWS(napi_get_element(env, keys, n, &key));
+      
+      napi_value column;
+      NAPI_STATUS_THROWS(napi_get_property(env, columns, key, &column));
+
+      rocksdb::ColumnFamilyOptions columnOptions;
+      NAPI_STATUS_THROWS(InitOptions(env, columnOptions, column));
+
+      column_families.emplace_back(ToString(env, key), columnOptions);
+    }
+  }
 
-  auto worker = new OpenWorker(env, database, callback, location, options, readOnly);
+  auto worker = new OpenWorker(env, database, callback, location, dbOptions, column_families);
   worker->Queue(env);
 
   return 0;
@@ -867,7 +865,13 @@ struct CloseWorker final : public Worker {
   CloseWorker(napi_env env, Database* database, napi_value callback)
       : Worker(env, database, callback, "leveldown.db.close") {}
 
-  rocksdb::Status Execute(Database& database) override { return database.db_->Close(); }
+  rocksdb::Status Execute(Database& database) override { 
+    for (auto it : database.columns_) {
+      database.db_->DestroyColumnFamilyHandle(it);
+    }
+
+    return database.db_->Close(); 
+  }
 };
 
 napi_value noop_callback(napi_env env, napi_callback_info info) {
@@ -1491,15 +1495,15 @@ struct NextWorker final : public Worker {
     napi_value result;
     NAPI_STATUS_RETURN(napi_create_array_with_length(env, size, &result));
 
-    for (size_t idx = 0; idx < cache_.size(); idx += 2) {
+    for (size_t n = 0; n < size; n += 2) {
       napi_value key;
       napi_value val;
 
-      NAPI_STATUS_RETURN(Convert(env, std::move(cache_[idx + 0]), iterator_->keyAsBuffer_, key));
-      NAPI_STATUS_RETURN(Convert(env, std::move(cache_[idx + 1]), iterator_->valueAsBuffer_, val));
+      NAPI_STATUS_RETURN(Convert(env, std::move(cache_[n + 0]), iterator_->keyAsBuffer_, key));
+      NAPI_STATUS_RETURN(Convert(env, std::move(cache_[n + 1]), iterator_->valueAsBuffer_, val));
 
-      NAPI_STATUS_RETURN(napi_set_element(env, result, static_cast<int>(idx + 0), key));
-      NAPI_STATUS_RETURN(napi_set_element(env, result, static_cast<int>(idx + 1), val));
+      NAPI_STATUS_RETURN(napi_set_element(env, result, static_cast<int>(n + 0), key));
+      NAPI_STATUS_RETURN(napi_set_element(env, result, static_cast<int>(n + 1), val));
     }
 
     cache_.clear();
@@ -1655,45 +1659,6 @@ NAPI_METHOD(batch_write) {
   return ToError(env, database->db_->Write(writeOptions, batch));
 }
 
-NAPI_METHOD(column_init) {
-  NAPI_ARGV(3);
-
-  Database* database;
-  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&database));
-
-  NapiSlice name;
-  NAPI_STATUS_THROWS(ToNapiSlice(env, argv[1], name));
-
-  rocksdb::ColumnFamilyOptions options;
-  NAPI_STATUS_THROWS(InitOptions(env, options, argv[2]));
-
-  rocksdb::ColumnFamilyHandle* handle;
-  ROCKS_STATUS_THROWS(database->db_->CreateColumnFamily(options, name.ToString(), &handle));
-
-  auto column = std::make_unique<Column>(database, handle);
-
-  napi_value result;
-  NAPI_STATUS_THROWS(napi_create_external(env, column.get(), Finalize<Column>, column.get(), &result));
-
-  // Prevent GC of JS object before the iterator is closed (explicitly or on
-  // db close) and keep track of non-closed iterators to end them on db close.
-  column.release()->Attach(env, result);
-
-  return result;
-}
-
-NAPI_METHOD(column_close) {
-  NAPI_ARGV(1);
-
-  Column* column;
-  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&column));
-
-  column->Detach(env);
-  column->Close();
-
-  return 0;
-}
-
 NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(db_init);
   NAPI_EXPORT_FUNCTION(db_open);
@@ -1705,9 +1670,6 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(db_clear);
   NAPI_EXPORT_FUNCTION(db_get_property);
 
-  NAPI_EXPORT_FUNCTION(column_init);
-  NAPI_EXPORT_FUNCTION(column_close);
-
   NAPI_EXPORT_FUNCTION(iterator_init);
   NAPI_EXPORT_FUNCTION(iterator_seek);
   NAPI_EXPORT_FUNCTION(iterator_close);

package/index.js

@@ -8,8 +8,8 @@ const { ChainedBatch } = require('./chained-batch')
 const { Iterator } = require('./iterator')
 
 const kContext = Symbol('context')
-const kLocation = Symbol('location')
 const kColumns = Symbol('columns')
+const kLocation = Symbol('location')
 
 class RocksLevel extends AbstractLevel {
   constructor (location, options, _) {
@@ -33,8 +33,6 @@ class RocksLevel extends AbstractLevel {
       createIfMissing: true,
       errorIfExists: true,
       additionalMethods: {
-        createColumn: true,
-        closeColumn: true,
         updates: true,
         query: true
       }
@@ -42,29 +40,37 @@ class RocksLevel extends AbstractLevel {
 
     this[kLocation] = location
     this[kContext] = binding.db_init()
-    this[kColumns] = new Set()
+    this[kColumns] = {}
   }
 
   get location () {
     return this[kLocation]
   }
 
+  get columns () {
+    return this[kColumns]
+  }
+
   _open (options, callback) {
+    const onOpen = (err, columns) => {
+      if (err) {
+        callback(err)
+      } else {
+        this[kColumns] = columns
+        callback(null)
+      }
+    }
     if (options.createIfMissing) {
       fs.mkdir(this[kLocation], { recursive: true }, (err) => {
         if (err) return callback(err)
-        binding.db_open(this[kContext], this[kLocation], options, callback)
+        binding.db_open(this[kContext], this[kLocation], options, onOpen)
       })
     } else {
-      binding.db_open(this[kContext], this[kLocation], options, callback)
+      binding.db_open(this[kContext], this[kLocation], options, onOpen)
     }
   }
 
   _close (callback) {
-    for (const column of this[kColumns]) {
-      binding.column_close(column)
-    }
-  
     binding.db_close(this[kContext], callback)
   }
 
@@ -115,99 +121,37 @@ class RocksLevel extends AbstractLevel {
     return binding.db_get_property(this[kContext], property)
   }
 
-  async createColumn (name, options) {
+  async query (options) {
     if (this.status !== 'open') {
       throw new ModuleError('Database is not open', {
         code: 'LEVEL_DATABASE_NOT_OPEN'
       })
     }
 
-    const column = binding.column_init(this[kContext], name, options || {})
-    
-    this[kColumns].add(column)
-    
-    return column
-  }
-
-  async * query (options) {
-    if (this.status !== 'open') {
-      throw new ModuleError('Database is not open', {
-        code: 'LEVEL_DATABASE_NOT_OPEN'
-      })
-    }
-
-    class Query {
-      constructor (db, options) {
-        this.context = binding.iterator_init(db[kContext], options)
-        this.closed = false
-        this.promise = null
-        this.sequence = binding.iterator_get_sequence(this.context)
-        this.db = db
-        this.db.attachResource(this)
-      }
-
-      async next () {
-        if (this.closed) {
-          return {}
-        }
-
-        this.promise = new Promise(resolve => binding.iterator_nextv(this.context, 1000, (err, rows, finished) => {
-          this.promise = null
-          if (err) {
-            resolve(Promise.reject(err))
-          } else {
-            resolve({
-              finished,
-              rows,
-              sequence: this.sequence
-            })
-          }
-        }))
-
-        return this.promise
-      }
-
-      async close (callback) {
-        try {
-          await this.promise
-        } catch {
-          // Do nothing...
-        }
-
-        try {
-          if (!this.closed) {
-            this.closed = true
-            binding.iterator_close(this.context)
-          }
-
-          if (callback) {
-            process.nextTick(callback)
-          }
-        } catch (err) {
-          if (callback) {
-            process.nextTick(callback, err)
-          } else {
-            throw err
-          }
-        } finally {
-          this.db.detachResource(this)
-        }
+    const context = binding.iterator_init(this[kContext], options)
+    const sequence = binding.iterator_get_sequence(context)
+    const resource = {
+      callback: null,
+      close (callback) {
+        this.callback = callback
       }
     }
 
-    const query = new Query(this, options)
     try {
-      while (true) {
-        const { finished, rows, sequence } = await query.next()
-
-        yield { rows, sequence }
-
-        if (finished) {
-          return
+      this.attachResource(resource)
+      return await new Promise((resolve, reject) => binding.iterator_nextv(context, options.limit || 1000, (err, rows) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve({ rows, sequence })
         }
-      }
+      }))
     } finally {
-      await query.close()
+      this.detachResource(resource)
+      binding.iterator_close(this.context)
+      if (resource.callback) {
+        resource.callback()
+      }
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "7.0.0-alpha.2",
+  "version": "7.0.0-alpha.3",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",