duckdb 0.4.1-dev155.0 → 0.4.1-dev1559.0

package/lib/duckdb.js

@@ -1,32 +1,154 @@
+/**
+ * @module duckdb
+ * @summary these jsdoc annotations are still a work in progress - feedback and suggestions are welcome!
+ */
+
 var duckdb = require('./duckdb-binding.js');
 module.exports = exports = duckdb;
 
+/**
+ * Check that errno attribute equals this to check for a duckdb error
+ * @constant {number}
+ */
+var ERROR = duckdb.ERROR;
+
+/**
+ * Open database in readonly mode
+ * @constant {number}
+ */
+var OPEN_READONLY = duckdb.OPEN_READONLY;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_READWRITE = duckdb.OPEN_READWRITE;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_CREATE = duckdb.OPEN_CREATE;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_FULLMUTEX = duckdb.OPEN_FULLMUTEX;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_SHAREDCACHE = duckdb.OPEN_SHAREDCACHE;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_PRIVATECACHE = duckdb.OPEN_PRIVATECACHE;
 
 // some wrappers for compatibilities sake
+/**
+ * Main database interface
+ */
 var Database = duckdb.Database;
+/**
+ * @class
+ */
 var Connection = duckdb.Connection;
+/**
+ * @class
+ */
 var Statement = duckdb.Statement;
+/**
+ * @class
+ */
+var QueryResult = duckdb.QueryResult;
 
+/**
+ * @method
+ * @return data chunk
+ */
+QueryResult.prototype.nextChunk;
 
-Connection.prototype.run = function(sql) {
+/**
+ * @name asyncIterator
+ * @memberof module:duckdb~QueryResult
+ * @method
+ * @instance
+ * @yields data chunks
+ */
+QueryResult.prototype[Symbol.asyncIterator] = async function*() {
+    let prefetch = this.nextChunk();
+    while (true) {
+        const chunk = await prefetch;
+        // Null chunk indicates end of stream
+        if (!chunk) {
+            return;
+        }
+        // Prefetch the next chunk while we're iterating
+        prefetch = this.nextChunk();
+        for (const row of chunk) {
+            yield row;
+        }
+    }
+}
+
+
+/**
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.run = function (sql) {
     var statement = new Statement(this, sql);
     return statement.run.apply(statement, arguments);
 }
 
-Connection.prototype.all = function(sql) {
-    var statement = new Statement(this,sql);
+/**
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.all = function (sql) {
+    var statement = new Statement(this, sql);
     return statement.all.apply(statement, arguments);
 }
 
-Connection.prototype.each = function(sql) {
+/**
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.each = function (sql) {
     var statement = new Statement(this, sql);
     return statement.each.apply(statement, arguments);
 }
 
-// this follows the wasm udfs somewhat but is simpler because we can pass data much more cleanly
-Connection.prototype.register = function(name, return_type, fun) {
+/**
+ * @arg sql
+ * @param {...*} params
+ * @yields row chunks
+ */
+Connection.prototype.stream = async function* (sql) {
+    const statement = new Statement(this, sql);
+    const queryResult = await statement.stream.apply(statement, arguments);
+    for await (const result of queryResult) {
+        yield result;
+    }
+}
+
+/**
+ * Register a User Defined Function
+ *
+ * @arg name
+ * @arg return_type
+ * @arg fun
+ * @return {void}
+ * @note this follows the wasm udfs somewhat but is simpler because we can pass data much more cleanly
+ */
+Connection.prototype.register = function (name, return_type, fun) {
     // TODO what if this throws an error somewhere? do we need a try/catch?
-    return this.register_bulk(name, return_type, function(desc) {
+    return this.register_bulk(name, return_type, function (desc) {
         try {
             // Build an argument resolver
             const buildResolver = (arg) => {
@@ -126,62 +248,236 @@ Connection.prototype.register = function(name, return_type, fun) {
                 desc.ret.data[i] = res;
                 desc.ret.validity[i] = res === undefined || res === null ? 0 : 1;
             }
-        } catch(error) { // work around recently fixed napi bug https://github.com/nodejs/node-addon-api/issues/912
+        } catch (error) { // work around recently fixed napi bug https://github.com/nodejs/node-addon-api/issues/912
             console.log(desc.ret);
             msg = error;
             if (typeof error == 'object' && 'message' in error) {
                 msg = error.message
             }
-            throw {name: 'DuckDB-UDF-Exception', message : msg};
+            throw { name: 'DuckDB-UDF-Exception', message: msg };
         }
     })
 }
 
-default_connection = function(o) {
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {Statement}
+ */
+Connection.prototype.prepare;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.exec;
+/**
+ * Register a User Defined Function
+ *
+ * @method
+ * @arg name
+ * @arg return_type
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.register_bulk;
+/**
+ * Unregister a User Defined Function
+ *
+ * @method
+ * @arg name
+ * @arg return_type
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.unregister;
+
+default_connection = function (o) {
     if (o.default_connection == undefined) {
         o.default_connection = new duckdb.Connection(o);
     }
-    return(o.default_connection);
+    return o.default_connection;
 }
 
-Database.prototype.prepare = function() {
+
+/**
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.close;
+
+/**
+ * @method
+ * @param callback
+ * TODO: what does this do?
+ * @return {void}
+ */
+Database.prototype.wait;
+
+/**
+ * TODO: what does this do?
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.serialize;
+
+/**
+ * TODO: what does this do?
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.parallelize;
+
+/**
+ * @method
+ * @arg path the database to connect to, either a file path, or `:memory:`
+ * @return {Connection}
+ */
+Database.prototype.connect;
+
+/**
+ * TODO: what does this do?
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.interrupt;
+
+/**
+ * @arg sql
+ * @return {Statement}
+ */
+Database.prototype.prepare = function () {
     return default_connection(this).prepare.apply(this.default_connection, arguments);
 }
 
-Database.prototype.run = function() {
+/**
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.run = function () {
     default_connection(this).run.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.each = function() {
+/**
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.each = function () {
     default_connection(this).each.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.all = function() {
+/**
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.all = function () {
     default_connection(this).all.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.exec = function() {
+/**
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.exec = function () {
     default_connection(this).exec.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.register = function() {
+/**
+ * Register a User Defined Function
+ *
+ * Convenience method for Connection#register
+ * @arg name
+ * @arg return_type
+ * @arg fun
+ * @return {this}
+ */
+Database.prototype.register = function () {
     default_connection(this).register.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.unregister = function() {
+/**
+ * Unregister a User Defined Function
+ *
+ * Convenience method for Connection#unregister
+ * @arg name
+ * @return {this}
+ */
+Database.prototype.unregister = function () {
     default_connection(this).unregister.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.get = function() {
+/**
+ * Not implemented
+ */
+Database.prototype.get = function () {
     throw "get() is not implemented because it's evil";
 }
 
-Statement.prototype.get = function() {
+/**
+ * Not implemented
+ */
+Statement.prototype.get = function () {
     throw "get() is not implemented because it's evil";
 }
+
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.run;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.all;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.each;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.finalize
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @yield callback
+ */
+Statement.prototype.stream;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev155.0",
+  "version": "0.4.1-dev1559.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {
@@ -26,6 +26,8 @@
   },
   "devDependencies": {
     "aws-sdk": "^2.790.0",
+    "chai": "^4.3.6",
+    "jsdoc3-parser": "^2.0.0",
     "mocha": "^8.3.0"
   },
   "repository": {

package/src/connection.cpp

@@ -207,7 +207,7 @@ struct RegisterTask : public Task {
 			// here we can do only DuckDB stuff because we do not have a functioning env
 
 			// Flatten all args to simplify udfs
-			args.Normalify();
+			args.Flatten();
 
 			JSArgs jsargs;
 			jsargs.rows = args.size();
@@ -321,19 +321,25 @@ struct ExecTask : public Task {
 		auto &connection = Get<Connection>();
 
 		success = true;
-		auto statements = connection.connection->ExtractStatements(sql);
-		if (statements.empty()) {
-			return;
-		}
+		try {
+			auto statements = connection.connection->ExtractStatements(sql);
+			if (statements.empty()) {
+				return;
+			}
 
-		// thanks Mark
-		for (duckdb::idx_t i = 0; i < statements.size(); i++) {
-			auto res = connection.connection->Query(move(statements[i]));
-			if (!res->success) {
-				success = false;
-				error = res->error;
-				break;
+			// thanks Mark
+			for (duckdb::idx_t i = 0; i < statements.size(); i++) {
+				auto res = connection.connection->Query(move(statements[i]));
+				if (!res->success) {
+					success = false;
+					error = res->error;
+					break;
+				}
 			}
+		} catch (duckdb::ParserException &e) {
+			success = false;
+			error = e.what();
+			return;
 		}
 	}
 

package/src/data_chunk.cpp

@@ -14,7 +14,7 @@ Napi::Array EncodeDataChunk(Napi::Env env, duckdb::DataChunk &chunk, bool with_t
 		// Make sure we only have flat vectors hereafter (for now)
 		auto &chunk_vec = chunk.data[col_idx];
 		if (with_data) {
-			chunk_vec.Normalify(chunk.size());
+			chunk_vec.Flatten(chunk.size());
 		}
 
 		// Do a post-order DFS traversal
@@ -52,7 +52,7 @@ Napi::Array EncodeDataChunk(Napi::Env env, duckdb::DataChunk &chunk, bool with_t
 
 			// Create validity vector
 			if (with_data) {
-				vec->Normalify(chunk.size());
+				vec->Flatten(chunk.size());
 				auto &validity = duckdb::FlatVector::Validity(*vec);
 				auto validity_buffer = Napi::Uint8Array::New(env, chunk.size());
 				for (idx_t row_idx = 0; row_idx < chunk.size(); row_idx++) {
@@ -182,4 +182,4 @@ Napi::Array EncodeDataChunk(Napi::Env env, duckdb::DataChunk &chunk, bool with_t
 	return col_descs;
 }
 
-} // namespace node_duckdb
+} // namespace node_duckdb

package/src/database.cpp

@@ -1,4 +1,5 @@
 #include "duckdb_node.hpp"
+#include "napi.h"
 #include "parquet-amalgamation.hpp"
 
 namespace node_duckdb {
@@ -22,17 +23,39 @@ Napi::Object Database::Init(Napi::Env env, Napi::Object exports) {
 }
 
 struct OpenTask : public Task {
-	OpenTask(Database &database_, std::string filename_, bool read_only_, Napi::Function callback_)
-	    : Task(database_, callback_), filename(filename_), read_only(read_only_) {
+	OpenTask(Database &database_, std::string filename_, duckdb::AccessMode access_mode_, Napi::Object config_,
+	         Napi::Function callback_)
+	    : Task(database_, callback_), filename(filename_) {
+
+		duckdb_config.options.access_mode = access_mode_;
+		Napi::Env env = database_.Env();
+		Napi::HandleScope scope(env);
+
+		if (!config_.IsUndefined()) {
+			const Napi::Array config_names = config_.GetPropertyNames();
+
+			for (duckdb::idx_t config_idx = 0; config_idx < config_names.Length(); config_idx++) {
+				std::string key = config_names.Get(config_idx).As<Napi::String>();
+				std::string val = config_.Get(key).As<Napi::String>();
+				auto config_property = duckdb::DBConfig::GetOptionByName(key);
+				if (!config_property) {
+					Napi::TypeError::New(env, "Unrecognized configuration property" + key).ThrowAsJavaScriptException();
+					return;
+				}
+				try {
+					duckdb_config.SetOption(*config_property, duckdb::Value(val));
+				} catch (std::exception &e) {
+					Napi::TypeError::New(env, "Failed to set configuration option " + key + ": " + e.what())
+					    .ThrowAsJavaScriptException();
+					return;
+				}
+			}
+		}
 	}
 
 	void DoWork() override {
 		try {
-			duckdb::DBConfig config;
-			if (read_only) {
-				config.access_mode = duckdb::AccessMode::READ_ONLY;
-			}
-			Get<Database>().database = duckdb::make_unique<duckdb::DuckDB>(filename, &config);
+			Get<Database>().database = duckdb::make_unique<duckdb::DuckDB>(filename, &duckdb_config);
 			duckdb::ParquetExtension extension;
 			extension.Load(*Get<Database>().database);
 			success = true;
@@ -59,22 +82,35 @@ struct OpenTask : public Task {
 	}
 
 	std::string filename;
-	bool read_only = false;
+	duckdb::DBConfig duckdb_config;
 	std::string error = "";
 	bool success = false;
 };
 
-Database::Database(const Napi::CallbackInfo &info) : Napi::ObjectWrap<Database>(info), task_inflight(false) {
+Database::Database(const Napi::CallbackInfo &info)
+    : Napi::ObjectWrap<Database>(info), task_inflight(false), env(info.Env()) {
 	auto env = info.Env();
+
 	if (info.Length() < 1 || !info[0].IsString()) {
 		Napi::TypeError::New(env, "Database location expected").ThrowAsJavaScriptException();
 		return;
 	}
 	std::string filename = info[0].As<Napi::String>();
 	unsigned int pos = 1;
+
+	duckdb::AccessMode access_mode = duckdb::AccessMode::AUTOMATIC;
+
 	int mode = 0;
 	if (info.Length() >= pos && info[pos].IsNumber() && Utils::OtherIsInt(info[pos].As<Napi::Number>())) {
 		mode = info[pos++].As<Napi::Number>().Int32Value();
+		if (mode == DUCKDB_NODEJS_READONLY) {
+			access_mode = duckdb::AccessMode::READ_ONLY;
+		}
+	}
+
+	Napi::Object config;
+	if (info.Length() >= pos && info[pos].IsObject() && !info[pos].IsFunction()) {
+		config = info[pos++].As<Napi::Object>();
 	}
 
 	Napi::Function callback;
@@ -82,7 +118,11 @@ Database::Database(const Napi::CallbackInfo &info) : Napi::ObjectWrap<Database>(
 		callback = info[pos++].As<Napi::Function>();
 	}
 
-	Schedule(env, duckdb::make_unique<OpenTask>(*this, filename, mode == DUCKDB_NODEJS_READONLY, callback));
+	Schedule(env, duckdb::make_unique<OpenTask>(*this, filename, access_mode, config, callback));
+}
+
+Database::~Database() {
+	Napi::MemoryManagement::AdjustExternalMemory(env, -bytes_allocated);
 }
 
 void Database::Schedule(Napi::Env env, std::unique_ptr<Task> task) {
@@ -101,9 +141,7 @@ static void TaskExecuteCallback(napi_env e, void *data) {
 static void TaskCompleteCallback(napi_env e, napi_status status, void *data) {
 	std::unique_ptr<TaskHolder> holder((TaskHolder *)data);
 	holder->db->TaskComplete(e);
-	if (holder->task->callback.Value().IsFunction()) {
-		holder->task->Callback();
-	}
+	holder->task->DoCallback();
 }
 
 void Database::TaskComplete(Napi::Env env) {
@@ -112,6 +150,16 @@ void Database::TaskComplete(Napi::Env env) {
 		task_inflight = false;
 	}
 	Process(env);
+
+	if (database) {
+		// Bookkeeping: tell node (and the node GC in particular) how much
+		// memory we're using, such that it can make better decisions on when to
+		// trigger collections.
+		auto &buffer_manager = duckdb::BufferManager::GetBufferManager(*database->instance);
+		auto current_bytes = buffer_manager.GetUsedMemory();
+		Napi::MemoryManagement::AdjustExternalMemory(env, current_bytes - bytes_allocated);
+		bytes_allocated = current_bytes;
+	}
 }
 
 void Database::Process(Napi::Env env) {