duckdb 0.4.1-dev960.0 → 0.5.1-dev2.0

package/lib/duckdb.js

@@ -1,32 +1,157 @@
+/**
+ * @module duckdb
+ * @summary these jsdoc annotations are still a work in progress - feedback and suggestions are welcome!
+ */
+
 var duckdb = require('./duckdb-binding.js');
 module.exports = exports = duckdb;
 
+/**
+ * Check that errno attribute equals this to check for a duckdb error
+ * @constant {number}
+ */
+var ERROR = duckdb.ERROR;
+
+/**
+ * Open database in readonly mode
+ * @constant {number}
+ */
+var OPEN_READONLY = duckdb.OPEN_READONLY;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_READWRITE = duckdb.OPEN_READWRITE;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_CREATE = duckdb.OPEN_CREATE;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_FULLMUTEX = duckdb.OPEN_FULLMUTEX;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_SHAREDCACHE = duckdb.OPEN_SHAREDCACHE;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_PRIVATECACHE = duckdb.OPEN_PRIVATECACHE;
 
 // some wrappers for compatibilities sake
+/**
+ * Main database interface
+ */
 var Database = duckdb.Database;
+/**
+ * @class
+ */
 var Connection = duckdb.Connection;
+/**
+ * @class
+ */
 var Statement = duckdb.Statement;
+/**
+ * @class
+ */
+var QueryResult = duckdb.QueryResult;
 
+/**
+ * @method
+ * @return data chunk
+ */
+QueryResult.prototype.nextChunk;
 
-Connection.prototype.run = function(sql) {
+/**
+ * @name asyncIterator
+ * @memberof module:duckdb~QueryResult
+ * @method
+ * @instance
+ * @yields data chunks
+ */
+QueryResult.prototype[Symbol.asyncIterator] = async function*() {
+    let prefetch = this.nextChunk();
+    while (true) {
+        const chunk = await prefetch;
+        // Null chunk indicates end of stream
+        if (!chunk) {
+            return;
+        }
+        // Prefetch the next chunk while we're iterating
+        prefetch = this.nextChunk();
+        for (const row of chunk) {
+            yield row;
+        }
+    }
+}
+
+
+/**
+ * Run a SQL statement and trigger a callback when done
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.run = function (sql) {
     var statement = new Statement(this, sql);
     return statement.run.apply(statement, arguments);
 }
 
-Connection.prototype.all = function(sql) {
-    var statement = new Statement(this,sql);
+/**
+ * Run a SQL query and triggers the callback once for all result rows
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.all = function (sql) {
+    var statement = new Statement(this, sql);
     return statement.all.apply(statement, arguments);
 }
 
-Connection.prototype.each = function(sql) {
+/**
+ * Runs a SQL query and triggers the callback for each result row
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.each = function (sql) {
     var statement = new Statement(this, sql);
     return statement.each.apply(statement, arguments);
 }
 
-// this follows the wasm udfs somewhat but is simpler because we can pass data much more cleanly
-Connection.prototype.register = function(name, return_type, fun) {
+/**
+ * @arg sql
+ * @param {...*} params
+ * @yields row chunks
+ */
+Connection.prototype.stream = async function* (sql) {
+    const statement = new Statement(this, sql);
+    const queryResult = await statement.stream.apply(statement, arguments);
+    for await (const result of queryResult) {
+        yield result;
+    }
+}
+
+/**
+ * Register a User Defined Function
+ *
+ * @arg name
+ * @arg return_type
+ * @arg fun
+ * @return {void}
+ * @note this follows the wasm udfs somewhat but is simpler because we can pass data much more cleanly
+ */
+Connection.prototype.register = function (name, return_type, fun) {
     // TODO what if this throws an error somewhere? do we need a try/catch?
-    return this.register_bulk(name, return_type, function(desc) {
+    return this.register_bulk(name, return_type, function (desc) {
         try {
             // Build an argument resolver
             const buildResolver = (arg) => {
@@ -126,62 +251,252 @@ Connection.prototype.register = function(name, return_type, fun) {
                 desc.ret.data[i] = res;
                 desc.ret.validity[i] = res === undefined || res === null ? 0 : 1;
             }
-        } catch(error) { // work around recently fixed napi bug https://github.com/nodejs/node-addon-api/issues/912
+        } catch (error) { // work around recently fixed napi bug https://github.com/nodejs/node-addon-api/issues/912
             console.log(desc.ret);
             msg = error;
             if (typeof error == 'object' && 'message' in error) {
                 msg = error.message
             }
-            throw {name: 'DuckDB-UDF-Exception', message : msg};
+            throw { name: 'DuckDB-UDF-Exception', message: msg };
         }
     })
 }
 
-default_connection = function(o) {
+/**
+ * Prepare a SQL query for execution
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {Statement}
+ */
+Connection.prototype.prepare;
+/**
+ * Execute a SQL query
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.exec;
+/**
+ * Register a User Defined Function
+ *
+ * @method
+ * @arg name
+ * @arg return_type
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.register_bulk;
+/**
+ * Unregister a User Defined Function
+ *
+ * @method
+ * @arg name
+ * @arg return_type
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.unregister;
+
+var default_connection = function (o) {
     if (o.default_connection == undefined) {
         o.default_connection = new duckdb.Connection(o);
     }
-    return(o.default_connection);
+    return o.default_connection;
 }
 
-Database.prototype.prepare = function() {
+
+/**
+ * Closes database instance
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.close = function() {
+    this.default_connection = null
+    this.close_internal.apply(this, arguments);
+};
+
+/**
+ * Internal method. Do not use, call Connection#close instead
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.close_internal;
+
+/**
+ * Triggers callback when all scheduled database tasks have completed.
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.wait;
+
+/**
+ * TODO: what does this do?
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.serialize;
+
+/**
+ * TODO: what does this do?
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.parallelize;
+
+/**
+ * Create a new database connection
+ * @method
+ * @arg path the database to connect to, either a file path, or `:memory:`
+ * @return {Connection}
+ */
+Database.prototype.connect;
+
+/**
+ * Supposedly interrupt queries, but currently does not do anything.
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.interrupt;
+
+/**
+ * Prepare a SQL query for execution
+ * @arg sql
+ * @return {Statement}
+ */
+Database.prototype.prepare = function () {
     return default_connection(this).prepare.apply(this.default_connection, arguments);
 }
 
-Database.prototype.run = function() {
+/**
+ * Convenience method for Connection#run using a built-in default connection
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.run = function () {
     default_connection(this).run.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.each = function() {
+/**
+ * Convenience method for Connection#each using a built-in default connection
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.each = function () {
     default_connection(this).each.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.all = function() {
+/**
+ * Convenience method for Connection#apply using a built-in default connection
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.all = function () {
     default_connection(this).all.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.exec = function() {
+/**
+ * Convenience method for Connection#exec using a built-in default connection
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.exec = function () {
     default_connection(this).exec.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.register = function() {
+/**
+ * Convenience method for Connection#register using a built-in default connection
+ * @arg name
+ * @arg return_type
+ * @arg fun
+ * @return {this}
+ */
+Database.prototype.register = function () {
     default_connection(this).register.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.unregister = function() {
+/**
+ * Convenience method for Connection#unregister using a built-in default connection
+ * @arg name
+ * @return {this}
+ */
+Database.prototype.unregister = function () {
     default_connection(this).unregister.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.get = function() {
+/**
+ * Not implemented
+ */
+Database.prototype.get = function () {
     throw "get() is not implemented because it's evil";
 }
 
-Statement.prototype.get = function() {
+/**
+ * Not implemented
+ */
+Statement.prototype.get = function () {
     throw "get() is not implemented because it's evil";
 }
+
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.run;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.all;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.each;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.finalize
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @yield callback
+ */
+Statement.prototype.stream;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev960.0",
+  "version": "0.5.1-dev2.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {
@@ -26,6 +26,8 @@
   },
   "devDependencies": {
     "aws-sdk": "^2.790.0",
+    "chai": "^4.3.6",
+    "jsdoc3-parser": "^2.0.0",
     "mocha": "^8.3.0"
   },
   "repository": {

package/src/connection.cpp

@@ -31,8 +31,29 @@ struct ConnectTask : public Task {
 
 	void DoWork() override {
 		auto &connection = Get<Connection>();
+		if (!connection.database_ref || !connection.database_ref->database) {
+			return;
+		}
 		connection.connection = duckdb::make_unique<duckdb::Connection>(*connection.database_ref->database);
+		success = true;
+	}
+	void Callback() override {
+		auto &connection = Get<Connection>();
+		Napi::Env env = connection.Env();
+
+		std::vector<napi_value> args;
+		if (!success) {
+			args.push_back(Utils::CreateError(env, "Invalid database object"));
+		} else {
+			args.push_back(env.Null());
+		}
+
+		Napi::HandleScope scope(env);
+
+		callback.Value().MakeCallback(connection.Value(), args);
 	}
+
+	bool success = false;
 };
 
 Connection::Connection(const Napi::CallbackInfo &info) : Napi::ObjectWrap<Connection>(info) {
@@ -78,7 +99,7 @@ struct JSArgs {
 	duckdb::DataChunk *args;
 	duckdb::Vector *result;
 	bool done;
-	std::string error;
+	duckdb::PreservedError error;
 };
 
 void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, std::nullptr_t *, JSArgs *jsargs) {
@@ -188,8 +209,10 @@ void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, std::nullptr_t *
 			}
 		}
 		}
+	} catch (const duckdb::Exception &e) {
+		jsargs->error = duckdb::PreservedError(e);
 	} catch (const std::exception &e) {
-		jsargs->error = e.what();
+		jsargs->error = duckdb::PreservedError(e);
 	}
 	jsargs->done = true;
 }
@@ -219,8 +242,8 @@ struct RegisterTask : public Task {
 			while (!jsargs.done) {
 				std::this_thread::yield();
 			}
-			if (!jsargs.error.empty()) {
-				throw duckdb::IOException(jsargs.error);
+			if (jsargs.error) {
+				jsargs.error.Throw();
 			}
 		};
 
@@ -327,18 +350,17 @@ struct ExecTask : public Task {
 				return;
 			}
 
-			// thanks Mark
 			for (duckdb::idx_t i = 0; i < statements.size(); i++) {
 				auto res = connection.connection->Query(move(statements[i]));
-				if (!res->success) {
+				if (res->HasError()) {
 					success = false;
-					error = res->error;
+					error = res->GetErrorObject();
 					break;
 				}
 			}
 		} catch (duckdb::ParserException &e) {
 			success = false;
-			error = e.what();
+			error = duckdb::PreservedError(e);
 			return;
 		}
 	}
@@ -346,12 +368,12 @@ struct ExecTask : public Task {
 	void Callback() override {
 		auto env = object.Env();
 		Napi::HandleScope scope(env);
-		callback.Value().MakeCallback(object.Value(), {success ? env.Null() : Napi::String::New(env, error)});
+		callback.Value().MakeCallback(object.Value(), {success ? env.Null() : Napi::String::New(env, error.Message())});
 	};
 
 	std::string sql;
 	bool success;
-	std::string error;
+	duckdb::PreservedError error;
 };
 
 Napi::Value Connection::Exec(const Napi::CallbackInfo &info) {

package/src/database.cpp

@@ -11,7 +11,7 @@ Napi::Object Database::Init(Napi::Env env, Napi::Object exports) {
 
 	Napi::Function t = DefineClass(
 	    env, "Database",
-	    {InstanceMethod("close", &Database::Close), InstanceMethod("wait", &Database::Wait),
+	    {InstanceMethod("close_internal", &Database::Close), InstanceMethod("wait", &Database::Wait),
 	     InstanceMethod("serialize", &Database::Serialize), InstanceMethod("parallelize", &Database::Parallelize),
 	     InstanceMethod("connect", &Database::Connect), InstanceMethod("interrupt", &Database::Interrupt)});
 
@@ -60,8 +60,10 @@ struct OpenTask : public Task {
 			extension.Load(*Get<Database>().database);
 			success = true;
 
+		} catch (const duckdb::Exception &ex) {
+			error = duckdb::PreservedError(ex);
 		} catch (std::exception &ex) {
-			error = ex.what();
+			error = duckdb::PreservedError(ex);
 		}
 	}
 
@@ -71,7 +73,7 @@ struct OpenTask : public Task {
 
 		std::vector<napi_value> args;
 		if (!success) {
-			args.push_back(Utils::CreateError(env, error));
+			args.push_back(Utils::CreateError(env, error.Message()));
 		} else {
 			args.push_back(env.Null());
 		}
@@ -83,7 +85,7 @@ struct OpenTask : public Task {
 
 	std::string filename;
 	duckdb::DBConfig duckdb_config;
-	std::string error = "";
+	duckdb::PreservedError error;
 	bool success = false;
 };
 
@@ -141,9 +143,7 @@ static void TaskExecuteCallback(napi_env e, void *data) {
 static void TaskCompleteCallback(napi_env e, napi_status status, void *data) {
 	std::unique_ptr<TaskHolder> holder((TaskHolder *)data);
 	holder->db->TaskComplete(e);
-	if (holder->task->callback.Value().IsFunction()) {
-		holder->task->Callback();
-	}
+	holder->task->DoCallback();
 }
 
 void Database::TaskComplete(Napi::Env env) {