duckdb 0.4.1-dev182.0 → 0.4.1-dev1896.0

package/src/statement.cpp

@@ -1,5 +1,8 @@
+#include "duckdb.hpp"
 #include "duckdb_node.hpp"
+#include "napi.h"
 
+#include <algorithm>
 #include <cassert>
 
 namespace node_duckdb {
@@ -12,7 +15,8 @@ Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) {
 	Napi::Function t =
 	    DefineClass(env, "Statement",
 	                {InstanceMethod("run", &Statement::Run), InstanceMethod("all", &Statement::All),
-	                 InstanceMethod("each", &Statement::Each), InstanceMethod("finalize", &Statement::Finish)});
+	                 InstanceMethod("each", &Statement::Each), InstanceMethod("finalize", &Statement::Finish),
+	                 InstanceMethod("stream", &Statement::Stream)});
 
 	constructor = Napi::Persistent(t);
 	constructor.SuppressDestruct();
@@ -36,8 +40,8 @@ struct PrepareTask : public Task {
 		Napi::HandleScope scope(env);
 
 		auto cb = callback.Value();
-		if (!statement.statement->success) {
-			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->error)});
+		if (statement.statement->HasError()) {
+			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->error.Message())});
 			return;
 		}
 		cb.MakeCallback(statement.Value(), {env.Null(), statement.Value()});
@@ -126,6 +130,10 @@ static duckdb::Value BindParameter(const Napi::Value source) {
 static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::LogicalTypeId id) {
 	Napi::Value value;
 
+	if (dval.IsNull()) {
+		return env.Null();
+	}
+
 	// TODO templateroo here
 	switch (id) {
 	case duckdb::LogicalTypeId::BOOLEAN: {
@@ -221,11 +229,6 @@ static Napi::Value convert_chunk(Napi::Env &env, std::vector<std::string> names,
 
 		for (duckdb::idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) {
 			duckdb::Value dval = chunk.GetValue(col_idx, row_idx);
-			if (dval.IsNull()) {
-				row_result.Set(node_names[col_idx], env.Null());
-				continue;
-			}
-
 			row_result.Set(node_names[col_idx], convert_col_val(env, dval, chunk.data[col_idx].GetType().id()));
 		}
 		result.Set(row_idx, row_result);
@@ -250,7 +253,7 @@ struct RunPreparedTask : public Task {
 	void DoWork() override {
 		auto &statement = Get<Statement>();
 		// ignorant folk arrive here without caring about the prepare callback error
-		if (!statement.statement || !statement.statement->success) {
+		if (!statement.statement || statement.statement->HasError()) {
 			return;
 		}
 
@@ -268,16 +271,12 @@ struct RunPreparedTask : public Task {
 			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, "statement was finalized")});
 			return;
 		}
-		if (!statement.statement->success) {
-			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->error)});
+		if (statement.statement->HasError()) {
+			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->GetError())});
 			return;
 		}
-		if (!statement.statement->success) {
-			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->error)});
-			return;
-		}
-		if (!result->success) {
-			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, result->error)});
+		if (result->HasError()) {
+			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, result->GetError())});
 			return;
 		}
 
@@ -288,6 +287,8 @@ struct RunPreparedTask : public Task {
 		case RunType::EACH: {
 			duckdb::idx_t count = 0;
 			while (true) {
+				Napi::HandleScope scope(env);
+
 				auto chunk = result->Fetch();
 				if (!chunk || chunk->size() == 0) {
 					break;
@@ -310,7 +311,7 @@ struct RunPreparedTask : public Task {
 		}
 		case RunType::ALL: {
 			auto materialized_result = (duckdb::MaterializedQueryResult *)result.get();
-			Napi::Array result_arr(Napi::Array::New(env, materialized_result->collection.Count()));
+			Napi::Array result_arr(Napi::Array::New(env, materialized_result->RowCount()));
 
 			duckdb::idx_t out_idx = 0;
 			while (true) {
@@ -338,6 +339,45 @@ struct RunPreparedTask : public Task {
 	RunType run_type;
 };
 
+struct RunQueryTask : public Task {
+	RunQueryTask(Statement &statement, duckdb::unique_ptr<StatementParam> params, Napi::Promise::Deferred deferred)
+	    : Task(statement), deferred(deferred), params(move(params)) {
+	}
+
+	void DoWork() override {
+		auto &statement = Get<Statement>();
+		if (!statement.statement || statement.statement->HasError()) {
+			return;
+		}
+
+		result = statement.statement->Execute(params->params, true);
+	}
+
+	void DoCallback() override {
+		auto &statement = Get<Statement>();
+		Napi::Env env = statement.Env();
+		Napi::HandleScope scope(env);
+
+		if (!statement.statement) {
+			deferred.Reject(Utils::CreateError(env, "statement was finalized"));
+		} else if (statement.statement->HasError()) {
+			deferred.Reject(Utils::CreateError(env, statement.statement->GetError()));
+		} else if (result->HasError()) {
+			deferred.Reject(Utils::CreateError(env, result->GetError()));
+		} else {
+			auto db = statement.connection_ref->database_ref->Value();
+			auto query_result = QueryResult::constructor.New({db});
+			auto unwrapped = Napi::ObjectWrap<QueryResult>::Unwrap(query_result);
+			unwrapped->result = move(result);
+			deferred.Resolve(query_result);
+		}
+	}
+
+	Napi::Promise::Deferred deferred;
+	std::unique_ptr<duckdb::QueryResult> result;
+	duckdb::unique_ptr<StatementParam> params;
+};
+
 duckdb::unique_ptr<StatementParam> Statement::HandleArgs(const Napi::CallbackInfo &info) {
 	size_t start_idx = ignore_first_param ? 1 : 0;
 	auto params = duckdb::make_unique<StatementParam>();
@@ -367,19 +407,24 @@ Napi::Value Statement::All(const Napi::CallbackInfo &info) {
 }
 
 Napi::Value Statement::Run(const Napi::CallbackInfo &info) {
-	auto params = HandleArgs(info);
 	connection_ref->database_ref->Schedule(info.Env(),
 	                                       duckdb::make_unique<RunPreparedTask>(*this, HandleArgs(info), RunType::RUN));
 	return info.This();
 }
 
 Napi::Value Statement::Each(const Napi::CallbackInfo &info) {
-	auto params = HandleArgs(info);
 	connection_ref->database_ref->Schedule(
 	    info.Env(), duckdb::make_unique<RunPreparedTask>(*this, HandleArgs(info), RunType::EACH));
 	return info.This();
 }
 
+Napi::Value Statement::Stream(const Napi::CallbackInfo &info) {
+	auto deferred = Napi::Promise::Deferred::New(info.Env());
+	connection_ref->database_ref->Schedule(info.Env(),
+	                                       duckdb::make_unique<RunQueryTask>(*this, HandleArgs(info), deferred));
+	return deferred.Promise();
+}
+
 struct FinishTask : public Task {
 	FinishTask(Statement &statement, Napi::Function callback) : Task(statement, callback) {
 	}
@@ -404,4 +449,67 @@ Napi::Value Statement::Finish(const Napi::CallbackInfo &info) {
 	return env.Null();
 }
 
+Napi::FunctionReference QueryResult::constructor;
+
+Napi::Object QueryResult::Init(Napi::Env env, Napi::Object exports) {
+	Napi::HandleScope scope(env);
+
+	Napi::Function t = DefineClass(env, "QueryResult", {InstanceMethod("nextChunk", &QueryResult::NextChunk)});
+
+	constructor = Napi::Persistent(t);
+	constructor.SuppressDestruct();
+
+	exports.Set("QueryResult", t);
+	return exports;
+}
+
+QueryResult::QueryResult(const Napi::CallbackInfo &info) : Napi::ObjectWrap<QueryResult>(info) {
+	database_ref = Napi::ObjectWrap<Database>::Unwrap(info[0].As<Napi::Object>());
+	database_ref->Ref();
+}
+
+QueryResult::~QueryResult() {
+	database_ref->Unref();
+	database_ref = nullptr;
+}
+
+struct GetChunkTask : public Task {
+	GetChunkTask(QueryResult &query_result, Napi::Promise::Deferred deferred) : Task(query_result), deferred(deferred) {
+	}
+
+	void DoWork() override {
+		auto &query_result = Get<QueryResult>();
+		chunk = query_result.result->Fetch();
+	}
+
+	void DoCallback() override {
+		auto &query_result = Get<QueryResult>();
+		Napi::Env env = query_result.Env();
+		Napi::HandleScope scope(env);
+
+		if (chunk == nullptr || chunk->size() == 0) {
+			deferred.Resolve(env.Null());
+			return;
+		}
+
+		auto chunk_converted = convert_chunk(env, query_result.result->names, *chunk).ToObject();
+		if (!chunk_converted.IsArray()) {
+			deferred.Reject(Utils::CreateError(env, "internal error: chunk is not array"));
+		} else {
+			deferred.Resolve(chunk_converted);
+		}
+	}
+
+	Napi::Promise::Deferred deferred;
+	std::unique_ptr<duckdb::DataChunk> chunk;
+};
+
+Napi::Value QueryResult::NextChunk(const Napi::CallbackInfo &info) {
+	auto env = info.Env();
+	auto deferred = Napi::Promise::Deferred::New(env);
+	database_ref->Schedule(env, duckdb::make_unique<GetChunkTask>(*this, deferred));
+
+	return deferred.Promise();
+}
+
 } // namespace node_duckdb

package/test/data_type_support.test.js

@@ -10,7 +10,7 @@ describe("data type support", function () {
   it("supports BOOLEAN values", function (done) {
     db.run("CREATE TABLE boolean_table (i BOOLEAN)");
     const stmt = db.prepare("INSERT INTO boolean_table VALUES (?)");
-    const values = [true, false];
+    const values = [true, false, null];
     values.forEach((bool) => {
       stmt.run(bool);
     });
@@ -21,7 +21,8 @@ describe("data type support", function () {
     });
   });
   it("supports INTERVAL values", function (done) {
-    db.prepare(`SELECT
+    db.prepare(
+      `SELECT
     INTERVAL 1 MINUTE as minutes,
     INTERVAL 5 DAY as days,
     INTERVAL 4 MONTH as months,
@@ -43,6 +44,7 @@ describe("data type support", function () {
       done();
     });
   });
+
   it("supports STRUCT values", function (done) {
     db.prepare(`SELECT {'x': 1, 'y': 2, 'z': {'a': 'b'}} as struct`).each(
       (err, row) => {
@@ -51,12 +53,76 @@ describe("data type support", function () {
       }
     );
   });
+
+  it("supports STRUCT values with NULL", function (done) {
+    db.run("CREATE TABLE struct_table (s STRUCT(a VARCHAR, b BOOLEAN))");
+    db.run("INSERT INTO struct_table VALUES ({'a': 'hello', 'b': true})");
+    db.run("INSERT INTO struct_table VALUES ({'a': 'goodbye', 'b': false})");
+    db.run("INSERT INTO struct_table VALUES ({'a': 'aloha', 'b': NULL})");
+    db.prepare("SELECT s from struct_table;").all((err, res) => {
+      assert(err === null);
+      assert.deepEqual(res, [
+        { s: { a: "hello", b: true } },
+        { s: { a: "goodbye", b: false } },
+        { s: { a: "aloha", b: null } },
+      ]);
+      done();
+    });
+  });
+
+  it("recursively supports NULL values", function (done) {
+    db.run(
+      `CREATE TABLE recursive_struct AS SELECT [
+      { 'a': 42, 'b': [1, 2, 3]},
+      NULL,
+      { 'a': NULL, 'b': [4, NULL, 6]},
+      {'a': 43, 'b': NULL}
+   ] l UNION ALL SELECT NULL`
+    );
+    db.prepare("SELECT l from recursive_struct").all((err, res) => {
+      assert(err === null);
+      assert.deepEqual(res, [
+        {
+          l: [
+            {
+              a: 42,
+              b: [1, 2, 3],
+            },
+            null,
+            {
+              a: null,
+              b: [4, null, 6],
+            },
+            {
+              a: 43,
+              b: null,
+            },
+          ],
+        },
+        {
+          l: null,
+        },
+      ]);
+      done();
+    });
+  });
+
   it("supports LIST values", function (done) {
     db.prepare(`SELECT ['duck', 'duck', 'goose'] as list`).each((err, row) => {
+      assert(err === null);
       assert.deepEqual(row.list, ["duck", "duck", "goose"]);
       done();
     });
   });
+
+  it("supports LIST with NULL values", function (done) {
+    db.prepare(`SELECT ['duck', 'duck', NULL] as list`).each((err, row) => {
+      assert(err === null);
+      assert.deepEqual(row.list, ["duck", "duck", null]);
+      done();
+    });
+  });
+
   it("supports DATE values", function (done) {
     db.prepare(`SELECT '2021-01-01'::DATE as dt;`).each((err, row) => {
       assert(err === null);
@@ -65,23 +131,27 @@ describe("data type support", function () {
     });
   });
   it("supports TIMESTAMP values", function (done) {
-    db.prepare(`SELECT '2021-01-01T00:00:00'::TIMESTAMP as ts;`).each((err, row) => {
-      assert(err === null);
-      assert.deepEqual(row.ts, new Date(Date.UTC(2021, 0, 1)));
-      done();
-    });
+    db.prepare(`SELECT '2021-01-01T00:00:00'::TIMESTAMP as ts;`).each(
+      (err, row) => {
+        assert(err === null);
+        assert.deepEqual(row.ts, new Date(Date.UTC(2021, 0, 1)));
+        done();
+      }
+    );
   });
   it("supports TIMESTAMP WITH TIME ZONE values", function (done) {
-    db.prepare(`SELECT '2021-01-01T00:00:00Z'::TIMESTAMPTZ as tstz;`).each((err, row) => {
-      assert(err === null);
-      assert.deepEqual(row.tstz, new Date(Date.UTC(2021, 0, 1)));
-      done();
-    });
+    db.prepare(`SELECT '2021-01-01T00:00:00Z'::TIMESTAMPTZ as tstz;`).each(
+      (err, row) => {
+        assert(err === null);
+        assert.deepEqual(row.tstz, new Date(Date.UTC(2021, 0, 1)));
+        done();
+      }
+    );
   });
   it("supports DECIMAL values", function (done) {
     db.run("CREATE TABLE decimal_table (d DECIMAL(24, 6))");
     const stmt = db.prepare("INSERT INTO decimal_table VALUES (?)");
-    const values = [0, -1, 23534642362547.543463];
+    const values = [0, -1, 23534642362547.543463, null];
     values.forEach((d) => {
       stmt.run(d);
     });

package/test/extension.test.js

@@ -67,7 +67,7 @@ describe('Extension loading', function() {
     var db;
 
     before(function(done) {
-        db = new duckdb.Database(':memory:', done);
+        db = new duckdb.Database(':memory:', {"allow_unsigned_extensions":"true"}, done);
     });
 
     for (ext of extension_paths) {

package/test/jsdoc.test.js

@@ -0,0 +1,60 @@
+/**
+ * Intended to be similar to stubtest for python
+ */
+
+var duckdb = require("..");
+var jsdoc = require("jsdoc3-parser");
+const { expect } = require('chai');
+const { promisify } = require('util');
+
+function lastDot(string) {
+  if (string.endsWith(')')) {
+    string = string.substr(0, string.length - 1);
+  }
+  const array = string.split('.');
+  return array[array.length - 1];
+}
+
+/**
+ * @typedef {Object} Node
+ * @property {string} name
+ * @property {string} memberof
+ * @property {string} longname
+ * @property {string} scope
+ */
+
+describe("JSDoc contains all methods", () => {
+  /**
+   * @type {Node[]}
+   */
+  let docs;
+  before(async () => {
+    docs = await promisify(jsdoc)(require.resolve("../lib/duckdb"));
+  })
+
+  function checkDocs(obj, scope) {
+    const symbols = Object.getOwnPropertySymbols(obj).map(i => lastDot(i.toString()));
+    const expected = Object
+      .getOwnPropertyNames(obj)
+      .concat(symbols)
+      .sort()
+      .filter(name => name !== 'constructor');
+
+    const actual = docs
+      .filter((node) => node.memberof === scope && !node.undocumented)
+      .map((node) => lastDot(node.name))
+      .sort();
+
+    expect(expected).to.deep.equals(actual, 'items missing from documentation');
+  }
+
+  for (const clazz of ['Database', 'QueryResult', 'Connection', 'Statement']) {
+    it(clazz, () => {
+      checkDocs(duckdb[clazz].prototype, `module:duckdb~${clazz}`);
+    });
+  }
+
+  it('module root', () => {
+    checkDocs(duckdb, 'module:duckdb');
+  });
+});

package/test/query_result.test.js

@@ -0,0 +1,23 @@
+var duckdb = require('..');
+var assert = require('assert');
+
+describe('QueryResult', () => {
+    const total = 1000;
+
+    let db;
+    let conn;
+    before((done) => {
+        db = new duckdb.Database(':memory:', () => {
+            conn = new duckdb.Connection(db, done);
+        });
+    });
+
+    it('streams results', async () => {
+        let retrieved = 0;
+        const stream = conn.stream('SELECT * FROM range(0, ?)', total);
+        for await (const row of stream) {
+            retrieved++;
+        }
+        assert.equal(total, retrieved)
+    })
+})

package/test/syntax_error.test.js

@@ -0,0 +1,16 @@
+var duckdb = require('..');
+var assert = require('assert');
+
+describe('exec', function() {
+    var db;
+    before(function(done) {
+        db = new duckdb.Database(':memory:', done);
+    });
+
+    it("doesn't crash on a syntax error", function(done) {
+        db.exec("syntax error", function(err) {
+            assert.notEqual(err, null, "Expected an error")
+            done();
+        });
+    });
+});