duckdb 0.4.1-dev223.0 → 0.4.1-dev2253.0

package/src/parquet-amalgamation.hpp

@@ -34,6 +34,10 @@ public:
 
 #include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
+#include "duckdb/planner/table_filter.hpp"
+#include "duckdb/planner/filter/constant_filter.hpp"
+#include "duckdb/planner/filter/null_filter.hpp"
+#include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/string_util.hpp"
@@ -6926,7 +6930,7 @@ struct ReadHead {
 	uint64_t size;
 
 	// Current info
-	unique_ptr<AllocatedData> data;
+	AllocatedData data;
 	bool data_isset = false;
 
 	idx_t GetEnd() const {
@@ -7021,7 +7025,7 @@ struct ReadAheadBuffer {
 				throw std::runtime_error("Prefetch registered requested for bytes outside file");
 			}
 
-			handle.Read(read_head.data->get(), read_head.size, read_head.location);
+			handle.Read(read_head.data.get(), read_head.size, read_head.location);
 			read_head.data_isset = true;
 		}
 	}
@@ -7043,16 +7047,16 @@ public:
 
 			if (!prefetch_buffer->data_isset) {
 				prefetch_buffer->Allocate(allocator);
-				handle.Read(prefetch_buffer->data->get(), prefetch_buffer->size, prefetch_buffer->location);
+				handle.Read(prefetch_buffer->data.get(), prefetch_buffer->size, prefetch_buffer->location);
 				prefetch_buffer->data_isset = true;
 			}
-			memcpy(buf, prefetch_buffer->data->get() + location - prefetch_buffer->location, len);
+			memcpy(buf, prefetch_buffer->data.get() + location - prefetch_buffer->location, len);
 		} else {
 			if (prefetch_mode && len < PREFETCH_FALLBACK_BUFFERSIZE && len > 0) {
 				Prefetch(location, MinValue<uint64_t>(PREFETCH_FALLBACK_BUFFERSIZE, handle.GetFileSize() - location));
 				auto prefetch_buffer_fallback = ra_buffer.GetReadHead(location);
 				D_ASSERT(location - prefetch_buffer_fallback->location + len <= prefetch_buffer_fallback->size);
-				memcpy(buf, prefetch_buffer_fallback->data->get() + location - prefetch_buffer_fallback->location, len);
+				memcpy(buf, prefetch_buffer_fallback->data.get() + location - prefetch_buffer_fallback->location, len);
 			} else {
 				handle.Read(buf, len, location);
 			}
@@ -7193,12 +7197,12 @@ public:
 		if (new_size > alloc_len) {
 			alloc_len = new_size;
 			allocated_data = allocator.Allocate(alloc_len);
-			ptr = (char *)allocated_data->get();
+			ptr = (char *)allocated_data.get();
 		}
 	}
 
 private:
-	unique_ptr<AllocatedData> allocated_data;
+	AllocatedData allocated_data;
 	idx_t alloc_len = 0;
 };
 
@@ -7701,7 +7705,6 @@ class FileMetaData;
 namespace duckdb {
 class Allocator;
 class ClientContext;
-class ChunkCollection;
 class BaseStatistics;
 class TableFilterSet;
 
@@ -7736,6 +7739,12 @@ struct ParquetOptions {
 	explicit ParquetOptions(ClientContext &context);
 
 	bool binary_as_string = false;
+	bool filename = false;
+	bool hive_partitioning = false;
+
+public:
+	void Serialize(FieldWriter &writer) const;
+	void Deserialize(FieldReader &reader);
 };
 
 class ParquetReader {
@@ -7832,7 +7841,7 @@ private:
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
-#include "duckdb/common/types/chunk_collection.hpp"
+#include "duckdb/common/types/column_data_collection.hpp"
 #endif
 
 
@@ -7853,7 +7862,7 @@ namespace duckdb {
 class BufferedSerializer;
 class ParquetWriter;
 class ColumnWriterPageState;
-class StandardColumnWriterState;
+class BasicColumnWriterState;
 
 class ColumnWriterState {
 public:
@@ -7875,9 +7884,6 @@ public:
 };
 
 class ColumnWriter {
-	//! We limit the uncompressed page size to 100MB
-	// The max size in Parquet is 2GB, but we choose a more conservative limit
-	static constexpr const idx_t MAX_UNCOMPRESSED_PAGE_SIZE = 100000000;
 
 public:
 	ColumnWriter(ParquetWriter &writer, idx_t schema_idx, vector<string> schema_path, idx_t max_repeat,
@@ -7901,46 +7907,35 @@ public:
 	                                                      idx_t max_repeat = 0, idx_t max_define = 1,
 	                                                      bool can_have_nulls = true);
 
-	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group);
-	virtual void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count);
-
-	virtual void BeginWrite(ColumnWriterState &state);
-	virtual void Write(ColumnWriterState &state, Vector &vector, idx_t count);
-	virtual void FinalizeWrite(ColumnWriterState &state);
-
-protected:
-	void HandleDefineLevels(ColumnWriterState &state, ColumnWriterState *parent, ValidityMask &validity, idx_t count,
-	                        uint16_t define_value, uint16_t null_value);
-	void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count, idx_t max_repeat);
+	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) = 0;
 
-	void WriteLevels(Serializer &temp_writer, const vector<uint16_t> &levels, idx_t max_value, idx_t start_offset,
-	                 idx_t count);
+	//! indicates whether the write need to analyse the data before preparing it
+	virtual bool HasAnalyze() {
+		return false;
+	}
 
-	virtual duckdb_parquet::format::Encoding::type GetEncoding();
+	virtual void Analyze(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) {
+		throw NotImplementedException("Writer does not need analysis");
+	}
 
-	void NextPage(ColumnWriterState &state_p);
-	void FlushPage(ColumnWriterState &state_p);
-	void WriteDictionary(ColumnWriterState &state_p, unique_ptr<BufferedSerializer> temp_writer, idx_t row_count);
+	//! Called after all data has been passed to Analyze
+	virtual void FinalizeAnalyze(ColumnWriterState &state) {
+		throw NotImplementedException("Writer does not need analysis");
+	}
 
-	virtual void FlushDictionary(ColumnWriterState &state, ColumnWriterStatistics *stats);
+	virtual void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) = 0;
 
-	//! Initializes the state used to track statistics during writing. Only used for scalar types.
-	virtual unique_ptr<ColumnWriterStatistics> InitializeStatsState();
-	//! Retrieves the row size of a vector at the specified location. Only used for scalar types.
-	virtual idx_t GetRowSize(Vector &vector, idx_t index);
-	//! Writes a (subset of a) vector to the specified serializer. Only used for scalar types.
-	virtual void WriteVector(Serializer &temp_writer, ColumnWriterStatistics *stats, ColumnWriterPageState *page_state,
-	                         Vector &vector, idx_t chunk_start, idx_t chunk_end);
+	virtual void BeginWrite(ColumnWriterState &state) = 0;
+	virtual void Write(ColumnWriterState &state, Vector &vector, idx_t count) = 0;
+	virtual void FinalizeWrite(ColumnWriterState &state) = 0;
 
-	//! Initialize the writer for a specific page. Only used for scalar types.
-	virtual unique_ptr<ColumnWriterPageState> InitializePageState();
-	//! Flushes the writer for a specific page. Only used for scalar types.
-	virtual void FlushPageState(Serializer &temp_writer, ColumnWriterPageState *state);
+protected:
+	void HandleDefineLevels(ColumnWriterState &state, ColumnWriterState *parent, ValidityMask &validity, idx_t count,
+	                        uint16_t define_value, uint16_t null_value);
+	void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count, idx_t max_repeat);
 
 	void CompressPage(BufferedSerializer &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
 	                  unique_ptr<data_t[]> &compressed_buf);
-
-	void SetParquetStatistics(StandardColumnWriterState &state, duckdb_parquet::format::ColumnChunk &column);
 };
 
 } // namespace duckdb
@@ -7953,6 +7948,7 @@ class FileOpener;
 
 class ParquetWriter {
 	friend class ColumnWriter;
+	friend class BasicColumnWriter;
 	friend class ListColumnWriter;
 	friend class StructColumnWriter;
 
@@ -7961,7 +7957,7 @@ public:
 	              vector<string> names, duckdb_parquet::format::CompressionCodec::type codec);
 
 public:
-	void Flush(ChunkCollection &buffer);
+	void Flush(ColumnDataCollection &buffer);
 	void Finalize();
 
 	static duckdb_parquet::format::Type::type DuckDBTypeToParquetType(const LogicalType &duckdb_type);

package/src/statement.cpp

@@ -1,5 +1,8 @@
+#include "duckdb.hpp"
 #include "duckdb_node.hpp"
+#include "napi.h"
 
+#include <algorithm>
 #include <cassert>
 
 namespace node_duckdb {
@@ -12,7 +15,8 @@ Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) {
 	Napi::Function t =
 	    DefineClass(env, "Statement",
 	                {InstanceMethod("run", &Statement::Run), InstanceMethod("all", &Statement::All),
-	                 InstanceMethod("each", &Statement::Each), InstanceMethod("finalize", &Statement::Finish)});
+	                 InstanceMethod("each", &Statement::Each), InstanceMethod("finalize", &Statement::Finish),
+	                 InstanceMethod("stream", &Statement::Stream)});
 
 	constructor = Napi::Persistent(t);
 	constructor.SuppressDestruct();
@@ -36,8 +40,8 @@ struct PrepareTask : public Task {
 		Napi::HandleScope scope(env);
 
 		auto cb = callback.Value();
-		if (!statement.statement->success) {
-			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->error)});
+		if (statement.statement->HasError()) {
+			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->error.Message())});
 			return;
 		}
 		cb.MakeCallback(statement.Value(), {env.Null(), statement.Value()});
@@ -126,6 +130,10 @@ static duckdb::Value BindParameter(const Napi::Value source) {
 static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::LogicalTypeId id) {
 	Napi::Value value;
 
+	if (dval.IsNull()) {
+		return env.Null();
+	}
+
 	// TODO templateroo here
 	switch (id) {
 	case duckdb::LogicalTypeId::BOOLEAN: {
@@ -221,11 +229,6 @@ static Napi::Value convert_chunk(Napi::Env &env, std::vector<std::string> names,
 
 		for (duckdb::idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) {
 			duckdb::Value dval = chunk.GetValue(col_idx, row_idx);
-			if (dval.IsNull()) {
-				row_result.Set(node_names[col_idx], env.Null());
-				continue;
-			}
-
 			row_result.Set(node_names[col_idx], convert_col_val(env, dval, chunk.data[col_idx].GetType().id()));
 		}
 		result.Set(row_idx, row_result);
@@ -250,7 +253,7 @@ struct RunPreparedTask : public Task {
 	void DoWork() override {
 		auto &statement = Get<Statement>();
 		// ignorant folk arrive here without caring about the prepare callback error
-		if (!statement.statement || !statement.statement->success) {
+		if (!statement.statement || statement.statement->HasError()) {
 			return;
 		}
 
@@ -268,16 +271,12 @@ struct RunPreparedTask : public Task {
 			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, "statement was finalized")});
 			return;
 		}
-		if (!statement.statement->success) {
-			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->error)});
+		if (statement.statement->HasError()) {
+			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->GetError())});
 			return;
 		}
-		if (!statement.statement->success) {
-			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, statement.statement->error)});
-			return;
-		}
-		if (!result->success) {
-			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, result->error)});
+		if (result->HasError()) {
+			cb.MakeCallback(statement.Value(), {Utils::CreateError(env, result->GetError())});
 			return;
 		}
 
@@ -288,6 +287,8 @@ struct RunPreparedTask : public Task {
 		case RunType::EACH: {
 			duckdb::idx_t count = 0;
 			while (true) {
+				Napi::HandleScope scope(env);
+
 				auto chunk = result->Fetch();
 				if (!chunk || chunk->size() == 0) {
 					break;
@@ -310,7 +311,7 @@ struct RunPreparedTask : public Task {
 		}
 		case RunType::ALL: {
 			auto materialized_result = (duckdb::MaterializedQueryResult *)result.get();
-			Napi::Array result_arr(Napi::Array::New(env, materialized_result->collection.Count()));
+			Napi::Array result_arr(Napi::Array::New(env, materialized_result->RowCount()));
 
 			duckdb::idx_t out_idx = 0;
 			while (true) {
@@ -338,6 +339,45 @@ struct RunPreparedTask : public Task {
 	RunType run_type;
 };
 
+struct RunQueryTask : public Task {
+	RunQueryTask(Statement &statement, duckdb::unique_ptr<StatementParam> params, Napi::Promise::Deferred deferred)
+	    : Task(statement), deferred(deferred), params(move(params)) {
+	}
+
+	void DoWork() override {
+		auto &statement = Get<Statement>();
+		if (!statement.statement || statement.statement->HasError()) {
+			return;
+		}
+
+		result = statement.statement->Execute(params->params, true);
+	}
+
+	void DoCallback() override {
+		auto &statement = Get<Statement>();
+		Napi::Env env = statement.Env();
+		Napi::HandleScope scope(env);
+
+		if (!statement.statement) {
+			deferred.Reject(Utils::CreateError(env, "statement was finalized"));
+		} else if (statement.statement->HasError()) {
+			deferred.Reject(Utils::CreateError(env, statement.statement->GetError()));
+		} else if (result->HasError()) {
+			deferred.Reject(Utils::CreateError(env, result->GetError()));
+		} else {
+			auto db = statement.connection_ref->database_ref->Value();
+			auto query_result = QueryResult::constructor.New({db});
+			auto unwrapped = Napi::ObjectWrap<QueryResult>::Unwrap(query_result);
+			unwrapped->result = move(result);
+			deferred.Resolve(query_result);
+		}
+	}
+
+	Napi::Promise::Deferred deferred;
+	std::unique_ptr<duckdb::QueryResult> result;
+	duckdb::unique_ptr<StatementParam> params;
+};
+
 duckdb::unique_ptr<StatementParam> Statement::HandleArgs(const Napi::CallbackInfo &info) {
 	size_t start_idx = ignore_first_param ? 1 : 0;
 	auto params = duckdb::make_unique<StatementParam>();
@@ -367,19 +407,24 @@ Napi::Value Statement::All(const Napi::CallbackInfo &info) {
 }
 
 Napi::Value Statement::Run(const Napi::CallbackInfo &info) {
-	auto params = HandleArgs(info);
 	connection_ref->database_ref->Schedule(info.Env(),
 	                                       duckdb::make_unique<RunPreparedTask>(*this, HandleArgs(info), RunType::RUN));
 	return info.This();
 }
 
 Napi::Value Statement::Each(const Napi::CallbackInfo &info) {
-	auto params = HandleArgs(info);
 	connection_ref->database_ref->Schedule(
 	    info.Env(), duckdb::make_unique<RunPreparedTask>(*this, HandleArgs(info), RunType::EACH));
 	return info.This();
 }
 
+Napi::Value Statement::Stream(const Napi::CallbackInfo &info) {
+	auto deferred = Napi::Promise::Deferred::New(info.Env());
+	connection_ref->database_ref->Schedule(info.Env(),
+	                                       duckdb::make_unique<RunQueryTask>(*this, HandleArgs(info), deferred));
+	return deferred.Promise();
+}
+
 struct FinishTask : public Task {
 	FinishTask(Statement &statement, Napi::Function callback) : Task(statement, callback) {
 	}
@@ -404,4 +449,67 @@ Napi::Value Statement::Finish(const Napi::CallbackInfo &info) {
 	return env.Null();
 }
 
+Napi::FunctionReference QueryResult::constructor;
+
+Napi::Object QueryResult::Init(Napi::Env env, Napi::Object exports) {
+	Napi::HandleScope scope(env);
+
+	Napi::Function t = DefineClass(env, "QueryResult", {InstanceMethod("nextChunk", &QueryResult::NextChunk)});
+
+	constructor = Napi::Persistent(t);
+	constructor.SuppressDestruct();
+
+	exports.Set("QueryResult", t);
+	return exports;
+}
+
+QueryResult::QueryResult(const Napi::CallbackInfo &info) : Napi::ObjectWrap<QueryResult>(info) {
+	database_ref = Napi::ObjectWrap<Database>::Unwrap(info[0].As<Napi::Object>());
+	database_ref->Ref();
+}
+
+QueryResult::~QueryResult() {
+	database_ref->Unref();
+	database_ref = nullptr;
+}
+
+struct GetChunkTask : public Task {
+	GetChunkTask(QueryResult &query_result, Napi::Promise::Deferred deferred) : Task(query_result), deferred(deferred) {
+	}
+
+	void DoWork() override {
+		auto &query_result = Get<QueryResult>();
+		chunk = query_result.result->Fetch();
+	}
+
+	void DoCallback() override {
+		auto &query_result = Get<QueryResult>();
+		Napi::Env env = query_result.Env();
+		Napi::HandleScope scope(env);
+
+		if (chunk == nullptr || chunk->size() == 0) {
+			deferred.Resolve(env.Null());
+			return;
+		}
+
+		auto chunk_converted = convert_chunk(env, query_result.result->names, *chunk).ToObject();
+		if (!chunk_converted.IsArray()) {
+			deferred.Reject(Utils::CreateError(env, "internal error: chunk is not array"));
+		} else {
+			deferred.Resolve(chunk_converted);
+		}
+	}
+
+	Napi::Promise::Deferred deferred;
+	std::unique_ptr<duckdb::DataChunk> chunk;
+};
+
+Napi::Value QueryResult::NextChunk(const Napi::CallbackInfo &info) {
+	auto env = info.Env();
+	auto deferred = Napi::Promise::Deferred::New(env);
+	database_ref->Schedule(env, duckdb::make_unique<GetChunkTask>(*this, deferred));
+
+	return deferred.Promise();
+}
+
 } // namespace node_duckdb

package/test/data_type_support.test.js

@@ -10,7 +10,7 @@ describe("data type support", function () {
   it("supports BOOLEAN values", function (done) {
     db.run("CREATE TABLE boolean_table (i BOOLEAN)");
     const stmt = db.prepare("INSERT INTO boolean_table VALUES (?)");
-    const values = [true, false];
+    const values = [true, false, null];
     values.forEach((bool) => {
       stmt.run(bool);
     });
@@ -21,7 +21,8 @@ describe("data type support", function () {
     });
   });
   it("supports INTERVAL values", function (done) {
-    db.prepare(`SELECT
+    db.prepare(
+      `SELECT
     INTERVAL 1 MINUTE as minutes,
     INTERVAL 5 DAY as days,
     INTERVAL 4 MONTH as months,
@@ -43,6 +44,7 @@ describe("data type support", function () {
       done();
     });
   });
+
   it("supports STRUCT values", function (done) {
     db.prepare(`SELECT {'x': 1, 'y': 2, 'z': {'a': 'b'}} as struct`).each(
       (err, row) => {
@@ -51,12 +53,76 @@ describe("data type support", function () {
       }
     );
   });
+
+  it("supports STRUCT values with NULL", function (done) {
+    db.run("CREATE TABLE struct_table (s STRUCT(a VARCHAR, b BOOLEAN))");
+    db.run("INSERT INTO struct_table VALUES ({'a': 'hello', 'b': true})");
+    db.run("INSERT INTO struct_table VALUES ({'a': 'goodbye', 'b': false})");
+    db.run("INSERT INTO struct_table VALUES ({'a': 'aloha', 'b': NULL})");
+    db.prepare("SELECT s from struct_table;").all((err, res) => {
+      assert(err === null);
+      assert.deepEqual(res, [
+        { s: { a: "hello", b: true } },
+        { s: { a: "goodbye", b: false } },
+        { s: { a: "aloha", b: null } },
+      ]);
+      done();
+    });
+  });
+
+  it("recursively supports NULL values", function (done) {
+    db.run(
+      `CREATE TABLE recursive_struct AS SELECT [
+      { 'a': 42, 'b': [1, 2, 3]},
+      NULL,
+      { 'a': NULL, 'b': [4, NULL, 6]},
+      {'a': 43, 'b': NULL}
+   ] l UNION ALL SELECT NULL`
+    );
+    db.prepare("SELECT l from recursive_struct").all((err, res) => {
+      assert(err === null);
+      assert.deepEqual(res, [
+        {
+          l: [
+            {
+              a: 42,
+              b: [1, 2, 3],
+            },
+            null,
+            {
+              a: null,
+              b: [4, null, 6],
+            },
+            {
+              a: 43,
+              b: null,
+            },
+          ],
+        },
+        {
+          l: null,
+        },
+      ]);
+      done();
+    });
+  });
+
   it("supports LIST values", function (done) {
     db.prepare(`SELECT ['duck', 'duck', 'goose'] as list`).each((err, row) => {
+      assert(err === null);
       assert.deepEqual(row.list, ["duck", "duck", "goose"]);
       done();
     });
   });
+
+  it("supports LIST with NULL values", function (done) {
+    db.prepare(`SELECT ['duck', 'duck', NULL] as list`).each((err, row) => {
+      assert(err === null);
+      assert.deepEqual(row.list, ["duck", "duck", null]);
+      done();
+    });
+  });
+
   it("supports DATE values", function (done) {
     db.prepare(`SELECT '2021-01-01'::DATE as dt;`).each((err, row) => {
       assert(err === null);
@@ -65,23 +131,27 @@ describe("data type support", function () {
     });
   });
   it("supports TIMESTAMP values", function (done) {
-    db.prepare(`SELECT '2021-01-01T00:00:00'::TIMESTAMP as ts;`).each((err, row) => {
-      assert(err === null);
-      assert.deepEqual(row.ts, new Date(Date.UTC(2021, 0, 1)));
-      done();
-    });
+    db.prepare(`SELECT '2021-01-01T00:00:00'::TIMESTAMP as ts;`).each(
+      (err, row) => {
+        assert(err === null);
+        assert.deepEqual(row.ts, new Date(Date.UTC(2021, 0, 1)));
+        done();
+      }
+    );
   });
   it("supports TIMESTAMP WITH TIME ZONE values", function (done) {
-    db.prepare(`SELECT '2021-01-01T00:00:00Z'::TIMESTAMPTZ as tstz;`).each((err, row) => {
-      assert(err === null);
-      assert.deepEqual(row.tstz, new Date(Date.UTC(2021, 0, 1)));
-      done();
-    });
+    db.prepare(`SELECT '2021-01-01T00:00:00Z'::TIMESTAMPTZ as tstz;`).each(
+      (err, row) => {
+        assert(err === null);
+        assert.deepEqual(row.tstz, new Date(Date.UTC(2021, 0, 1)));
+        done();
+      }
+    );
   });
   it("supports DECIMAL values", function (done) {
     db.run("CREATE TABLE decimal_table (d DECIMAL(24, 6))");
     const stmt = db.prepare("INSERT INTO decimal_table VALUES (?)");
-    const values = [0, -1, 23534642362547.543463];
+    const values = [0, -1, 23534642362547.543463, null];
     values.forEach((d) => {
       stmt.run(d);
     });

package/test/extension.test.js

@@ -67,7 +67,7 @@ describe('Extension loading', function() {
     var db;
 
     before(function(done) {
-        db = new duckdb.Database(':memory:', done);
+        db = new duckdb.Database(':memory:', {"allow_unsigned_extensions":"true"}, done);
     });
 
     for (ext of extension_paths) {

package/test/jsdoc.test.js

@@ -0,0 +1,60 @@
+/**
+ * Intended to be similar to stubtest for python
+ */
+
+var duckdb = require("..");
+var jsdoc = require("jsdoc3-parser");
+const { expect } = require('chai');
+const { promisify } = require('util');
+
+function lastDot(string) {
+  if (string.endsWith(')')) {
+    string = string.substr(0, string.length - 1);
+  }
+  const array = string.split('.');
+  return array[array.length - 1];
+}
+
+/**
+ * @typedef {Object} Node
+ * @property {string} name
+ * @property {string} memberof
+ * @property {string} longname
+ * @property {string} scope
+ */
+
+describe("JSDoc contains all methods", () => {
+  /**
+   * @type {Node[]}
+   */
+  let docs;
+  before(async () => {
+    docs = await promisify(jsdoc)(require.resolve("../lib/duckdb"));
+  })
+
+  function checkDocs(obj, scope) {
+    const symbols = Object.getOwnPropertySymbols(obj).map(i => lastDot(i.toString()));
+    const expected = Object
+      .getOwnPropertyNames(obj)
+      .concat(symbols)
+      .sort()
+      .filter(name => name !== 'constructor');
+
+    const actual = docs
+      .filter((node) => node.memberof === scope && !node.undocumented)
+      .map((node) => lastDot(node.name))
+      .sort();
+
+    expect(expected).to.deep.equals(actual, 'items missing from documentation');
+  }
+
+  for (const clazz of ['Database', 'QueryResult', 'Connection', 'Statement']) {
+    it(clazz, () => {
+      checkDocs(duckdb[clazz].prototype, `module:duckdb~${clazz}`);
+    });
+  }
+
+  it('module root', () => {
+    checkDocs(duckdb, 'module:duckdb');
+  });
+});

package/test/query_result.test.js

@@ -0,0 +1,23 @@
+var duckdb = require('..');
+var assert = require('assert');
+
+describe('QueryResult', () => {
+    const total = 1000;
+
+    let db;
+    let conn;
+    before((done) => {
+        db = new duckdb.Database(':memory:', () => {
+            conn = new duckdb.Connection(db, done);
+        });
+    });
+
+    it('streams results', async () => {
+        let retrieved = 0;
+        const stream = conn.stream('SELECT * FROM range(0, ?)', total);
+        for await (const row of stream) {
+            retrieved++;
+        }
+        assert.equal(total, retrieved)
+    })
+})

package/test/syntax_error.test.js

@@ -0,0 +1,16 @@
+var duckdb = require('..');
+var assert = require('assert');
+
+describe('exec', function() {
+    var db;
+    before(function(done) {
+        db = new duckdb.Database(':memory:', done);
+    });
+
+    it("doesn't crash on a syntax error", function(done) {
+        db.exec("syntax error", function(err) {
+            assert.notEqual(err, null, "Expected an error")
+            done();
+        });
+    });
+});