duckdb 0.4.1-dev13.0 → 0.4.1-dev1328.0

package/src/connection.cpp

@@ -1,5 +1,8 @@
+#include "duckdb.hpp"
 #include "duckdb_node.hpp"
+#include "napi.h"
 
+#include <iostream>
 #include <thread>
 
 namespace node_duckdb {
@@ -23,7 +26,7 @@ Napi::Object Connection::Init(Napi::Env env, Napi::Object exports) {
 }
 
 struct ConnectTask : public Task {
-	ConnectTask(Connection &connection_, Napi::Function callback_) : Task(connection_, callback_) {
+	ConnectTask(Connection &connection, Napi::Function callback) : Task(connection, callback) {
 	}
 
 	void DoWork() override {
@@ -78,91 +81,25 @@ struct JSArgs {
 	std::string error;
 };
 
-static Napi::Value transform_vector(Napi::Env env, duckdb::Vector &vec, duckdb::idx_t rows, bool copy) {
-	Napi::EscapableHandleScope scope(env);
-
-	Napi::Array data_buffers(Napi::Array::New(env));
-
-	auto data_buffer_index = -1;
-	auto length_buffer_index = -1;
-	auto validity_buffer_index = -1;
-
-	auto validity_buffer = Napi::Buffer<uint8_t>::New(env, rows);
-	auto validity = duckdb::FlatVector::Validity(vec);
-	auto validity_ptr = validity_buffer.Data();
-	for (duckdb::idx_t row_idx = 0; row_idx < rows; row_idx++) {
-		validity_ptr[row_idx] = validity.RowIsValid(row_idx);
-	}
-	data_buffers.Set(data_buffers.Length(), validity_buffer);
-	validity_buffer_index = data_buffers.Length() - 1;
-
-	auto &vec_type = vec.GetType();
-
-	switch (vec_type.id()) {
-	case duckdb::LogicalTypeId::BOOLEAN:
-	case duckdb::LogicalTypeId::UTINYINT:
-	case duckdb::LogicalTypeId::TINYINT:
-	case duckdb::LogicalTypeId::USMALLINT:
-	case duckdb::LogicalTypeId::SMALLINT:
-	case duckdb::LogicalTypeId::INTEGER:
-	case duckdb::LogicalTypeId::UINTEGER:
-	case duckdb::LogicalTypeId::FLOAT:
-	case duckdb::LogicalTypeId::DOUBLE: {
-		auto data_buffer = Napi::Buffer<uint8_t>::New(env, rows * duckdb::GetTypeIdSize(vec_type.InternalType()));
-		// TODO this is technically not neccessary but it fixes a crash in Node 14. Some weird data ownership issue.
-		memcpy(data_buffer.Data(), duckdb::FlatVector::GetData<uint8_t>(vec),
-		       rows * duckdb::GetTypeIdSize(vec_type.InternalType()));
-
-		data_buffers.Set(data_buffers.Length(), data_buffer);
-		data_buffer_index = data_buffers.Length() - 1;
-		break;
-	}
-	case duckdb::LogicalTypeId::VARCHAR: {
-		Napi::Array string_buffers(Napi::Array::New(env, rows));
-		if (copy) {
-			auto string_vec_ptr = duckdb::FlatVector::GetData<duckdb::string_t>(vec);
-			for (duckdb::idx_t row_idx = 0; row_idx < rows; row_idx++) {
-				string_buffers.Set(row_idx, validity_ptr[row_idx]
-				                                ? Napi::String::New(env, string_vec_ptr[row_idx].GetDataUnsafe(),
-				                                                    string_vec_ptr[row_idx].GetSize())
-				                                : Napi::Value());
-			}
-		}
-		data_buffers.Set(data_buffers.Length(), string_buffers);
-		data_buffer_index = data_buffers.Length() - 1;
-		break;
-	}
-	default:
-		throw duckdb::NotImplementedException(vec_type.ToString());
-	}
-
-	Napi::Object desc(Napi::Object::New(env));
-	desc.Set("logical_type", vec_type.ToString());
-	desc.Set("physical_type", TypeIdToString(vec_type.InternalType()));
-	desc.Set("validity_buffer", validity_buffer_index);
-	desc.Set("data_buffer", data_buffer_index);
-	desc.Set("length_buffer", length_buffer_index);
-	desc.Set("data_buffers", data_buffers);
-
-	return scope.Escape(desc);
-}
-
-void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, nullptr_t *, JSArgs *jsargs) {
+void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, std::nullptr_t *, JSArgs *jsargs) {
 	try { // if we dont catch exceptions here we terminate node if one happens ^^
+		Napi::EscapableHandleScope scope(env);
 
-		// set up descriptor and data arrays
-		Napi::Array args_descr(Napi::Array::New(env, jsargs->args->ColumnCount()));
-		for (duckdb::idx_t col_idx = 0; col_idx < jsargs->args->ColumnCount(); col_idx++) {
-			auto &vec = jsargs->args->data[col_idx];
-			auto arg_descr = transform_vector(env, vec, jsargs->rows, true);
-			args_descr.Set(col_idx, arg_descr);
-		}
-		auto ret_descr = transform_vector(env, *jsargs->result, jsargs->rows, false);
-
-		Napi::Object descr(Napi::Object::New(env));
+		// Set up descriptor and data arrays
+		auto descr = Napi::Object::New(env);
+		auto chunk = EncodeDataChunk(env, *jsargs->args, true, true);
+		descr.Set("args", scope.Escape(chunk));
 		descr.Set("rows", jsargs->rows);
-		descr.Set("args", args_descr);
-		descr.Set("ret", ret_descr);
+		auto ret = Napi::Object::New(env);
+		ret.Set("sqlType", jsargs->result->GetType().ToString());
+		auto ret_type = jsargs->result->GetType().InternalType();
+#if NAPI_VERSION <= 5
+		if (ret_type == duckdb::PhysicalType::INT64 || ret_type == duckdb::PhysicalType::UINT64) {
+			ret_type = duckdb::PhysicalType::DOUBLE;
+		}
+#endif
+		ret.Set("physicalType", TypeIdToString(ret_type));
+		descr.Set("ret", ret);
 
 		// actually call the UDF, or rather its vectorized wrapper from duckdb.js/Connection.prototype.register wrapper
 		jsudf({descr});
@@ -181,39 +118,75 @@ void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, nullptr_t *, JSA
 		}
 
 		// transform the result back to a vector
-		auto return_validity = ret_descr.ToObject()
-		                           .Get("data_buffers")
-		                           .ToObject()
-		                           .Get(ret_descr.ToObject().Get("validity_buffer"))
-		                           .As<Napi::Buffer<uint8_t>>();
+		auto return_validity = ret.ToObject().Get("validity").As<Napi::Uint8Array>();
 		for (duckdb::idx_t row_idx = 0; row_idx < jsargs->rows; row_idx++) {
 			duckdb::FlatVector::SetNull(*jsargs->result, row_idx, !return_validity[row_idx]);
 		}
 
-		if (jsargs->result->GetType().id() == duckdb::LogicalTypeId::VARCHAR) {
-			auto return_string_array = ret_descr.ToObject()
-			                               .Get("data_buffers")
-			                               .ToObject()
-			                               .Get(ret_descr.ToObject().Get("data_buffer"))
-			                               .ToObject();
-			auto return_string_vec_ptr = duckdb::FlatVector::GetData<duckdb::string_t>(*jsargs->result);
-
+		switch (jsargs->result->GetType().id()) {
+		case duckdb::LogicalTypeId::TINYINT: {
+			auto data = ret.Get("data").As<Napi::Int8Array>();
+			auto out = duckdb::FlatVector::GetData<int8_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::SMALLINT: {
+			auto data = ret.Get("data").As<Napi::Int16Array>();
+			auto out = duckdb::FlatVector::GetData<int16_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::INTEGER: {
+			auto data = ret.Get("data").As<Napi::Int32Array>();
+			auto out = duckdb::FlatVector::GetData<int32_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::DOUBLE: {
+			auto data = ret.Get("data").As<Napi::Float64Array>();
+			auto out = duckdb::FlatVector::GetData<double>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::TIME:
+		case duckdb::LogicalTypeId::TIMESTAMP:
+		case duckdb::LogicalTypeId::TIMESTAMP_MS:
+		case duckdb::LogicalTypeId::TIMESTAMP_SEC:
+		case duckdb::LogicalTypeId::TIMESTAMP_NS:
+		case duckdb::LogicalTypeId::BIGINT: {
+#if NAPI_VERSION > 5
+			auto data = ret.Get("data").As<Napi::BigInt64Array>();
+#else
+			auto data = ret.Get("data").As<Napi::Float64Array>();
+#endif
+			auto out = duckdb::FlatVector::GetData<int64_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::UBIGINT: {
+#if NAPI_VERSION > 5
+			auto data = ret.Get("data").As<Napi::BigUint64Array>();
+#else
+			auto data = ret.Get("data").As<Napi::Float64Array>();
+#endif
+			auto out = duckdb::FlatVector::GetData<uint64_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::BLOB:
+		case duckdb::LogicalTypeId::VARCHAR: {
+			auto data = ret.Get("data").As<Napi::Array>();
+			auto out = duckdb::FlatVector::GetData<duckdb::string_t>(*jsargs->result);
+			for (size_t i = 0; i < data.Length(); ++i) {
+				out[i] = duckdb::string_t(data.Get(i).ToString());
+			}
+			break;
+		}
+		default: {
 			for (duckdb::idx_t row_idx = 0; row_idx < jsargs->rows; row_idx++) {
-				if (!return_validity[row_idx]) {
-					duckdb::FlatVector::SetNull(*jsargs->result, row_idx, true);
-				} else {
-					auto str = return_string_array.Get(row_idx).As<Napi::String>();
-					return_string_vec_ptr[row_idx] = duckdb::StringVector::AddString(*jsargs->result, str);
-				}
+				duckdb::FlatVector::SetNull(*jsargs->result, row_idx, false);
 			}
-		} else {
-			auto return_data = ret_descr.ToObject()
-			                       .Get("data_buffers")
-			                       .ToObject()
-			                       .Get(ret_descr.ToObject().Get("data_buffer"))
-			                       .As<Napi::Buffer<uint8_t>>();
-			memcpy(duckdb::FlatVector::GetData<uint8_t>(*jsargs->result), return_data.Data(),
-			       jsargs->rows * duckdb::GetTypeIdSize(jsargs->result->GetType().InternalType()));
+		}
 		}
 	} catch (const std::exception &e) {
 		jsargs->error = e.what();
@@ -222,8 +195,8 @@ void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, nullptr_t *, JSA
 }
 
 struct RegisterTask : public Task {
-	RegisterTask(Connection &connection_, std::string name_, std::string return_type_name_, Napi::Function callback_)
-	    : Task(connection_, callback_), name(name_), return_type_name(return_type_name_) {
+	RegisterTask(Connection &connection, std::string name, std::string return_type_name, Napi::Function callback)
+	    : Task(connection, callback), name(std::move(name)), return_type_name(std::move(return_type_name)) {
 	}
 
 	void DoWork() override {
@@ -234,7 +207,7 @@ struct RegisterTask : public Task {
 			// here we can do only DuckDB stuff because we do not have a functioning env
 
 			// Flatten all args to simplify udfs
-			args.Normalify();
+			args.Flatten();
 
 			JSArgs jsargs;
 			jsargs.rows = args.size();
@@ -282,8 +255,8 @@ Napi::Value Connection::Register(const Napi::CallbackInfo &info) {
 		return env.Null();
 	}
 
-	auto udf = DuckDBNodeUDFFUnction::New(env, udf_callback, "duckdb_node_udf" + name, 0, 1, nullptr,
-	                                      [](Napi::Env, void *, nullptr_t *ctx) {});
+	auto udf = duckdb_node_udf_function_t::New(env, udf_callback, "duckdb_node_udf" + name, 0, 1, nullptr,
+	                                           [](Napi::Env, void *, std::nullptr_t *ctx) {});
 
 	// we have to unref the udf because otherwise there is a circular ref with the connection somehow(?)
 	// this took far too long to figure out
@@ -297,8 +270,8 @@ Napi::Value Connection::Register(const Napi::CallbackInfo &info) {
 }
 
 struct UnregisterTask : public Task {
-	UnregisterTask(Connection &connection_, std::string name_, Napi::Function callback_)
-	    : Task(connection_, callback_), name(name_) {
+	UnregisterTask(Connection &connection, std::string name, Napi::Function callback)
+	    : Task(connection, callback), name(std::move(name)) {
 	}
 
 	void DoWork() override {
@@ -340,27 +313,33 @@ Napi::Value Connection::Unregister(const Napi::CallbackInfo &info) {
 }
 
 struct ExecTask : public Task {
-	ExecTask(Connection &connection_, std::string sql_, Napi::Function callback_)
-	    : Task(connection_, callback_), sql(sql_) {
+	ExecTask(Connection &connection, std::string sql, Napi::Function callback)
+	    : Task(connection, callback), sql(std::move(sql)) {
 	}
 
 	void DoWork() override {
 		auto &connection = Get<Connection>();
 
 		success = true;
-		auto statements = connection.connection->ExtractStatements(sql);
-		if (statements.size() == 0) {
-			return;
-		}
+		try {
+			auto statements = connection.connection->ExtractStatements(sql);
+			if (statements.empty()) {
+				return;
+			}
 
-		// thanks Mark
-		for (duckdb::idx_t i = 0; i < statements.size(); i++) {
-			auto res = connection.connection->Query(move(statements[i]));
-			if (!res->success) {
-				success = false;
-				error = res->error;
-				break;
+			// thanks Mark
+			for (duckdb::idx_t i = 0; i < statements.size(); i++) {
+				auto res = connection.connection->Query(move(statements[i]));
+				if (!res->success) {
+					success = false;
+					error = res->error;
+					break;
+				}
 			}
+		} catch (duckdb::ParserException &e) {
+			success = false;
+			error = e.what();
+			return;
 		}
 	}
 

package/src/data_chunk.cpp

@@ -0,0 +1,185 @@
+#include "duckdb.hpp"
+#include "duckdb_node.hpp"
+#include "napi.h"
+
+#include <thread>
+
+namespace node_duckdb {
+
+Napi::Array EncodeDataChunk(Napi::Env env, duckdb::DataChunk &chunk, bool with_types, bool with_data) {
+	Napi::Array col_descs(Napi::Array::New(env, chunk.ColumnCount()));
+	for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) {
+		auto col_desc = Napi::Object::New(env);
+
+		// Make sure we only have flat vectors hereafter (for now)
+		auto &chunk_vec = chunk.data[col_idx];
+		if (with_data) {
+			chunk_vec.Flatten(chunk.size());
+		}
+
+		// Do a post-order DFS traversal
+		std::vector<std::tuple<bool, duckdb::Vector *, Napi::Object, size_t, size_t>> pending;
+		pending.emplace_back(false, &chunk_vec, Napi::Object::New(env), 0, 0);
+
+		while (!pending.empty()) {
+			// Unpack DFS node
+			auto &back = pending.back();
+			auto &visited = std::get<0>(back);
+			auto &vec = std::get<1>(back);
+			auto &desc = std::get<2>(back);
+			auto &parent_idx = std::get<3>(back);
+			auto &idx_in_parent = std::get<4>(back);
+
+			// Already visited?
+			if (visited) {
+				if (pending.size() == 1) {
+					col_desc = desc;
+					break;
+				}
+				std::get<2>(pending[parent_idx]).Get("children").As<Napi::Array>().Set(idx_in_parent, desc);
+				pending.pop_back();
+				continue;
+			}
+			visited = true;
+			auto current_idx = pending.size() - 1;
+
+			// Store types
+			auto &vec_type = vec->GetType();
+			if (with_types) {
+				desc.Set("sqlType", vec_type.ToString());
+				desc.Set("physicalType", TypeIdToString(vec_type.InternalType()));
+			}
+
+			// Create validity vector
+			if (with_data) {
+				vec->Flatten(chunk.size());
+				auto &validity = duckdb::FlatVector::Validity(*vec);
+				auto validity_buffer = Napi::Uint8Array::New(env, chunk.size());
+				for (idx_t row_idx = 0; row_idx < chunk.size(); row_idx++) {
+					validity_buffer[row_idx] = validity.RowIsValid(row_idx);
+				}
+				desc.Set("validity", validity_buffer);
+			}
+
+			// Create data buffer
+			switch (vec_type.id()) {
+			case duckdb::LogicalTypeId::TINYINT: {
+				if (with_data) {
+					auto array = Napi::Int8Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<int8_t>(*vec);
+					for (size_t i = 0; i < chunk.size(); ++i) {
+						array[i] = data[i];
+					}
+					desc.Set("data", array);
+				}
+				break;
+			}
+			case duckdb::LogicalTypeId::SMALLINT: {
+				if (with_data) {
+					auto array = Napi::Int16Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<int16_t>(*vec);
+					for (size_t i = 0; i < chunk.size(); ++i) {
+						array[i] = data[i];
+					}
+					desc.Set("data", array);
+				}
+				break;
+			}
+			case duckdb::LogicalTypeId::INTEGER: {
+				if (with_data) {
+					auto array = Napi::Int32Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<int32_t>(*vec);
+					for (size_t i = 0; i < chunk.size(); ++i) {
+						array[i] = data[i];
+					}
+					desc.Set("data", array);
+				}
+				break;
+			}
+			case duckdb::LogicalTypeId::DOUBLE: {
+				if (with_data) {
+					auto array = Napi::Float64Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<double>(*vec);
+					for (size_t i = 0; i < chunk.size(); ++i) {
+						array[i] = data[i];
+					}
+					desc.Set("data", array);
+				}
+				break;
+			}
+			case duckdb::LogicalTypeId::BIGINT:
+			case duckdb::LogicalTypeId::TIME:
+			case duckdb::LogicalTypeId::TIME_TZ:
+			case duckdb::LogicalTypeId::TIMESTAMP_MS:
+			case duckdb::LogicalTypeId::TIMESTAMP_NS:
+			case duckdb::LogicalTypeId::TIMESTAMP_SEC:
+			case duckdb::LogicalTypeId::TIMESTAMP: {
+				if (with_data) {
+#if NAPI_VERSION > 5
+					auto array = Napi::BigInt64Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<int64_t>(*vec);
+#else
+					auto array = Napi::Float64Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<int64_t>(*vec);
+#endif
+					for (size_t i = 0; i < chunk.size(); ++i) {
+						array[i] = data[i];
+					}
+					desc.Set("data", array);
+				}
+				break;
+			}
+			case duckdb::LogicalTypeId::UBIGINT: {
+				if (with_data) {
+#if NAPI_VERSION > 5
+					auto array = Napi::BigUint64Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<uint64_t>(*vec);
+#else
+					auto array = Napi::Float64Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<int64_t>(*vec);
+#endif
+					for (size_t i = 0; i < chunk.size(); ++i) {
+						array[i] = data[i];
+					}
+					desc.Set("data", array);
+				}
+				break;
+			}
+			case duckdb::LogicalTypeId::BLOB:
+			case duckdb::LogicalTypeId::VARCHAR: {
+				if (with_data) {
+					auto array = Napi::Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<duckdb::string_t>(*vec);
+					for (size_t i = 0; i < chunk.size(); ++i) {
+						array.Set(i, data[i].GetString());
+					}
+					desc.Set("data", array);
+				}
+				break;
+			}
+			case duckdb::LogicalTypeId::STRUCT: {
+				auto child_count = duckdb::StructType::GetChildCount(vec_type);
+				auto &entries = duckdb::StructVector::GetEntries(*vec);
+				desc.Set("children", Napi::Array::New(env, child_count));
+				for (size_t i = 0; i < child_count; ++i) {
+					auto c = child_count - 1 - i;
+					auto &entry = entries[c];
+					auto desc = Napi::Object::New(env);
+					auto name = duckdb::StructType::GetChildName(vec_type, c);
+					desc.Set("name", name);
+					pending.emplace_back(false, entry.get(), desc, current_idx, i);
+				}
+				break;
+			}
+			default:
+				Napi::TypeError::New(env, "Unsupported UDF argument type " + vec->GetType().ToString())
+				    .ThrowAsJavaScriptException();
+				break;
+			}
+		}
+		col_descs.Set(col_idx, col_desc);
+	}
+	return col_descs;
+}
+
+} // namespace node_duckdb

package/src/database.cpp

@@ -1,4 +1,5 @@
 #include "duckdb_node.hpp"
+#include "napi.h"
 #include "parquet-amalgamation.hpp"
 
 namespace node_duckdb {
@@ -22,17 +23,39 @@ Napi::Object Database::Init(Napi::Env env, Napi::Object exports) {
 }
 
 struct OpenTask : public Task {
-	OpenTask(Database &database_, std::string filename_, bool read_only_, Napi::Function callback_)
-	    : Task(database_, callback_), filename(filename_), read_only(read_only_) {
+	OpenTask(Database &database_, std::string filename_, duckdb::AccessMode access_mode_, Napi::Object config_,
+	         Napi::Function callback_)
+	    : Task(database_, callback_), filename(filename_) {
+
+		duckdb_config.options.access_mode = access_mode_;
+		Napi::Env env = database_.Env();
+		Napi::HandleScope scope(env);
+
+		if (!config_.IsUndefined()) {
+			const Napi::Array config_names = config_.GetPropertyNames();
+
+			for (duckdb::idx_t config_idx = 0; config_idx < config_names.Length(); config_idx++) {
+				std::string key = config_names.Get(config_idx).As<Napi::String>();
+				std::string val = config_.Get(key).As<Napi::String>();
+				auto config_property = duckdb::DBConfig::GetOptionByName(key);
+				if (!config_property) {
+					Napi::TypeError::New(env, "Unrecognized configuration property" + key).ThrowAsJavaScriptException();
+					return;
+				}
+				try {
+					duckdb_config.SetOption(*config_property, duckdb::Value(val));
+				} catch (std::exception &e) {
+					Napi::TypeError::New(env, "Failed to set configuration option " + key + ": " + e.what())
+					    .ThrowAsJavaScriptException();
+					return;
+				}
+			}
+		}
 	}
 
 	void DoWork() override {
 		try {
-			duckdb::DBConfig config;
-			if (read_only) {
-				config.access_mode = duckdb::AccessMode::READ_ONLY;
-			}
-			Get<Database>().database = duckdb::make_unique<duckdb::DuckDB>(filename, &config);
+			Get<Database>().database = duckdb::make_unique<duckdb::DuckDB>(filename, &duckdb_config);
 			duckdb::ParquetExtension extension;
 			extension.Load(*Get<Database>().database);
 			success = true;
@@ -59,22 +82,35 @@ struct OpenTask : public Task {
 	}
 
 	std::string filename;
-	bool read_only = false;
+	duckdb::DBConfig duckdb_config;
 	std::string error = "";
 	bool success = false;
 };
 
-Database::Database(const Napi::CallbackInfo &info) : Napi::ObjectWrap<Database>(info), task_inflight(false) {
+Database::Database(const Napi::CallbackInfo &info)
+    : Napi::ObjectWrap<Database>(info), task_inflight(false), env(info.Env()) {
 	auto env = info.Env();
+
 	if (info.Length() < 1 || !info[0].IsString()) {
 		Napi::TypeError::New(env, "Database location expected").ThrowAsJavaScriptException();
 		return;
 	}
 	std::string filename = info[0].As<Napi::String>();
 	unsigned int pos = 1;
+
+	duckdb::AccessMode access_mode = duckdb::AccessMode::AUTOMATIC;
+
 	int mode = 0;
 	if (info.Length() >= pos && info[pos].IsNumber() && Utils::OtherIsInt(info[pos].As<Napi::Number>())) {
 		mode = info[pos++].As<Napi::Number>().Int32Value();
+		if (mode == DUCKDB_NODEJS_READONLY) {
+			access_mode = duckdb::AccessMode::READ_ONLY;
+		}
+	}
+
+	Napi::Object config;
+	if (info.Length() >= pos && info[pos].IsObject() && !info[pos].IsFunction()) {
+		config = info[pos++].As<Napi::Object>();
 	}
 
 	Napi::Function callback;
@@ -82,7 +118,11 @@ Database::Database(const Napi::CallbackInfo &info) : Napi::ObjectWrap<Database>(
 		callback = info[pos++].As<Napi::Function>();
 	}
 
-	Schedule(env, duckdb::make_unique<OpenTask>(*this, filename, mode == DUCKDB_NODEJS_READONLY, callback));
+	Schedule(env, duckdb::make_unique<OpenTask>(*this, filename, access_mode, config, callback));
+}
+
+Database::~Database() {
+	Napi::MemoryManagement::AdjustExternalMemory(env, -bytes_allocated);
 }
 
 void Database::Schedule(Napi::Env env, std::unique_ptr<Task> task) {
@@ -93,17 +133,15 @@ void Database::Schedule(Napi::Env env, std::unique_ptr<Task> task) {
 	Process(env);
 }
 
-static void task_execute(napi_env e, void *data) {
+static void TaskExecuteCallback(napi_env e, void *data) {
 	auto holder = (TaskHolder *)data;
 	holder->task->DoWork();
 }
 
-static void task_complete(napi_env e, napi_status status, void *data) {
+static void TaskCompleteCallback(napi_env e, napi_status status, void *data) {
 	std::unique_ptr<TaskHolder> holder((TaskHolder *)data);
 	holder->db->TaskComplete(e);
-	if (holder->task->callback.Value().IsFunction()) {
-		holder->task->Callback();
-	}
+	holder->task->DoCallback();
 }
 
 void Database::TaskComplete(Napi::Env env) {
@@ -112,6 +150,16 @@ void Database::TaskComplete(Napi::Env env) {
 		task_inflight = false;
 	}
 	Process(env);
+
+	if (database) {
+		// Bookkeeping: tell node (and the node GC in particular) how much
+		// memory we're using, such that it can make better decisions on when to
+		// trigger collections.
+		auto &buffer_manager = duckdb::BufferManager::GetBufferManager(*database->instance);
+		auto current_bytes = buffer_manager.GetUsedMemory();
+		Napi::MemoryManagement::AdjustExternalMemory(env, current_bytes - bytes_allocated);
+		bytes_allocated = current_bytes;
+	}
 }
 
 void Database::Process(Napi::Env env) {
@@ -131,8 +179,8 @@ void Database::Process(Napi::Env env) {
 	holder->task = move(task);
 	holder->db = this;
 
-	napi_create_async_work(env, NULL, Napi::String::New(env, "duckdb.Database.Task"), task_execute, task_complete,
-	                       holder, &holder->request);
+	napi_create_async_work(env, nullptr, Napi::String::New(env, "duckdb.Database.Task"), TaskExecuteCallback,
+	                       TaskCompleteCallback, holder, &holder->request);
 
 	napi_queue_async_work(env, holder->request);
 }
@@ -157,7 +205,7 @@ Napi::Value Database::Serialize(const Napi::CallbackInfo &info) {
 }
 
 struct WaitTask : public Task {
-	WaitTask(Database &database_, Napi::Function callback_) : Task(database_, callback_) {
+	WaitTask(Database &database, Napi::Function callback) : Task(database, callback) {
 	}
 
 	void DoWork() override {
@@ -171,7 +219,7 @@ Napi::Value Database::Wait(const Napi::CallbackInfo &info) {
 }
 
 struct CloseTask : public Task {
-	CloseTask(Database &database_, Napi::Function callback_) : Task(database_, callback_) {
+	CloseTask(Database &database, Napi::Function callback) : Task(database, callback) {
 	}
 
 	void DoWork() override {