duckdb 0.8.2-dev4142.0 → 0.8.2-dev4314.0

package/src/duckdb_node.hpp

@@ -11,6 +11,20 @@
 
 using duckdb::vector;
 
+class NodeDuckDB : public Napi::Addon<NodeDuckDB> {
+public:
+	NodeDuckDB(Napi::Env env, Napi::Object exports);
+
+	static NodeDuckDB *GetData(Napi::Env env) {
+		return env.GetInstanceData<NodeDuckDB>();
+	}
+
+	Napi::FunctionReference database_constructor;
+	Napi::FunctionReference connection_constructor;
+	Napi::FunctionReference statement_constructor;
+	Napi::FunctionReference query_result_constructor;
+};
+
 namespace node_duckdb {
 
 struct Task {
@@ -70,7 +84,7 @@ class Database : public Napi::ObjectWrap<Database> {
 public:
 	explicit Database(const Napi::CallbackInfo &info);
 	~Database() override;
-	static Napi::Object Init(Napi::Env env, Napi::Object exports);
+	static Napi::FunctionReference Init(Napi::Env env, Napi::Object exports);
 	void Process(Napi::Env env);
 	void TaskComplete(Napi::Env env);
 
@@ -82,6 +96,7 @@ public:
 		if (!val.IsObject()) {
 			return false;
 		}
+		auto &constructor = NodeDuckDB::GetData(env)->database_constructor;
 		Napi::Object obj = val.As<Napi::Object>();
 		return obj.InstanceOf(constructor.Value());
 	}
@@ -105,7 +120,6 @@ private:
 	std::queue<duckdb::unique_ptr<Task>> task_queue;
 	std::mutex task_mutex;
 	bool task_inflight;
-	static Napi::FunctionReference constructor;
 	Napi::Env env;
 	int64_t bytes_allocated = 0;
 	int replacement_scan_count = 0;
@@ -120,7 +134,8 @@ class Connection : public Napi::ObjectWrap<Connection> {
 public:
 	explicit Connection(const Napi::CallbackInfo &info);
 	~Connection() override;
-	static Napi::Object Init(Napi::Env env, Napi::Object exports);
+	static Napi::FunctionReference Init(Napi::Env env, Napi::Object exports);
+	static Napi::Object NewInstance(const Napi::Value &db);
 
 public:
 	Napi::Value Prepare(const Napi::CallbackInfo &info);
@@ -136,12 +151,12 @@ public:
 		if (!val.IsObject()) {
 			return false;
 		}
+		auto &constructor = NodeDuckDB::GetData(env)->connection_constructor;
 		Napi::Object obj = val.As<Napi::Object>();
 		return obj.InstanceOf(constructor.Value());
 	}
 
 public:
-	static Napi::FunctionReference constructor;
 	duckdb::unique_ptr<duckdb::Connection> connection;
 	Database *database_ref;
 	std::unordered_map<std::string, duckdb_node_udf_function_t> udfs;
@@ -154,12 +169,13 @@ class Statement : public Napi::ObjectWrap<Statement> {
 public:
 	explicit Statement(const Napi::CallbackInfo &info);
 	~Statement() override;
-	static Napi::Object Init(Napi::Env env, Napi::Object exports);
+	static Napi::FunctionReference Init(Napi::Env env, Napi::Object exports);
 	void SetProcessFirstParam() {
 		ignore_first_param = false;
 	}
 
 public:
+	static Napi::Object NewInstance(Napi::Env env, const vector<napi_value> &args);
 	Napi::Value All(const Napi::CallbackInfo &info);
 	Napi::Value ArrowIPCAll(const Napi::CallbackInfo &info);
 	Napi::Value Each(const Napi::CallbackInfo &info);
@@ -169,7 +185,6 @@ public:
 	Napi::Value Columns(const Napi::CallbackInfo &info);
 
 public:
-	static Napi::FunctionReference constructor;
 	duckdb::unique_ptr<duckdb::PreparedStatement> statement;
 	Connection *connection_ref;
 	bool ignore_first_param = true;
@@ -183,11 +198,11 @@ class QueryResult : public Napi::ObjectWrap<QueryResult> {
 public:
 	explicit QueryResult(const Napi::CallbackInfo &info);
 	~QueryResult() override;
-	static Napi::Object Init(Napi::Env env, Napi::Object exports);
+	static Napi::FunctionReference Init(Napi::Env env, Napi::Object exports);
+	static Napi::Object NewInstance(const Napi::Object &db);
 	duckdb::unique_ptr<duckdb::QueryResult> result;
 
 public:
-	static Napi::FunctionReference constructor;
 	Napi::Value NextChunk(const Napi::CallbackInfo &info);
 	Napi::Value NextIpcBuffer(const Napi::CallbackInfo &info);
 	duckdb::shared_ptr<ArrowSchema> cschema;
@@ -208,11 +223,6 @@ public:
 	static Napi::Object CreateError(Napi::Env env, std::string msg);
 	static bool OtherIsInt(Napi::Number source);
 
-	template <class T>
-	static T *NewUnwrap(vector<napi_value> args) {
-		auto obj = T::constructor.New(args);
-		return Napi::ObjectWrap<T>::Unwrap(obj);
-	}
 	static duckdb::Value BindParameter(const Napi::Value source);
 };
 

package/src/statement.cpp

@@ -17,9 +17,7 @@ using duckdb::vector;
 
 namespace node_duckdb {
 
-Napi::FunctionReference Statement::constructor;
-
-Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) {
+Napi::FunctionReference Statement::Init(Napi::Env env, Napi::Object exports) {
 	Napi::HandleScope scope(env);
 
 	Napi::Function t =
@@ -29,11 +27,9 @@ Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) {
 	                 InstanceMethod("finalize", &Statement::Finish), InstanceMethod("stream", &Statement::Stream),
 	                 InstanceMethod("columns", &Statement::Columns)});
 
-	constructor = Napi::Persistent(t);
-	constructor.SuppressDestruct();
-
 	exports.Set("Statement", t);
-	return exports;
+
+	return Napi::Persistent(t);
 }
 
 static unique_ptr<duckdb::PreparedStatement> PrepareManyInternal(Statement &statement) {
@@ -447,8 +443,8 @@ struct RunQueryTask : public Task {
 			deferred.Reject(Utils::CreateError(env, result->GetErrorObject()));
 		} else {
 			auto db = statement.connection_ref->database_ref->Value();
-			auto query_result = QueryResult::constructor.New({db});
-			auto unwrapped = Napi::ObjectWrap<QueryResult>::Unwrap(query_result);
+			auto query_result = QueryResult::NewInstance(db);
+			auto unwrapped = QueryResult::Unwrap(query_result);
 			unwrapped->result = std::move(result);
 			deferred.Resolve(query_result);
 		}
@@ -624,21 +620,20 @@ Napi::Value Statement::Finish(const Napi::CallbackInfo &info) {
 	connection_ref->database_ref->Schedule(env, duckdb::make_uniq<FinishTask>(*this, callback));
 	return env.Null();
 }
+Napi::Object Statement::NewInstance(Napi::Env env, const vector<napi_value> &args) {
+	return NodeDuckDB::GetData(env)->statement_constructor.New(args);
+}
 
-Napi::FunctionReference QueryResult::constructor;
-
-Napi::Object QueryResult::Init(Napi::Env env, Napi::Object exports) {
+Napi::FunctionReference QueryResult::Init(Napi::Env env, Napi::Object exports) {
 	Napi::HandleScope scope(env);
 
 	Napi::Function t = DefineClass(env, "QueryResult",
 	                               {InstanceMethod("nextChunk", &QueryResult::NextChunk),
 	                                InstanceMethod("nextIpcBuffer", &QueryResult::NextIpcBuffer)});
 
-	constructor = Napi::Persistent(t);
-	constructor.SuppressDestruct();
-
 	exports.Set("QueryResult", t);
-	return exports;
+
+	return Napi::Persistent(t);
 }
 
 QueryResult::QueryResult(const Napi::CallbackInfo &info) : Napi::ObjectWrap<QueryResult>(info) {
@@ -740,4 +735,8 @@ Napi::Value QueryResult::NextIpcBuffer(const Napi::CallbackInfo &info) {
 	return deferred.Promise();
 }
 
+Napi::Object QueryResult::NewInstance(const Napi::Object &db) {
+	return NodeDuckDB::GetData(db.Env())->query_result_constructor.New({db});
+}
+
 } // namespace node_duckdb

package/test/database_fail.test.ts

@@ -1,6 +1,7 @@
 import * as sqlite3 from '..';
 import * as assert from 'assert';
 import {DuckDbError, RowData} from "..";
+import {Worker} from 'worker_threads';
 
 describe('error handling', function() {
     var db: sqlite3.Database;
@@ -143,4 +144,23 @@ describe('error handling', function() {
             }
         });
     });
+
+    it('should not error when multiple instances are started in one process', async () => {
+      async function run_worker() {
+        return new Promise((resolve, reject) => {
+          const worker = new Worker(__dirname + '/worker.js', { workerData: 'test' });
+
+          worker.on('message', resolve);
+          worker.on('error', reject);
+          worker.on('exit', (code: number) => {
+            if (code !== 0) {
+              console.log(new Error(`Worker stopped with exit code ${code}`));
+            }
+          })
+        });
+      }
+
+      await run_worker(); // first should always succeed
+      await run_worker(); // second fails without thread safety
+    })
 });

package/test/worker.js

@@ -0,0 +1,7 @@
+const { parentPort, workerData } = require('worker_threads');
+
+const duckdb = require('..'); // The presence of this line alone causes the issue
+
+new duckdb.Database(":memory:").all("SELECT 42 AS answer", function (err, res) {
+    parentPort.postMessage(JSON.stringify({workerData, pid: process.pid, err, res}));
+});

package/src/duckdb/src/common/field_writer.cpp

@@ -1,97 +0,0 @@
-#include "duckdb/common/field_writer.hpp"
-
-namespace duckdb {
-
-//===--------------------------------------------------------------------===//
-// Field Writer
-//===--------------------------------------------------------------------===//
-FieldWriter::FieldWriter(Serializer &serializer_p)
-    : serializer(serializer_p), buffer(make_uniq<BufferedSerializer>()), field_count(0), finalized(false) {
-	buffer->SetVersion(serializer.GetVersion());
-}
-
-FieldWriter::~FieldWriter() {
-	if (Exception::UncaughtException()) {
-		return;
-	}
-	D_ASSERT(finalized);
-	// finalize should always have been called, unless this is destroyed as part of stack unwinding
-	D_ASSERT(!buffer);
-}
-
-void FieldWriter::WriteData(const_data_ptr_t buffer_ptr, idx_t write_size) {
-	D_ASSERT(buffer);
-	buffer->WriteData(buffer_ptr, write_size);
-}
-
-template <>
-void FieldWriter::Write(const string &val) {
-	Write<uint32_t>((uint32_t)val.size());
-	if (!val.empty()) {
-		WriteData(const_data_ptr_cast(val.c_str()), val.size());
-	}
-}
-
-void FieldWriter::Finalize() {
-	D_ASSERT(buffer);
-	D_ASSERT(!finalized);
-	finalized = true;
-	serializer.Write<uint32_t>(field_count);
-	serializer.Write<uint64_t>(buffer->blob.size);
-	serializer.WriteData(buffer->blob.data.get(), buffer->blob.size);
-
-	buffer.reset();
-}
-
-//===--------------------------------------------------------------------===//
-// Field Deserializer
-//===--------------------------------------------------------------------===//
-FieldDeserializer::FieldDeserializer(Deserializer &root) : root(root), remaining_data(idx_t(-1)) {
-	SetVersion(root.GetVersion());
-}
-
-void FieldDeserializer::ReadData(data_ptr_t buffer, idx_t read_size) {
-	D_ASSERT(remaining_data != idx_t(-1));
-	D_ASSERT(read_size <= remaining_data);
-	root.ReadData(buffer, read_size);
-	remaining_data -= read_size;
-}
-
-idx_t FieldDeserializer::RemainingData() {
-	return remaining_data;
-}
-
-void FieldDeserializer::SetRemainingData(idx_t remaining_data) {
-	this->remaining_data = remaining_data;
-}
-
-//===--------------------------------------------------------------------===//
-// Field Reader
-//===--------------------------------------------------------------------===//
-FieldReader::FieldReader(Deserializer &source_p) : source(source_p), field_count(0), finalized(false) {
-	max_field_count = source_p.Read<uint32_t>();
-	total_size = source_p.Read<uint64_t>();
-	D_ASSERT(max_field_count > 0);
-	D_ASSERT(total_size > 0);
-	source.SetRemainingData(total_size);
-}
-
-FieldReader::~FieldReader() {
-	if (Exception::UncaughtException()) {
-		return;
-	}
-	D_ASSERT(finalized);
-}
-
-void FieldReader::Finalize() {
-	D_ASSERT(!finalized);
-	finalized = true;
-	if (field_count < max_field_count) {
-		// we can handle this case by calling source.ReadData(buffer, source.RemainingData())
-		throw SerializationException("Not all fields were read. This file might have been written with a newer version "
-		                             "of DuckDB and is incompatible with this version of DuckDB.");
-	}
-	D_ASSERT(source.RemainingData() == 0);
-}
-
-} // namespace duckdb

package/src/duckdb/src/common/serializer/buffered_deserializer.cpp

@@ -1,27 +0,0 @@
-#include "duckdb/common/serializer/buffered_deserializer.hpp"
-
-#include <cstring>
-
-namespace duckdb {
-
-BufferedDeserializer::BufferedDeserializer(data_ptr_t ptr, idx_t data_size) : ptr(ptr), endptr(ptr + data_size) {
-}
-
-BufferedDeserializer::BufferedDeserializer(BufferedSerializer &serializer)
-    : BufferedDeserializer(serializer.data, serializer.maximum_size) {
-	SetVersion(serializer.GetVersion());
-}
-
-void BufferedDeserializer::ReadData(data_ptr_t buffer, idx_t read_size) {
-	if (ptr + read_size > endptr) {
-		throw SerializationException("Failed to deserialize: not enough data in buffer to fulfill read request");
-	}
-	memcpy(buffer, ptr, read_size);
-	ptr += read_size;
-}
-
-ClientContext &BufferedContextDeserializer::GetContext() {
-	return context;
-}
-
-} // namespace duckdb

package/src/duckdb/src/common/serializer/buffered_serializer.cpp

@@ -1,36 +0,0 @@
-#include "duckdb/common/serializer/buffered_serializer.hpp"
-
-#include <cstring>
-
-namespace duckdb {
-
-BufferedSerializer::BufferedSerializer(idx_t maximum_size)
-    : BufferedSerializer(make_unsafe_uniq_array<data_t>(maximum_size), maximum_size) {
-}
-
-BufferedSerializer::BufferedSerializer(unsafe_unique_array<data_t> data, idx_t size)
-    : maximum_size(size), data(data.get()) {
-	blob.size = 0;
-	blob.data = std::move(data);
-}
-
-BufferedSerializer::BufferedSerializer(data_ptr_t data, idx_t size) : maximum_size(size), data(data) {
-	blob.size = 0;
-}
-
-void BufferedSerializer::WriteData(const_data_ptr_t buffer, idx_t write_size) {
-	if (blob.size + write_size >= maximum_size) {
-		do {
-			maximum_size *= 2;
-		} while (blob.size + write_size > maximum_size);
-		auto new_data = new data_t[maximum_size];
-		memcpy(new_data, data, blob.size);
-		data = new_data;
-		blob.data = unsafe_unique_array<data_t>(new_data);
-	}
-
-	memcpy(data + blob.size, buffer, write_size);
-	blob.size += write_size;
-}
-
-} // namespace duckdb

package/src/duckdb/src/common/serializer.cpp

@@ -1,24 +0,0 @@
-#include "duckdb/common/serializer.hpp"
-
-namespace duckdb {
-
-template <>
-string Deserializer::Read() {
-	uint32_t size = Read<uint32_t>();
-	if (size == 0) {
-		return string();
-	}
-	auto buffer = make_unsafe_uniq_array<data_t>(size);
-	ReadData(buffer.get(), size);
-	return string(const_char_ptr_cast(buffer.get()), size);
-}
-
-void Deserializer::ReadStringVector(vector<string> &list) {
-	uint32_t sz = Read<uint32_t>();
-	list.resize(sz);
-	for (idx_t i = 0; i < sz; i++) {
-		list[i] = Read<string>();
-	}
-}
-
-} // namespace duckdb