duckdb 0.4.1-dev9.0 → 0.4.1-dev953.0

package/Makefile

@@ -7,10 +7,10 @@ src/duckdb.cpp:
 	npm install --build-from-source
 
 build: ./node_modules src/duckdb.cpp
-	./node_modules/.bin/node-pre-gyp build --loglevel=silent
+	./node_modules/.bin/node-pre-gyp build -j max --loglevel=silent
 
 debug: ./node_modules src/duckdb.cpp
-	./node_modules/.bin/node-pre-gyp build --debug --verbose
+	./node_modules/.bin/node-pre-gyp build -j max --debug --verbose
 
 clean:
 	@rm -rf ./build

package/binding.gyp

@@ -5,6 +5,7 @@
             "sources": [
                 "src/duckdb_node.cpp",
                 "src/database.cpp",
+                "src/data_chunk.cpp",
                 "src/connection.cpp",
                 "src/statement.cpp",
                 "src/utils.cpp",
@@ -20,18 +21,20 @@
             ],
             "cflags_cc": [
                 "-frtti",
-                "-fexceptions"
+                "-fexceptions",
+                "-Wno-redundant-move",
             ],
             "cflags_cc!": [
-                "-fno-rrti"
+                "-fno-rrti",
                 "-fno-exceptions",
             ],
             "cflags": [
                 "-frtti",
-                "-fexceptions"
+                "-fexceptions",
+                "-Wno-redundant-move",
             ],
             "cflags!": [
-                "-fno-rrti"
+                "-fno-rrti",
                 "-fno-exceptions",
             ],
             "xcode_settings": {
@@ -40,7 +43,7 @@
                 "CLANG_CXX_LIBRARY": "libc++",
                 "MACOSX_DEPLOYMENT_TARGET": "10.15",
                 'CLANG_CXX_LANGUAGE_STANDARD':'c++11',
-                'OTHER_CFLAGS' : ['-fexceptions', '-frtti']
+                'OTHER_CFLAGS' : ['-fexceptions', '-frtti', '-Wno-redundant-move']
 
             },
             "msvs_settings": {

package/lib/duckdb.js

@@ -23,83 +23,111 @@ Connection.prototype.each = function(sql) {
     return statement.each.apply(statement, arguments);
 }
 
-function ptr_to_arr(buffer, ptype, n) {
-    // TODO can we create those on the C++ side of things already?
-    switch(ptype) {
-        case 'BOOL':
-        case 'UINT8':
-            return new Uint8Array(buffer.buffer, 0, n);
-        case 'INT8':
-            return new Int8Array(buffer.buffer, 0, n);
-        case 'INT16':
-            return new Int16Array(buffer.buffer, 0, n);
-        case 'UINT16':
-            return new UInt16Array(buffer.buffer, 0, n);
-        case 'INT32':
-            return new Int32Array(buffer.buffer, 0, n);
-        case 'UINT32':
-            return new UInt32Array(buffer.buffer, 0, n);
-        case 'FLOAT':
-            return new Float32Array(buffer.buffer, 0, n);
-        case 'DOUBLE':
-            return new Float64Array(buffer.buffer, 0, n);
-        case 'VARCHAR':  // we already have created a string array on the C++ side for this
-            return buffer;
-        default:
-            return new Array<string>(0); // cough
-    }
-}
-
 // this follows the wasm udfs somewhat but is simpler because we can pass data much more cleanly
 Connection.prototype.register = function(name, return_type, fun) {
     // TODO what if this throws an error somewhere? do we need a try/catch?
-    return this.register_bulk(name, return_type, function(descr) {
+    return this.register_bulk(name, return_type, function(desc) {
         try {
-            const data_arr = [];
-            const validity_arr = [];
+            // Build an argument resolver
+            const buildResolver = (arg) => {
+                let validity = arg.validity || null;
+                switch (arg.physicalType) {
+                    case 'STRUCT': {
+                        const tmp = {};
+                        const children = [];
+                        for (let j = 0; j < (arg.children.length || 0); ++j) {
+                            const attr = arg.children[j];
+                            const child = buildResolver(attr);
+                            children.push((row) => {
+                                tmp[attr.name] = child(row);
+                            });
+                        }
+                        if (validity != null) {
+                            return (row) => {
+                                if (!validity[row]) {
+                                    return null;
+                                }
+                                for (const resolver of children) {
+                                    resolver(row);
+                                }
+                                return tmp;
+                            };
+                        } else {
+                            return (row) => {
+                                for (const resolver of children) {
+                                    resolver(row);
+                                }
+                                return tmp;
+                            };
+                        }
+                    }
+                    default: {
+                        if (arg.data === undefined) {
+                            throw new Error(
+                                'malformed data view, expected data buffer for argument of type: ' + arg.physicalType,
+                            );
+                        }
+                        const data = arg.data;
+                        if (validity != null) {
+                            return (row) => (!validity[row] ? null : data[row]);
+                        } else {
+                            return (row) => data[row];
+                        }
+                    }
+                }
+            };
 
-            for (const idx in descr.args) {
-                const arg = descr.args[idx];
-                validity_arr.push(arg.data_buffers[arg.validity_buffer]);
-                data_arr.push(ptr_to_arr(arg.data_buffers[arg.data_buffer], arg.physical_type, descr.rows));
+            // Translate argument data
+            const argResolvers = [];
+            for (let i = 0; i < desc.args.length; ++i) {
+                argResolvers.push(buildResolver(desc.args[i]));
+            }
+            const args = [];
+            for (let i = 0; i < desc.args.length; ++i) {
+                args.push(null);
             }
 
-            const out_data = ptr_to_arr(descr.ret.data_buffers[descr.ret.data_buffer], descr.ret.physical_type, descr.rows);
-            const out_validity = descr.ret.data_buffers[descr.ret.validity_buffer];
-
-            switch (descr.args.length) {
-                case 0:
-                    for (let i = 0; i < descr.rows; ++i) {
-                        const res = fun();
-                        out_data[i] = res;
-                        out_validity[i] = res == undefined || res == null ? 0 : 1;
-                    }
+            // Return type
+            desc.ret.validity = new Uint8Array(desc.rows);
+            switch (desc.ret.physicalType) {
+                case 'INT8':
+                    desc.ret.data = new Int8Array(desc.rows);
                     break;
-                case 1:
-                    for (let i = 0; i < descr.rows; ++i) {
-                        const res = fun(validity_arr[0][i] ? data_arr[0][i] : undefined);
-                        out_data[i] = res;
-                        out_validity[i] = res == undefined || res == null ? 0 : 1;
-                    }
+                case 'INT16':
+                    desc.ret.data = new Int16Array(desc.rows);
                     break;
-                case 2:
-                    for (let i = 0; i < descr.rows; ++i) {
-                        const res = fun(validity_arr[0][i] ? data_arr[0][i] : undefined, validity_arr[1][i] ? data_arr[1][i] : undefined);
-                        out_data[i] = res;
-                        out_validity[i] = res == undefined || res == null ? 0 : 1;
-                    }
+                case 'INT32':
+                    desc.ret.data = new Int32Array(desc.rows);
                     break;
-                case 3:
-                    for (let i = 0; i < descr.rows; ++i) {
-                        const res = fun(validity_arr[0][i] ? data_arr[0][i] : undefined, validity_arr[1][i] ? data_arr[1][i] : undefined, validity_arr[2][i] ? data_arr[2][i] : undefined);
-                        out_data[i] = res;
-                        out_validity[i] = res == undefined || res == null ? 0 : 1;
-                    }
+                case 'DOUBLE':
+                    desc.ret.data = new Float64Array(desc.rows);
+                    break;
+                case 'DATE64':
+                case 'TIME64':
+                case 'TIMESTAMP':
+                case 'INT64':
+                    desc.ret.data = new BigInt64Array(desc.rows);
                     break;
-                default:
-                    throw "Unsupported argument count";
+                case 'UINT64':
+                    desc.ret.data = new BigUint64Array(desc.rows);
+                    break;
+                case 'BLOB':
+                case 'VARCHAR':
+                    desc.ret.data = new Array(desc.rows);
+                    break;
+            }
+
+            // Call the function
+            for (let i = 0; i < desc.rows; ++i) {
+                for (let j = 0; j < desc.args.length; ++j) {
+                    args[j] = argResolvers[j](i);
+                }
+                const res = fun(...args);
+                desc.ret.data[i] = res;
+                desc.ret.validity[i] = res === undefined || res === null ? 0 : 1;
             }
         } catch(error) { // work around recently fixed napi bug https://github.com/nodejs/node-addon-api/issues/912
+            console.log(desc.ret);
             msg = error;
             if (typeof error == 'object' && 'message' in error) {
                 msg = error.message

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev9.0",
+  "version": "0.4.1-dev953.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/connection.cpp

@@ -1,5 +1,8 @@
+#include "duckdb.hpp"
 #include "duckdb_node.hpp"
+#include "napi.h"
 
+#include <iostream>
 #include <thread>
 
 namespace node_duckdb {
@@ -23,7 +26,7 @@ Napi::Object Connection::Init(Napi::Env env, Napi::Object exports) {
 }
 
 struct ConnectTask : public Task {
-	ConnectTask(Connection &connection_, Napi::Function callback_) : Task(connection_, callback_) {
+	ConnectTask(Connection &connection, Napi::Function callback) : Task(connection, callback) {
 	}
 
 	void DoWork() override {
@@ -78,91 +81,25 @@ struct JSArgs {
 	std::string error;
 };
 
-static Napi::Value transform_vector(Napi::Env env, duckdb::Vector &vec, duckdb::idx_t rows, bool copy) {
-	Napi::EscapableHandleScope scope(env);
-
-	Napi::Array data_buffers(Napi::Array::New(env));
-
-	auto data_buffer_index = -1;
-	auto length_buffer_index = -1;
-	auto validity_buffer_index = -1;
-
-	auto validity_buffer = Napi::Buffer<uint8_t>::New(env, rows);
-	auto validity = duckdb::FlatVector::Validity(vec);
-	auto validity_ptr = validity_buffer.Data();
-	for (duckdb::idx_t row_idx = 0; row_idx < rows; row_idx++) {
-		validity_ptr[row_idx] = validity.RowIsValid(row_idx);
-	}
-	data_buffers.Set(data_buffers.Length(), validity_buffer);
-	validity_buffer_index = data_buffers.Length() - 1;
-
-	auto &vec_type = vec.GetType();
-
-	switch (vec_type.id()) {
-	case duckdb::LogicalTypeId::BOOLEAN:
-	case duckdb::LogicalTypeId::UTINYINT:
-	case duckdb::LogicalTypeId::TINYINT:
-	case duckdb::LogicalTypeId::USMALLINT:
-	case duckdb::LogicalTypeId::SMALLINT:
-	case duckdb::LogicalTypeId::INTEGER:
-	case duckdb::LogicalTypeId::UINTEGER:
-	case duckdb::LogicalTypeId::FLOAT:
-	case duckdb::LogicalTypeId::DOUBLE: {
-		auto data_buffer = Napi::Buffer<uint8_t>::New(env, rows * duckdb::GetTypeIdSize(vec_type.InternalType()));
-		// TODO this is technically not neccessary but it fixes a crash in Node 14. Some weird data ownership issue.
-		memcpy(data_buffer.Data(), duckdb::FlatVector::GetData<uint8_t>(vec),
-		       rows * duckdb::GetTypeIdSize(vec_type.InternalType()));
-
-		data_buffers.Set(data_buffers.Length(), data_buffer);
-		data_buffer_index = data_buffers.Length() - 1;
-		break;
-	}
-	case duckdb::LogicalTypeId::VARCHAR: {
-		Napi::Array string_buffers(Napi::Array::New(env, rows));
-		if (copy) {
-			auto string_vec_ptr = duckdb::FlatVector::GetData<duckdb::string_t>(vec);
-			for (duckdb::idx_t row_idx = 0; row_idx < rows; row_idx++) {
-				string_buffers.Set(row_idx, validity_ptr[row_idx]
-				                                ? Napi::String::New(env, string_vec_ptr[row_idx].GetDataUnsafe(),
-				                                                    string_vec_ptr[row_idx].GetSize())
-				                                : Napi::Value());
-			}
-		}
-		data_buffers.Set(data_buffers.Length(), string_buffers);
-		data_buffer_index = data_buffers.Length() - 1;
-		break;
-	}
-	default:
-		throw duckdb::NotImplementedException(vec_type.ToString());
-	}
-
-	Napi::Object desc(Napi::Object::New(env));
-	desc.Set("logical_type", vec_type.ToString());
-	desc.Set("physical_type", TypeIdToString(vec_type.InternalType()));
-	desc.Set("validity_buffer", validity_buffer_index);
-	desc.Set("data_buffer", data_buffer_index);
-	desc.Set("length_buffer", length_buffer_index);
-	desc.Set("data_buffers", data_buffers);
-
-	return scope.Escape(desc);
-}
-
-void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, nullptr_t *, JSArgs *jsargs) {
+void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, std::nullptr_t *, JSArgs *jsargs) {
 	try { // if we dont catch exceptions here we terminate node if one happens ^^
+		Napi::EscapableHandleScope scope(env);
 
-		// set up descriptor and data arrays
-		Napi::Array args_descr(Napi::Array::New(env, jsargs->args->ColumnCount()));
-		for (duckdb::idx_t col_idx = 0; col_idx < jsargs->args->ColumnCount(); col_idx++) {
-			auto &vec = jsargs->args->data[col_idx];
-			auto arg_descr = transform_vector(env, vec, jsargs->rows, true);
-			args_descr.Set(col_idx, arg_descr);
-		}
-		auto ret_descr = transform_vector(env, *jsargs->result, jsargs->rows, false);
-
-		Napi::Object descr(Napi::Object::New(env));
+		// Set up descriptor and data arrays
+		auto descr = Napi::Object::New(env);
+		auto chunk = EncodeDataChunk(env, *jsargs->args, true, true);
+		descr.Set("args", scope.Escape(chunk));
 		descr.Set("rows", jsargs->rows);
-		descr.Set("args", args_descr);
-		descr.Set("ret", ret_descr);
+		auto ret = Napi::Object::New(env);
+		ret.Set("sqlType", jsargs->result->GetType().ToString());
+		auto ret_type = jsargs->result->GetType().InternalType();
+#if NAPI_VERSION <= 5
+		if (ret_type == duckdb::PhysicalType::INT64 || ret_type == duckdb::PhysicalType::UINT64) {
+			ret_type = duckdb::PhysicalType::DOUBLE;
+		}
+#endif
+		ret.Set("physicalType", TypeIdToString(ret_type));
+		descr.Set("ret", ret);
 
 		// actually call the UDF, or rather its vectorized wrapper from duckdb.js/Connection.prototype.register wrapper
 		jsudf({descr});
@@ -181,39 +118,75 @@ void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, nullptr_t *, JSA
 		}
 
 		// transform the result back to a vector
-		auto return_validity = ret_descr.ToObject()
-		                           .Get("data_buffers")
-		                           .ToObject()
-		                           .Get(ret_descr.ToObject().Get("validity_buffer"))
-		                           .As<Napi::Buffer<uint8_t>>();
+		auto return_validity = ret.ToObject().Get("validity").As<Napi::Uint8Array>();
 		for (duckdb::idx_t row_idx = 0; row_idx < jsargs->rows; row_idx++) {
 			duckdb::FlatVector::SetNull(*jsargs->result, row_idx, !return_validity[row_idx]);
 		}
 
-		if (jsargs->result->GetType().id() == duckdb::LogicalTypeId::VARCHAR) {
-			auto return_string_array = ret_descr.ToObject()
-			                               .Get("data_buffers")
-			                               .ToObject()
-			                               .Get(ret_descr.ToObject().Get("data_buffer"))
-			                               .ToObject();
-			auto return_string_vec_ptr = duckdb::FlatVector::GetData<duckdb::string_t>(*jsargs->result);
-
+		switch (jsargs->result->GetType().id()) {
+		case duckdb::LogicalTypeId::TINYINT: {
+			auto data = ret.Get("data").As<Napi::Int8Array>();
+			auto out = duckdb::FlatVector::GetData<int8_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::SMALLINT: {
+			auto data = ret.Get("data").As<Napi::Int16Array>();
+			auto out = duckdb::FlatVector::GetData<int16_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::INTEGER: {
+			auto data = ret.Get("data").As<Napi::Int32Array>();
+			auto out = duckdb::FlatVector::GetData<int32_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::DOUBLE: {
+			auto data = ret.Get("data").As<Napi::Float64Array>();
+			auto out = duckdb::FlatVector::GetData<double>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::TIME:
+		case duckdb::LogicalTypeId::TIMESTAMP:
+		case duckdb::LogicalTypeId::TIMESTAMP_MS:
+		case duckdb::LogicalTypeId::TIMESTAMP_SEC:
+		case duckdb::LogicalTypeId::TIMESTAMP_NS:
+		case duckdb::LogicalTypeId::BIGINT: {
+#if NAPI_VERSION > 5
+			auto data = ret.Get("data").As<Napi::BigInt64Array>();
+#else
+			auto data = ret.Get("data").As<Napi::Float64Array>();
+#endif
+			auto out = duckdb::FlatVector::GetData<int64_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::UBIGINT: {
+#if NAPI_VERSION > 5
+			auto data = ret.Get("data").As<Napi::BigUint64Array>();
+#else
+			auto data = ret.Get("data").As<Napi::Float64Array>();
+#endif
+			auto out = duckdb::FlatVector::GetData<uint64_t>(*jsargs->result);
+			memcpy(out, data.Data(), jsargs->rows * duckdb::GetTypeIdSize(ret_type));
+			break;
+		}
+		case duckdb::LogicalTypeId::BLOB:
+		case duckdb::LogicalTypeId::VARCHAR: {
+			auto data = ret.Get("data").As<Napi::Array>();
+			auto out = duckdb::FlatVector::GetData<duckdb::string_t>(*jsargs->result);
+			for (size_t i = 0; i < data.Length(); ++i) {
+				out[i] = duckdb::string_t(data.Get(i).ToString());
+			}
+			break;
+		}
+		default: {
 			for (duckdb::idx_t row_idx = 0; row_idx < jsargs->rows; row_idx++) {
-				if (!return_validity[row_idx]) {
-					duckdb::FlatVector::SetNull(*jsargs->result, row_idx, true);
-				} else {
-					auto str = return_string_array.Get(row_idx).As<Napi::String>();
-					return_string_vec_ptr[row_idx] = duckdb::StringVector::AddString(*jsargs->result, str);
-				}
+				duckdb::FlatVector::SetNull(*jsargs->result, row_idx, false);
 			}
-		} else {
-			auto return_data = ret_descr.ToObject()
-			                       .Get("data_buffers")
-			                       .ToObject()
-			                       .Get(ret_descr.ToObject().Get("data_buffer"))
-			                       .As<Napi::Buffer<uint8_t>>();
-			memcpy(duckdb::FlatVector::GetData<uint8_t>(*jsargs->result), return_data.Data(),
-			       jsargs->rows * duckdb::GetTypeIdSize(jsargs->result->GetType().InternalType()));
+		}
 		}
 	} catch (const std::exception &e) {
 		jsargs->error = e.what();
@@ -222,8 +195,8 @@ void DuckDBNodeUDFLauncher(Napi::Env env, Napi::Function jsudf, nullptr_t *, JSA
 }
 
 struct RegisterTask : public Task {
-	RegisterTask(Connection &connection_, std::string name_, std::string return_type_name_, Napi::Function callback_)
-	    : Task(connection_, callback_), name(name_), return_type_name(return_type_name_) {
+	RegisterTask(Connection &connection, std::string name, std::string return_type_name, Napi::Function callback)
+	    : Task(connection, callback), name(std::move(name)), return_type_name(std::move(return_type_name)) {
 	}
 
 	void DoWork() override {
@@ -234,7 +207,7 @@ struct RegisterTask : public Task {
 			// here we can do only DuckDB stuff because we do not have a functioning env
 
 			// Flatten all args to simplify udfs
-			args.Normalify();
+			args.Flatten();
 
 			JSArgs jsargs;
 			jsargs.rows = args.size();
@@ -282,8 +255,8 @@ Napi::Value Connection::Register(const Napi::CallbackInfo &info) {
 		return env.Null();
 	}
 
-	auto udf = DuckDBNodeUDFFUnction::New(env, udf_callback, "duckdb_node_udf" + name, 0, 1, nullptr,
-	                                      [](Napi::Env, void *, nullptr_t *ctx) {});
+	auto udf = duckdb_node_udf_function_t::New(env, udf_callback, "duckdb_node_udf" + name, 0, 1, nullptr,
+	                                           [](Napi::Env, void *, std::nullptr_t *ctx) {});
 
 	// we have to unref the udf because otherwise there is a circular ref with the connection somehow(?)
 	// this took far too long to figure out
@@ -297,8 +270,8 @@ Napi::Value Connection::Register(const Napi::CallbackInfo &info) {
 }
 
 struct UnregisterTask : public Task {
-	UnregisterTask(Connection &connection_, std::string name_, Napi::Function callback_)
-	    : Task(connection_, callback_), name(name_) {
+	UnregisterTask(Connection &connection, std::string name, Napi::Function callback)
+	    : Task(connection, callback), name(std::move(name)) {
 	}
 
 	void DoWork() override {
@@ -340,27 +313,33 @@ Napi::Value Connection::Unregister(const Napi::CallbackInfo &info) {
 }
 
 struct ExecTask : public Task {
-	ExecTask(Connection &connection_, std::string sql_, Napi::Function callback_)
-	    : Task(connection_, callback_), sql(sql_) {
+	ExecTask(Connection &connection, std::string sql, Napi::Function callback)
+	    : Task(connection, callback), sql(std::move(sql)) {
 	}
 
 	void DoWork() override {
 		auto &connection = Get<Connection>();
 
 		success = true;
-		auto statements = connection.connection->ExtractStatements(sql);
-		if (statements.size() == 0) {
-			return;
-		}
+		try {
+			auto statements = connection.connection->ExtractStatements(sql);
+			if (statements.empty()) {
+				return;
+			}
 
-		// thanks Mark
-		for (duckdb::idx_t i = 0; i < statements.size(); i++) {
-			auto res = connection.connection->Query(move(statements[i]));
-			if (!res->success) {
-				success = false;
-				error = res->error;
-				break;
+			// thanks Mark
+			for (duckdb::idx_t i = 0; i < statements.size(); i++) {
+				auto res = connection.connection->Query(move(statements[i]));
+				if (!res->success) {
+					success = false;
+					error = res->error;
+					break;
+				}
 			}
+		} catch (duckdb::ParserException &e) {
+			success = false;
+			error = e.what();
+			return;
 		}
 	}