duckdb 0.4.1-dev98.0 → 0.5.1-dev5.0

package/Makefile

@@ -7,10 +7,10 @@ src/duckdb.cpp:
 	npm install --build-from-source
 
 build: ./node_modules src/duckdb.cpp
-	./node_modules/.bin/node-pre-gyp build --loglevel=silent
+	./node_modules/.bin/node-pre-gyp build -j max --loglevel=silent
 
 debug: ./node_modules src/duckdb.cpp
-	./node_modules/.bin/node-pre-gyp build --debug --verbose
+	./node_modules/.bin/node-pre-gyp build -j max --debug --verbose
 
 clean:
 	@rm -rf ./build

package/binding.gyp

@@ -5,6 +5,7 @@
             "sources": [
                 "src/duckdb_node.cpp",
                 "src/database.cpp",
+                "src/data_chunk.cpp",
                 "src/connection.cpp",
                 "src/statement.cpp",
                 "src/utils.cpp",
@@ -20,18 +21,20 @@
             ],
             "cflags_cc": [
                 "-frtti",
-                "-fexceptions"
+                "-fexceptions",
+                "-Wno-redundant-move",
             ],
             "cflags_cc!": [
-                "-fno-rrti"
+                "-fno-rrti",
                 "-fno-exceptions",
             ],
             "cflags": [
                 "-frtti",
-                "-fexceptions"
+                "-fexceptions",
+                "-Wno-redundant-move",
             ],
             "cflags!": [
-                "-fno-rrti"
+                "-fno-rrti",
                 "-fno-exceptions",
             ],
             "xcode_settings": {
@@ -40,7 +43,7 @@
                 "CLANG_CXX_LIBRARY": "libc++",
                 "MACOSX_DEPLOYMENT_TARGET": "10.15",
                 'CLANG_CXX_LANGUAGE_STANDARD':'c++11',
-                'OTHER_CFLAGS' : ['-fexceptions', '-frtti']
+                'OTHER_CFLAGS' : ['-fexceptions', '-frtti', '-Wno-redundant-move']
 
             },
             "msvs_settings": {

package/lib/duckdb.js

@@ -1,159 +1,502 @@
+/**
+ * @module duckdb
+ * @summary these jsdoc annotations are still a work in progress - feedback and suggestions are welcome!
+ */
+
 var duckdb = require('./duckdb-binding.js');
 module.exports = exports = duckdb;
 
+/**
+ * Check that errno attribute equals this to check for a duckdb error
+ * @constant {number}
+ */
+var ERROR = duckdb.ERROR;
+
+/**
+ * Open database in readonly mode
+ * @constant {number}
+ */
+var OPEN_READONLY = duckdb.OPEN_READONLY;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_READWRITE = duckdb.OPEN_READWRITE;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_CREATE = duckdb.OPEN_CREATE;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_FULLMUTEX = duckdb.OPEN_FULLMUTEX;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_SHAREDCACHE = duckdb.OPEN_SHAREDCACHE;
+/**
+ * Currently ignored
+ * @constant {number}
+ */
+var OPEN_PRIVATECACHE = duckdb.OPEN_PRIVATECACHE;
 
 // some wrappers for compatibilities sake
+/**
+ * Main database interface
+ */
 var Database = duckdb.Database;
+/**
+ * @class
+ */
 var Connection = duckdb.Connection;
+/**
+ * @class
+ */
 var Statement = duckdb.Statement;
+/**
+ * @class
+ */
+var QueryResult = duckdb.QueryResult;
+
+/**
+ * @method
+ * @return data chunk
+ */
+QueryResult.prototype.nextChunk;
+
+/**
+ * @name asyncIterator
+ * @memberof module:duckdb~QueryResult
+ * @method
+ * @instance
+ * @yields data chunks
+ */
+QueryResult.prototype[Symbol.asyncIterator] = async function*() {
+    let prefetch = this.nextChunk();
+    while (true) {
+        const chunk = await prefetch;
+        // Null chunk indicates end of stream
+        if (!chunk) {
+            return;
+        }
+        // Prefetch the next chunk while we're iterating
+        prefetch = this.nextChunk();
+        for (const row of chunk) {
+            yield row;
+        }
+    }
+}
 
 
-Connection.prototype.run = function(sql) {
+/**
+ * Run a SQL statement and trigger a callback when done
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.run = function (sql) {
     var statement = new Statement(this, sql);
     return statement.run.apply(statement, arguments);
 }
 
-Connection.prototype.all = function(sql) {
-    var statement = new Statement(this,sql);
+/**
+ * Run a SQL query and triggers the callback once for all result rows
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.all = function (sql) {
+    var statement = new Statement(this, sql);
     return statement.all.apply(statement, arguments);
 }
 
-Connection.prototype.each = function(sql) {
+/**
+ * Runs a SQL query and triggers the callback for each result row
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.each = function (sql) {
     var statement = new Statement(this, sql);
     return statement.each.apply(statement, arguments);
 }
 
-function ptr_to_arr(buffer, ptype, n) {
-    // TODO can we create those on the C++ side of things already?
-    switch(ptype) {
-        case 'BOOL':
-        case 'UINT8':
-            return new Uint8Array(buffer.buffer, 0, n);
-        case 'INT8':
-            return new Int8Array(buffer.buffer, 0, n);
-        case 'INT16':
-            return new Int16Array(buffer.buffer, 0, n);
-        case 'UINT16':
-            return new UInt16Array(buffer.buffer, 0, n);
-        case 'INT32':
-            return new Int32Array(buffer.buffer, 0, n);
-        case 'UINT32':
-            return new UInt32Array(buffer.buffer, 0, n);
-        case 'FLOAT':
-            return new Float32Array(buffer.buffer, 0, n);
-        case 'DOUBLE':
-            return new Float64Array(buffer.buffer, 0, n);
-        case 'VARCHAR':  // we already have created a string array on the C++ side for this
-            return buffer;
-        default:
-            return new Array<string>(0); // cough
+/**
+ * @arg sql
+ * @param {...*} params
+ * @yields row chunks
+ */
+Connection.prototype.stream = async function* (sql) {
+    const statement = new Statement(this, sql);
+    const queryResult = await statement.stream.apply(statement, arguments);
+    for await (const result of queryResult) {
+        yield result;
     }
 }
 
-// this follows the wasm udfs somewhat but is simpler because we can pass data much more cleanly
-Connection.prototype.register = function(name, return_type, fun) {
+/**
+ * Register a User Defined Function
+ *
+ * @arg name
+ * @arg return_type
+ * @arg fun
+ * @return {void}
+ * @note this follows the wasm udfs somewhat but is simpler because we can pass data much more cleanly
+ */
+Connection.prototype.register = function (name, return_type, fun) {
     // TODO what if this throws an error somewhere? do we need a try/catch?
-    return this.register_bulk(name, return_type, function(descr) {
+    return this.register_bulk(name, return_type, function (desc) {
         try {
-            const data_arr = [];
-            const validity_arr = [];
+            // Build an argument resolver
+            const buildResolver = (arg) => {
+                let validity = arg.validity || null;
+                switch (arg.physicalType) {
+                    case 'STRUCT': {
+                        const tmp = {};
+                        const children = [];
+                        for (let j = 0; j < (arg.children.length || 0); ++j) {
+                            const attr = arg.children[j];
+                            const child = buildResolver(attr);
+                            children.push((row) => {
+                                tmp[attr.name] = child(row);
+                            });
+                        }
+                        if (validity != null) {
+                            return (row) => {
+                                if (!validity[row]) {
+                                    return null;
+                                }
+                                for (const resolver of children) {
+                                    resolver(row);
+                                }
+                                return tmp;
+                            };
+                        } else {
+                            return (row) => {
+                                for (const resolver of children) {
+                                    resolver(row);
+                                }
+                                return tmp;
+                            };
+                        }
+                    }
+                    default: {
+                        if (arg.data === undefined) {
+                            throw new Error(
+                                'malformed data view, expected data buffer for argument of type: ' + arg.physicalType,
+                            );
+                        }
+                        const data = arg.data;
+                        if (validity != null) {
+                            return (row) => (!validity[row] ? null : data[row]);
+                        } else {
+                            return (row) => data[row];
+                        }
+                    }
+                }
+            };
 
-            for (const idx in descr.args) {
-                const arg = descr.args[idx];
-                validity_arr.push(arg.data_buffers[arg.validity_buffer]);
-                data_arr.push(ptr_to_arr(arg.data_buffers[arg.data_buffer], arg.physical_type, descr.rows));
+            // Translate argument data
+            const argResolvers = [];
+            for (let i = 0; i < desc.args.length; ++i) {
+                argResolvers.push(buildResolver(desc.args[i]));
+            }
+            const args = [];
+            for (let i = 0; i < desc.args.length; ++i) {
+                args.push(null);
             }
 
-            const out_data = ptr_to_arr(descr.ret.data_buffers[descr.ret.data_buffer], descr.ret.physical_type, descr.rows);
-            const out_validity = descr.ret.data_buffers[descr.ret.validity_buffer];
-
-            switch (descr.args.length) {
-                case 0:
-                    for (let i = 0; i < descr.rows; ++i) {
-                        const res = fun();
-                        out_data[i] = res;
-                        out_validity[i] = res == undefined || res == null ? 0 : 1;
-                    }
+            // Return type
+            desc.ret.validity = new Uint8Array(desc.rows);
+            switch (desc.ret.physicalType) {
+                case 'INT8':
+                    desc.ret.data = new Int8Array(desc.rows);
                     break;
-                case 1:
-                    for (let i = 0; i < descr.rows; ++i) {
-                        const res = fun(validity_arr[0][i] ? data_arr[0][i] : undefined);
-                        out_data[i] = res;
-                        out_validity[i] = res == undefined || res == null ? 0 : 1;
-                    }
+                case 'INT16':
+                    desc.ret.data = new Int16Array(desc.rows);
                     break;
-                case 2:
-                    for (let i = 0; i < descr.rows; ++i) {
-                        const res = fun(validity_arr[0][i] ? data_arr[0][i] : undefined, validity_arr[1][i] ? data_arr[1][i] : undefined);
-                        out_data[i] = res;
-                        out_validity[i] = res == undefined || res == null ? 0 : 1;
-                    }
+                case 'INT32':
+                    desc.ret.data = new Int32Array(desc.rows);
                     break;
-                case 3:
-                    for (let i = 0; i < descr.rows; ++i) {
-                        const res = fun(validity_arr[0][i] ? data_arr[0][i] : undefined, validity_arr[1][i] ? data_arr[1][i] : undefined, validity_arr[2][i] ? data_arr[2][i] : undefined);
-                        out_data[i] = res;
-                        out_validity[i] = res == undefined || res == null ? 0 : 1;
-                    }
+                case 'DOUBLE':
+                    desc.ret.data = new Float64Array(desc.rows);
+                    break;
+                case 'DATE64':
+                case 'TIME64':
+                case 'TIMESTAMP':
+                case 'INT64':
+                    desc.ret.data = new BigInt64Array(desc.rows);
+                    break;
+                case 'UINT64':
+                    desc.ret.data = new BigUint64Array(desc.rows);
+                    break;
+                case 'BLOB':
+                case 'VARCHAR':
+                    desc.ret.data = new Array(desc.rows);
                     break;
-                default:
-                    throw "Unsupported argument count";
             }
-        } catch(error) { // work around recently fixed napi bug https://github.com/nodejs/node-addon-api/issues/912
+
+            // Call the function
+            for (let i = 0; i < desc.rows; ++i) {
+                for (let j = 0; j < desc.args.length; ++j) {
+                    args[j] = argResolvers[j](i);
+                }
+                const res = fun(...args);
+                desc.ret.data[i] = res;
+                desc.ret.validity[i] = res === undefined || res === null ? 0 : 1;
+            }
+        } catch (error) { // work around recently fixed napi bug https://github.com/nodejs/node-addon-api/issues/912
+            console.log(desc.ret);
             msg = error;
             if (typeof error == 'object' && 'message' in error) {
                 msg = error.message
             }
-            throw {name: 'DuckDB-UDF-Exception', message : msg};
+            throw { name: 'DuckDB-UDF-Exception', message: msg };
         }
     })
 }
 
-default_connection = function(o) {
+/**
+ * Prepare a SQL query for execution
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {Statement}
+ */
+Connection.prototype.prepare;
+/**
+ * Execute a SQL query
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.exec;
+/**
+ * Register a User Defined Function
+ *
+ * @method
+ * @arg name
+ * @arg return_type
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.register_bulk;
+/**
+ * Unregister a User Defined Function
+ *
+ * @method
+ * @arg name
+ * @arg return_type
+ * @param callback
+ * @return {void}
+ */
+Connection.prototype.unregister;
+
+var default_connection = function (o) {
     if (o.default_connection == undefined) {
         o.default_connection = new duckdb.Connection(o);
     }
-    return(o.default_connection);
+    return o.default_connection;
 }
 
-Database.prototype.prepare = function() {
+
+/**
+ * Closes database instance
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.close = function() {
+    this.default_connection = null
+    this.close_internal.apply(this, arguments);
+};
+
+/**
+ * Internal method. Do not use, call Connection#close instead
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.close_internal;
+
+/**
+ * Triggers callback when all scheduled database tasks have completed.
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.wait;
+
+/**
+ * TODO: what does this do?
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.serialize;
+
+/**
+ * TODO: what does this do?
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.parallelize;
+
+/**
+ * Create a new database connection
+ * @method
+ * @arg path the database to connect to, either a file path, or `:memory:`
+ * @return {Connection}
+ */
+Database.prototype.connect;
+
+/**
+ * Supposedly interrupt queries, but currently does not do anything.
+ * @method
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.interrupt;
+
+/**
+ * Prepare a SQL query for execution
+ * @arg sql
+ * @return {Statement}
+ */
+Database.prototype.prepare = function () {
     return default_connection(this).prepare.apply(this.default_connection, arguments);
 }
 
-Database.prototype.run = function() {
+/**
+ * Convenience method for Connection#run using a built-in default connection
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.run = function () {
     default_connection(this).run.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.each = function() {
+/**
+ * Convenience method for Connection#each using a built-in default connection
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.each = function () {
     default_connection(this).each.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.all = function() {
+/**
+ * Convenience method for Connection#apply using a built-in default connection
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.all = function () {
     default_connection(this).all.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.exec = function() {
+/**
+ * Convenience method for Connection#exec using a built-in default connection
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Database.prototype.exec = function () {
     default_connection(this).exec.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.register = function() {
+/**
+ * Convenience method for Connection#register using a built-in default connection
+ * @arg name
+ * @arg return_type
+ * @arg fun
+ * @return {this}
+ */
+Database.prototype.register = function () {
     default_connection(this).register.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.unregister = function() {
+/**
+ * Convenience method for Connection#unregister using a built-in default connection
+ * @arg name
+ * @return {this}
+ */
+Database.prototype.unregister = function () {
     default_connection(this).unregister.apply(this.default_connection, arguments);
     return this;
 }
 
-Database.prototype.get = function() {
+/**
+ * Not implemented
+ */
+Database.prototype.get = function () {
     throw "get() is not implemented because it's evil";
 }
 
-Statement.prototype.get = function() {
+/**
+ * Not implemented
+ */
+Statement.prototype.get = function () {
     throw "get() is not implemented because it's evil";
 }
+
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.run;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.all;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.each;
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @param callback
+ * @return {void}
+ */
+Statement.prototype.finalize
+/**
+ * @method
+ * @arg sql
+ * @param {...*} params
+ * @yield callback
+ */
+Statement.prototype.stream;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev98.0",
+  "version": "0.5.1-dev5.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {
@@ -26,6 +26,8 @@
   },
   "devDependencies": {
     "aws-sdk": "^2.790.0",
+    "chai": "^4.3.6",
+    "jsdoc3-parser": "^2.0.0",
     "mocha": "^8.3.0"
   },
   "repository": {