@duckdb/node-api 1.4.2-r.1 → 1.4.3-r.2

package/README.md

@@ -925,6 +925,17 @@ for (let stmtIndex = 0; stmtIndex < statementCount; stmtIndex++) {
 
 ### Control Evaluation of Tasks
 
+DuckDB splits work into relatively short tasks. By controlling the
+evaluation of these tasks explicitly, better cooperative multithreading
+can be accomplished. This is especially important for maximum throughput
+in the Node environment, which has a small, fixed number of threads for
+running asynchronous work.
+(See https://docs.libuv.org/en/stable/threadpool.html)
+
+Below is a low-level way of controlling the evaluation of processing.
+See the "startThenRead" and "startStreamThenRead" methods for helpers
+that do much of this for you.
+
 ```ts
 import { DuckDBPendingResultState } from '@duckdb/node-api';
 
@@ -935,7 +946,7 @@ async function sleep(ms) {
 }
 
 const prepared = await connection.prepare('from range(10_000_000)');
-const pending = prepared.start();
+const pending = prepared.startStream();
 while (pending.runTask() !== DuckDBPendingResultState.RESULT_READY) {
   console.log('not ready');
   await sleep(1);
@@ -945,6 +956,12 @@ const result = await pending.getResult();
 // ...
 ```
 
+Equivalently, using `startStreamThenRead`:
+
+```ts
+const reader = await connection.startStreamThenRead(sql);
+```
+
 ### Ways to run SQL
 
 ```ts
@@ -1030,11 +1047,51 @@ const pending = await connection.start(sql);
 const pending = await connection.start(sql, values);
 const pending = await connection.start(sql, values, types);
 
+// The methods beginning with "startThenRead" provide some, but not full,
+// cooperative multithreading. They use pending results to split processing
+// into short tasks, but they fully materialize the result, which can
+// take some time (and memory). For full cooperative multithreading,
+// see the "startStreamThenRead" methods below.
+const reader = await connection.startThenRead(sql);
+const reader = await connection.startThenRead(sql, values);
+const reader = await connection.startThenRead(sql, values, types);
+
+const reader = await connection.startThenReadAll(sql);
+const reader = await connection.startThenReadAll(sql, values);
+const reader = await connection.startThenReadAll(sql, values, types);
+
+const reader = await connection.startThenReadUntil(sql, targetRowCount);
+const reader =
+  await connection.startThenReadUntil(sql, targetRowCount, values);
+const reader =
+  await connection.startThenReadUntil(sql, targetRowCount, values, types);
+
 // Create a pending, streaming result.
 const pending = await connection.startStream(sql);
 const pending = await connection.startStream(sql, values);
 const pending = await connection.startStream(sql, values, types);
 
+// The methods beginning with "startStreamThenRead" are the best options
+// for cooperative multithreading. By creating a streaming result, they
+// prevent the result from being fully materialized. By using a pending
+// result, they split processing into short tasks, preventing any single
+// task from occupying a thread for too long.
+const reader = await connection.startStreamThenRead(sql);
+const reader = await connection.startStreamThenRead(sql, values);
+const reader = await connection.startStreamThenRead(sql, values, types);
+
+const reader = await connection.startStreamThenReadAll(sql);
+const reader = await connection.startStreamThenReadAll(sql, values);
+const reader =
+  await connection.startStreamThenReadAll(sql, values, types);
+
+const reader =
+  await connection.startStreamThenReadUntil(sql, targetRowCount);
+const reader =
+  await connection.startStreamThenReadUntil(sql, targetRowCount, values);
+const reader = await connection.startStreamThenReadUntil(
+  sql, targetRowCount, values, types);
+
 // Create a pending result from a prepared statement.
 const pending = await prepared.start();
 const pending = await prepared.startStream();
@@ -1043,6 +1100,9 @@ while (pending.runTask() !== DuckDBPendingResultState.RESULT_READY) {
   // optionally sleep or do other work between tasks
 }
 
+// Or, run tasks (cooperatively) until the result is ready.
+await pending.runAllTasks();
+
 // Retrieve the result. If not yet READY, will run until it is.
 const result = await pending.getResult();
 
@@ -1058,14 +1118,30 @@ const reader = await pending.readUntil(targetRowCount);
 
 // Asynchronously retrieve data for all rows:
 const columns = await result.getColumns();
+const columnsJS = await result.getColumnsJS();
 const columnsJson = await result.getColumnsJson();
 const columnsObject = await result.getColumnsObject();
+const columnsObjectJS = await result.getColumnsObjectJS();
 const columnsObjectJson = await result.getColumnsObjectJson();
 const rows = await result.getRows();
+const rowsJS = await result.getRowsJS();
 const rowsJson = await result.getRowsJson();
 const rowObjects = await result.getRowObjects();
+const rowObjectsJS = await result.getRowObjectsJS();
 const rowObjectsJson = await result.getRowObjectsJson();
 
+// Asynchronous iterators can be used to retrieve partial data:
+for await (const chunk of result) {
+  // ...
+}
+
+// Each chunk can be converted to rows:
+for await (const rows of result.yieldRows()) {
+  // ...
+}
+// See also variations of "yieldRow" for returning row objects
+// (instead of row arrays), and with JS, JSON, or custom conversion.
+
 // From a reader
 
 // First, (asynchronously) read some rows:
@@ -1075,12 +1151,16 @@ await reader.readUntil(targetRowCount);
 
 // Then, (synchronously) get result data for the rows read:
 const columns = reader.getColumns();
+const columnsJS = reader.getColumnsJS();
 const columnsJson = reader.getColumnsJson();
 const columnsObject = reader.getColumnsObject();
+const columnsObjectJS = reader.getColumnsObjectJS();
 const columnsObjectJson = reader.getColumnsObjectJson();
 const rows = reader.getRows();
+const rowsJS = reader.getRowsJS();
 const rowsJson = reader.getRowsJson();
 const rowObjects = reader.getRowObjects();
+const rowObjectsJS = reader.getRowObjectsJS();
 const rowObjectsJson = reader.getRowObjectsJson();
 
 // Individual values can also be read directly:

package/lib/DuckDBConnection.d.ts

@@ -31,7 +31,13 @@ export declare class DuckDBConnection {
     streamAndReadAll(sql: string, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBResultReader>;
     streamAndReadUntil(sql: string, targetRowCount: number, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBResultReader>;
     start(sql: string, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBPendingResult>;
+    startThenRead(sql: string, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBResultReader>;
+    startThenReadAll(sql: string, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBResultReader>;
+    startThenReadUntil(sql: string, targetRowCount: number, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBResultReader>;
     startStream(sql: string, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBPendingResult>;
+    startStreamThenRead(sql: string, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBResultReader>;
+    startStreamThenReadAll(sql: string, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBResultReader>;
+    startStreamThenReadUntil(sql: string, targetRowCount: number, values?: DuckDBValue[] | Record<string, DuckDBValue>, types?: DuckDBType[] | Record<string, DuckDBType | undefined>): Promise<DuckDBResultReader>;
     prepare(sql: string): Promise<DuckDBPreparedStatement>;
     private createPrepared;
     extractStatements(sql: string): Promise<DuckDBExtractedStatements>;

package/lib/DuckDBConnection.js

@@ -63,12 +63,12 @@ class DuckDBConnection {
         return new DuckDBResultReader_1.DuckDBResultReader(await this.run(sql, values, types));
     }
     async runAndReadAll(sql, values, types) {
-        const reader = new DuckDBResultReader_1.DuckDBResultReader(await this.run(sql, values, types));
+        const reader = await this.runAndRead(sql, values, types);
         await reader.readAll();
         return reader;
     }
     async runAndReadUntil(sql, targetRowCount, values, types) {
-        const reader = new DuckDBResultReader_1.DuckDBResultReader(await this.run(sql, values, types));
+        const reader = await this.runAndRead(sql, values, types);
         await reader.readUntil(targetRowCount);
         return reader;
     }
@@ -89,12 +89,12 @@ class DuckDBConnection {
         return new DuckDBResultReader_1.DuckDBResultReader(await this.stream(sql, values, types));
     }
     async streamAndReadAll(sql, values, types) {
-        const reader = new DuckDBResultReader_1.DuckDBResultReader(await this.stream(sql, values, types));
+        const reader = await this.streamAndRead(sql, values, types);
         await reader.readAll();
         return reader;
     }
     async streamAndReadUntil(sql, targetRowCount, values, types) {
-        const reader = new DuckDBResultReader_1.DuckDBResultReader(await this.stream(sql, values, types));
+        const reader = await this.streamAndRead(sql, values, types);
         await reader.readUntil(targetRowCount);
         return reader;
     }
@@ -110,6 +110,21 @@ class DuckDBConnection {
             prepared.destroySync();
         }
     }
+    async startThenRead(sql, values, types) {
+        const pending = await this.start(sql, values, types);
+        await pending.runAllTasks();
+        return new DuckDBResultReader_1.DuckDBResultReader(await pending.getResult());
+    }
+    async startThenReadAll(sql, values, types) {
+        const reader = await this.startThenRead(sql, values, types);
+        reader.readAll();
+        return reader;
+    }
+    async startThenReadUntil(sql, targetRowCount, values, types) {
+        const reader = await this.startThenRead(sql, values, types);
+        reader.readUntil(targetRowCount);
+        return reader;
+    }
     async startStream(sql, values, types) {
         const prepared = await this.runUntilLast(sql);
         try {
@@ -122,6 +137,21 @@ class DuckDBConnection {
             prepared.destroySync();
         }
     }
+    async startStreamThenRead(sql, values, types) {
+        const pending = await this.startStream(sql, values, types);
+        await pending.runAllTasks();
+        return new DuckDBResultReader_1.DuckDBResultReader(await pending.getResult());
+    }
+    async startStreamThenReadAll(sql, values, types) {
+        const reader = await this.startStreamThenRead(sql, values, types);
+        reader.readAll();
+        return reader;
+    }
+    async startStreamThenReadUntil(sql, targetRowCount, values, types) {
+        const reader = await this.startStreamThenRead(sql, values, types);
+        reader.readUntil(targetRowCount);
+        return reader;
+    }
     async prepare(sql) {
         const prepared = await this.createPrepared(sql);
         this.preparedStatements.add(prepared);

package/lib/DuckDBPendingResult.d.ts

@@ -10,6 +10,7 @@ export declare class DuckDBPendingResult {
     private readonly pending_result;
     constructor(pending_result: duckdb.PendingResult);
     runTask(): DuckDBPendingResultState;
+    runAllTasks(): Promise<void>;
     getResult(): Promise<DuckDBResult>;
     read(): Promise<DuckDBResultReader>;
     readAll(): Promise<DuckDBResultReader>;

package/lib/DuckDBPendingResult.js

@@ -7,6 +7,7 @@ exports.DuckDBPendingResult = exports.DuckDBPendingResultState = void 0;
 const node_bindings_1 = __importDefault(require("@duckdb/node-bindings"));
 const createResult_1 = require("./createResult");
 const DuckDBResultReader_1 = require("./DuckDBResultReader");
+const sleep_1 = require("./sleep");
 // Values match similar enum in C API.
 var DuckDBPendingResultState;
 (function (DuckDBPendingResultState) {
@@ -34,6 +35,11 @@ class DuckDBPendingResult {
                 throw new Error(`Unexpected pending state: ${pending_state}`);
         }
     }
+    async runAllTasks() {
+        while (this.runTask() !== DuckDBPendingResultState.RESULT_READY) {
+            await (0, sleep_1.sleep)(1);
+        }
+    }
     async getResult() {
         return (0, createResult_1.createResult)(await node_bindings_1.default.execute_pending(this.pending_result));
     }

package/lib/sleep.d.ts

@@ -0,0 +1 @@
+export declare function sleep(ms: number): Promise<unknown>;

package/lib/sleep.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sleep = sleep;
+async function sleep(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@duckdb/node-api",
-  "version": "1.4.2-r.1",
+  "version": "1.4.3-r.2",
   "license": "MIT",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
   "dependencies": {
-    "@duckdb/node-bindings": "1.4.2-r.1"
+    "@duckdb/node-bindings": "1.4.3-r.2"
   },
   "repository": {
     "type": "git",