@flowblade/sqlduck 0.4.0 → 0.5.0

package/dist/index.cjs

@@ -50,6 +50,25 @@ async function* rowsToColumnsChunks(rows, chunkSize) {
 	}
 	if (rowsInChunk > 0) yield columns;
 }
+const isOnDataAppendedAsyncCb = (v) => {
+	return v.constructor.name === "AsyncFunction";
+};
+const createOnDataAppendedCollector = () => {
+	let lastCallbackTimeStart = Date.now();
+	let appendedTotalRows = 0;
+	return (currentTotalRows) => {
+		const cbTimeMs = Math.round(Date.now() - lastCallbackTimeStart);
+		const cbTotalRows = currentTotalRows - appendedTotalRows;
+		const payload = {
+			rowsCount: cbTotalRows,
+			timeMs: cbTimeMs,
+			rowsPerSecond: Math.round(cbTotalRows / cbTimeMs * 1e3)
+		};
+		appendedTotalRows = currentTotalRows;
+		lastCallbackTimeStart = Date.now();
+		return payload;
+	};
+};
 const createMap = {
 	CREATE: "CREATE TABLE",
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
@@ -124,10 +143,8 @@ var SqlDuck = class {
 		this.#logger = params.logger;
 	}
 	toTable = async (params) => {
-		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended, onDataAppendedBatchSize } = params;
+		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
 		if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
-		const callbackBatchSize = onDataAppendedBatchSize ?? chunkSize;
-		if (!Number.isSafeInteger(callbackBatchSize) || callbackBatchSize < 1) throw new Error("onDataAppendedBatchSize must be a number greater than 0");
 		const timeStart = Date.now();
 		const { columnTypes, ddl } = await createTableFromZod({
 			conn: this.#duck,
@@ -138,6 +155,7 @@ var SqlDuck = class {
 		const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
 		const chunkTypes = columnTypes.map((v) => v[1]);
 		let totalRows = 0;
+		const dataAppendedCollector = createOnDataAppendedCollector();
 		const columnStream = rowsToColumnsChunks(rowStream, chunkSize);
 		for await (const dataChunk of columnStream) {
 			const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
@@ -146,9 +164,12 @@ var SqlDuck = class {
 			chunk.setColumns(dataChunk);
 			appender.appendDataChunk(chunk);
 			appender.flushSync();
-			if (onDataAppended !== void 0 && totalRows % callbackBatchSize === 0) onDataAppended({ total: totalRows });
+			if (onDataAppended !== void 0) {
+				const payload = dataAppendedCollector(totalRows);
+				if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
+				else onDataAppended(payload);
+			}
 		}
-		if (onDataAppended !== void 0 && totalRows % callbackBatchSize !== 0) onDataAppended({ total: totalRows });
 		return {
 			timeMs: Math.round(Date.now() - timeStart),
 			totalRows,

package/dist/index.d.cts

@@ -2,6 +2,25 @@ import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
 import * as z from "zod";
 import { ZodObject } from "zod";
 
+//#region src/appender/data-appender-callback.d.ts
+type OnDataAppendedParams = {
+  /**
+   * Total number of rows appended so far
+   */
+  rowsCount: number;
+  /**
+   * Time taken to append the last batch in milliseconds
+   */
+  timeMs: number;
+  /**
+   * Estimated rows per seconds
+   */
+  rowsPerSecond: number;
+};
+type OnDataAppendedSyncCb = (params: OnDataAppendedParams) => void;
+type OnDataAppendedAsyncCb = (params: OnDataAppendedParams) => Promise<void>;
+type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
+//#endregion
 //#region src/table/table.d.ts
 /**
  * Fully qualified table information
@@ -71,21 +90,9 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
    */
   createOptions?: TableCreateOptions;
   /**
-   * Callback called each time data is appended to the table
-   * See also `onDataAppendedBatchSize` to limit the number of calls
-   * when appending a lot of data
-   */
-  onDataAppended?: (params: {
-    /**
-     * Total number of rows appended so far
-     */
-    total: number;
-  }) => void;
-  /**
-   * Number of rows appended before calling `onDataAppended` callback
-   * @default chunkSize
+   * Callback called each time a datachunk is appended to the table
    */
-  onDataAppendedBatchSize?: number;
+  onDataAppended?: OnDataAppendedCb;
 };
 type ToTableResult = {
   /**

package/dist/index.d.mts

@@ -2,6 +2,25 @@ import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
 import * as z from "zod";
 import { ZodObject } from "zod";
 
+//#region src/appender/data-appender-callback.d.ts
+type OnDataAppendedParams = {
+  /**
+   * Total number of rows appended so far
+   */
+  rowsCount: number;
+  /**
+   * Time taken to append the last batch in milliseconds
+   */
+  timeMs: number;
+  /**
+   * Estimated rows per seconds
+   */
+  rowsPerSecond: number;
+};
+type OnDataAppendedSyncCb = (params: OnDataAppendedParams) => void;
+type OnDataAppendedAsyncCb = (params: OnDataAppendedParams) => Promise<void>;
+type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
+//#endregion
 //#region src/table/table.d.ts
 /**
  * Fully qualified table information
@@ -71,21 +90,9 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
    */
   createOptions?: TableCreateOptions;
   /**
-   * Callback called each time data is appended to the table
-   * See also `onDataAppendedBatchSize` to limit the number of calls
-   * when appending a lot of data
-   */
-  onDataAppended?: (params: {
-    /**
-     * Total number of rows appended so far
-     */
-    total: number;
-  }) => void;
-  /**
-   * Number of rows appended before calling `onDataAppended` callback
-   * @default chunkSize
+   * Callback called each time a datachunk is appended to the table
    */
-  onDataAppendedBatchSize?: number;
+  onDataAppended?: OnDataAppendedCb;
 };
 type ToTableResult = {
   /**

package/dist/index.mjs

@@ -29,6 +29,25 @@ async function* rowsToColumnsChunks(rows, chunkSize) {
 	}
 	if (rowsInChunk > 0) yield columns;
 }
+const isOnDataAppendedAsyncCb = (v) => {
+	return v.constructor.name === "AsyncFunction";
+};
+const createOnDataAppendedCollector = () => {
+	let lastCallbackTimeStart = Date.now();
+	let appendedTotalRows = 0;
+	return (currentTotalRows) => {
+		const cbTimeMs = Math.round(Date.now() - lastCallbackTimeStart);
+		const cbTotalRows = currentTotalRows - appendedTotalRows;
+		const payload = {
+			rowsCount: cbTotalRows,
+			timeMs: cbTimeMs,
+			rowsPerSecond: Math.round(cbTotalRows / cbTimeMs * 1e3)
+		};
+		appendedTotalRows = currentTotalRows;
+		lastCallbackTimeStart = Date.now();
+		return payload;
+	};
+};
 const createMap = {
 	CREATE: "CREATE TABLE",
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
@@ -103,10 +122,8 @@ var SqlDuck = class {
 		this.#logger = params.logger;
 	}
 	toTable = async (params) => {
-		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended, onDataAppendedBatchSize } = params;
+		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
 		if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
-		const callbackBatchSize = onDataAppendedBatchSize ?? chunkSize;
-		if (!Number.isSafeInteger(callbackBatchSize) || callbackBatchSize < 1) throw new Error("onDataAppendedBatchSize must be a number greater than 0");
 		const timeStart = Date.now();
 		const { columnTypes, ddl } = await createTableFromZod({
 			conn: this.#duck,
@@ -117,6 +134,7 @@ var SqlDuck = class {
 		const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
 		const chunkTypes = columnTypes.map((v) => v[1]);
 		let totalRows = 0;
+		const dataAppendedCollector = createOnDataAppendedCollector();
 		const columnStream = rowsToColumnsChunks(rowStream, chunkSize);
 		for await (const dataChunk of columnStream) {
 			const chunk = DuckDBDataChunk.create(chunkTypes);
@@ -125,9 +143,12 @@ var SqlDuck = class {
 			chunk.setColumns(dataChunk);
 			appender.appendDataChunk(chunk);
 			appender.flushSync();
-			if (onDataAppended !== void 0 && totalRows % callbackBatchSize === 0) onDataAppended({ total: totalRows });
+			if (onDataAppended !== void 0) {
+				const payload = dataAppendedCollector(totalRows);
+				if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
+				else onDataAppended(payload);
+			}
 		}
-		if (onDataAppended !== void 0 && totalRows % callbackBatchSize !== 0) onDataAppended({ total: totalRows });
 		return {
 			timeMs: Math.round(Date.now() - timeStart),
 			totalRows,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -96,7 +96,7 @@
     "tarn": "3.0.2",
     "tedious": "19.2.0",
     "testcontainers": "11.11.0",
-    "tsdown": "0.18.3",
+    "tsdown": "0.18.4",
     "tsx": "4.21.0",
     "typedoc": "0.28.15",
     "typedoc-plugin-markdown": "4.9.0",