@flowblade/sqlduck 0.16.0 → 0.17.0

package/README.md

@@ -58,7 +58,7 @@ const result = await sqlDuck.toTable({
     rowStream: getUsers(), // The async iterable that yields rows
     // 👇Optional:
     chunkSize: 2048, // Number of rows to append when using duckdb appender. Default is 2048
-    onDataAppended: ({ timeMs, totalRows, rowsPerSecond }) => {
+    onChunkAppended: ({ timeMs, totalRows, rowsPerSecond }) => {
         console.log(
             `Appended ${totalRows} in time ${timeMs}ms, est: ${rowsPerSecond} rows/s`
         );
@@ -105,8 +105,8 @@ const result = sqlDuck.toTable({
   rowStream: getUserRows(), // The async iterable that yields rows
   // 👇Optional:
   chunkSize: 2048, // Number of rows to append when using duckdb appender. Default is 2048
-  onDataAppended: ({ total }) => {
-    console.log(`Appended ${total} rows so far`);
+  onChunkAppended: ({ totalRows }) => {
+    console.log(`Appended ${totalRows} rows so far`);
   },
   // Optional table creation options
   createOptions: {
@@ -131,67 +131,72 @@ const queryResult = await dbDuckDbMemoryConn.query<{
 ### Node 24
 
 ```
+
  RUN  v4.1.4 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 3412ms
+ ✓ bench/appender.bench.ts > appender benches 2910ms
      name                                                        hz     min     max    mean     p75     p99    p995    p999       rme  samples
-   · duckdb appender memory, count: 100000, chunk size 2048  2.8872  295.84  396.87  346.35  396.87  396.87  396.87  396.87  ±185.36%        2
-   · duckdb appender file, count: 100000, chunk size 2048    1.7908  558.40  558.40  558.40  558.40  558.40  558.40  558.40    ±0.00%        1
-   · duckdb appender, count: 100000, chunk size 1024         1.9967  500.82  500.82  500.82  500.82  500.82  500.82  500.82    ±0.00%        1
+   · duckdb appender memory, count: 100000, chunk size 2048  3.5446  265.91  298.32  282.12  298.32  298.32  298.32  298.32   ±72.99%        2
+   · duckdb appender file, count: 100000, chunk size 2048    2.6130  355.30  410.10  382.70  410.10  410.10  410.10  410.10   ±91.00%        2
+   · duckdb appender, count: 100000, chunk size 1024         3.8027  226.52  299.42  262.97  299.42  299.42  299.42  299.42  ±176.17%        2
 
- ✓ bench/stream.bench.ts > Bench stream 2140ms
-     name                                                        hz      min     max     mean      p75     p99    p995    p999      rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  11.8099  63.0430  172.82  84.6748  81.4757  172.82  172.82  172.82  ±27.12%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 100000)   10.0442  79.5805  125.81  99.5603   112.35  125.81  125.81  125.81  ±12.04%       10
+ ✓ bench/stream.bench.ts > Bench rowsToColumnsChunks 2998ms
+     name                                                                         hz      min     max     mean      p75     p99    p995    p999      rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)                   11.3182  60.6404  190.56  88.3532  79.4999  190.56  190.56  190.56  ±30.64%       10
+   · rowToColumnsChunk with transformer with chunkSize 2048 (count: 100000)  13.4430  63.9716  102.73  74.3883  75.2151  102.73  102.73  102.73  ±10.40%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)                    10.7785  84.3040  117.57  92.7773  97.1595  117.57  117.57  117.57   ±7.61%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 617ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 615ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  27,583.03  0.0226  2.7818  0.0363  0.0403  0.1185  0.1583  0.4321  ±1.66%    13792
+   · getTableCreateFromZod  27,048.71  0.0239  3.2029  0.0370  0.0404  0.1310  0.1980  0.5674  ±2.08%    13525
 
  BENCH  Summary
 
-  duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.45x faster than duckdb appender, count: 100000, chunk size 1024
-    1.61x faster than duckdb appender file, count: 100000, chunk size 2048
+  duckdb appender, count: 100000, chunk size 1024 - bench/appender.bench.ts > appender benches
+    1.07x faster than duckdb appender memory, count: 100000, chunk size 2048
+    1.46x faster than duckdb appender file, count: 100000, chunk size 2048
 
-  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
-    1.18x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
+  rowToColumnsChunk with transformer with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench rowsToColumnsChunks
+    1.19x faster than rowToColumnsChunk with chunkSize 2048 (count: 100000)
+    1.25x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 ```
 
-### Bun 1.3.11
+### Bun 1.3.12
 
 ```
  RUN  v4.1.4 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 3357ms
+ ✓ bench/appender.bench.ts > appender benches 2811ms
      name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
-   · duckdb appender memory, count: 100000, chunk size 2048  2.9741  315.71  356.77  336.24  356.77  356.77  356.77  356.77  ±77.59%        2
-   · duckdb appender file, count: 100000, chunk size 2048    1.8953  527.62  527.62  527.62  527.62  527.62  527.62  527.62   ±0.00%        1
-   · duckdb appender, count: 100000, chunk size 1024         1.7803  561.70  561.70  561.70  561.70  561.70  561.70  561.70   ±0.00%        1
+   · duckdb appender memory, count: 100000, chunk size 2048  3.9242  224.75  285.38  254.83  285.38  285.38  285.38  285.38  ±29.56%        3
+   · duckdb appender file, count: 100000, chunk size 2048    3.8209  256.09  267.34  261.72  267.34  267.34  267.34  267.34  ±27.31%        2
+   · duckdb appender, count: 100000, chunk size 1024         4.6118  196.77  234.22  216.84  234.22  234.22  234.22  234.22  ±21.62%        3
 
- ✓ bench/stream.bench.ts > Bench stream 2058ms
-     name                                                        hz      min     max     mean      p75     p99    p995    p999      rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  12.0130  60.7081  111.32  83.2432  99.7116  111.32  111.32  111.32  ±14.15%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 100000)   10.9710  76.4253  145.14  91.1493  91.1424  145.14  145.14  145.14  ±16.04%       10
+ ✓ bench/stream.bench.ts > Bench rowsToColumnsChunks 2667ms
+     name                                                                         hz      min      max     mean      p75      p99     p995     p999     rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)                   14.4994  57.7717  81.8690  68.9683  78.5234  81.8690  81.8690  81.8690  ±9.43%       10
+   · rowToColumnsChunk with transformer with chunkSize 2048 (count: 100000)  13.2052  68.7139  96.8400  75.7275  79.4380  96.8400  96.8400  96.8400  ±8.22%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)                    12.7827  73.9127  85.0696  78.2310  82.5510  85.0696  85.0696  85.0696  ±3.69%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 621ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 624ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  27,472.91  0.0185  4.6226  0.0364  0.0369  0.1137  0.1476  1.9138  ±5.20%    13737
+   · getTableCreateFromZod  28,477.04  0.0191  6.4836  0.0351  0.0335  0.1071  0.1530  2.4823  ±6.21%    14239
 
  BENCH  Summary
 
-  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
-    1.09x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
+  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench rowsToColumnsChunks
+    1.10x faster than rowToColumnsChunk with transformer with chunkSize 2048 (count: 100000)
+    1.13x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 
-  duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.57x faster than duckdb appender file, count: 100000, chunk size 2048
-    1.67x faster than duckdb appender, count: 100000, chunk size 1024
+  duckdb appender, count: 100000, chunk size 1024 - bench/appender.bench.ts > appender benches
+    1.18x faster than duckdb appender memory, count: 100000, chunk size 2048
+    1.21x faster than duckdb appender file, count: 100000, chunk size 2048
 
 ```
 

package/dist/index.d.mts

@@ -7,7 +7,7 @@ import * as z from "zod";
 import { ZodObject } from "zod";
 
 //#region src/appender/data-appender-callback.d.ts
-type OnDataAppendedStats = {
+type OnChunkAppendedStats = {
   /**
    * Total number of rows appended so far (all batches included)
    */
@@ -21,9 +21,9 @@ type OnDataAppendedStats = {
    */
   rowsPerSecond: number;
 };
-type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
-type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
-type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
+type OnChunkAppendedSyncCb = (stats: OnChunkAppendedStats) => void;
+type OnChunkAppendedAsyncCb = (stats: OnChunkAppendedStats) => Promise<void>;
+type OnChunkAppendedCb = OnChunkAppendedSyncCb | OnChunkAppendedAsyncCb;
 //#endregion
 //#region src/helpers/duck-memory.d.ts
 declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
@@ -113,41 +113,65 @@ type SqlDuckParams = {
 type RowStream<T> = AsyncIterableIterator<T> | AsyncGenerator<T> | Generator<T>;
 type ToTableParams<TSchema extends TableSchemaZod> = {
   /**
-   * Used to create and fill the data into the table
+   * The target table where the data will be inserted.
+   * This object contains the table name and optionally the schema and database name.
    */
   table: Table;
   /**
-   * Schema describing the table structure and rowStream content
+   * A Zod schema that defines the structure of the table and the expected format of the rows in the `rowStream`.
+   * The schema is used to generate the `CREATE TABLE` DDL and to convert row values to DuckDB types.
    */
   schema: TSchema;
   /**
-   * Stream of rows to insert into the table
+   * An iterable (async or sync) or generator that yields rows to be inserted.
+   * Each row must match the structure defined in the `schema`.
    */
   rowStream: RowStream<z.infer<TSchema>>;
   /**
-   * Chunk size when using appender to insert data.
-   * Valid numbers between 1 and 2048.
+   * The number of rows to accumulate before appending them to the DuckDB table as a single data chunk.
+   * Tuning this value can impact memory usage and insertion performance.
+   * Valid values are between 1 and 2048.
    * @default 2048
    */
   chunkSize?: number;
   /**
-   * Extra options when creating the table
+   * Configuration options for the `CREATE TABLE` statement (e.g., `IF NOT EXISTS`, `CREATE OR REPLACE`).
+   * If omitted, a standard `CREATE TABLE` statement is used.
    */
   createOptions?: TableCreateOptions;
   /**
-   * Callback called each time a datachunk is appended to the table
+   * An optional callback invoked after each data chunk is successfully appended to the table.
+   * Useful for tracking progress, logging statistics, or implementing custom hooks during the insertion process.
    */
-  onDataAppended?: OnDataAppendedCb;
+  onChunkAppended?: OnChunkAppendedCb;
   /**
-   * Automatically checkpoint the table after all chunks have been appended.
+   * Specifies the frequency (in number of chunks) at which the `onChunkAppended` callback should be triggered.
+   *
+   * For example, if `chunkSize` is 2048 and `onChunkAppendedFrequency` is 5,
+   * the callback will be called every 10,240 rows (5 chunks * 2048 rows/chunk).
+   *
+   * @default 1
+   */
+  onChunkAppendedFrequency?: number;
+  /**
+   * Specifies the frequency (in number of chunks) at which the `appender.flushSync()` should be called.
+   * Calling `flushSync()` can help to clear internal buffers and make the data visible.
+   *
+   * For example, if `chunkSize` is 2048 and `flushSyncFrequency` is 5,
+   * the appender will be flushed every 10,240 rows (5 chunks * 2048 rows/chunk).
+   */
+  flushSyncFrequency?: number;
+  /**
+   * If set to `true`, a checkpoint is automatically performed after all rows from the `rowStream` have been processed.
+   * This ensures that all data is persisted and WAL is cleared.
    * @default true
    */
   autoCheckpoint?: boolean;
   /**
-   * Checkpoint the table after 'n' chunks have been appended
+   * Specifies the frequency (in number of chunks) at which a checkpoint should be triggered.
    *
-   * For example if the chunkSize is 2048, setting frequency to 2
-   * will checkpoint the table every 4096 rows (2x chunksize)
+   * For example, if `chunkSize` is 2048 and `checkpointChunksFrequency` is 5,
+   * a checkpoint will occur every 10,240 rows (5 chunks * 2048 rows/chunk).
    */
   checkpointChunksFrequency?: number;
 };
@@ -193,9 +217,13 @@ declare class SqlDuck {
    *  schema: userSchema,
    *  rowStream: getUserRows(),
    *  chunkSize: 2048,
-   *  onDataAppended: ({ total }) => {
-   *    console.log(`Appended ${total} rows so far`);
+   *  flushSyncFrequency: 10, // flush after every 10 chunks
+   *  onChunkAppendedFrequency: 1, // multiple of chunks
+   *  onChunkAppended: ({ totalRows }) => {
+   *    console.log(`Appended ${totalRows} rows so far`);
    *  },
+   *  autoCheckpoint: true,
+   *  checkpointChunksFrequency: 100, // checkpoint after every 100 chunks
    *  createOptions: {
    *    create: 'CREATE_OR_REPLACE',
    *  },
@@ -302,4 +330,4 @@ declare const flowbladeLogtapeSqlduckConfig: {
 //#region src/logger/sqlduck-default-logtape-logger.d.ts
 declare const sqlduckDefaultLogtapeLogger: _$_logtape_logtape0.Logger;
 //#endregion
-export { Database, type DuckConnectionParams, DuckDatabaseManager, DuckMemory, type DuckMemoryTag, type DuckdbReservedKeywords, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
+export { Database, type DuckConnectionParams, DuckDatabaseManager, DuckMemory, type DuckMemoryTag, type DuckdbReservedKeywords, type OnChunkAppendedCb, type OnChunkAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/dist/index.mjs

@@ -104,10 +104,12 @@ var DuckMemory = class {
 };
 //#endregion
 //#region src/appender/data-appender-callback.ts
-const isOnDataAppendedAsyncCb = (v) => {
-	return v.constructor.name === "AsyncFunction";
+const isOnChunkAppendedAsyncCb = (v) => {
+	return v.constructor.name === "AsyncFunction" || v.constructor === (async () => {
+		await Promise.resolve();
+	}).constructor;
 };
-const createOnDataAppendedCollector = () => {
+const createOnChunkAppendedCollector = () => {
 	let lastCallbackTimeStart = Date.now();
 	let appendedTotalRows = 0;
 	return (currentTotalRows) => {
@@ -188,7 +190,7 @@ var DuckValueConverter = class {
 //#endregion
 //#region src/converter/create-duck-column-converters.ts
 const createDuckColumnConverters = (duckTypes) => {
-	const convMap = /* @__PURE__ */ new Map();
+	const convMap = {};
 	const converter = new DuckValueConverter();
 	for (const [key, duckType] of Object.entries(duckTypes)) {
 		let conv;
@@ -223,7 +225,7 @@ const createDuckColumnConverters = (duckTypes) => {
 				break;
 			default: throw new Error(`Unsupported duck type ${duckTypeId} / ${duckType.toString()} for column '${key}'`);
 		}
-		if (conv !== false) convMap.set(key, conv);
+		if (conv !== false) convMap[key] = conv;
 	}
 	return convMap;
 };
@@ -650,13 +652,42 @@ const createTableFromZod = async (params) => {
 //#endregion
 //#region src/utils/rows-to-columns-chunks.ts
 /**
-* Similar to `rowsToColumns` but yields results in chunks to avoid buffering
-* the entire dataset in memory. Each yielded item is a columns array for up to
-* `chunkSize` rows.
+* Converts a stream of rows (row-oriented) into a stream of column-oriented chunks.
 *
-* Example for chunkSize = 2:
-*   input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
-*   yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
+* This function processes row data incrementally using an async generator, which prevents
+* loading the entire dataset into memory. Each yielded chunk is an object where keys are
+* column names and values are arrays of up to `chunkSize` elements.
+*
+* This is particularly useful for DuckDB's Appender API or other columnar processing
+* engines that expect data in chunks of columns.
+*
+* @param params - Configuration for the transformation.
+* @param params.rows - An async or sync iterable of rows.
+* @param params.chunkSize - The maximum number of rows per yielded chunk. Must be a positive integer.
+* @param params.transformers - Optional mappers for specific columns to transform values before chunking.
+*
+* @returns An async iterator yielding chunks of column-oriented data.
+*
+* @example
+* ```typescript
+*  async function* generateRows() {
+*    yield { id: 1, name: 'A' };
+*    yield { id: 2, name: 'B' };
+*    yield { id: 3, name: 'C' };
+*  }
+*
+*  const columnChunks = rowsToColumnsChunks({
+*    rows: generateRows(),
+*    chunkSize: 2,
+*  })
+*
+* for await (const chunk of columnChunks) {
+*   console.log(chunk);
+* }
+* // Output:
+* // { id: [1, 2], name: ['A', 'B'] } // first chunk
+* // { id: [3], name: ['C'] } // second chunk
+* ```
 */
 async function* rowsToColumnsChunks(params) {
 	const { rows, chunkSize, transformers } = params;
@@ -664,27 +695,46 @@ async function* rowsToColumnsChunks(params) {
 	const first = await rows.next();
 	if (first.done) return;
 	const keys = Object.keys(first.value);
-	let columns = keys.map(() => []);
+	const numKeys = keys.length;
+	const mappers = new Array(numKeys);
+	if (transformers !== void 0) {
+		const unknownKeys = Object.keys(transformers).filter((k) => !keys.includes(k));
+		if (unknownKeys.length > 0) throw new Error(`transformers parameter contains unknown row ids: ${unknownKeys.join(", ")}`);
+		for (let i = 0; i < numKeys; i++) mappers[i] = transformers[keys[i]];
+	}
+	function createColumns() {
+		const obj = {};
+		for (let i = 0; i < numKeys; i++) {
+			const k = keys[i];
+			obj[k] = [];
+		}
+		return obj;
+	}
+	let columns = createColumns();
 	let rowsInChunk = 0;
-	keys.forEach((k, i) => {
-		const fn = transformers?.get(k);
-		columns[i].push(fn === void 0 ? first.value[k] : fn(first.value[k]));
-	});
+	for (let i = 0; i < numKeys; i++) {
+		const k = keys[i];
+		const fn = mappers[i];
+		const val = first.value[k];
+		columns[k].push(fn === void 0 ? val : fn(val));
+	}
 	rowsInChunk++;
 	if (rowsInChunk >= chunkSize) {
 		yield columns;
-		columns = keys.map(() => []);
+		columns = createColumns();
 		rowsInChunk = 0;
 	}
 	for await (const row of rows) {
-		keys.forEach((k, i) => {
-			const fn = transformers?.get(k);
-			columns[i].push(fn === void 0 ? row[k] : fn(row[k]));
-		});
+		for (let i = 0; i < numKeys; i++) {
+			const k = keys[i];
+			const fn = mappers[i];
+			const val = row[k];
+			columns[k].push(fn === void 0 ? val : fn(val));
+		}
 		rowsInChunk++;
 		if (rowsInChunk >= chunkSize) {
 			yield columns;
-			columns = keys.map(() => []);
+			columns = createColumns();
 			rowsInChunk = 0;
 		}
 	}
@@ -724,9 +774,13 @@ var SqlDuck = class {
 	*  schema: userSchema,
 	*  rowStream: getUserRows(),
 	*  chunkSize: 2048,
-	*  onDataAppended: ({ total }) => {
-	*    console.log(`Appended ${total} rows so far`);
+	*  flushSyncFrequency: 10, // flush after every 10 chunks
+	*  onChunkAppendedFrequency: 1, // multiple of chunks
+	*  onChunkAppended: ({ totalRows }) => {
+	*    console.log(`Appended ${totalRows} rows so far`);
 	*  },
+	*  autoCheckpoint: true,
+	*  checkpointChunksFrequency: 100, // checkpoint after every 100 chunks
 	*  createOptions: {
 	*    create: 'CREATE_OR_REPLACE',
 	*  },
@@ -737,11 +791,13 @@ var SqlDuck = class {
 	* ```
 	*/
 	toTable = async (params) => {
-		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended, autoCheckpoint = true, checkpointChunksFrequency = 10 } = params;
+		const { table, schema, chunkSize = 2048, rowStream, createOptions, onChunkAppended, onChunkAppendedFrequency, flushSyncFrequency = 10, autoCheckpoint = true, checkpointChunksFrequency } = params;
 		if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
 		if (autoCheckpoint && typeof table.databaseName !== "string") throw new Error("autoCheckpoint requires table.databaseName to be provided.");
-		if (checkpointChunksFrequency && typeof table.databaseName !== "string") throw new Error("checkpointChunksFrequency requires table.databaseName to be provided.");
-		if (checkpointChunksFrequency !== void 0 && checkpointChunksFrequency < 1) throw new Error("checkpointChunksFrequency must be a positive number.");
+		if (checkpointChunksFrequency !== void 0 && typeof table.databaseName !== "string") throw new Error("checkpointChunksFrequency requires table.databaseName to be provided.");
+		if (checkpointChunksFrequency !== void 0 && (checkpointChunksFrequency < 1 || checkpointChunksFrequency > 1e5)) throw new Error("checkpointChunksFrequency must be a number between 1 and 100_000.");
+		if (onChunkAppendedFrequency !== void 0 && (onChunkAppendedFrequency < 1 || onChunkAppendedFrequency > 1e5)) throw new Error("onChunkAppendedFrequency must be a number between 1 and 100_000.");
+		if (flushSyncFrequency !== void 0 && (flushSyncFrequency < 1 || flushSyncFrequency > 1e5)) throw new Error("flushSyncFrequency must be a number between 1 and 100_000.");
 		const dbManager = new DuckDatabaseManager(this.#conn);
 		const timeStart = Date.now();
 		const { columnTypes, ddl } = await createTableFromZod({
@@ -752,46 +808,56 @@ var SqlDuck = class {
 		});
 		const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
 		const chunkTypes = Array.from(columnTypes.values());
-		const transformers = createDuckColumnConverters(Object.fromEntries(Array.from(columnTypes).map(([key, duckType]) => {
-			return [key, duckType];
-		})));
+		const columnTypeIds = {};
+		const columnKeys = [];
+		for (const [key, duckType] of columnTypes) {
+			columnKeys.push(key);
+			columnTypeIds[key] = duckType;
+		}
+		const numColumns = columnKeys.length;
+		const transformers = createDuckColumnConverters(columnTypeIds);
 		let totalRows = 0;
-		const dataAppendedCollector = createOnDataAppendedCollector();
+		const chunkAppendedCollector = createOnChunkAppendedCollector();
 		const columnStream = rowsToColumnsChunks({
 			rows: rowStream,
 			chunkSize,
 			transformers
 		});
 		let appendedChunkCount = 0;
+		const tableFullName = table.getFullName();
+		const tableName = table.tableName;
 		try {
+			const isAsyncCb = onChunkAppended !== void 0 && isOnChunkAppendedAsyncCb(onChunkAppended);
 			for await (const dataChunk of columnStream) {
 				const chunk = DuckDBDataChunk.create(chunkTypes);
-				this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
-				totalRows += dataChunk?.[0]?.length ?? 0;
-				chunk.setColumns(dataChunk);
+				const columns = new Array(numColumns);
+				for (let i = 0; i < numColumns; i++) columns[i] = dataChunk[columnKeys[i]];
+				totalRows += columns[0]?.length ?? 0;
+				chunk.setColumns(columns);
 				appender.appendDataChunk(chunk);
-				appender.flushSync();
 				appendedChunkCount += 1;
-				if (onDataAppended !== void 0) {
-					const payload = dataAppendedCollector(totalRows);
-					if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
-					else onDataAppended(payload);
+				if (onChunkAppended !== void 0 && (onChunkAppendedFrequency === void 0 || appendedChunkCount % onChunkAppendedFrequency === 0)) {
+					const payload = chunkAppendedCollector(totalRows);
+					if (isAsyncCb) await onChunkAppended(payload);
+					else onChunkAppended(payload);
 				}
+				if (flushSyncFrequency !== void 0 && appendedChunkCount % flushSyncFrequency === 0) appender.flushSync();
 				if (checkpointChunksFrequency !== void 0 && appendedChunkCount % checkpointChunksFrequency === 0 && typeof table.databaseName === "string") try {
 					await dbManager.checkpoint(table.databaseName);
 				} catch (e) {
-					this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending chunk into table '${table.getFullName()}' - ${e?.message ?? ""}`, { table: table.getFullName() });
+					this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending chunk into table '${tableName}' - ${e?.message ?? ""}`, { table: tableFullName });
 				}
 			}
+			appender.flushSync();
 			appender.closeSync();
 			if (autoCheckpoint && typeof table.databaseName === "string") try {
 				await dbManager.checkpoint(table.databaseName);
 			} catch (e) {
-				this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending data into table '${table.getFullName()}' - ${e?.message ?? ""}`, { table: table.getFullName() });
+				this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending data into table '${tableName}' - ${e?.message ?? ""}`, { table: tableFullName });
 			}
 			const timeMs = Math.round(Date.now() - timeStart);
 			this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
-				table: table.getFullName(),
+				table: tableFullName,
 				timeMs,
 				totalRows
 			});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -57,7 +57,7 @@
   },
   "dependencies": {
     "@flowblade/core": "^0.2.26",
-    "@flowblade/source-duckdb": "^0.20.1",
+    "@flowblade/source-duckdb": "^0.20.2",
     "@flowblade/sql-tag": "^0.3.2",
     "@httpx/assert": "^0.16.9",
     "@httpx/dsn-parser": "^1.9.9",
@@ -69,7 +69,7 @@
     "zod": "^4.3.6"
   },
   "peerDependencies": {
-    "@duckdb/node-api": "^1.5.0-r.1",
+    "@duckdb/node-api": "^1.5.1-r.2",
     "valibot": "^1.3.1"
   },
   "peerDependenciesMeta": {
@@ -79,20 +79,20 @@
   },
   "devDependencies": {
     "@belgattitude/eslint-config-bases": "8.12.0",
-    "@dotenvx/dotenvx": "1.59.1",
-    "@duckdb/node-api": "1.5.1-r.1",
+    "@dotenvx/dotenvx": "1.61.0",
+    "@duckdb/node-api": "1.5.1-r.2",
     "@faker-js/faker": "10.4.0",
     "@flowblade/source-kysely": "^1.3.0",
     "@httpx/assert": "0.16.9",
     "@mitata/counters": "0.0.8",
     "@size-limit/esbuild": "12.0.1",
     "@size-limit/file": "12.0.1",
-    "@testcontainers/mssqlserver": "11.13.0",
+    "@testcontainers/mssqlserver": "11.14.0",
     "@total-typescript/ts-reset": "0.6.1",
     "@types/node": "25.5.2",
     "@typescript-eslint/eslint-plugin": "8.58.1",
     "@typescript-eslint/parser": "8.58.1",
-    "@typescript/native-preview": "7.0.0-dev.20260406.1",
+    "@typescript/native-preview": "7.0.0-dev.20260410.1",
     "@vitest/coverage-v8": "4.1.4",
     "@vitest/ui": "4.1.4",
     "ansis": "4.2.0",
@@ -105,10 +105,10 @@
     "eslint": "8.57.1",
     "execa": "9.6.1",
     "is-in-ci": "2.0.0",
-    "kysely": "0.28.15",
+    "kysely": "0.28.16",
     "mitata": "1.0.34",
     "npm-run-all2": "8.0.4",
-    "prettier": "3.8.1",
+    "prettier": "3.8.2",
     "publint": "0.3.18",
     "regexp.escape": "2.0.1",
     "rimraf": "6.1.3",
@@ -116,7 +116,7 @@
     "sql-formatter": "15.7.3",
     "tarn": "3.0.2",
     "tedious": "19.2.1",
-    "testcontainers": "11.13.0",
+    "testcontainers": "11.14.0",
     "tsdown": "0.21.7",
     "tsx": "4.21.0",
     "typedoc": "0.28.18",