@flowblade/sqlduck 0.15.0 → 0.17.0

package/README.md

@@ -58,7 +58,7 @@ const result = await sqlDuck.toTable({
     rowStream: getUsers(), // The async iterable that yields rows
     // 👇Optional:
     chunkSize: 2048, // Number of rows to append when using duckdb appender. Default is 2048
-    onDataAppended: ({ timeMs, totalRows, rowsPerSecond }) => {
+    onChunkAppended: ({ timeMs, totalRows, rowsPerSecond }) => {
         console.log(
             `Appended ${totalRows} in time ${timeMs}ms, est: ${rowsPerSecond} rows/s`
         );
@@ -105,8 +105,8 @@ const result = sqlDuck.toTable({
   rowStream: getUserRows(), // The async iterable that yields rows
   // 👇Optional:
   chunkSize: 2048, // Number of rows to append when using duckdb appender. Default is 2048
-  onDataAppended: ({ total }) => {
-    console.log(`Appended ${total} rows so far`);
+  onChunkAppended: ({ totalRows }) => {
+    console.log(`Appended ${totalRows} rows so far`);
   },
   // Optional table creation options
   createOptions: {
@@ -131,67 +131,72 @@ const queryResult = await dbDuckDbMemoryConn.query<{
 ### Node 24
 
 ```
- RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
 
+ RUN  v4.1.4 /home/sebastien/github/flowblade/packages/sqlduck
 
- ✓ bench/appender.bench.ts > appender benches 4157ms
-     name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
-   · duckdb appender memory, count: 100000, chunk size 2048  2.6950  357.08  385.05  371.06  385.05  385.05  385.05  385.05  ±47.90%        2
-   · duckdb appender file, count: 100000, chunk size 2048    1.4218  703.35  703.35  703.35  703.35  703.35  703.35  703.35   ±0.00%        1
-   · duckdb appender, count: 100000, chunk size 1024         2.5157  391.12  403.89  397.50  403.89  403.89  403.89  403.89  ±20.41%        2
 
- ✓ bench/stream.bench.ts > Bench stream 2809ms
-     name                                                       hz      min     max    mean     p75     p99    p995    p999      rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.2627  87.7271  151.56  107.96  116.92  151.56  151.56  151.56  ±15.98%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.1479   125.04  168.13  139.90  152.97  168.13  168.13  168.13   ±7.78%       10
+ ✓ bench/appender.bench.ts > appender benches 2910ms
+     name                                                        hz     min     max    mean     p75     p99    p995    p999       rme  samples
+   · duckdb appender memory, count: 100000, chunk size 2048  3.5446  265.91  298.32  282.12  298.32  298.32  298.32  298.32   ±72.99%        2
+   · duckdb appender file, count: 100000, chunk size 2048    2.6130  355.30  410.10  382.70  410.10  410.10  410.10  410.10   ±91.00%        2
+   · duckdb appender, count: 100000, chunk size 1024         3.8027  226.52  299.42  262.97  299.42  299.42  299.42  299.42  ±176.17%        2
+
+ ✓ bench/stream.bench.ts > Bench rowsToColumnsChunks 2998ms
+     name                                                                         hz      min     max     mean      p75     p99    p995    p999      rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)                   11.3182  60.6404  190.56  88.3532  79.4999  190.56  190.56  190.56  ±30.64%       10
+   · rowToColumnsChunk with transformer with chunkSize 2048 (count: 100000)  13.4430  63.9716  102.73  74.3883  75.2151  102.73  102.73  102.73  ±10.40%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)                    10.7785  84.3040  117.57  92.7773  97.1595  117.57  117.57  117.57   ±7.61%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 614ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 615ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  18,899.24  0.0334  5.3351  0.0529  0.0546  0.1943  0.3087  0.7214  ±2.72%     9450
+   · getTableCreateFromZod  27,048.71  0.0239  3.2029  0.0370  0.0404  0.1310  0.1980  0.5674  ±2.08%    13525
 
  BENCH  Summary
 
-  duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.07x faster than duckdb appender, count: 100000, chunk size 1024
-    1.90x faster than duckdb appender file, count: 100000, chunk size 2048
+  duckdb appender, count: 100000, chunk size 1024 - bench/appender.bench.ts > appender benches
+    1.07x faster than duckdb appender memory, count: 100000, chunk size 2048
+    1.46x faster than duckdb appender file, count: 100000, chunk size 2048
 
-  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
-    1.30x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
+  rowToColumnsChunk with transformer with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench rowsToColumnsChunks
+    1.19x faster than rowToColumnsChunk with chunkSize 2048 (count: 100000)
+    1.25x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 ```
 
-### Bun 1.3.11
+### Bun 1.3.12
 
 ```
- RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
+ RUN  v4.1.4 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 4159ms
+ ✓ bench/appender.bench.ts > appender benches 2811ms
      name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
-   · duckdb appender memory, count: 100000, chunk size 2048  2.6465  375.34  380.38  377.86  380.38  380.38  380.38  380.38   ±8.48%        2
-   · duckdb appender file, count: 100000, chunk size 2048    1.5016  665.98  665.98  665.98  665.98  665.98  665.98  665.98   ±0.00%        1
-   · duckdb appender, count: 100000, chunk size 1024         2.2828  413.11  463.01  438.06  463.01  463.01  463.01  463.01  ±72.39%        2
+   · duckdb appender memory, count: 100000, chunk size 2048  3.9242  224.75  285.38  254.83  285.38  285.38  285.38  285.38  ±29.56%        3
+   · duckdb appender file, count: 100000, chunk size 2048    3.8209  256.09  267.34  261.72  267.34  267.34  267.34  267.34  ±27.31%        2
+   · duckdb appender, count: 100000, chunk size 1024         4.6118  196.77  234.22  216.84  234.22  234.22  234.22  234.22  ±21.62%        3
 
- ✓ bench/stream.bench.ts > Bench stream 2690ms
-     name                                                       hz      min     max    mean     p75     p99    p995    p999     rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.5675  95.6610  114.11  104.52  107.75  114.11  114.11  114.11  ±3.46%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.6895   117.83  138.26  130.05  137.51  138.26  138.26  138.26  ±4.05%       10
+ ✓ bench/stream.bench.ts > Bench rowsToColumnsChunks 2667ms
+     name                                                                         hz      min      max     mean      p75      p99     p995     p999     rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)                   14.4994  57.7717  81.8690  68.9683  78.5234  81.8690  81.8690  81.8690  ±9.43%       10
+   · rowToColumnsChunk with transformer with chunkSize 2048 (count: 100000)  13.2052  68.7139  96.8400  75.7275  79.4380  96.8400  96.8400  96.8400  ±8.22%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)                    12.7827  73.9127  85.0696  78.2310  82.5510  85.0696  85.0696  85.0696  ±3.69%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 629ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 624ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  18,892.06  0.0281  7.5844  0.0529  0.0516  0.1893  0.2477  3.2823  ±6.26%     9447
+   · getTableCreateFromZod  28,477.04  0.0191  6.4836  0.0351  0.0335  0.1071  0.1530  2.4823  ±6.21%    14239
 
  BENCH  Summary
 
-  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
-    1.24x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
+  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench rowsToColumnsChunks
+    1.10x faster than rowToColumnsChunk with transformer with chunkSize 2048 (count: 100000)
+    1.13x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 
-  duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.16x faster than duckdb appender, count: 100000, chunk size 1024
-    1.76x faster than duckdb appender file, count: 100000, chunk size 2048
+  duckdb appender, count: 100000, chunk size 1024 - bench/appender.bench.ts > appender benches
+    1.18x faster than duckdb appender memory, count: 100000, chunk size 2048
+    1.21x faster than duckdb appender file, count: 100000, chunk size 2048
 
 ```
 

package/dist/index.d.mts

@@ -7,7 +7,7 @@ import * as z from "zod";
 import { ZodObject } from "zod";
 
 //#region src/appender/data-appender-callback.d.ts
-type OnDataAppendedStats = {
+type OnChunkAppendedStats = {
   /**
    * Total number of rows appended so far (all batches included)
    */
@@ -21,9 +21,9 @@ type OnDataAppendedStats = {
    */
   rowsPerSecond: number;
 };
-type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
-type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
-type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
+type OnChunkAppendedSyncCb = (stats: OnChunkAppendedStats) => void;
+type OnChunkAppendedAsyncCb = (stats: OnChunkAppendedStats) => Promise<void>;
+type OnChunkAppendedCb = OnChunkAppendedSyncCb | OnChunkAppendedAsyncCb;
 //#endregion
 //#region src/helpers/duck-memory.d.ts
 declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
@@ -81,7 +81,7 @@ declare class Table {
 }
 //#endregion
 //#region src/table/table-schema-zod.type.d.ts
-type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID;
+type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID | z.ZodEnum;
 type TableSchemaZod = z.ZodObject<Record<string, ZodSchemaSupportedTypes | z.ZodNullable<ZodSchemaSupportedTypes> | z.ZodCodec | z.ZodNullable<z.ZodCodec>>>;
 //#endregion
 //#region src/table/get-table-create-from-zod.d.ts
@@ -113,41 +113,65 @@ type SqlDuckParams = {
 type RowStream<T> = AsyncIterableIterator<T> | AsyncGenerator<T> | Generator<T>;
 type ToTableParams<TSchema extends TableSchemaZod> = {
   /**
-   * Used to create and fill the data into the table
+   * The target table where the data will be inserted.
+   * This object contains the table name and optionally the schema and database name.
    */
   table: Table;
   /**
-   * Schema describing the table structure and rowStream content
+   * A Zod schema that defines the structure of the table and the expected format of the rows in the `rowStream`.
+   * The schema is used to generate the `CREATE TABLE` DDL and to convert row values to DuckDB types.
    */
   schema: TSchema;
   /**
-   * Stream of rows to insert into the table
+   * An iterable (async or sync) or generator that yields rows to be inserted.
+   * Each row must match the structure defined in the `schema`.
    */
   rowStream: RowStream<z.infer<TSchema>>;
   /**
-   * Chunk size when using appender to insert data.
-   * Valid numbers between 1 and 2048.
+   * The number of rows to accumulate before appending them to the DuckDB table as a single data chunk.
+   * Tuning this value can impact memory usage and insertion performance.
+   * Valid values are between 1 and 2048.
    * @default 2048
    */
   chunkSize?: number;
   /**
-   * Extra options when creating the table
+   * Configuration options for the `CREATE TABLE` statement (e.g., `IF NOT EXISTS`, `CREATE OR REPLACE`).
+   * If omitted, a standard `CREATE TABLE` statement is used.
    */
   createOptions?: TableCreateOptions;
   /**
-   * Callback called each time a datachunk is appended to the table
+   * An optional callback invoked after each data chunk is successfully appended to the table.
+   * Useful for tracking progress, logging statistics, or implementing custom hooks during the insertion process.
    */
-  onDataAppended?: OnDataAppendedCb;
+  onChunkAppended?: OnChunkAppendedCb;
   /**
-   * Automatically checkpoint the table after all chunks have been appended.
+   * Specifies the frequency (in number of chunks) at which the `onChunkAppended` callback should be triggered.
+   *
+   * For example, if `chunkSize` is 2048 and `onChunkAppendedFrequency` is 5,
+   * the callback will be called every 10,240 rows (5 chunks * 2048 rows/chunk).
+   *
+   * @default 1
+   */
+  onChunkAppendedFrequency?: number;
+  /**
+   * Specifies the frequency (in number of chunks) at which the `appender.flushSync()` should be called.
+   * Calling `flushSync()` can help to clear internal buffers and make the data visible.
+   *
+   * For example, if `chunkSize` is 2048 and `flushSyncFrequency` is 5,
+   * the appender will be flushed every 10,240 rows (5 chunks * 2048 rows/chunk).
+   */
+  flushSyncFrequency?: number;
+  /**
+   * If set to `true`, a checkpoint is automatically performed after all rows from the `rowStream` have been processed.
+   * This ensures that all data is persisted and WAL is cleared.
    * @default true
    */
   autoCheckpoint?: boolean;
   /**
-   * Checkpoint the table after 'n' chunks have been appended
+   * Specifies the frequency (in number of chunks) at which a checkpoint should be triggered.
    *
-   * For example if the chunkSize is 2048, setting frequency to 2
-   * will checkpoint the table every 4096 rows (2x chunksize)
+   * For example, if `chunkSize` is 2048 and `checkpointChunksFrequency` is 5,
+   * a checkpoint will occur every 10,240 rows (5 chunks * 2048 rows/chunk).
    */
   checkpointChunksFrequency?: number;
 };
@@ -193,9 +217,13 @@ declare class SqlDuck {
    *  schema: userSchema,
    *  rowStream: getUserRows(),
    *  chunkSize: 2048,
-   *  onDataAppended: ({ total }) => {
-   *    console.log(`Appended ${total} rows so far`);
+   *  flushSyncFrequency: 10, // flush after every 10 chunks
+   *  onChunkAppendedFrequency: 1, // multiple of chunks
+   *  onChunkAppended: ({ totalRows }) => {
+   *    console.log(`Appended ${totalRows} rows so far`);
    *  },
+   *  autoCheckpoint: true,
+   *  checkpointChunksFrequency: 100, // checkpoint after every 100 chunks
    *  createOptions: {
    *    create: 'CREATE_OR_REPLACE',
    *  },
@@ -302,4 +330,4 @@ declare const flowbladeLogtapeSqlduckConfig: {
 //#region src/logger/sqlduck-default-logtape-logger.d.ts
 declare const sqlduckDefaultLogtapeLogger: _$_logtape_logtape0.Logger;
 //#endregion
-export { Database, type DuckConnectionParams, DuckDatabaseManager, DuckMemory, type DuckMemoryTag, type DuckdbReservedKeywords, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
+export { Database, type DuckConnectionParams, DuckDatabaseManager, DuckMemory, type DuckMemoryTag, type DuckdbReservedKeywords, type OnChunkAppendedCb, type OnChunkAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/dist/index.mjs

@@ -1,6 +1,6 @@
 import { t as duckReservedKeywords } from "./duck-reserved-keywords-B8XUjnaY.mjs";
 import { n as assertValidAliasName, o as duckConnectionParamsZodSchema, s as duckValidatorsZod } from "./zod-CuPjTLv8.mjs";
-import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBInstanceCache, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
+import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBInstanceCache, DuckDBTimestampMillisecondsValue, DuckDBTypeId, ENUM, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TIMESTAMP_MS, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
 import { getLogger } from "@logtape/logtape";
 import fs from "node:fs";
 import { basename, dirname } from "node:path";
@@ -104,10 +104,12 @@ var DuckMemory = class {
 };
 //#endregion
 //#region src/appender/data-appender-callback.ts
-const isOnDataAppendedAsyncCb = (v) => {
-	return v.constructor.name === "AsyncFunction";
+const isOnChunkAppendedAsyncCb = (v) => {
+	return v.constructor.name === "AsyncFunction" || v.constructor === (async () => {
+		await Promise.resolve();
+	}).constructor;
 };
-const createOnDataAppendedCollector = () => {
+const createOnChunkAppendedCollector = () => {
 	let lastCallbackTimeStart = Date.now();
 	let appendedTotalRows = 0;
 	return (currentTotalRows) => {
@@ -124,6 +126,110 @@ const createOnDataAppendedCollector = () => {
 	};
 };
 //#endregion
+//#region src/converter/duck-value-converter.ts
+const stringTimestampRegexp = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d{3,6})?Z?$/i;
+const dateRegexp = /^\d{4}-\d{2}-\d{2}$/;
+const createDuckValueConverterTypeError = (params) => {
+	let serializableValue;
+	try {
+		serializableValue = JSON.stringify(params.value);
+	} catch {
+		serializableValue = "<unserializable>";
+	}
+	return /* @__PURE__ */ new TypeError(`[DuckValueConverter.${params.method}]: Unsupported type ${typeof params.value} with value ${serializableValue}`);
+};
+var DuckValueConverter = class {
+	toUUID = (value) => {
+		if (typeof value === "bigint") return value;
+		else if (typeof value === "string") return BigInt("0x" + value.replaceAll("-", ""));
+		if (value === void 0 || value === null) return null;
+		throw createDuckValueConverterTypeError({
+			method: "toUUID",
+			value
+		});
+	};
+	toStringEnum = (value) => {
+		if (typeof value === "string") return value;
+		if (value === void 0 || value === null) return null;
+		throw createDuckValueConverterTypeError({
+			method: "toStringEnum",
+			value
+		});
+	};
+	toBigIntString = (value) => {
+		if (typeof value === "string") return value;
+		if (typeof value === "number" || typeof value === "bigint") return value.toString(10);
+		if (value === void 0 || value === null) return null;
+		throw createDuckValueConverterTypeError({
+			method: "toBigIntString",
+			value
+		});
+	};
+	toTimestampMs = (value) => {
+		if (value instanceof Date) return new DuckDBTimestampMillisecondsValue(BigInt(value.getTime()));
+		if (value === void 0 || value === null) return null;
+		if (typeof value === "string") {
+			const len = value.length;
+			if (len > 18 && len < 31 && stringTimestampRegexp.test(value)) {
+				const date = /* @__PURE__ */ new Date(value + (value.endsWith("Z") ? "" : "Z"));
+				return new DuckDBTimestampMillisecondsValue(BigInt(date.getTime()));
+			}
+			if (len === 10 && dateRegexp.test(value)) {
+				const date = /* @__PURE__ */ new Date(value + "T00:00:00Z");
+				return new DuckDBTimestampMillisecondsValue(BigInt(date.getTime()));
+			}
+		}
+		if (typeof value === "bigint") return new DuckDBTimestampMillisecondsValue(value);
+		if (typeof value === "number") return new DuckDBTimestampMillisecondsValue(BigInt(value));
+		throw createDuckValueConverterTypeError({
+			method: "toTimestampMs",
+			value
+		});
+	};
+};
+//#endregion
+//#region src/converter/create-duck-column-converters.ts
+const createDuckColumnConverters = (duckTypes) => {
+	const convMap = {};
+	const converter = new DuckValueConverter();
+	for (const [key, duckType] of Object.entries(duckTypes)) {
+		let conv;
+		const duckTypeId = duckType.typeId;
+		switch (duckTypeId) {
+			case DuckDBTypeId.TIMESTAMP_MS:
+				conv = converter.toTimestampMs;
+				break;
+			case DuckDBTypeId.BIGINT:
+			case DuckDBTypeId.UBIGINT:
+			case DuckDBTypeId.HUGEINT:
+			case DuckDBTypeId.UHUGEINT:
+			case DuckDBTypeId.INTEGER:
+			case DuckDBTypeId.UINTEGER:
+			case DuckDBTypeId.BIGNUM:
+				conv = converter.toBigIntString;
+				break;
+			case DuckDBTypeId.ENUM:
+				conv = converter.toStringEnum;
+				break;
+			case DuckDBTypeId.UUID:
+				conv = converter.toUUID;
+				break;
+			case DuckDBTypeId.BIT:
+			case DuckDBTypeId.BOOLEAN:
+			case DuckDBTypeId.TINYINT:
+			case DuckDBTypeId.USMALLINT:
+			case DuckDBTypeId.UTINYINT:
+			case DuckDBTypeId.VARCHAR:
+			case DuckDBTypeId.SMALLINT:
+				conv = false;
+				break;
+			default: throw new Error(`Unsupported duck type ${duckTypeId} / ${duckType.toString()} for column '${key}'`);
+		}
+		if (conv !== false) convMap[key] = conv;
+	}
+	return convMap;
+};
+//#endregion
 //#region src/config/flowblade-logtape-sqlduck.config.ts
 const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
 //#endregion
@@ -442,7 +548,10 @@ const createOptions = {
 const duckDbTypesMap = new Map([
 	["VARCHAR", VARCHAR],
 	["BIGINT", BIGINT],
+	["UBIGINT", UBIGINT],
+	["HUGEINT", HUGEINT],
 	["TIMESTAMP", TIMESTAMP],
+	["TIMESTAMP_MS", TIMESTAMP_MS],
 	["UUID", UUID],
 	["BOOLEAN", BOOLEAN],
 	["INTEGER", INTEGER],
@@ -466,9 +575,10 @@ const getTableCreateFromZod = (params) => {
 		if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
 		else switch (type) {
 			case "string":
-				switch (format) {
+				if (Array.isArray(def.enum)) c.duckdbType = ENUM(def.enum);
+				else switch (format) {
 					case "date-time":
-						c.duckdbType = TIMESTAMP;
+						c.duckdbType = TIMESTAMP_MS;
 						break;
 					case "int64":
 						c.duckdbType = BIGINT;
@@ -541,41 +651,90 @@ const createTableFromZod = async (params) => {
 };
 //#endregion
 //#region src/utils/rows-to-columns-chunks.ts
-const toDuckValue = (value) => {
-	if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
-	if (typeof value === "bigint") return value.toString(10);
-	return value === void 0 ? null : value;
-};
 /**
-* Similar to `rowsToColumns` but yields results in chunks to avoid buffering
-* the entire dataset in memory. Each yielded item is a columns array for up to
-* `chunkSize` rows.
+* Converts a stream of rows (row-oriented) into a stream of column-oriented chunks.
+*
+* This function processes row data incrementally using an async generator, which prevents
+* loading the entire dataset into memory. Each yielded chunk is an object where keys are
+* column names and values are arrays of up to `chunkSize` elements.
+*
+* This is particularly useful for DuckDB's Appender API or other columnar processing
+* engines that expect data in chunks of columns.
+*
+* @param params - Configuration for the transformation.
+* @param params.rows - An async or sync iterable of rows.
+* @param params.chunkSize - The maximum number of rows per yielded chunk. Must be a positive integer.
+* @param params.transformers - Optional mappers for specific columns to transform values before chunking.
 *
-* Example for chunkSize = 2:
-*   input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
-*   yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
+* @returns An async iterator yielding chunks of column-oriented data.
+*
+* @example
+* ```typescript
+*  async function* generateRows() {
+*    yield { id: 1, name: 'A' };
+*    yield { id: 2, name: 'B' };
+*    yield { id: 3, name: 'C' };
+*  }
+*
+*  const columnChunks = rowsToColumnsChunks({
+*    rows: generateRows(),
+*    chunkSize: 2,
+*  })
+*
+* for await (const chunk of columnChunks) {
+*   console.log(chunk);
+* }
+* // Output:
+* // { id: [1, 2], name: ['A', 'B'] } // first chunk
+* // { id: [3], name: ['C'] } // second chunk
+* ```
 */
 async function* rowsToColumnsChunks(params) {
-	const { rows, chunkSize } = params;
+	const { rows, chunkSize, transformers } = params;
 	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
 	const first = await rows.next();
 	if (first.done) return;
 	const keys = Object.keys(first.value);
-	let columns = keys.map(() => []);
+	const numKeys = keys.length;
+	const mappers = new Array(numKeys);
+	if (transformers !== void 0) {
+		const unknownKeys = Object.keys(transformers).filter((k) => !keys.includes(k));
+		if (unknownKeys.length > 0) throw new Error(`transformers parameter contains unknown row ids: ${unknownKeys.join(", ")}`);
+		for (let i = 0; i < numKeys; i++) mappers[i] = transformers[keys[i]];
+	}
+	function createColumns() {
+		const obj = {};
+		for (let i = 0; i < numKeys; i++) {
+			const k = keys[i];
+			obj[k] = [];
+		}
+		return obj;
+	}
+	let columns = createColumns();
 	let rowsInChunk = 0;
-	keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
+	for (let i = 0; i < numKeys; i++) {
+		const k = keys[i];
+		const fn = mappers[i];
+		const val = first.value[k];
+		columns[k].push(fn === void 0 ? val : fn(val));
+	}
 	rowsInChunk++;
 	if (rowsInChunk >= chunkSize) {
 		yield columns;
-		columns = keys.map(() => []);
+		columns = createColumns();
 		rowsInChunk = 0;
 	}
 	for await (const row of rows) {
-		keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
+		for (let i = 0; i < numKeys; i++) {
+			const k = keys[i];
+			const fn = mappers[i];
+			const val = row[k];
+			columns[k].push(fn === void 0 ? val : fn(val));
+		}
 		rowsInChunk++;
 		if (rowsInChunk >= chunkSize) {
 			yield columns;
-			columns = keys.map(() => []);
+			columns = createColumns();
 			rowsInChunk = 0;
 		}
 	}
@@ -615,9 +774,13 @@ var SqlDuck = class {
 	*  schema: userSchema,
 	*  rowStream: getUserRows(),
 	*  chunkSize: 2048,
-	*  onDataAppended: ({ total }) => {
-	*    console.log(`Appended ${total} rows so far`);
+	*  flushSyncFrequency: 10, // flush after every 10 chunks
+	*  onChunkAppendedFrequency: 1, // multiple of chunks
+	*  onChunkAppended: ({ totalRows }) => {
+	*    console.log(`Appended ${totalRows} rows so far`);
 	*  },
+	*  autoCheckpoint: true,
+	*  checkpointChunksFrequency: 100, // checkpoint after every 100 chunks
 	*  createOptions: {
 	*    create: 'CREATE_OR_REPLACE',
 	*  },
@@ -628,11 +791,13 @@ var SqlDuck = class {
 	* ```
 	*/
 	toTable = async (params) => {
-		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended, autoCheckpoint = true, checkpointChunksFrequency = 10 } = params;
+		const { table, schema, chunkSize = 2048, rowStream, createOptions, onChunkAppended, onChunkAppendedFrequency, flushSyncFrequency = 10, autoCheckpoint = true, checkpointChunksFrequency } = params;
 		if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
 		if (autoCheckpoint && typeof table.databaseName !== "string") throw new Error("autoCheckpoint requires table.databaseName to be provided.");
-		if (checkpointChunksFrequency && typeof table.databaseName !== "string") throw new Error("checkpointChunksFrequency requires table.databaseName to be provided.");
-		if (checkpointChunksFrequency !== void 0 && checkpointChunksFrequency < 1) throw new Error("checkpointChunksFrequency must be a positive number.");
+		if (checkpointChunksFrequency !== void 0 && typeof table.databaseName !== "string") throw new Error("checkpointChunksFrequency requires table.databaseName to be provided.");
+		if (checkpointChunksFrequency !== void 0 && (checkpointChunksFrequency < 1 || checkpointChunksFrequency > 1e5)) throw new Error("checkpointChunksFrequency must be a number between 1 and 100_000.");
+		if (onChunkAppendedFrequency !== void 0 && (onChunkAppendedFrequency < 1 || onChunkAppendedFrequency > 1e5)) throw new Error("onChunkAppendedFrequency must be a number between 1 and 100_000.");
+		if (flushSyncFrequency !== void 0 && (flushSyncFrequency < 1 || flushSyncFrequency > 1e5)) throw new Error("flushSyncFrequency must be a number between 1 and 100_000.");
 		const dbManager = new DuckDatabaseManager(this.#conn);
 		const timeStart = Date.now();
 		const { columnTypes, ddl } = await createTableFromZod({
@@ -643,42 +808,56 @@ var SqlDuck = class {
 		});
 		const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
 		const chunkTypes = Array.from(columnTypes.values());
+		const columnTypeIds = {};
+		const columnKeys = [];
+		for (const [key, duckType] of columnTypes) {
+			columnKeys.push(key);
+			columnTypeIds[key] = duckType;
+		}
+		const numColumns = columnKeys.length;
+		const transformers = createDuckColumnConverters(columnTypeIds);
 		let totalRows = 0;
-		const dataAppendedCollector = createOnDataAppendedCollector();
+		const chunkAppendedCollector = createOnChunkAppendedCollector();
 		const columnStream = rowsToColumnsChunks({
 			rows: rowStream,
-			chunkSize
+			chunkSize,
+			transformers
 		});
 		let appendedChunkCount = 0;
+		const tableFullName = table.getFullName();
+		const tableName = table.tableName;
 		try {
+			const isAsyncCb = onChunkAppended !== void 0 && isOnChunkAppendedAsyncCb(onChunkAppended);
 			for await (const dataChunk of columnStream) {
 				const chunk = DuckDBDataChunk.create(chunkTypes);
-				this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
-				totalRows += dataChunk?.[0]?.length ?? 0;
-				chunk.setColumns(dataChunk);
+				const columns = new Array(numColumns);
+				for (let i = 0; i < numColumns; i++) columns[i] = dataChunk[columnKeys[i]];
+				totalRows += columns[0]?.length ?? 0;
+				chunk.setColumns(columns);
 				appender.appendDataChunk(chunk);
-				appender.flushSync();
 				appendedChunkCount += 1;
-				if (onDataAppended !== void 0) {
-					const payload = dataAppendedCollector(totalRows);
-					if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
-					else onDataAppended(payload);
+				if (onChunkAppended !== void 0 && (onChunkAppendedFrequency === void 0 || appendedChunkCount % onChunkAppendedFrequency === 0)) {
+					const payload = chunkAppendedCollector(totalRows);
+					if (isAsyncCb) await onChunkAppended(payload);
+					else onChunkAppended(payload);
 				}
+				if (flushSyncFrequency !== void 0 && appendedChunkCount % flushSyncFrequency === 0) appender.flushSync();
 				if (checkpointChunksFrequency !== void 0 && appendedChunkCount % checkpointChunksFrequency === 0 && typeof table.databaseName === "string") try {
 					await dbManager.checkpoint(table.databaseName);
 				} catch (e) {
-					this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending chunk into table '${table.getFullName()}' - ${e?.message ?? ""}`, { table: table.getFullName() });
+					this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending chunk into table '${tableName}' - ${e?.message ?? ""}`, { table: tableFullName });
 				}
 			}
+			appender.flushSync();
 			appender.closeSync();
 			if (autoCheckpoint && typeof table.databaseName === "string") try {
 				await dbManager.checkpoint(table.databaseName);
 			} catch (e) {
-				this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending data into table '${table.getFullName()}' - ${e?.message ?? ""}`, { table: table.getFullName() });
+				this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending data into table '${tableName}' - ${e?.message ?? ""}`, { table: tableFullName });
 			}
 			const timeMs = Math.round(Date.now() - timeStart);
 			this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
-				table: table.getFullName(),
+				table: tableFullName,
 				timeMs,
 				totalRows
 			});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.15.0",
+  "version": "0.17.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -57,7 +57,7 @@
   },
   "dependencies": {
     "@flowblade/core": "^0.2.26",
-    "@flowblade/source-duckdb": "^0.20.1",
+    "@flowblade/source-duckdb": "^0.20.2",
     "@flowblade/sql-tag": "^0.3.2",
     "@httpx/assert": "^0.16.9",
     "@httpx/dsn-parser": "^1.9.9",
@@ -65,11 +65,11 @@
     "@logtape/logtape": "^2.0.5",
     "@standard-schema/spec": "^1.1.0",
     "is-safe-filename": "0.1.1",
-    "p-queue": "9.1.1",
+    "p-queue": "9.1.2",
     "zod": "^4.3.6"
   },
   "peerDependencies": {
-    "@duckdb/node-api": "^1.5.0-r.1",
+    "@duckdb/node-api": "^1.5.1-r.2",
     "valibot": "^1.3.1"
   },
   "peerDependenciesMeta": {
@@ -79,22 +79,22 @@
   },
   "devDependencies": {
     "@belgattitude/eslint-config-bases": "8.12.0",
-    "@dotenvx/dotenvx": "1.59.1",
-    "@duckdb/node-api": "1.5.1-r.1",
+    "@dotenvx/dotenvx": "1.61.0",
+    "@duckdb/node-api": "1.5.1-r.2",
     "@faker-js/faker": "10.4.0",
     "@flowblade/source-kysely": "^1.3.0",
     "@httpx/assert": "0.16.9",
     "@mitata/counters": "0.0.8",
     "@size-limit/esbuild": "12.0.1",
     "@size-limit/file": "12.0.1",
-    "@testcontainers/mssqlserver": "11.13.0",
+    "@testcontainers/mssqlserver": "11.14.0",
     "@total-typescript/ts-reset": "0.6.1",
     "@types/node": "25.5.2",
     "@typescript-eslint/eslint-plugin": "8.58.1",
     "@typescript-eslint/parser": "8.58.1",
-    "@typescript/native-preview": "7.0.0-dev.20260406.1",
-    "@vitest/coverage-v8": "4.1.3",
-    "@vitest/ui": "4.1.3",
+    "@typescript/native-preview": "7.0.0-dev.20260410.1",
+    "@vitest/coverage-v8": "4.1.4",
+    "@vitest/ui": "4.1.4",
     "ansis": "4.2.0",
     "browserslist-to-esbuild": "2.1.1",
     "core-js": "3.49.0",
@@ -105,10 +105,10 @@
     "eslint": "8.57.1",
     "execa": "9.6.1",
     "is-in-ci": "2.0.0",
-    "kysely": "0.28.15",
+    "kysely": "0.28.16",
     "mitata": "1.0.34",
     "npm-run-all2": "8.0.4",
-    "prettier": "3.8.1",
+    "prettier": "3.8.2",
     "publint": "0.3.18",
     "regexp.escape": "2.0.1",
     "rimraf": "6.1.3",
@@ -116,14 +116,14 @@
     "sql-formatter": "15.7.3",
     "tarn": "3.0.2",
     "tedious": "19.2.1",
-    "testcontainers": "11.13.0",
+    "testcontainers": "11.14.0",
     "tsdown": "0.21.7",
     "tsx": "4.21.0",
     "typedoc": "0.28.18",
     "typedoc-plugin-markdown": "4.11.0",
     "typescript": "6.0.2",
     "valibot": "1.3.1",
-    "vitest": "4.1.3"
+    "vitest": "4.1.4"
   },
   "files": [
     "dist"