@flowblade/sqlduck 0.14.0 → 0.16.0

package/README.md

@@ -131,32 +131,32 @@ const queryResult = await dbDuckDbMemoryConn.query<{
 ### Node 24
 
 ```
- RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
+ RUN  v4.1.4 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 4157ms
-     name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
-   · duckdb appender memory, count: 100000, chunk size 2048  2.6950  357.08  385.05  371.06  385.05  385.05  385.05  385.05  ±47.90%        2
-   · duckdb appender file, count: 100000, chunk size 2048    1.4218  703.35  703.35  703.35  703.35  703.35  703.35  703.35   ±0.00%        1
-   · duckdb appender, count: 100000, chunk size 1024         2.5157  391.12  403.89  397.50  403.89  403.89  403.89  403.89  ±20.41%        2
+ ✓ bench/appender.bench.ts > appender benches 3412ms
+     name                                                        hz     min     max    mean     p75     p99    p995    p999       rme  samples
+   · duckdb appender memory, count: 100000, chunk size 2048  2.8872  295.84  396.87  346.35  396.87  396.87  396.87  396.87  ±185.36%        2
+   · duckdb appender file, count: 100000, chunk size 2048    1.7908  558.40  558.40  558.40  558.40  558.40  558.40  558.40    ±0.00%        1
+   · duckdb appender, count: 100000, chunk size 1024         1.9967  500.82  500.82  500.82  500.82  500.82  500.82  500.82    ±0.00%        1
 
- ✓ bench/stream.bench.ts > Bench stream 2809ms
-     name                                                       hz      min     max    mean     p75     p99    p995    p999      rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.2627  87.7271  151.56  107.96  116.92  151.56  151.56  151.56  ±15.98%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.1479   125.04  168.13  139.90  152.97  168.13  168.13  168.13   ±7.78%       10
+ ✓ bench/stream.bench.ts > Bench stream 2140ms
+     name                                                        hz      min     max     mean      p75     p99    p995    p999      rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  11.8099  63.0430  172.82  84.6748  81.4757  172.82  172.82  172.82  ±27.12%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)   10.0442  79.5805  125.81  99.5603   112.35  125.81  125.81  125.81  ±12.04%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 614ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 617ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  18,899.24  0.0334  5.3351  0.0529  0.0546  0.1943  0.3087  0.7214  ±2.72%     9450
+   · getTableCreateFromZod  27,583.03  0.0226  2.7818  0.0363  0.0403  0.1185  0.1583  0.4321  ±1.66%    13792
 
  BENCH  Summary
 
   duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.07x faster than duckdb appender, count: 100000, chunk size 1024
-    1.90x faster than duckdb appender file, count: 100000, chunk size 2048
+    1.45x faster than duckdb appender, count: 100000, chunk size 1024
+    1.61x faster than duckdb appender file, count: 100000, chunk size 2048
 
   rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
-    1.30x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
+    1.18x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 ```
@@ -164,34 +164,34 @@ const queryResult = await dbDuckDbMemoryConn.query<{
 ### Bun 1.3.11
 
 ```
- RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
+ RUN  v4.1.4 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 4159ms
+ ✓ bench/appender.bench.ts > appender benches 3357ms
      name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
-   · duckdb appender memory, count: 100000, chunk size 2048  2.6465  375.34  380.38  377.86  380.38  380.38  380.38  380.38   ±8.48%        2
-   · duckdb appender file, count: 100000, chunk size 2048    1.5016  665.98  665.98  665.98  665.98  665.98  665.98  665.98   ±0.00%        1
-   · duckdb appender, count: 100000, chunk size 1024         2.2828  413.11  463.01  438.06  463.01  463.01  463.01  463.01  ±72.39%        2
+   · duckdb appender memory, count: 100000, chunk size 2048  2.9741  315.71  356.77  336.24  356.77  356.77  356.77  356.77  ±77.59%        2
+   · duckdb appender file, count: 100000, chunk size 2048    1.8953  527.62  527.62  527.62  527.62  527.62  527.62  527.62   ±0.00%        1
+   · duckdb appender, count: 100000, chunk size 1024         1.7803  561.70  561.70  561.70  561.70  561.70  561.70  561.70   ±0.00%        1
 
- ✓ bench/stream.bench.ts > Bench stream 2690ms
-     name                                                       hz      min     max    mean     p75     p99    p995    p999     rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.5675  95.6610  114.11  104.52  107.75  114.11  114.11  114.11  ±3.46%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.6895   117.83  138.26  130.05  137.51  138.26  138.26  138.26  ±4.05%       10
+ ✓ bench/stream.bench.ts > Bench stream 2058ms
+     name                                                        hz      min     max     mean      p75     p99    p995    p999      rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  12.0130  60.7081  111.32  83.2432  99.7116  111.32  111.32  111.32  ±14.15%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)   10.9710  76.4253  145.14  91.1493  91.1424  145.14  145.14  145.14  ±16.04%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 629ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 621ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  18,892.06  0.0281  7.5844  0.0529  0.0516  0.1893  0.2477  3.2823  ±6.26%     9447
+   · getTableCreateFromZod  27,472.91  0.0185  4.6226  0.0364  0.0369  0.1137  0.1476  1.9138  ±5.20%    13737
 
  BENCH  Summary
 
   rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
-    1.24x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
+    1.09x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 
   duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.16x faster than duckdb appender, count: 100000, chunk size 1024
-    1.76x faster than duckdb appender file, count: 100000, chunk size 2048
+    1.57x faster than duckdb appender file, count: 100000, chunk size 2048
+    1.67x faster than duckdb appender, count: 100000, chunk size 1024
 
 ```
 

package/dist/{duck-reserved-keywords-CUmaTbrD.mjs → duck-reserved-keywords-B8XUjnaY.mjs}

@@ -3,12 +3,20 @@ const duckIdentifierNameRegex = /^[a-z_]\w*$/i;
 const duckStorageVersionRegexp = /^v?\d{1,4}\.\d{1,4}\.\d{1,4}$/;
 //#endregion
 //#region src/validation/core/duck-connections-options.ts
-const duckConnectionsOptions = { types: [
-	"DUCKDB",
-	"SQLITE",
-	"MYSQL",
-	"PostgreSQL"
-] };
+const duckConnectionsOptions = {
+	types: [
+		"DUCKDB",
+		"SQLITE",
+		"MYSQL",
+		"PostgreSQL"
+	],
+	accessModes: ["READ_ONLY", "READ_WRITE"],
+	encryptionCiphers: [
+		"CBC",
+		"CTR",
+		"GCM"
+	]
+};
 //#endregion
 //#region src/validation/core/duck-reserved-keywords.ts
 /**

package/dist/index.d.mts

@@ -81,7 +81,7 @@ declare class Table {
 }
 //#endregion
 //#region src/table/table-schema-zod.type.d.ts
-type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID;
+type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID | z.ZodEnum;
 type TableSchemaZod = z.ZodObject<Record<string, ZodSchemaSupportedTypes | z.ZodNullable<ZodSchemaSupportedTypes> | z.ZodCodec | z.ZodNullable<z.ZodCodec>>>;
 //#endregion
 //#region src/table/get-table-create-from-zod.d.ts
@@ -283,6 +283,15 @@ declare class DuckDatabaseManager {
   analyze: () => Promise<boolean>;
   checkpoint: (dbAlias: string) => Promise<boolean>;
   vacuum: () => Promise<boolean>;
+  /**
+   * Helper to create an initial database file.
+   */
+  createDatabaseFile: (params: {
+    path: string;
+    createDirectory?: boolean;
+  }) => Promise<{
+    status: "exists" | "created";
+  }>;
 }
 //#endregion
 //#region src/config/flowblade-logtape-sqlduck.config.d.ts

package/dist/index.mjs

@@ -1,7 +1,9 @@
-import { t as duckReservedKeywords } from "./duck-reserved-keywords-CUmaTbrD.mjs";
-import { n as assertValidAliasName, o as duckConnectionParamsZodSchema, s as duckValidatorsZod } from "./zod-BRUUIdGW.mjs";
-import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
+import { t as duckReservedKeywords } from "./duck-reserved-keywords-B8XUjnaY.mjs";
+import { n as assertValidAliasName, o as duckConnectionParamsZodSchema, s as duckValidatorsZod } from "./zod-CuPjTLv8.mjs";
+import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBInstanceCache, DuckDBTimestampMillisecondsValue, DuckDBTypeId, ENUM, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TIMESTAMP_MS, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
 import { getLogger } from "@logtape/logtape";
+import fs from "node:fs";
+import { basename, dirname } from "node:path";
 import * as z from "zod";
 import { assertNever } from "@httpx/assert";
 import { isPlainObject } from "@httpx/plain-object";
@@ -122,12 +124,191 @@ const createOnDataAppendedCollector = () => {
 	};
 };
 //#endregion
+//#region src/converter/duck-value-converter.ts
+const stringTimestampRegexp = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d{3,6})?Z?$/i;
+const dateRegexp = /^\d{4}-\d{2}-\d{2}$/;
+const createDuckValueConverterTypeError = (params) => {
+	let serializableValue;
+	try {
+		serializableValue = JSON.stringify(params.value);
+	} catch {
+		serializableValue = "<unserializable>";
+	}
+	return /* @__PURE__ */ new TypeError(`[DuckValueConverter.${params.method}]: Unsupported type ${typeof params.value} with value ${serializableValue}`);
+};
+var DuckValueConverter = class {
+	toUUID = (value) => {
+		if (typeof value === "bigint") return value;
+		else if (typeof value === "string") return BigInt("0x" + value.replaceAll("-", ""));
+		if (value === void 0 || value === null) return null;
+		throw createDuckValueConverterTypeError({
+			method: "toUUID",
+			value
+		});
+	};
+	toStringEnum = (value) => {
+		if (typeof value === "string") return value;
+		if (value === void 0 || value === null) return null;
+		throw createDuckValueConverterTypeError({
+			method: "toStringEnum",
+			value
+		});
+	};
+	toBigIntString = (value) => {
+		if (typeof value === "string") return value;
+		if (typeof value === "number" || typeof value === "bigint") return value.toString(10);
+		if (value === void 0 || value === null) return null;
+		throw createDuckValueConverterTypeError({
+			method: "toBigIntString",
+			value
+		});
+	};
+	toTimestampMs = (value) => {
+		if (value instanceof Date) return new DuckDBTimestampMillisecondsValue(BigInt(value.getTime()));
+		if (value === void 0 || value === null) return null;
+		if (typeof value === "string") {
+			const len = value.length;
+			if (len > 18 && len < 31 && stringTimestampRegexp.test(value)) {
+				const date = /* @__PURE__ */ new Date(value + (value.endsWith("Z") ? "" : "Z"));
+				return new DuckDBTimestampMillisecondsValue(BigInt(date.getTime()));
+			}
+			if (len === 10 && dateRegexp.test(value)) {
+				const date = /* @__PURE__ */ new Date(value + "T00:00:00Z");
+				return new DuckDBTimestampMillisecondsValue(BigInt(date.getTime()));
+			}
+		}
+		if (typeof value === "bigint") return new DuckDBTimestampMillisecondsValue(value);
+		if (typeof value === "number") return new DuckDBTimestampMillisecondsValue(BigInt(value));
+		throw createDuckValueConverterTypeError({
+			method: "toTimestampMs",
+			value
+		});
+	};
+};
+//#endregion
+//#region src/converter/create-duck-column-converters.ts
+const createDuckColumnConverters = (duckTypes) => {
+	const convMap = /* @__PURE__ */ new Map();
+	const converter = new DuckValueConverter();
+	for (const [key, duckType] of Object.entries(duckTypes)) {
+		let conv;
+		const duckTypeId = duckType.typeId;
+		switch (duckTypeId) {
+			case DuckDBTypeId.TIMESTAMP_MS:
+				conv = converter.toTimestampMs;
+				break;
+			case DuckDBTypeId.BIGINT:
+			case DuckDBTypeId.UBIGINT:
+			case DuckDBTypeId.HUGEINT:
+			case DuckDBTypeId.UHUGEINT:
+			case DuckDBTypeId.INTEGER:
+			case DuckDBTypeId.UINTEGER:
+			case DuckDBTypeId.BIGNUM:
+				conv = converter.toBigIntString;
+				break;
+			case DuckDBTypeId.ENUM:
+				conv = converter.toStringEnum;
+				break;
+			case DuckDBTypeId.UUID:
+				conv = converter.toUUID;
+				break;
+			case DuckDBTypeId.BIT:
+			case DuckDBTypeId.BOOLEAN:
+			case DuckDBTypeId.TINYINT:
+			case DuckDBTypeId.USMALLINT:
+			case DuckDBTypeId.UTINYINT:
+			case DuckDBTypeId.VARCHAR:
+			case DuckDBTypeId.SMALLINT:
+				conv = false;
+				break;
+			default: throw new Error(`Unsupported duck type ${duckTypeId} / ${duckType.toString()} for column '${key}'`);
+		}
+		if (conv !== false) convMap.set(key, conv);
+	}
+	return convMap;
+};
+//#endregion
 //#region src/config/flowblade-logtape-sqlduck.config.ts
 const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
 //#endregion
 //#region src/logger/sqlduck-default-logtape-logger.ts
 const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
 //#endregion
+//#region src/filesystem/file-system-utils.ts
+var FileSystemUtils = class {
+	#logger;
+	constructor(params) {
+		this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "FileSystemUtils" });
+	}
+	/**
+	* Create a directory recursively if it doesn't exist
+	*
+	* @throws Error if it can't be created
+	*/
+	createDirectory = (path) => {
+		try {
+			fs.mkdirSync(path, { recursive: true });
+		} catch (err) {
+			if (err.code !== "EEXIST") throw err;
+		}
+	};
+	/**
+	* Create a directory recursively if it doesn't exist and ensure it's writable
+	*/
+	createAndEnsureWritableDirectory = (label, path) => {
+		if (path === void 0) return;
+		if (!fs.existsSync(path)) try {
+			this.createDirectory(path);
+		} catch (e) {
+			throw new Error(`Failed to create ${label} '${path}' - ${e?.message ?? ""}`);
+		}
+		if (!fs.statSync(path).isDirectory()) throw new Error(`${label} '${path}' must be a directory`);
+		try {
+			fs.accessSync(path, fs.constants.W_OK);
+		} catch {
+			throw new Error(`${label} '${path}' must be writable`);
+		}
+	};
+	/**
+	* Check if a path is a regular file and exists
+	*/
+	isFile = (path) => {
+		try {
+			return fs.statSync(path).isFile();
+		} catch {
+			return false;
+		}
+	};
+	/**
+	* Check if a path is a directory and exists
+	*/
+	isDirectory = (path) => {
+		try {
+			return fs.statSync(path).isDirectory();
+		} catch {
+			return false;
+		}
+	};
+	parsePath = (path) => {
+		const dir = dirname(path);
+		if (dir.trim() === "") throw new Error(`Invalid path, missing directory '${path}'`);
+		const base = basename(path);
+		if (base.trim() === "") throw new Error(`Invalid path, missing filename '${path}'`);
+		return {
+			directory: dir,
+			filename: base
+		};
+	};
+	removeFileIfExists = (path) => {
+		try {
+			fs.rmSync(path);
+			return true;
+		} catch {
+			return false;
+		}
+	};
+};
+//#endregion
 //#region src/objects/database.ts
 var Database = class {
 	#params;
@@ -207,6 +388,7 @@ var DuckDatabaseAttachCommand = class {
 var DuckDatabaseManager = class {
 	#conn;
 	#logger;
+	#fs;
 	constructor(conn, params) {
 		this.#conn = conn;
 		this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
@@ -229,7 +411,12 @@ var DuckDatabaseManager = class {
 	attach = async (dbParams, options) => {
 		const params = duckConnectionParamsZodSchema.parse(dbParams);
 		const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
-		await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
+		const { behaviour } = options ?? {};
+		await this.#executeRawSqlCommand([
+			`attach(${params.alias}`,
+			behaviour ?? null,
+			")"
+		].filter(Boolean).join(","), rawSql);
 		return new Database({ alias: params.alias });
 	};
 	attachOrReplace = async (dbParams) => {
@@ -272,6 +459,32 @@ var DuckDatabaseManager = class {
 		await this.#executeRawSqlCommand("vacuum()", "VACUUM");
 		return true;
 	};
+	/**
+	* Helper to create an initial database file.
+	*/
+	createDatabaseFile = async (params) => {
+		const startTime = Date.now();
+		const { path, createDirectory = true } = params;
+		const fs = this.#getFs();
+		if (fs.isFile(path)) return { status: "exists" };
+		if (createDirectory) {
+			const { directory } = fs.parsePath(path);
+			fs.createAndEnsureWritableDirectory("database file directory", directory);
+		}
+		const instanceCache = new DuckDBInstanceCache();
+		try {
+			(await (await instanceCache.getOrCreateInstance(path)).connect()).closeSync();
+			const timeMs = Math.round(Date.now() - startTime);
+			this.#logger.info(`DuckDatabaseManager.createDatabaseFile('${path}') in ${timeMs}ms`, {
+				timeMs,
+				path
+			});
+		} catch (e) {
+			this.#logger.error(`DuckDatabaseManager.createDatabaseFile('${path}') failed - ${e?.message ?? ""}`, { path });
+			throw e;
+		}
+		return { status: "created" };
+	};
 	#executeRawSqlCommand = async (name, rawSql) => {
 		const startTime = Date.now();
 		try {
@@ -291,6 +504,10 @@ var DuckDatabaseManager = class {
 			throw new Error(msg, { cause: e });
 		}
 	};
+	#getFs = () => {
+		if (this.#fs === void 0) this.#fs = new FileSystemUtils({ logger: this.#logger });
+		return this.#fs;
+	};
 };
 //#endregion
 //#region src/table/get-duckdb-number-column-type.ts
@@ -329,7 +546,10 @@ const createOptions = {
 const duckDbTypesMap = new Map([
 	["VARCHAR", VARCHAR],
 	["BIGINT", BIGINT],
+	["UBIGINT", UBIGINT],
+	["HUGEINT", HUGEINT],
 	["TIMESTAMP", TIMESTAMP],
+	["TIMESTAMP_MS", TIMESTAMP_MS],
 	["UUID", UUID],
 	["BOOLEAN", BOOLEAN],
 	["INTEGER", INTEGER],
@@ -353,9 +573,10 @@ const getTableCreateFromZod = (params) => {
 		if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
 		else switch (type) {
 			case "string":
-				switch (format) {
+				if (Array.isArray(def.enum)) c.duckdbType = ENUM(def.enum);
+				else switch (format) {
 					case "date-time":
-						c.duckdbType = TIMESTAMP;
+						c.duckdbType = TIMESTAMP_MS;
 						break;
 					case "int64":
 						c.duckdbType = BIGINT;
@@ -428,11 +649,6 @@ const createTableFromZod = async (params) => {
 };
 //#endregion
 //#region src/utils/rows-to-columns-chunks.ts
-const toDuckValue = (value) => {
-	if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
-	if (typeof value === "bigint") return value.toString(10);
-	return value === void 0 ? null : value;
-};
 /**
 * Similar to `rowsToColumns` but yields results in chunks to avoid buffering
 * the entire dataset in memory. Each yielded item is a columns array for up to
@@ -443,14 +659,17 @@ const toDuckValue = (value) => {
 *   yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
 */
 async function* rowsToColumnsChunks(params) {
-	const { rows, chunkSize } = params;
+	const { rows, chunkSize, transformers } = params;
 	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
 	const first = await rows.next();
 	if (first.done) return;
 	const keys = Object.keys(first.value);
 	let columns = keys.map(() => []);
 	let rowsInChunk = 0;
-	keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
+	keys.forEach((k, i) => {
+		const fn = transformers?.get(k);
+		columns[i].push(fn === void 0 ? first.value[k] : fn(first.value[k]));
+	});
 	rowsInChunk++;
 	if (rowsInChunk >= chunkSize) {
 		yield columns;
@@ -458,7 +677,10 @@ async function* rowsToColumnsChunks(params) {
 		rowsInChunk = 0;
 	}
 	for await (const row of rows) {
-		keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
+		keys.forEach((k, i) => {
+			const fn = transformers?.get(k);
+			columns[i].push(fn === void 0 ? row[k] : fn(row[k]));
+		});
 		rowsInChunk++;
 		if (rowsInChunk >= chunkSize) {
 			yield columns;
@@ -530,11 +752,15 @@ var SqlDuck = class {
 		});
 		const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
 		const chunkTypes = Array.from(columnTypes.values());
+		const transformers = createDuckColumnConverters(Object.fromEntries(Array.from(columnTypes).map(([key, duckType]) => {
+			return [key, duckType];
+		})));
 		let totalRows = 0;
 		const dataAppendedCollector = createOnDataAppendedCollector();
 		const columnStream = rowsToColumnsChunks({
 			rows: rowStream,
-			chunkSize
+			chunkSize,
+			transformers
 		});
 		let appendedChunkCount = 0;
 		try {

package/dist/validation/valibot/index.d.mts

@@ -2,42 +2,42 @@ import * as v from "valibot";
 
 //#region src/validation/valibot/duck-connection-params-valibot-schema.d.ts
 declare const duckAllConnectionOptionsValibotSchema: v.ObjectSchema<{
-  readonly accessMode: v.OptionalSchema<v.PicklistSchema<["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
+  readonly accessMode: v.OptionalSchema<v.PicklistSchema<readonly ["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
   readonly compress: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
   readonly type: v.OptionalSchema<v.PicklistSchema<readonly ["DUCKDB", "SQLITE", "MYSQL", "PostgreSQL"], undefined>, undefined>;
   readonly blockSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 16384, undefined>, v.MaxValueAction<number, 262144, undefined>]>, undefined>;
   readonly rowGroupSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 1, undefined>]>, undefined>;
   readonly storageVersion: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.StartsWithAction<string, "v", undefined>, v.RegexAction<string, undefined>]>, undefined>;
   readonly encryptionKey: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 8, undefined>]>, undefined>;
-  readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<["CBC", "CTR", "GCM"], undefined>, undefined>;
+  readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<readonly ["CBC", "CTR", "GCM"], undefined>, undefined>;
 }, undefined>;
 type DuckAllConnectionOptionsValibotSchema = v.InferOutput<typeof duckAllConnectionOptionsValibotSchema>;
 declare const duckConnectionParamsValibotSchema: v.VariantSchema<"type", [v.ObjectSchema<{
   readonly type: v.LiteralSchema<"memory", undefined>;
   readonly alias: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>, v.MaxLengthAction<string, 120, undefined>, v.RegexAction<string, "Identifier must start with a letter or underscore, and contain only letters, numbers and underscores">, v.CheckAction<string, "Identifier value is a DuckDB reserved keyword and cannot be used as an identifier">]>;
   readonly options: v.OptionalSchema<v.ObjectSchema<{
-    readonly accessMode: v.OptionalSchema<v.PicklistSchema<["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
+    readonly accessMode: v.OptionalSchema<v.PicklistSchema<readonly ["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
     readonly compress: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
     readonly type: v.OptionalSchema<v.PicklistSchema<readonly ["DUCKDB", "SQLITE", "MYSQL", "PostgreSQL"], undefined>, undefined>;
     readonly blockSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 16384, undefined>, v.MaxValueAction<number, 262144, undefined>]>, undefined>;
     readonly rowGroupSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 1, undefined>]>, undefined>;
     readonly storageVersion: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.StartsWithAction<string, "v", undefined>, v.RegexAction<string, undefined>]>, undefined>;
     readonly encryptionKey: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 8, undefined>]>, undefined>;
-    readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<["CBC", "CTR", "GCM"], undefined>, undefined>;
+    readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<readonly ["CBC", "CTR", "GCM"], undefined>, undefined>;
   }, undefined>, undefined>;
 }, undefined>, v.ObjectSchema<{
   readonly type: v.LiteralSchema<"filesystem", undefined>;
   readonly path: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.CheckAction<string, "Invalid database filename - it must be a safe filename (no path traversal, no absolute paths, no reserved names, etc.)">, v.CheckAction<string, "Invalid database pathname - it must be an absolute path with no traversal">]>;
   readonly alias: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>, v.MaxLengthAction<string, 120, undefined>, v.RegexAction<string, "Identifier must start with a letter or underscore, and contain only letters, numbers and underscores">, v.CheckAction<string, "Identifier value is a DuckDB reserved keyword and cannot be used as an identifier">]>;
   readonly options: v.OptionalSchema<v.ObjectSchema<{
-    readonly accessMode: v.OptionalSchema<v.PicklistSchema<["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
+    readonly accessMode: v.OptionalSchema<v.PicklistSchema<readonly ["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
     readonly compress: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
     readonly type: v.OptionalSchema<v.PicklistSchema<readonly ["DUCKDB", "SQLITE", "MYSQL", "PostgreSQL"], undefined>, undefined>;
     readonly blockSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 16384, undefined>, v.MaxValueAction<number, 262144, undefined>]>, undefined>;
     readonly rowGroupSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 1, undefined>]>, undefined>;
     readonly storageVersion: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.StartsWithAction<string, "v", undefined>, v.RegexAction<string, undefined>]>, undefined>;
     readonly encryptionKey: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 8, undefined>]>, undefined>;
-    readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<["CBC", "CTR", "GCM"], undefined>, undefined>;
+    readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<readonly ["CBC", "CTR", "GCM"], undefined>, undefined>;
   }, undefined>, undefined>;
 }, undefined>], undefined>;
 //#endregion
@@ -73,28 +73,28 @@ declare const duckDsnValibotSchema: v.SchemaWithPipe<readonly [v.StringSchema<un
   readonly type: v.LiteralSchema<"memory", undefined>;
   readonly alias: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>, v.MaxLengthAction<string, 120, undefined>, v.RegexAction<string, "Identifier must start with a letter or underscore, and contain only letters, numbers and underscores">, v.CheckAction<string, "Identifier value is a DuckDB reserved keyword and cannot be used as an identifier">]>;
   readonly options: v.OptionalSchema<v.ObjectSchema<{
-    readonly accessMode: v.OptionalSchema<v.PicklistSchema<["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
+    readonly accessMode: v.OptionalSchema<v.PicklistSchema<readonly ["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
     readonly compress: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
     readonly type: v.OptionalSchema<v.PicklistSchema<readonly ["DUCKDB", "SQLITE", "MYSQL", "PostgreSQL"], undefined>, undefined>;
     readonly blockSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 16384, undefined>, v.MaxValueAction<number, 262144, undefined>]>, undefined>;
     readonly rowGroupSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 1, undefined>]>, undefined>;
     readonly storageVersion: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.StartsWithAction<string, "v", undefined>, v.RegexAction<string, undefined>]>, undefined>;
     readonly encryptionKey: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 8, undefined>]>, undefined>;
-    readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<["CBC", "CTR", "GCM"], undefined>, undefined>;
+    readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<readonly ["CBC", "CTR", "GCM"], undefined>, undefined>;
   }, undefined>, undefined>;
 }, undefined>, v.ObjectSchema<{
   readonly type: v.LiteralSchema<"filesystem", undefined>;
   readonly path: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.CheckAction<string, "Invalid database filename - it must be a safe filename (no path traversal, no absolute paths, no reserved names, etc.)">, v.CheckAction<string, "Invalid database pathname - it must be an absolute path with no traversal">]>;
   readonly alias: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>, v.MaxLengthAction<string, 120, undefined>, v.RegexAction<string, "Identifier must start with a letter or underscore, and contain only letters, numbers and underscores">, v.CheckAction<string, "Identifier value is a DuckDB reserved keyword and cannot be used as an identifier">]>;
   readonly options: v.OptionalSchema<v.ObjectSchema<{
-    readonly accessMode: v.OptionalSchema<v.PicklistSchema<["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
+    readonly accessMode: v.OptionalSchema<v.PicklistSchema<readonly ["READ_ONLY", "READ_WRITE"], undefined>, undefined>;
     readonly compress: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
     readonly type: v.OptionalSchema<v.PicklistSchema<readonly ["DUCKDB", "SQLITE", "MYSQL", "PostgreSQL"], undefined>, undefined>;
     readonly blockSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 16384, undefined>, v.MaxValueAction<number, 262144, undefined>]>, undefined>;
     readonly rowGroupSize: v.OptionalSchema<v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 1, undefined>]>, undefined>;
     readonly storageVersion: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.StartsWithAction<string, "v", undefined>, v.RegexAction<string, undefined>]>, undefined>;
     readonly encryptionKey: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 8, undefined>]>, undefined>;
-    readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<["CBC", "CTR", "GCM"], undefined>, undefined>;
+    readonly encryptionCipher: v.OptionalSchema<v.PicklistSchema<readonly ["CBC", "CTR", "GCM"], undefined>, undefined>;
   }, undefined>, undefined>;
 }, undefined>], undefined>]>;
 //#endregion

package/dist/validation/valibot/index.mjs

@@ -1,4 +1,4 @@
-import { a as duckStorageVersionRegexp, i as duckIdentifierNameRegex, n as duckdbReservedKeywordsSet, r as duckConnectionsOptions } from "../../duck-reserved-keywords-CUmaTbrD.mjs";
+import { a as duckStorageVersionRegexp, i as duckIdentifierNameRegex, n as duckdbReservedKeywordsSet, r as duckConnectionsOptions } from "../../duck-reserved-keywords-B8XUjnaY.mjs";
 import isSafeFilename from "is-safe-filename";
 import { parseDsn, parseDsnOrThrow } from "@httpx/dsn-parser";
 import * as v from "valibot";
@@ -26,18 +26,14 @@ const duckValidatorsValibot = {
 //#endregion
 //#region src/validation/valibot/duck-connection-params-valibot-schema.ts
 const duckAllConnectionOptionsValibotSchema = v.object({
-	accessMode: v.optional(v.picklist(["READ_ONLY", "READ_WRITE"])),
+	accessMode: v.optional(v.picklist(duckConnectionsOptions.accessModes)),
 	compress: v.optional(v.boolean()),
 	type: v.optional(v.picklist(duckConnectionsOptions.types)),
 	blockSize: v.optional(v.pipe(v.number(), v.integer(), v.minValue(16384), v.maxValue(262144))),
 	rowGroupSize: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1))),
 	storageVersion: v.optional(v.pipe(v.string(), v.startsWith("v"), v.regex(duckStorageVersionRegexp))),
 	encryptionKey: v.optional(v.pipe(v.string(), v.minLength(8))),
-	encryptionCipher: v.optional(v.picklist([
-		"CBC",
-		"CTR",
-		"GCM"
-	]))
+	encryptionCipher: v.optional(v.picklist(duckConnectionsOptions.encryptionCiphers))
 });
 const duckConnectionParamsValibotSchema = v.variant("type", [v.object({
 	type: v.literal("memory"),

package/dist/validation/zod/index.mjs

@@ -1,2 +1,2 @@
-import { a as duckAllConnectionOptionsZodSchema, i as assertValidTableName, n as assertValidAliasName, o as duckConnectionParamsZodSchema, r as assertValidSchemaName, s as duckValidatorsZod, t as duckDsnZodSchema } from "../../zod-BRUUIdGW.mjs";
+import { a as duckAllConnectionOptionsZodSchema, i as assertValidTableName, n as assertValidAliasName, o as duckConnectionParamsZodSchema, r as assertValidSchemaName, s as duckValidatorsZod, t as duckDsnZodSchema } from "../../zod-CuPjTLv8.mjs";
 export { assertValidAliasName, assertValidSchemaName, assertValidTableName, duckAllConnectionOptionsZodSchema, duckConnectionParamsZodSchema, duckDsnZodSchema, duckValidatorsZod };

package/dist/{zod-BRUUIdGW.mjs → zod-CuPjTLv8.mjs}

@@ -1,4 +1,4 @@
-import { a as duckStorageVersionRegexp, i as duckIdentifierNameRegex, n as duckdbReservedKeywordsSet, r as duckConnectionsOptions } from "./duck-reserved-keywords-CUmaTbrD.mjs";
+import { a as duckStorageVersionRegexp, i as duckIdentifierNameRegex, n as duckdbReservedKeywordsSet, r as duckConnectionsOptions } from "./duck-reserved-keywords-B8XUjnaY.mjs";
 import isSafeFilename from "is-safe-filename";
 import * as z from "zod";
 import { parseDsn } from "@httpx/dsn-parser";
@@ -28,18 +28,14 @@ const duckValidatorsZod = {
 //#endregion
 //#region src/validation/zod/duck-connection-params-zod-schema.ts
 const duckAllConnectionOptionsZodSchema = z.strictObject({
-	accessMode: z.optional(z.enum(["READ_ONLY", "READ_WRITE"])),
+	accessMode: z.optional(z.enum(duckConnectionsOptions.accessModes)),
 	compress: z.optional(z.boolean()),
 	type: z.optional(z.enum(duckConnectionsOptions.types)),
 	blockSize: z.optional(z.int32().min(16384).max(262144)),
 	rowGroupSize: z.optional(z.int32().positive()),
 	storageVersion: z.optional(z.string().startsWith("v").regex(duckStorageVersionRegexp)),
 	encryptionKey: z.optional(z.string().min(8)),
-	encryptionCipher: z.optional(z.enum([
-		"CBC",
-		"CTR",
-		"GCM"
-	]))
+	encryptionCipher: z.optional(z.enum(duckConnectionsOptions.encryptionCiphers))
 });
 const duckConnectionParamsZodSchema = z.discriminatedUnion("type", [z.strictObject({
 	type: z.literal("memory"),
@@ -52,7 +48,7 @@ const duckConnectionParamsZodSchema = z.discriminatedUnion("type", [z.strictObje
 		return typeof filename === "string" && isSafeFilename(filename);
 	}, { message: "Invalid database filename - it must be a safe filename (no path traversal, no absolute paths, no reserved names, etc.)" }).refine((path) => {
 		const pathname = "/" + path.replace("\\", "/").split("/").slice(0, -1).filter(Boolean).join("/");
-		return pathname.length > 0 && pathname.startsWith("/") && pathname.includes("..") === false;
+		return pathname.length > 0 && pathname.startsWith("/") && !pathname.includes("..");
 	}, { message: "Invalid database pathname - it must be an absolute path with no traversal" }),
 	alias: duckValidatorsZod.aliasName,
 	options: z.optional(duckAllConnectionOptionsZodSchema)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.14.0",
+  "version": "0.16.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -53,7 +53,7 @@
     "fix-staged": "lint-staged --allow-empty",
     "check-dist": "es-check --config=.escheckrc.json",
     "check-pub": "publint",
-    "check-size": "size-limit"
+    "check-size-disabled": "size-limit"
   },
   "dependencies": {
     "@flowblade/core": "^0.2.26",
@@ -65,7 +65,7 @@
     "@logtape/logtape": "^2.0.5",
     "@standard-schema/spec": "^1.1.0",
     "is-safe-filename": "0.1.1",
-    "p-queue": "9.1.1",
+    "p-queue": "9.1.2",
     "zod": "^4.3.6"
   },
   "peerDependencies": {
@@ -90,11 +90,11 @@
     "@testcontainers/mssqlserver": "11.13.0",
     "@total-typescript/ts-reset": "0.6.1",
     "@types/node": "25.5.2",
-    "@typescript-eslint/eslint-plugin": "8.58.0",
-    "@typescript-eslint/parser": "8.58.0",
+    "@typescript-eslint/eslint-plugin": "8.58.1",
+    "@typescript-eslint/parser": "8.58.1",
     "@typescript/native-preview": "7.0.0-dev.20260406.1",
-    "@vitest/coverage-v8": "4.1.2",
-    "@vitest/ui": "4.1.2",
+    "@vitest/coverage-v8": "4.1.4",
+    "@vitest/ui": "4.1.4",
     "ansis": "4.2.0",
     "browserslist-to-esbuild": "2.1.1",
     "core-js": "3.49.0",
@@ -123,7 +123,7 @@
     "typedoc-plugin-markdown": "4.11.0",
     "typescript": "6.0.2",
     "valibot": "1.3.1",
-    "vitest": "4.1.2"
+    "vitest": "4.1.4"
   },
   "files": [
     "dist"