@flowblade/sqlduck 0.15.0 → 0.16.0

package/README.md

@@ -131,32 +131,32 @@ const queryResult = await dbDuckDbMemoryConn.query<{
 ### Node 24
 
 ```
- RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
+ RUN  v4.1.4 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 4157ms
-     name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
-   · duckdb appender memory, count: 100000, chunk size 2048  2.6950  357.08  385.05  371.06  385.05  385.05  385.05  385.05  ±47.90%        2
-   · duckdb appender file, count: 100000, chunk size 2048    1.4218  703.35  703.35  703.35  703.35  703.35  703.35  703.35   ±0.00%        1
-   · duckdb appender, count: 100000, chunk size 1024         2.5157  391.12  403.89  397.50  403.89  403.89  403.89  403.89  ±20.41%        2
+ ✓ bench/appender.bench.ts > appender benches 3412ms
+     name                                                        hz     min     max    mean     p75     p99    p995    p999       rme  samples
+   · duckdb appender memory, count: 100000, chunk size 2048  2.8872  295.84  396.87  346.35  396.87  396.87  396.87  396.87  ±185.36%        2
+   · duckdb appender file, count: 100000, chunk size 2048    1.7908  558.40  558.40  558.40  558.40  558.40  558.40  558.40    ±0.00%        1
+   · duckdb appender, count: 100000, chunk size 1024         1.9967  500.82  500.82  500.82  500.82  500.82  500.82  500.82    ±0.00%        1
 
- ✓ bench/stream.bench.ts > Bench stream 2809ms
-     name                                                       hz      min     max    mean     p75     p99    p995    p999      rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.2627  87.7271  151.56  107.96  116.92  151.56  151.56  151.56  ±15.98%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.1479   125.04  168.13  139.90  152.97  168.13  168.13  168.13   ±7.78%       10
+ ✓ bench/stream.bench.ts > Bench stream 2140ms
+     name                                                        hz      min     max     mean      p75     p99    p995    p999      rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  11.8099  63.0430  172.82  84.6748  81.4757  172.82  172.82  172.82  ±27.12%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)   10.0442  79.5805  125.81  99.5603   112.35  125.81  125.81  125.81  ±12.04%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 614ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 617ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  18,899.24  0.0334  5.3351  0.0529  0.0546  0.1943  0.3087  0.7214  ±2.72%     9450
+   · getTableCreateFromZod  27,583.03  0.0226  2.7818  0.0363  0.0403  0.1185  0.1583  0.4321  ±1.66%    13792
 
  BENCH  Summary
 
   duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.07x faster than duckdb appender, count: 100000, chunk size 1024
-    1.90x faster than duckdb appender file, count: 100000, chunk size 2048
+    1.45x faster than duckdb appender, count: 100000, chunk size 1024
+    1.61x faster than duckdb appender file, count: 100000, chunk size 2048
 
   rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
-    1.30x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
+    1.18x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 ```
@@ -164,34 +164,34 @@ const queryResult = await dbDuckDbMemoryConn.query<{
 ### Bun 1.3.11
 
 ```
- RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
+ RUN  v4.1.4 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 4159ms
+ ✓ bench/appender.bench.ts > appender benches 3357ms
      name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
-   · duckdb appender memory, count: 100000, chunk size 2048  2.6465  375.34  380.38  377.86  380.38  380.38  380.38  380.38   ±8.48%        2
-   · duckdb appender file, count: 100000, chunk size 2048    1.5016  665.98  665.98  665.98  665.98  665.98  665.98  665.98   ±0.00%        1
-   · duckdb appender, count: 100000, chunk size 1024         2.2828  413.11  463.01  438.06  463.01  463.01  463.01  463.01  ±72.39%        2
+   · duckdb appender memory, count: 100000, chunk size 2048  2.9741  315.71  356.77  336.24  356.77  356.77  356.77  356.77  ±77.59%        2
+   · duckdb appender file, count: 100000, chunk size 2048    1.8953  527.62  527.62  527.62  527.62  527.62  527.62  527.62   ±0.00%        1
+   · duckdb appender, count: 100000, chunk size 1024         1.7803  561.70  561.70  561.70  561.70  561.70  561.70  561.70   ±0.00%        1
 
- ✓ bench/stream.bench.ts > Bench stream 2690ms
-     name                                                       hz      min     max    mean     p75     p99    p995    p999     rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.5675  95.6610  114.11  104.52  107.75  114.11  114.11  114.11  ±3.46%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.6895   117.83  138.26  130.05  137.51  138.26  138.26  138.26  ±4.05%       10
+ ✓ bench/stream.bench.ts > Bench stream 2058ms
+     name                                                        hz      min     max     mean      p75     p99    p995    p999      rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  12.0130  60.7081  111.32  83.2432  99.7116  111.32  111.32  111.32  ±14.15%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)   10.9710  76.4253  145.14  91.1493  91.1424  145.14  145.14  145.14  ±16.04%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 629ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 621ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  18,892.06  0.0281  7.5844  0.0529  0.0516  0.1893  0.2477  3.2823  ±6.26%     9447
+   · getTableCreateFromZod  27,472.91  0.0185  4.6226  0.0364  0.0369  0.1137  0.1476  1.9138  ±5.20%    13737
 
  BENCH  Summary
 
   rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
-    1.24x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
+    1.09x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 
   duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.16x faster than duckdb appender, count: 100000, chunk size 1024
-    1.76x faster than duckdb appender file, count: 100000, chunk size 2048
+    1.57x faster than duckdb appender file, count: 100000, chunk size 2048
+    1.67x faster than duckdb appender, count: 100000, chunk size 1024
 
 ```
 

package/dist/index.d.mts

@@ -81,7 +81,7 @@ declare class Table {
 }
 //#endregion
 //#region src/table/table-schema-zod.type.d.ts
-type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID;
+type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID | z.ZodEnum;
 type TableSchemaZod = z.ZodObject<Record<string, ZodSchemaSupportedTypes | z.ZodNullable<ZodSchemaSupportedTypes> | z.ZodCodec | z.ZodNullable<z.ZodCodec>>>;
 //#endregion
 //#region src/table/get-table-create-from-zod.d.ts

package/dist/index.mjs

@@ -1,6 +1,6 @@
 import { t as duckReservedKeywords } from "./duck-reserved-keywords-B8XUjnaY.mjs";
 import { n as assertValidAliasName, o as duckConnectionParamsZodSchema, s as duckValidatorsZod } from "./zod-CuPjTLv8.mjs";
-import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBInstanceCache, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
+import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBInstanceCache, DuckDBTimestampMillisecondsValue, DuckDBTypeId, ENUM, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TIMESTAMP_MS, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
 import { getLogger } from "@logtape/logtape";
 import fs from "node:fs";
 import { basename, dirname } from "node:path";
@@ -124,6 +124,110 @@ const createOnDataAppendedCollector = () => {
 	};
 };
 //#endregion
+//#region src/converter/duck-value-converter.ts
+const stringTimestampRegexp = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d{3,6})?Z?$/i;
+const dateRegexp = /^\d{4}-\d{2}-\d{2}$/;
+const createDuckValueConverterTypeError = (params) => {
+	let serializableValue;
+	try {
+		serializableValue = JSON.stringify(params.value);
+	} catch {
+		serializableValue = "<unserializable>";
+	}
+	return /* @__PURE__ */ new TypeError(`[DuckValueConverter.${params.method}]: Unsupported type ${typeof params.value} with value ${serializableValue}`);
+};
+var DuckValueConverter = class {
+	toUUID = (value) => {
+		if (typeof value === "bigint") return value;
+		else if (typeof value === "string") return BigInt("0x" + value.replaceAll("-", ""));
+		if (value === void 0 || value === null) return null;
+		throw createDuckValueConverterTypeError({
+			method: "toUUID",
+			value
+		});
+	};
+	toStringEnum = (value) => {
+		if (typeof value === "string") return value;
+		if (value === void 0 || value === null) return null;
+		throw createDuckValueConverterTypeError({
+			method: "toStringEnum",
+			value
+		});
+	};
+	toBigIntString = (value) => {
+		if (typeof value === "string") return value;
+		if (typeof value === "number" || typeof value === "bigint") return value.toString(10);
+		if (value === void 0 || value === null) return null;
+		throw createDuckValueConverterTypeError({
+			method: "toBigIntString",
+			value
+		});
+	};
+	toTimestampMs = (value) => {
+		if (value instanceof Date) return new DuckDBTimestampMillisecondsValue(BigInt(value.getTime()));
+		if (value === void 0 || value === null) return null;
+		if (typeof value === "string") {
+			const len = value.length;
+			if (len > 18 && len < 31 && stringTimestampRegexp.test(value)) {
+				const date = /* @__PURE__ */ new Date(value + (value.endsWith("Z") ? "" : "Z"));
+				return new DuckDBTimestampMillisecondsValue(BigInt(date.getTime()));
+			}
+			if (len === 10 && dateRegexp.test(value)) {
+				const date = /* @__PURE__ */ new Date(value + "T00:00:00Z");
+				return new DuckDBTimestampMillisecondsValue(BigInt(date.getTime()));
+			}
+		}
+		if (typeof value === "bigint") return new DuckDBTimestampMillisecondsValue(value);
+		if (typeof value === "number") return new DuckDBTimestampMillisecondsValue(BigInt(value));
+		throw createDuckValueConverterTypeError({
+			method: "toTimestampMs",
+			value
+		});
+	};
+};
+//#endregion
+//#region src/converter/create-duck-column-converters.ts
+const createDuckColumnConverters = (duckTypes) => {
+	const convMap = /* @__PURE__ */ new Map();
+	const converter = new DuckValueConverter();
+	for (const [key, duckType] of Object.entries(duckTypes)) {
+		let conv;
+		const duckTypeId = duckType.typeId;
+		switch (duckTypeId) {
+			case DuckDBTypeId.TIMESTAMP_MS:
+				conv = converter.toTimestampMs;
+				break;
+			case DuckDBTypeId.BIGINT:
+			case DuckDBTypeId.UBIGINT:
+			case DuckDBTypeId.HUGEINT:
+			case DuckDBTypeId.UHUGEINT:
+			case DuckDBTypeId.INTEGER:
+			case DuckDBTypeId.UINTEGER:
+			case DuckDBTypeId.BIGNUM:
+				conv = converter.toBigIntString;
+				break;
+			case DuckDBTypeId.ENUM:
+				conv = converter.toStringEnum;
+				break;
+			case DuckDBTypeId.UUID:
+				conv = converter.toUUID;
+				break;
+			case DuckDBTypeId.BIT:
+			case DuckDBTypeId.BOOLEAN:
+			case DuckDBTypeId.TINYINT:
+			case DuckDBTypeId.USMALLINT:
+			case DuckDBTypeId.UTINYINT:
+			case DuckDBTypeId.VARCHAR:
+			case DuckDBTypeId.SMALLINT:
+				conv = false;
+				break;
+			default: throw new Error(`Unsupported duck type ${duckTypeId} / ${duckType.toString()} for column '${key}'`);
+		}
+		if (conv !== false) convMap.set(key, conv);
+	}
+	return convMap;
+};
+//#endregion
 //#region src/config/flowblade-logtape-sqlduck.config.ts
 const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
 //#endregion
@@ -442,7 +546,10 @@ const createOptions = {
 const duckDbTypesMap = new Map([
 	["VARCHAR", VARCHAR],
 	["BIGINT", BIGINT],
+	["UBIGINT", UBIGINT],
+	["HUGEINT", HUGEINT],
 	["TIMESTAMP", TIMESTAMP],
+	["TIMESTAMP_MS", TIMESTAMP_MS],
 	["UUID", UUID],
 	["BOOLEAN", BOOLEAN],
 	["INTEGER", INTEGER],
@@ -466,9 +573,10 @@ const getTableCreateFromZod = (params) => {
 		if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
 		else switch (type) {
 			case "string":
-				switch (format) {
+				if (Array.isArray(def.enum)) c.duckdbType = ENUM(def.enum);
+				else switch (format) {
 					case "date-time":
-						c.duckdbType = TIMESTAMP;
+						c.duckdbType = TIMESTAMP_MS;
 						break;
 					case "int64":
 						c.duckdbType = BIGINT;
@@ -541,11 +649,6 @@ const createTableFromZod = async (params) => {
 };
 //#endregion
 //#region src/utils/rows-to-columns-chunks.ts
-const toDuckValue = (value) => {
-	if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
-	if (typeof value === "bigint") return value.toString(10);
-	return value === void 0 ? null : value;
-};
 /**
 * Similar to `rowsToColumns` but yields results in chunks to avoid buffering
 * the entire dataset in memory. Each yielded item is a columns array for up to
@@ -556,14 +659,17 @@ const toDuckValue = (value) => {
 *   yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
 */
 async function* rowsToColumnsChunks(params) {
-	const { rows, chunkSize } = params;
+	const { rows, chunkSize, transformers } = params;
 	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
 	const first = await rows.next();
 	if (first.done) return;
 	const keys = Object.keys(first.value);
 	let columns = keys.map(() => []);
 	let rowsInChunk = 0;
-	keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
+	keys.forEach((k, i) => {
+		const fn = transformers?.get(k);
+		columns[i].push(fn === void 0 ? first.value[k] : fn(first.value[k]));
+	});
 	rowsInChunk++;
 	if (rowsInChunk >= chunkSize) {
 		yield columns;
@@ -571,7 +677,10 @@ async function* rowsToColumnsChunks(params) {
 		rowsInChunk = 0;
 	}
 	for await (const row of rows) {
-		keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
+		keys.forEach((k, i) => {
+			const fn = transformers?.get(k);
+			columns[i].push(fn === void 0 ? row[k] : fn(row[k]));
+		});
 		rowsInChunk++;
 		if (rowsInChunk >= chunkSize) {
 			yield columns;
@@ -643,11 +752,15 @@ var SqlDuck = class {
 		});
 		const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
 		const chunkTypes = Array.from(columnTypes.values());
+		const transformers = createDuckColumnConverters(Object.fromEntries(Array.from(columnTypes).map(([key, duckType]) => {
+			return [key, duckType];
+		})));
 		let totalRows = 0;
 		const dataAppendedCollector = createOnDataAppendedCollector();
 		const columnStream = rowsToColumnsChunks({
 			rows: rowStream,
-			chunkSize
+			chunkSize,
+			transformers
 		});
 		let appendedChunkCount = 0;
 		try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -65,7 +65,7 @@
     "@logtape/logtape": "^2.0.5",
     "@standard-schema/spec": "^1.1.0",
     "is-safe-filename": "0.1.1",
-    "p-queue": "9.1.1",
+    "p-queue": "9.1.2",
     "zod": "^4.3.6"
   },
   "peerDependencies": {
@@ -93,8 +93,8 @@
     "@typescript-eslint/eslint-plugin": "8.58.1",
     "@typescript-eslint/parser": "8.58.1",
     "@typescript/native-preview": "7.0.0-dev.20260406.1",
-    "@vitest/coverage-v8": "4.1.3",
-    "@vitest/ui": "4.1.3",
+    "@vitest/coverage-v8": "4.1.4",
+    "@vitest/ui": "4.1.4",
     "ansis": "4.2.0",
     "browserslist-to-esbuild": "2.1.1",
     "core-js": "3.49.0",
@@ -123,7 +123,7 @@
     "typedoc-plugin-markdown": "4.11.0",
     "typescript": "6.0.2",
     "valibot": "1.3.1",
-    "vitest": "4.1.3"
+    "vitest": "4.1.4"
   },
   "files": [
     "dist"