@flowblade/sqlduck 0.8.3 → 0.10.0

package/README.md

@@ -54,6 +54,72 @@ const queryResult = await dbDuckDbMemoryConn.query<{
 `);
 ```
 
+## Benchmarks
+
+### Node 24
+
+```
+ RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
+
+
+ ✓ bench/appender.bench.ts > appender benches 66030ms
+     name                                                  hz        min        max       mean        p75        p99       p995       p999     rme  samples
+   · duckdb appender, count: 1000000, chunk size 2048  0.0642  15,577.01  15,577.01  15,577.01  15,577.01  15,577.01  15,577.01  15,577.01  ±0.00%        1
+   · duckdb appender, count: 1000000, chunk size 1024  0.0579  17,263.44  17,263.44  17,263.44  17,263.44  17,263.44  17,263.44  17,263.44  ±0.00%        1
+
+ ✓ bench/stream.bench.ts > Bench stream 22923ms
+     name                                                        hz       min       max      mean       p75       p99      p995      p999     rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 1000000)  1.2126    742.59    906.65    824.67    863.91    906.65    906.65    906.65  ±4.26%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 1000000)   0.8049  1,123.18  1,498.68  1,242.43  1,257.91  1,498.68  1,498.68  1,498.68  ±6.40%       10
+
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 615ms
+     name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
+   · getTableCreateFromZod  16,562.93  0.0242  2.5902  0.0604  0.0734  0.2555  0.3741  0.8135  ±2.32%     8282
+
+ BENCH  Summary
+
+  duckdb appender, count: 1000000, chunk size 2048 - bench/appender.bench.ts > appender benches
+    1.11x faster than duckdb appender, count: 1000000, chunk size 1024
+
+  rowToColumnsChunk with chunkSize 2048 (count: 1000000) - bench/stream.bench.ts > Bench stream
+    1.51x faster than mapFakeRowStream with chunkSize 2048 (count: 1000000)
+
+  getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
+
+```
+
+### Bun 1.3.11
+
+```
+ RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
+
+
+ ✓ bench/appender.bench.ts > appender benches 36627ms
+     name                                                  hz       min       max      mean       p75       p99      p995      p999     rme  samples
+   · duckdb appender, count: 1000000, chunk size 2048  0.1177  8,495.41  8,495.41  8,495.41  8,495.41  8,495.41  8,495.41  8,495.41  ±0.00%        1
+   · duckdb appender, count: 1000000, chunk size 1024  0.1064  9,397.97  9,397.97  9,397.97  9,397.97  9,397.97  9,397.97  9,397.97  ±0.00%        1
+
+ ✓ bench/stream.bench.ts > Bench stream 23421ms
+     name                                                        hz       min       max      mean       p75       p99      p995      p999     rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 1000000)  1.1378    801.60  1,080.22    878.91    910.91  1,080.22  1,080.22  1,080.22  ±6.85%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 1000000)   0.8118  1,130.36  1,448.99  1,231.78  1,268.45  1,448.99  1,448.99  1,448.99  ±5.34%       10
+
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 622ms
+     name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
+   · getTableCreateFromZod  22,447.94  0.0210  5.4621  0.0445  0.0442  0.1657  0.2167  2.5852  ±5.37%    11224
+
+ BENCH  Summary
+
+  rowToColumnsChunk with chunkSize 2048 (count: 1000000) - bench/stream.bench.ts > Bench stream
+    1.40x faster than mapFakeRowStream with chunkSize 2048 (count: 1000000)
+
+  getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
+
+  duckdb appender, count: 1000000, chunk size 2048 - bench/appender.bench.ts > appender benches
+    1.11x faster than duckdb appender, count: 1000000, chunk size 1024
+
+```
+
 ### Local scripts
 
 | Name              | Description                    |

package/dist/index.cjs

@@ -21,9 +21,13 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 	enumerable: true
 }) : target, mod));
 //#endregion
+let _logtape_logtape = require("@logtape/logtape");
 let _duckdb_node_api = require("@duckdb/node-api");
 let zod = require("zod");
 zod = __toESM(zod);
+//#region src/config/flowblade-logtape-sqlduck.config.ts
+const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
+//#endregion
 //#region src/helpers/duck-exec.ts
 var DuckExec = class {
 	#conn;
@@ -120,6 +124,9 @@ var DuckMemory = class {
 	};
 };
 //#endregion
+//#region src/logger/sqlduck-default-logtape-logger.ts
+const sqlduckDefaultLogtapeLogger = (0, _logtape_logtape.getLogger)(flowbladeLogtapeSqlduckConfig.categories);
+//#endregion
 //#region src/appender/data-appender-callback.ts
 const isOnDataAppendedAsyncCb = (v) => {
 	return v.constructor.name === "AsyncFunction";
@@ -141,23 +148,66 @@ const createOnDataAppendedCollector = () => {
 	};
 };
 //#endregion
+//#region src/table/get-duckdb-number-column-type.ts
+const isFloatValue = (value) => {
+	if (!Number.isFinite(value)) return true;
+	if (Math.abs(value) > Number.MAX_SAFE_INTEGER) return true;
+	return !Number.isInteger(value);
+};
+const getDuckdbNumberColumnType = (params) => {
+	const { minimum, maximum } = params;
+	if (minimum === void 0 || maximum === void 0) return _duckdb_node_api.BIGINT;
+	if (isFloatValue(minimum) || isFloatValue(maximum)) {
+		if (minimum >= -34028235e31 && maximum <= 34028235e31) return _duckdb_node_api.FLOAT;
+		return _duckdb_node_api.DOUBLE;
+	}
+	if (minimum >= 0) {
+		if (maximum <= 255) return _duckdb_node_api.UTINYINT;
+		if (maximum <= 65535) return _duckdb_node_api.USMALLINT;
+		if (maximum <= 4294967295) return _duckdb_node_api.UINTEGER;
+		if (maximum <= 18446744073709551615n) return _duckdb_node_api.UBIGINT;
+		return _duckdb_node_api.UHUGEINT;
+	}
+	if (minimum >= -128 && maximum <= 127) return _duckdb_node_api.TINYINT;
+	if (minimum >= -32768 && maximum <= 32767) return _duckdb_node_api.SMALLINT;
+	if (minimum >= -2147483648 && maximum <= 2147483647) return _duckdb_node_api.INTEGER;
+	if (minimum >= -9223372036854775808n && maximum <= 9223372036854775807n) return _duckdb_node_api.BIGINT;
+	return _duckdb_node_api.HUGEINT;
+};
+//#endregion
 //#region src/table/get-table-create-from-zod.ts
-const createMap = {
+const createOptions = {
 	CREATE: "CREATE TABLE",
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
 	IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
 };
-const getTableCreateFromZod = (table, schema, options) => {
+const duckDbTypes = [
+	["VARCHAR", _duckdb_node_api.VARCHAR],
+	["BIGINT", _duckdb_node_api.BIGINT],
+	["TIMESTAMP", _duckdb_node_api.TIMESTAMP],
+	["UUID", _duckdb_node_api.UUID],
+	["BOOLEAN", _duckdb_node_api.BOOLEAN],
+	["INTEGER", _duckdb_node_api.INTEGER],
+	["DOUBLE", _duckdb_node_api.DOUBLE],
+	["FLOAT", _duckdb_node_api.FLOAT]
+];
+const duckDbTypesMap = new Map(duckDbTypes);
+const getTableCreateFromZod = (params) => {
+	const { table, schema, options } = params;
 	const { create = "CREATE" } = options ?? {};
 	const fqTable = table.getFullName();
-	const json = schema.toJSONSchema({ target: "openapi-3.0" });
+	const json = schema.toJSONSchema({
+		target: "openapi-3.0",
+		unrepresentable: "throw"
+	});
 	const columns = [];
 	if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
-	const columnTypes = [];
+	const columnTypesMap = /* @__PURE__ */ new Map();
 	for (const [columnName, def] of Object.entries(json.properties)) {
-		const { type, nullable, format, primaryKey } = def;
+		const { type, duckdbType, nullable, format, primaryKey, minimum, maximum } = def;
 		const c = { name: columnName };
-		switch (type) {
+		if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
+		else switch (type) {
 			case "string":
 				switch (format) {
 					case "date-time":
@@ -166,22 +216,37 @@ const getTableCreateFromZod = (table, schema, options) => {
 					case "int64":
 						c.duckdbType = _duckdb_node_api.BIGINT;
 						break;
+					case "uuid":
+						c.duckdbType = _duckdb_node_api.UUID;
+						break;
 					default: c.duckdbType = _duckdb_node_api.VARCHAR;
 				}
 				break;
 			case "number":
-				c.duckdbType = _duckdb_node_api.INTEGER;
+				c.duckdbType = getDuckdbNumberColumnType({
+					minimum,
+					maximum
+				});
 				break;
-			default: throw new Error("Not a supported type");
+			case "integer":
+				c.duckdbType = getDuckdbNumberColumnType({
+					minimum,
+					maximum
+				});
+				break;
+			case "boolean":
+				c.duckdbType = _duckdb_node_api.BOOLEAN;
+				break;
+			default: throw new Error(`Cannot guess '${columnName}' type - ${JSON.stringify(def)}`);
 		}
 		if (primaryKey === true) c.constraint = "PRIMARY KEY";
 		else if (nullable !== true) c.constraint = "NOT NULL";
-		columnTypes.push([columnName, c.duckdbType]);
+		columnTypesMap.set(columnName, c.duckdbType);
 		columns.push(c);
 	}
 	return {
 		ddl: [
-			`${createMap[create]} ${fqTable} (\n`,
+			`${createOptions[create]} ${fqTable} (\n`,
 			columns.map((colDDL) => {
 				const { name, duckdbType, constraint } = colDDL;
 				return `  ${[
@@ -192,17 +257,24 @@ const getTableCreateFromZod = (table, schema, options) => {
 			}).join(",\n"),
 			"\n)"
 		].join(""),
-		columnTypes
+		columnTypes: columnTypesMap
 	};
 };
 //#endregion
 //#region src/table/create-table-from-zod.ts
 const createTableFromZod = async (params) => {
-	const { conn, table, schema, options } = params;
-	const { ddl, columnTypes } = getTableCreateFromZod(table, schema, options);
+	const { conn, table, schema, options, logger = sqlduckDefaultLogtapeLogger } = params;
+	const { ddl, columnTypes } = getTableCreateFromZod({
+		table,
+		schema,
+		options
+	});
+	logger.debug(`Generate DDL for table '${table.getFullName()}'`, { ddl });
 	try {
 		await conn.run(ddl);
+		logger.info(`Table '${table.getFullName()}' successfully created`, { ddl });
 	} catch (e) {
+		logger.error(`Failed to create table '${table.getFullName()}': ${e.message}`, { ddl });
 		throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
 	}
 	return {
@@ -214,6 +286,7 @@ const createTableFromZod = async (params) => {
 //#region src/utils/rows-to-columns-chunks.ts
 const toDuckValue = (value) => {
 	if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
+	if (typeof value === "bigint") return value.toString(10);
 	return value === void 0 ? null : value;
 };
 /**
@@ -225,7 +298,8 @@ const toDuckValue = (value) => {
 *   input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
 *   yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
 */
-async function* rowsToColumnsChunks(rows, chunkSize) {
+async function* rowsToColumnsChunks(params) {
+	const { rows, chunkSize } = params;
 	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
 	const first = await rows.next();
 	if (first.done) return;
@@ -257,12 +331,11 @@ var SqlDuck = class {
 	#logger;
 	constructor(params) {
 		this.#duck = params.conn;
-		this.#logger = params.logger;
+		this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
 	}
 	/**
 	* Create a table from a Zod schema and fill it with data from a row stream.
 	*
-	*
 	* @example
 	* ```typescript
 	* import * as z from 'zod';
@@ -308,28 +381,45 @@ var SqlDuck = class {
 			options: createOptions
 		});
 		const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
-		const chunkTypes = columnTypes.map((v) => v[1]);
+		const chunkTypes = Array.from(columnTypes.values());
 		let totalRows = 0;
 		const dataAppendedCollector = createOnDataAppendedCollector();
-		const columnStream = rowsToColumnsChunks(rowStream, chunkSize);
-		for await (const dataChunk of columnStream) {
-			const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
-			if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
-			totalRows += dataChunk?.[0]?.length ?? 0;
-			chunk.setColumns(dataChunk);
-			appender.appendDataChunk(chunk);
-			appender.flushSync();
-			if (onDataAppended !== void 0) {
-				const payload = dataAppendedCollector(totalRows);
-				if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
-				else onDataAppended(payload);
+		const columnStream = rowsToColumnsChunks({
+			rows: rowStream,
+			chunkSize
+		});
+		try {
+			for await (const dataChunk of columnStream) {
+				const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
+				this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
+				totalRows += dataChunk?.[0]?.length ?? 0;
+				chunk.setColumns(dataChunk);
+				appender.appendDataChunk(chunk);
+				appender.flushSync();
+				if (onDataAppended !== void 0) {
+					const payload = dataAppendedCollector(totalRows);
+					if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
+					else onDataAppended(payload);
+				}
 			}
+			appender.closeSync();
+			const timeMs = Math.round(Date.now() - timeStart);
+			this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
+				table: table.getFullName(),
+				timeMs,
+				totalRows
+			});
+			return {
+				timeMs,
+				totalRows,
+				createTableDDL: ddl
+			};
+		} catch (e) {
+			appender.closeSync();
+			const msg = `Failed to append data into table '${table.getFullName()}' - ${e?.message ?? ""}`;
+			this.#logger.error(msg, { table: table.getFullName() });
+			throw new Error(msg, { cause: e });
 		}
-		return {
-			timeMs: Math.round(Date.now() - timeStart),
-			totalRows,
-			createTableDDL: ddl
-		};
 	};
 };
 //#endregion
@@ -390,5 +480,7 @@ const zodCodecs = {
 exports.DuckMemory = DuckMemory;
 exports.SqlDuck = SqlDuck;
 exports.Table = Table;
+exports.flowbladeLogtapeSqlduckConfig = flowbladeLogtapeSqlduckConfig;
 exports.getTableCreateFromZod = getTableCreateFromZod;
+exports.sqlduckDefaultLogtapeLogger = sqlduckDefaultLogtapeLogger;
 exports.zodCodecs = zodCodecs;

package/dist/index.d.cts

@@ -1,4 +1,6 @@
 import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
+import * as _logtape_logtape0 from "@logtape/logtape";
+import { Logger } from "@logtape/logtape";
 import * as z from "zod";
 import { ZodObject } from "zod";
 
@@ -21,6 +23,11 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
 type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
 type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
 //#endregion
+//#region src/config/flowblade-logtape-sqlduck.config.d.ts
+declare const flowbladeLogtapeSqlduckConfig: {
+  categories: string[];
+};
+//#endregion
 //#region src/helpers/duck-memory.d.ts
 declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
 type DuckMemoryTag = (typeof duckMemoryTags)[number];
@@ -49,6 +56,9 @@ declare class DuckMemory {
   getSummary: () => Promise<DuckMemorySummary>;
 }
 //#endregion
+//#region src/logger/sqlduck-default-logtape-logger.d.ts
+declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
+//#endregion
 //#region src/table/table.d.ts
 /**
  * Fully qualified table information
@@ -76,24 +86,37 @@ declare class Table {
   withSchema: (schema: string) => Table;
 }
 //#endregion
+//#region src/table/table-schema-zod.type.d.ts
+type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID;
+type TableSchemaZod = z.ZodObject<Record<string, ZodSchemaSupportedTypes | z.ZodNullable<ZodSchemaSupportedTypes> | z.ZodCodec | z.ZodNullable<z.ZodCodec>>>;
+//#endregion
 //#region src/table/get-table-create-from-zod.d.ts
 type TableCreateOptions = {
   create?: 'CREATE' | 'CREATE_OR_REPLACE' | 'IF_NOT_EXISTS';
 };
-declare const getTableCreateFromZod: <T extends ZodObject>(table: Table, schema: T, options?: TableCreateOptions) => {
+type DuckdbColumnTypeMap<TKeys extends string> = Map<TKeys, DuckDBType>;
+type TableCreateFromZodResult<TSchema extends TableSchemaZod> = {
   ddl: string;
-  columnTypes: [name: string, type: DuckDBType][];
+  columnTypes: DuckdbColumnTypeMap<Exclude<keyof TSchema['shape'], symbol | number>>;
 };
-//#endregion
-//#region src/table/table-schema-zod.type.d.ts
-type TableSchemaZod = ZodObject;
+type GetTableCreateFromZodParams<TSchema extends TableSchemaZod> = {
+  table: Table;
+  schema: TSchema;
+  options?: TableCreateOptions;
+};
+declare const getTableCreateFromZod: <TSchema extends TableSchemaZod>(params: GetTableCreateFromZodParams<TSchema>) => TableCreateFromZodResult<TSchema>;
 //#endregion
 //#region src/sql-duck.d.ts
 type SqlDuckParams = {
   conn: DuckDBConnection;
-  logger?: (msg: string) => void;
+  /**
+   * Optional logtape/logger to use for logging.
+   * If not provided, a default logger will be used.
+   * @see {@link https://github.com/logtape/logtape}
+   */
+  logger?: Logger;
 };
-type RowStream<T> = AsyncIterableIterator<T>;
+type RowStream<T> = AsyncIterableIterator<T> | AsyncGenerator<T> | Generator<T>;
 type ToTableParams<TSchema extends TableSchemaZod> = {
   /**
    * Used to create and fill the data into the table
@@ -142,7 +165,6 @@ declare class SqlDuck {
   /**
    * Create a table from a Zod schema and fill it with data from a row stream.
    *
-   *
    * @example
    * ```typescript
    * import * as z from 'zod';
@@ -186,4 +208,4 @@ declare const zodCodecs: {
   readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
 };
 //#endregion
-export { DuckMemory, DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
+export { DuckMemory, DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/dist/index.d.mts

@@ -1,3 +1,5 @@
+import * as _logtape_logtape0 from "@logtape/logtape";
+import { Logger } from "@logtape/logtape";
 import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
 import * as z from "zod";
 import { ZodObject } from "zod";
@@ -21,6 +23,11 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
 type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
 type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
 //#endregion
+//#region src/config/flowblade-logtape-sqlduck.config.d.ts
+declare const flowbladeLogtapeSqlduckConfig: {
+  categories: string[];
+};
+//#endregion
 //#region src/helpers/duck-memory.d.ts
 declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
 type DuckMemoryTag = (typeof duckMemoryTags)[number];
@@ -49,6 +56,9 @@ declare class DuckMemory {
   getSummary: () => Promise<DuckMemorySummary>;
 }
 //#endregion
+//#region src/logger/sqlduck-default-logtape-logger.d.ts
+declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
+//#endregion
 //#region src/table/table.d.ts
 /**
  * Fully qualified table information
@@ -76,24 +86,37 @@ declare class Table {
   withSchema: (schema: string) => Table;
 }
 //#endregion
+//#region src/table/table-schema-zod.type.d.ts
+type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID;
+type TableSchemaZod = z.ZodObject<Record<string, ZodSchemaSupportedTypes | z.ZodNullable<ZodSchemaSupportedTypes> | z.ZodCodec | z.ZodNullable<z.ZodCodec>>>;
+//#endregion
 //#region src/table/get-table-create-from-zod.d.ts
 type TableCreateOptions = {
   create?: 'CREATE' | 'CREATE_OR_REPLACE' | 'IF_NOT_EXISTS';
 };
-declare const getTableCreateFromZod: <T extends ZodObject>(table: Table, schema: T, options?: TableCreateOptions) => {
+type DuckdbColumnTypeMap<TKeys extends string> = Map<TKeys, DuckDBType>;
+type TableCreateFromZodResult<TSchema extends TableSchemaZod> = {
   ddl: string;
-  columnTypes: [name: string, type: DuckDBType][];
+  columnTypes: DuckdbColumnTypeMap<Exclude<keyof TSchema['shape'], symbol | number>>;
 };
-//#endregion
-//#region src/table/table-schema-zod.type.d.ts
-type TableSchemaZod = ZodObject;
+type GetTableCreateFromZodParams<TSchema extends TableSchemaZod> = {
+  table: Table;
+  schema: TSchema;
+  options?: TableCreateOptions;
+};
+declare const getTableCreateFromZod: <TSchema extends TableSchemaZod>(params: GetTableCreateFromZodParams<TSchema>) => TableCreateFromZodResult<TSchema>;
 //#endregion
 //#region src/sql-duck.d.ts
 type SqlDuckParams = {
   conn: DuckDBConnection;
-  logger?: (msg: string) => void;
+  /**
+   * Optional logtape/logger to use for logging.
+   * If not provided, a default logger will be used.
+   * @see {@link https://github.com/logtape/logtape}
+   */
+  logger?: Logger;
 };
-type RowStream<T> = AsyncIterableIterator<T>;
+type RowStream<T> = AsyncIterableIterator<T> | AsyncGenerator<T> | Generator<T>;
 type ToTableParams<TSchema extends TableSchemaZod> = {
   /**
    * Used to create and fill the data into the table
@@ -142,7 +165,6 @@ declare class SqlDuck {
   /**
    * Create a table from a Zod schema and fill it with data from a row stream.
    *
-   *
    * @example
    * ```typescript
    * import * as z from 'zod';
@@ -186,4 +208,4 @@ declare const zodCodecs: {
   readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
 };
 //#endregion
-export { DuckMemory, type DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
+export { DuckMemory, type DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/dist/index.mjs

@@ -1,5 +1,9 @@
-import { BIGINT, DuckDBDataChunk, DuckDBTimestampValue, INTEGER, TIMESTAMP, VARCHAR } from "@duckdb/node-api";
+import { getLogger } from "@logtape/logtape";
+import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
 import * as z from "zod";
+//#region src/config/flowblade-logtape-sqlduck.config.ts
+const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
+//#endregion
 //#region src/helpers/duck-exec.ts
 var DuckExec = class {
 	#conn;
@@ -96,6 +100,9 @@ var DuckMemory = class {
 	};
 };
 //#endregion
+//#region src/logger/sqlduck-default-logtape-logger.ts
+const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
+//#endregion
 //#region src/appender/data-appender-callback.ts
 const isOnDataAppendedAsyncCb = (v) => {
 	return v.constructor.name === "AsyncFunction";
@@ -117,23 +124,66 @@ const createOnDataAppendedCollector = () => {
 	};
 };
 //#endregion
+//#region src/table/get-duckdb-number-column-type.ts
+const isFloatValue = (value) => {
+	if (!Number.isFinite(value)) return true;
+	if (Math.abs(value) > Number.MAX_SAFE_INTEGER) return true;
+	return !Number.isInteger(value);
+};
+const getDuckdbNumberColumnType = (params) => {
+	const { minimum, maximum } = params;
+	if (minimum === void 0 || maximum === void 0) return BIGINT;
+	if (isFloatValue(minimum) || isFloatValue(maximum)) {
+		if (minimum >= -34028235e31 && maximum <= 34028235e31) return FLOAT;
+		return DOUBLE;
+	}
+	if (minimum >= 0) {
+		if (maximum <= 255) return UTINYINT;
+		if (maximum <= 65535) return USMALLINT;
+		if (maximum <= 4294967295) return UINTEGER;
+		if (maximum <= 18446744073709551615n) return UBIGINT;
+		return UHUGEINT;
+	}
+	if (minimum >= -128 && maximum <= 127) return TINYINT;
+	if (minimum >= -32768 && maximum <= 32767) return SMALLINT;
+	if (minimum >= -2147483648 && maximum <= 2147483647) return INTEGER;
+	if (minimum >= -9223372036854775808n && maximum <= 9223372036854775807n) return BIGINT;
+	return HUGEINT;
+};
+//#endregion
 //#region src/table/get-table-create-from-zod.ts
-const createMap = {
+const createOptions = {
 	CREATE: "CREATE TABLE",
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
 	IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
 };
-const getTableCreateFromZod = (table, schema, options) => {
+const duckDbTypes = [
+	["VARCHAR", VARCHAR],
+	["BIGINT", BIGINT],
+	["TIMESTAMP", TIMESTAMP],
+	["UUID", UUID],
+	["BOOLEAN", BOOLEAN],
+	["INTEGER", INTEGER],
+	["DOUBLE", DOUBLE],
+	["FLOAT", FLOAT]
+];
+const duckDbTypesMap = new Map(duckDbTypes);
+const getTableCreateFromZod = (params) => {
+	const { table, schema, options } = params;
 	const { create = "CREATE" } = options ?? {};
 	const fqTable = table.getFullName();
-	const json = schema.toJSONSchema({ target: "openapi-3.0" });
+	const json = schema.toJSONSchema({
+		target: "openapi-3.0",
+		unrepresentable: "throw"
+	});
 	const columns = [];
 	if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
-	const columnTypes = [];
+	const columnTypesMap = /* @__PURE__ */ new Map();
 	for (const [columnName, def] of Object.entries(json.properties)) {
-		const { type, nullable, format, primaryKey } = def;
+		const { type, duckdbType, nullable, format, primaryKey, minimum, maximum } = def;
 		const c = { name: columnName };
-		switch (type) {
+		if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
+		else switch (type) {
 			case "string":
 				switch (format) {
 					case "date-time":
@@ -142,22 +192,37 @@ const getTableCreateFromZod = (table, schema, options) => {
 					case "int64":
 						c.duckdbType = BIGINT;
 						break;
+					case "uuid":
+						c.duckdbType = UUID;
+						break;
 					default: c.duckdbType = VARCHAR;
 				}
 				break;
 			case "number":
-				c.duckdbType = INTEGER;
+				c.duckdbType = getDuckdbNumberColumnType({
+					minimum,
+					maximum
+				});
 				break;
-			default: throw new Error("Not a supported type");
+			case "integer":
+				c.duckdbType = getDuckdbNumberColumnType({
+					minimum,
+					maximum
+				});
+				break;
+			case "boolean":
+				c.duckdbType = BOOLEAN;
+				break;
+			default: throw new Error(`Cannot guess '${columnName}' type - ${JSON.stringify(def)}`);
 		}
 		if (primaryKey === true) c.constraint = "PRIMARY KEY";
 		else if (nullable !== true) c.constraint = "NOT NULL";
-		columnTypes.push([columnName, c.duckdbType]);
+		columnTypesMap.set(columnName, c.duckdbType);
 		columns.push(c);
 	}
 	return {
 		ddl: [
-			`${createMap[create]} ${fqTable} (\n`,
+			`${createOptions[create]} ${fqTable} (\n`,
 			columns.map((colDDL) => {
 				const { name, duckdbType, constraint } = colDDL;
 				return `  ${[
@@ -168,17 +233,24 @@ const getTableCreateFromZod = (table, schema, options) => {
 			}).join(",\n"),
 			"\n)"
 		].join(""),
-		columnTypes
+		columnTypes: columnTypesMap
 	};
 };
 //#endregion
 //#region src/table/create-table-from-zod.ts
 const createTableFromZod = async (params) => {
-	const { conn, table, schema, options } = params;
-	const { ddl, columnTypes } = getTableCreateFromZod(table, schema, options);
+	const { conn, table, schema, options, logger = sqlduckDefaultLogtapeLogger } = params;
+	const { ddl, columnTypes } = getTableCreateFromZod({
+		table,
+		schema,
+		options
+	});
+	logger.debug(`Generate DDL for table '${table.getFullName()}'`, { ddl });
 	try {
 		await conn.run(ddl);
+		logger.info(`Table '${table.getFullName()}' successfully created`, { ddl });
 	} catch (e) {
+		logger.error(`Failed to create table '${table.getFullName()}': ${e.message}`, { ddl });
 		throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
 	}
 	return {
@@ -190,6 +262,7 @@ const createTableFromZod = async (params) => {
 //#region src/utils/rows-to-columns-chunks.ts
 const toDuckValue = (value) => {
 	if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
+	if (typeof value === "bigint") return value.toString(10);
 	return value === void 0 ? null : value;
 };
 /**
@@ -201,7 +274,8 @@ const toDuckValue = (value) => {
 *   input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
 *   yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
 */
-async function* rowsToColumnsChunks(rows, chunkSize) {
+async function* rowsToColumnsChunks(params) {
+	const { rows, chunkSize } = params;
 	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
 	const first = await rows.next();
 	if (first.done) return;
@@ -233,12 +307,11 @@ var SqlDuck = class {
 	#logger;
 	constructor(params) {
 		this.#duck = params.conn;
-		this.#logger = params.logger;
+		this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
 	}
 	/**
 	* Create a table from a Zod schema and fill it with data from a row stream.
 	*
-	*
 	* @example
 	* ```typescript
 	* import * as z from 'zod';
@@ -284,28 +357,45 @@ var SqlDuck = class {
 			options: createOptions
 		});
 		const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
-		const chunkTypes = columnTypes.map((v) => v[1]);
+		const chunkTypes = Array.from(columnTypes.values());
 		let totalRows = 0;
 		const dataAppendedCollector = createOnDataAppendedCollector();
-		const columnStream = rowsToColumnsChunks(rowStream, chunkSize);
-		for await (const dataChunk of columnStream) {
-			const chunk = DuckDBDataChunk.create(chunkTypes);
-			if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
-			totalRows += dataChunk?.[0]?.length ?? 0;
-			chunk.setColumns(dataChunk);
-			appender.appendDataChunk(chunk);
-			appender.flushSync();
-			if (onDataAppended !== void 0) {
-				const payload = dataAppendedCollector(totalRows);
-				if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
-				else onDataAppended(payload);
+		const columnStream = rowsToColumnsChunks({
+			rows: rowStream,
+			chunkSize
+		});
+		try {
+			for await (const dataChunk of columnStream) {
+				const chunk = DuckDBDataChunk.create(chunkTypes);
+				this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
+				totalRows += dataChunk?.[0]?.length ?? 0;
+				chunk.setColumns(dataChunk);
+				appender.appendDataChunk(chunk);
+				appender.flushSync();
+				if (onDataAppended !== void 0) {
+					const payload = dataAppendedCollector(totalRows);
+					if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
+					else onDataAppended(payload);
+				}
 			}
+			appender.closeSync();
+			const timeMs = Math.round(Date.now() - timeStart);
+			this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
+				table: table.getFullName(),
+				timeMs,
+				totalRows
+			});
+			return {
+				timeMs,
+				totalRows,
+				createTableDDL: ddl
+			};
+		} catch (e) {
+			appender.closeSync();
+			const msg = `Failed to append data into table '${table.getFullName()}' - ${e?.message ?? ""}`;
+			this.#logger.error(msg, { table: table.getFullName() });
+			throw new Error(msg, { cause: e });
 		}
-		return {
-			timeMs: Math.round(Date.now() - timeStart),
-			totalRows,
-			createTableDDL: ddl
-		};
 	};
 };
 //#endregion
@@ -363,4 +453,4 @@ const zodCodecs = {
 	})
 };
 //#endregion
-export { DuckMemory, SqlDuck, Table, getTableCreateFromZod, zodCodecs };
+export { DuckMemory, SqlDuck, Table, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.8.3",
+  "version": "0.10.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -35,8 +35,8 @@
     "docgen-typedoc": "rimraf ./docs/api && typedoc --plugin typedoc-plugin-markdown --out ./docs/api",
     "bench": "vitest bench --run",
     "bench-bun": "bun --bun run vitest bench --run",
-    "bench-mitata": "node --experimental-strip-types --expose-gc ./bench/stream.mitata.ts ",
-    "bench-mitata-bun": "bun --expose-gc ./bench/stream.mitata.ts ",
+    "bench-mitata": "node --experimental-strip-types --expose-gc bench/run_mitata.ts",
+    "bench-mitata-bun": "bun --expose-gc bench/run_mitata.ts",
     "bench-codspeed": "cross-env CODSPEED=1 vitest bench --run",
     "bench-watch": "vitest bench",
     "test": "vitest run",
@@ -57,6 +57,7 @@
     "@flowblade/core": "^0.2.26",
     "@flowblade/source-duckdb": "^0.20.1",
     "@flowblade/sql-tag": "^0.3.2",
+    "@logtape/logtape": "2.0.4",
     "@standard-schema/spec": "^1.1.0",
     "p-mutex": "^1.0.0",
     "valibot": "^1.3.1",
@@ -67,7 +68,7 @@
   },
   "devDependencies": {
     "@belgattitude/eslint-config-bases": "8.10.0",
-    "@dotenvx/dotenvx": "1.55.1",
+    "@dotenvx/dotenvx": "1.57.2",
     "@duckdb/node-api": "1.5.1-r.1",
     "@faker-js/faker": "10.3.0",
     "@flowblade/source-kysely": "^1.3.0",