@flowblade/sqlduck 0.9.0 → 0.10.0

package/dist/index.cjs

@@ -21,9 +21,13 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 	enumerable: true
 }) : target, mod));
 //#endregion
+let _logtape_logtape = require("@logtape/logtape");
 let _duckdb_node_api = require("@duckdb/node-api");
 let zod = require("zod");
 zod = __toESM(zod);
+//#region src/config/flowblade-logtape-sqlduck.config.ts
+const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
+//#endregion
 //#region src/helpers/duck-exec.ts
 var DuckExec = class {
 	#conn;
@@ -120,6 +124,9 @@ var DuckMemory = class {
 	};
 };
 //#endregion
+//#region src/logger/sqlduck-default-logtape-logger.ts
+const sqlduckDefaultLogtapeLogger = (0, _logtape_logtape.getLogger)(flowbladeLogtapeSqlduckConfig.categories);
+//#endregion
 //#region src/appender/data-appender-callback.ts
 const isOnDataAppendedAsyncCb = (v) => {
 	return v.constructor.name === "AsyncFunction";
@@ -174,6 +181,17 @@ const createOptions = {
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
 	IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
 };
+const duckDbTypes = [
+	["VARCHAR", _duckdb_node_api.VARCHAR],
+	["BIGINT", _duckdb_node_api.BIGINT],
+	["TIMESTAMP", _duckdb_node_api.TIMESTAMP],
+	["UUID", _duckdb_node_api.UUID],
+	["BOOLEAN", _duckdb_node_api.BOOLEAN],
+	["INTEGER", _duckdb_node_api.INTEGER],
+	["DOUBLE", _duckdb_node_api.DOUBLE],
+	["FLOAT", _duckdb_node_api.FLOAT]
+];
+const duckDbTypesMap = new Map(duckDbTypes);
 const getTableCreateFromZod = (params) => {
 	const { table, schema, options } = params;
 	const { create = "CREATE" } = options ?? {};
@@ -186,9 +204,10 @@ const getTableCreateFromZod = (params) => {
 	if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
 	const columnTypesMap = /* @__PURE__ */ new Map();
 	for (const [columnName, def] of Object.entries(json.properties)) {
-		const { type, nullable, format, primaryKey, minimum, maximum } = def;
+		const { type, duckdbType, nullable, format, primaryKey, minimum, maximum } = def;
 		const c = { name: columnName };
-		switch (type) {
+		if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
+		else switch (type) {
 			case "string":
 				switch (format) {
 					case "date-time":
@@ -244,15 +263,18 @@ const getTableCreateFromZod = (params) => {
 //#endregion
 //#region src/table/create-table-from-zod.ts
 const createTableFromZod = async (params) => {
-	const { conn, table, schema, options } = params;
+	const { conn, table, schema, options, logger = sqlduckDefaultLogtapeLogger } = params;
 	const { ddl, columnTypes } = getTableCreateFromZod({
 		table,
 		schema,
 		options
 	});
+	logger.debug(`Generate DDL for table '${table.getFullName()}'`, { ddl });
 	try {
 		await conn.run(ddl);
+		logger.info(`Table '${table.getFullName()}' successfully created`, { ddl });
 	} catch (e) {
+		logger.error(`Failed to create table '${table.getFullName()}': ${e.message}`, { ddl });
 		throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
 	}
 	return {
@@ -309,7 +331,7 @@ var SqlDuck = class {
 	#logger;
 	constructor(params) {
 		this.#duck = params.conn;
-		this.#logger = params.logger;
+		this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
 	}
 	/**
 	* Create a table from a Zod schema and fill it with data from a row stream.
@@ -366,25 +388,38 @@ var SqlDuck = class {
 			rows: rowStream,
 			chunkSize
 		});
-		for await (const dataChunk of columnStream) {
-			const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
-			if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
-			totalRows += dataChunk?.[0]?.length ?? 0;
-			chunk.setColumns(dataChunk);
-			appender.appendDataChunk(chunk);
-			appender.flushSync();
-			if (onDataAppended !== void 0) {
-				const payload = dataAppendedCollector(totalRows);
-				if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
-				else onDataAppended(payload);
+		try {
+			for await (const dataChunk of columnStream) {
+				const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
+				this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
+				totalRows += dataChunk?.[0]?.length ?? 0;
+				chunk.setColumns(dataChunk);
+				appender.appendDataChunk(chunk);
+				appender.flushSync();
+				if (onDataAppended !== void 0) {
+					const payload = dataAppendedCollector(totalRows);
+					if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
+					else onDataAppended(payload);
+				}
 			}
+			appender.closeSync();
+			const timeMs = Math.round(Date.now() - timeStart);
+			this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
+				table: table.getFullName(),
+				timeMs,
+				totalRows
+			});
+			return {
+				timeMs,
+				totalRows,
+				createTableDDL: ddl
+			};
+		} catch (e) {
+			appender.closeSync();
+			const msg = `Failed to append data into table '${table.getFullName()}' - ${e?.message ?? ""}`;
+			this.#logger.error(msg, { table: table.getFullName() });
+			throw new Error(msg, { cause: e });
 		}
-		appender.closeSync();
-		return {
-			timeMs: Math.round(Date.now() - timeStart),
-			totalRows,
-			createTableDDL: ddl
-		};
 	};
 };
 //#endregion
@@ -445,5 +480,7 @@ const zodCodecs = {
 exports.DuckMemory = DuckMemory;
 exports.SqlDuck = SqlDuck;
 exports.Table = Table;
+exports.flowbladeLogtapeSqlduckConfig = flowbladeLogtapeSqlduckConfig;
 exports.getTableCreateFromZod = getTableCreateFromZod;
+exports.sqlduckDefaultLogtapeLogger = sqlduckDefaultLogtapeLogger;
 exports.zodCodecs = zodCodecs;

package/dist/index.d.cts

@@ -1,4 +1,6 @@
 import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
+import * as _logtape_logtape0 from "@logtape/logtape";
+import { Logger } from "@logtape/logtape";
 import * as z from "zod";
 import { ZodObject } from "zod";
 
@@ -21,6 +23,11 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
 type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
 type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
 //#endregion
+//#region src/config/flowblade-logtape-sqlduck.config.d.ts
+declare const flowbladeLogtapeSqlduckConfig: {
+  categories: string[];
+};
+//#endregion
 //#region src/helpers/duck-memory.d.ts
 declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
 type DuckMemoryTag = (typeof duckMemoryTags)[number];
@@ -49,6 +56,9 @@ declare class DuckMemory {
   getSummary: () => Promise<DuckMemorySummary>;
 }
 //#endregion
+//#region src/logger/sqlduck-default-logtape-logger.d.ts
+declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
+//#endregion
 //#region src/table/table.d.ts
 /**
  * Fully qualified table information
@@ -99,9 +109,14 @@ declare const getTableCreateFromZod: <TSchema extends TableSchemaZod>(params: Ge
 //#region src/sql-duck.d.ts
 type SqlDuckParams = {
   conn: DuckDBConnection;
-  logger?: (msg: string) => void;
+  /**
+   * Optional logtape/logger to use for logging.
+   * If not provided, a default logger will be used.
+   * @see {@link https://github.com/logtape/logtape}
+   */
+  logger?: Logger;
 };
-type RowStream<T> = AsyncIterableIterator<T>;
+type RowStream<T> = AsyncIterableIterator<T> | AsyncGenerator<T> | Generator<T>;
 type ToTableParams<TSchema extends TableSchemaZod> = {
   /**
    * Used to create and fill the data into the table
@@ -193,4 +208,4 @@ declare const zodCodecs: {
   readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
 };
 //#endregion
-export { DuckMemory, DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
+export { DuckMemory, DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/dist/index.d.mts

@@ -1,3 +1,5 @@
+import * as _logtape_logtape0 from "@logtape/logtape";
+import { Logger } from "@logtape/logtape";
 import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
 import * as z from "zod";
 import { ZodObject } from "zod";
@@ -21,6 +23,11 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
 type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
 type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
 //#endregion
+//#region src/config/flowblade-logtape-sqlduck.config.d.ts
+declare const flowbladeLogtapeSqlduckConfig: {
+  categories: string[];
+};
+//#endregion
 //#region src/helpers/duck-memory.d.ts
 declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
 type DuckMemoryTag = (typeof duckMemoryTags)[number];
@@ -49,6 +56,9 @@ declare class DuckMemory {
   getSummary: () => Promise<DuckMemorySummary>;
 }
 //#endregion
+//#region src/logger/sqlduck-default-logtape-logger.d.ts
+declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
+//#endregion
 //#region src/table/table.d.ts
 /**
  * Fully qualified table information
@@ -99,9 +109,14 @@ declare const getTableCreateFromZod: <TSchema extends TableSchemaZod>(params: Ge
 //#region src/sql-duck.d.ts
 type SqlDuckParams = {
   conn: DuckDBConnection;
-  logger?: (msg: string) => void;
+  /**
+   * Optional logtape/logger to use for logging.
+   * If not provided, a default logger will be used.
+   * @see {@link https://github.com/logtape/logtape}
+   */
+  logger?: Logger;
 };
-type RowStream<T> = AsyncIterableIterator<T>;
+type RowStream<T> = AsyncIterableIterator<T> | AsyncGenerator<T> | Generator<T>;
 type ToTableParams<TSchema extends TableSchemaZod> = {
   /**
    * Used to create and fill the data into the table
@@ -193,4 +208,4 @@ declare const zodCodecs: {
   readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
 };
 //#endregion
-export { DuckMemory, type DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
+export { DuckMemory, type DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/dist/index.mjs

@@ -1,5 +1,9 @@
+import { getLogger } from "@logtape/logtape";
 import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
 import * as z from "zod";
+//#region src/config/flowblade-logtape-sqlduck.config.ts
+const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
+//#endregion
 //#region src/helpers/duck-exec.ts
 var DuckExec = class {
 	#conn;
@@ -96,6 +100,9 @@ var DuckMemory = class {
 	};
 };
 //#endregion
+//#region src/logger/sqlduck-default-logtape-logger.ts
+const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
+//#endregion
 //#region src/appender/data-appender-callback.ts
 const isOnDataAppendedAsyncCb = (v) => {
 	return v.constructor.name === "AsyncFunction";
@@ -150,6 +157,17 @@ const createOptions = {
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
 	IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
 };
+const duckDbTypes = [
+	["VARCHAR", VARCHAR],
+	["BIGINT", BIGINT],
+	["TIMESTAMP", TIMESTAMP],
+	["UUID", UUID],
+	["BOOLEAN", BOOLEAN],
+	["INTEGER", INTEGER],
+	["DOUBLE", DOUBLE],
+	["FLOAT", FLOAT]
+];
+const duckDbTypesMap = new Map(duckDbTypes);
 const getTableCreateFromZod = (params) => {
 	const { table, schema, options } = params;
 	const { create = "CREATE" } = options ?? {};
@@ -162,9 +180,10 @@ const getTableCreateFromZod = (params) => {
 	if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
 	const columnTypesMap = /* @__PURE__ */ new Map();
 	for (const [columnName, def] of Object.entries(json.properties)) {
-		const { type, nullable, format, primaryKey, minimum, maximum } = def;
+		const { type, duckdbType, nullable, format, primaryKey, minimum, maximum } = def;
 		const c = { name: columnName };
-		switch (type) {
+		if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
+		else switch (type) {
 			case "string":
 				switch (format) {
 					case "date-time":
@@ -220,15 +239,18 @@ const getTableCreateFromZod = (params) => {
 //#endregion
 //#region src/table/create-table-from-zod.ts
 const createTableFromZod = async (params) => {
-	const { conn, table, schema, options } = params;
+	const { conn, table, schema, options, logger = sqlduckDefaultLogtapeLogger } = params;
 	const { ddl, columnTypes } = getTableCreateFromZod({
 		table,
 		schema,
 		options
 	});
+	logger.debug(`Generate DDL for table '${table.getFullName()}'`, { ddl });
 	try {
 		await conn.run(ddl);
+		logger.info(`Table '${table.getFullName()}' successfully created`, { ddl });
 	} catch (e) {
+		logger.error(`Failed to create table '${table.getFullName()}': ${e.message}`, { ddl });
 		throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
 	}
 	return {
@@ -285,7 +307,7 @@ var SqlDuck = class {
 	#logger;
 	constructor(params) {
 		this.#duck = params.conn;
-		this.#logger = params.logger;
+		this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
 	}
 	/**
 	* Create a table from a Zod schema and fill it with data from a row stream.
@@ -342,25 +364,38 @@ var SqlDuck = class {
 			rows: rowStream,
 			chunkSize
 		});
-		for await (const dataChunk of columnStream) {
-			const chunk = DuckDBDataChunk.create(chunkTypes);
-			if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
-			totalRows += dataChunk?.[0]?.length ?? 0;
-			chunk.setColumns(dataChunk);
-			appender.appendDataChunk(chunk);
-			appender.flushSync();
-			if (onDataAppended !== void 0) {
-				const payload = dataAppendedCollector(totalRows);
-				if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
-				else onDataAppended(payload);
+		try {
+			for await (const dataChunk of columnStream) {
+				const chunk = DuckDBDataChunk.create(chunkTypes);
+				this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
+				totalRows += dataChunk?.[0]?.length ?? 0;
+				chunk.setColumns(dataChunk);
+				appender.appendDataChunk(chunk);
+				appender.flushSync();
+				if (onDataAppended !== void 0) {
+					const payload = dataAppendedCollector(totalRows);
+					if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
+					else onDataAppended(payload);
+				}
 			}
+			appender.closeSync();
+			const timeMs = Math.round(Date.now() - timeStart);
+			this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
+				table: table.getFullName(),
+				timeMs,
+				totalRows
+			});
+			return {
+				timeMs,
+				totalRows,
+				createTableDDL: ddl
+			};
+		} catch (e) {
+			appender.closeSync();
+			const msg = `Failed to append data into table '${table.getFullName()}' - ${e?.message ?? ""}`;
+			this.#logger.error(msg, { table: table.getFullName() });
+			throw new Error(msg, { cause: e });
 		}
-		appender.closeSync();
-		return {
-			timeMs: Math.round(Date.now() - timeStart),
-			totalRows,
-			createTableDDL: ddl
-		};
 	};
 };
 //#endregion
@@ -418,4 +453,4 @@ const zodCodecs = {
 	})
 };
 //#endregion
-export { DuckMemory, SqlDuck, Table, getTableCreateFromZod, zodCodecs };
+export { DuckMemory, SqlDuck, Table, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -57,6 +57,7 @@
     "@flowblade/core": "^0.2.26",
     "@flowblade/source-duckdb": "^0.20.1",
     "@flowblade/sql-tag": "^0.3.2",
+    "@logtape/logtape": "2.0.4",
     "@standard-schema/spec": "^1.1.0",
     "p-mutex": "^1.0.0",
     "valibot": "^1.3.1",
@@ -67,7 +68,7 @@
   },
   "devDependencies": {
     "@belgattitude/eslint-config-bases": "8.10.0",
-    "@dotenvx/dotenvx": "1.55.1",
+    "@dotenvx/dotenvx": "1.57.2",
     "@duckdb/node-api": "1.5.1-r.1",
     "@faker-js/faker": "10.3.0",
     "@flowblade/source-kysely": "^1.3.0",