@flowblade/sqlduck 0.8.0 → 0.8.2

package/dist/index.cjs

@@ -1,3 +1,5 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+//#region \0rolldown/runtime.js
 var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
@@ -18,9 +20,11 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 	value: mod,
 	enumerable: true
 }) : target, mod));
+//#endregion
 let _duckdb_node_api = require("@duckdb/node-api");
 let zod = require("zod");
 zod = __toESM(zod);
+//#region src/helpers/duck-exec.ts
 var DuckExec = class {
 	#conn;
 	constructor(duckConn) {
@@ -48,6 +52,8 @@ var DuckExec = class {
 		if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
 	};
 };
+//#endregion
+//#region src/helpers/duck-memory.ts
 const duckMemoryTags = [
 	"BASE_TABLE",
 	"HASH_TABLE",
@@ -62,7 +68,9 @@ const duckMemoryTags = [
 	"ALLOCATOR",
 	"EXTENSION",
 	"TRANSACTION",
-	"EXTERNAL_FILE_CACHE"
+	"EXTERNAL_FILE_CACHE",
+	"WINDOW",
+	"OBJECT_CACHE"
 ];
 const orderByParams = {
 	memory_usage_bytes_desc: "memory_usage_bytes DESC",
@@ -111,6 +119,8 @@ var DuckMemory = class {
 		return `${query} ORDER BY ${orderByClause}`;
 	};
 };
+//#endregion
+//#region src/appender/data-appender-callback.ts
 const isOnDataAppendedAsyncCb = (v) => {
 	return v.constructor.name === "AsyncFunction";
 };
@@ -130,6 +140,8 @@ const createOnDataAppendedCollector = () => {
 		return stats;
 	};
 };
+//#endregion
+//#region src/table/get-table-create-from-zod.ts
 const createMap = {
 	CREATE: "CREATE TABLE",
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
@@ -183,6 +195,8 @@ const getTableCreateFromZod = (table, schema, options) => {
 		columnTypes
 	};
 };
+//#endregion
+//#region src/table/create-table-from-zod.ts
 const createTableFromZod = async (params) => {
 	const { conn, table, schema, options } = params;
 	const { ddl, columnTypes } = getTableCreateFromZod(table, schema, options);
@@ -196,10 +210,21 @@ const createTableFromZod = async (params) => {
 		columnTypes
 	};
 };
+//#endregion
+//#region src/utils/rows-to-columns-chunks.ts
 const toDuckValue = (value) => {
 	if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
 	return value === void 0 ? null : value;
 };
+/**
+* Similar to `rowsToColumns` but yields results in chunks to avoid buffering
+* the entire dataset in memory. Each yielded item is a columns array for up to
+* `chunkSize` rows.
+*
+* Example for chunkSize = 2:
+*   input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
+*   yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
+*/
 async function* rowsToColumnsChunks(rows, chunkSize) {
 	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
 	const first = await rows.next();
@@ -225,6 +250,8 @@ async function* rowsToColumnsChunks(rows, chunkSize) {
 	}
 	if (rowsInChunk > 0) yield columns;
 }
+//#endregion
+//#region src/sql-duck.ts
 var SqlDuck = class {
 	#duck;
 	#logger;
@@ -232,6 +259,44 @@ var SqlDuck = class {
 		this.#duck = params.conn;
 		this.#logger = params.logger;
 	}
+	/**
+	* Create a table from a Zod schema and fill it with data from a row stream.
+	*
+	*
+	* @example
+	* ```typescript
+	* import * as z from 'zod';
+	*
+	* const sqlDuck = new SqlDuck({ conn: duckDbConnection });
+	*
+	* // Schema of the table, not that you can use meta to add information
+	* const userSchema = z.object({
+	*  id: z.number().int().meta({ primaryKey: true }),
+	*  name: z.string(),
+	* });
+	*
+	* // Async generator function that yields rows to insert
+	* async function* getUserRows(): AsyncIterableIterator<z.infer<typeof userSchema>> {
+	*   // database or api call
+	* }
+	*
+	* const result = sqlDuck.toTable({
+	*  table: new Table({ name: 'user', database: 'mydb' }),
+	*  schema: userSchema,
+	*  rowStream: getUserRows(),
+	*  chunkSize: 2048,
+	*  onDataAppended: ({ total }) => {
+	*    console.log(`Appended ${total} rows so far`);
+	*  },
+	*  createOptions: {
+	*    create: 'CREATE_OR_REPLACE',
+	*  },
+	* });
+	*
+	* console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
+	* console.log(`Table created with DDL: ${result.createTableDDL}`);
+	* ```
+	*/
 	toTable = async (params) => {
 		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
 		if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
@@ -267,6 +332,8 @@ var SqlDuck = class {
 		};
 	};
 };
+//#endregion
+//#region src/table/table.ts
 var Table = class Table {
 	#fqTable;
 	get tableName() {
@@ -281,6 +348,10 @@ var Table = class Table {
 	constructor(fqTableOrName) {
 		this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
 	}
+	/**
+	* Return fully qualified table name by concatenating
+	* database, schema and table with a 'dot' separator.
+	*/
 	getFullName = (options) => {
 		const { defaultDatabase, defaultSchema } = options ?? {};
 		const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
@@ -303,6 +374,8 @@ var Table = class Table {
 		});
 	};
 };
+//#endregion
+//#region src/utils/zod-codecs.ts
 const zodCodecs = {
 	dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
 		decode: (date) => date.toISOString(),
@@ -313,6 +386,7 @@ const zodCodecs = {
 		encode: BigInt
 	})
 };
+//#endregion
 exports.DuckMemory = DuckMemory;
 exports.SqlDuck = SqlDuck;
 exports.Table = Table;

package/dist/index.d.cts

@@ -22,7 +22,7 @@ type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
 type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
 //#endregion
 //#region src/helpers/duck-memory.d.ts
-declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE"];
+declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
 type DuckMemoryTag = (typeof duckMemoryTags)[number];
 type DuckMemoryRow = {
   tag: DuckMemoryTag;

package/dist/index.d.mts

@@ -22,7 +22,7 @@ type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
 type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
 //#endregion
 //#region src/helpers/duck-memory.d.ts
-declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE"];
+declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
 type DuckMemoryTag = (typeof duckMemoryTags)[number];
 type DuckMemoryRow = {
   tag: DuckMemoryTag;

package/dist/index.mjs

@@ -1,5 +1,6 @@
 import { BIGINT, DuckDBDataChunk, DuckDBTimestampValue, INTEGER, TIMESTAMP, VARCHAR } from "@duckdb/node-api";
 import * as z from "zod";
+//#region src/helpers/duck-exec.ts
 var DuckExec = class {
 	#conn;
 	constructor(duckConn) {
@@ -27,6 +28,8 @@ var DuckExec = class {
 		if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
 	};
 };
+//#endregion
+//#region src/helpers/duck-memory.ts
 const duckMemoryTags = [
 	"BASE_TABLE",
 	"HASH_TABLE",
@@ -41,7 +44,9 @@ const duckMemoryTags = [
 	"ALLOCATOR",
 	"EXTENSION",
 	"TRANSACTION",
-	"EXTERNAL_FILE_CACHE"
+	"EXTERNAL_FILE_CACHE",
+	"WINDOW",
+	"OBJECT_CACHE"
 ];
 const orderByParams = {
 	memory_usage_bytes_desc: "memory_usage_bytes DESC",
@@ -90,6 +95,8 @@ var DuckMemory = class {
 		return `${query} ORDER BY ${orderByClause}`;
 	};
 };
+//#endregion
+//#region src/appender/data-appender-callback.ts
 const isOnDataAppendedAsyncCb = (v) => {
 	return v.constructor.name === "AsyncFunction";
 };
@@ -109,6 +116,8 @@ const createOnDataAppendedCollector = () => {
 		return stats;
 	};
 };
+//#endregion
+//#region src/table/get-table-create-from-zod.ts
 const createMap = {
 	CREATE: "CREATE TABLE",
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
@@ -162,6 +171,8 @@ const getTableCreateFromZod = (table, schema, options) => {
 		columnTypes
 	};
 };
+//#endregion
+//#region src/table/create-table-from-zod.ts
 const createTableFromZod = async (params) => {
 	const { conn, table, schema, options } = params;
 	const { ddl, columnTypes } = getTableCreateFromZod(table, schema, options);
@@ -175,10 +186,21 @@ const createTableFromZod = async (params) => {
 		columnTypes
 	};
 };
+//#endregion
+//#region src/utils/rows-to-columns-chunks.ts
 const toDuckValue = (value) => {
 	if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
 	return value === void 0 ? null : value;
 };
+/**
+* Similar to `rowsToColumns` but yields results in chunks to avoid buffering
+* the entire dataset in memory. Each yielded item is a columns array for up to
+* `chunkSize` rows.
+*
+* Example for chunkSize = 2:
+*   input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
+*   yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
+*/
 async function* rowsToColumnsChunks(rows, chunkSize) {
 	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
 	const first = await rows.next();
@@ -204,6 +226,8 @@ async function* rowsToColumnsChunks(rows, chunkSize) {
 	}
 	if (rowsInChunk > 0) yield columns;
 }
+//#endregion
+//#region src/sql-duck.ts
 var SqlDuck = class {
 	#duck;
 	#logger;
@@ -211,6 +235,44 @@ var SqlDuck = class {
 		this.#duck = params.conn;
 		this.#logger = params.logger;
 	}
+	/**
+	* Create a table from a Zod schema and fill it with data from a row stream.
+	*
+	*
+	* @example
+	* ```typescript
+	* import * as z from 'zod';
+	*
+	* const sqlDuck = new SqlDuck({ conn: duckDbConnection });
+	*
+	* // Schema of the table, not that you can use meta to add information
+	* const userSchema = z.object({
+	*  id: z.number().int().meta({ primaryKey: true }),
+	*  name: z.string(),
+	* });
+	*
+	* // Async generator function that yields rows to insert
+	* async function* getUserRows(): AsyncIterableIterator<z.infer<typeof userSchema>> {
+	*   // database or api call
+	* }
+	*
+	* const result = sqlDuck.toTable({
+	*  table: new Table({ name: 'user', database: 'mydb' }),
+	*  schema: userSchema,
+	*  rowStream: getUserRows(),
+	*  chunkSize: 2048,
+	*  onDataAppended: ({ total }) => {
+	*    console.log(`Appended ${total} rows so far`);
+	*  },
+	*  createOptions: {
+	*    create: 'CREATE_OR_REPLACE',
+	*  },
+	* });
+	*
+	* console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
+	* console.log(`Table created with DDL: ${result.createTableDDL}`);
+	* ```
+	*/
 	toTable = async (params) => {
 		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
 		if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
@@ -246,6 +308,8 @@ var SqlDuck = class {
 		};
 	};
 };
+//#endregion
+//#region src/table/table.ts
 var Table = class Table {
 	#fqTable;
 	get tableName() {
@@ -260,6 +324,10 @@ var Table = class Table {
 	constructor(fqTableOrName) {
 		this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
 	}
+	/**
+	* Return fully qualified table name by concatenating
+	* database, schema and table with a 'dot' separator.
+	*/
 	getFullName = (options) => {
 		const { defaultDatabase, defaultSchema } = options ?? {};
 		const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
@@ -282,6 +350,8 @@ var Table = class Table {
 		});
 	};
 };
+//#endregion
+//#region src/utils/zod-codecs.ts
 const zodCodecs = {
 	dateToString: z.codec(z.date(), z.iso.datetime(), {
 		decode: (date) => date.toISOString(),
@@ -292,4 +362,5 @@ const zodCodecs = {
 		encode: BigInt
 	})
 };
+//#endregion
 export { DuckMemory, SqlDuck, Table, getTableCreateFromZod, zodCodecs };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -54,63 +54,62 @@
     "check-size": "size-limit"
   },
   "dependencies": {
-    "@flowblade/core": "^0.2.25",
-    "@flowblade/source-duckdb": "^0.18.0",
-    "@flowblade/sql-tag": "^0.3.1",
+    "@flowblade/core": "^0.2.26",
+    "@flowblade/source-duckdb": "^0.20.0",
+    "@flowblade/sql-tag": "^0.3.2",
     "@standard-schema/spec": "^1.1.0",
     "p-mutex": "^1.0.0",
     "valibot": "^1.2.0",
     "zod": "^4.3.6"
   },
   "peerDependencies": {
-    "@duckdb/node-api": "^1.4.4-r.1"
+    "@duckdb/node-api": "^1.5.0-r.1"
   },
   "devDependencies": {
-    "@belgattitude/eslint-config-bases": "8.9.0",
-    "@dotenvx/dotenvx": "1.52.0",
-    "@duckdb/node-api": "1.4.4-r.1",
-    "@faker-js/faker": "10.2.0",
-    "@flowblade/source-kysely": "^1.2.2",
-    "@httpx/assert": "0.16.7",
+    "@belgattitude/eslint-config-bases": "8.10.0",
+    "@dotenvx/dotenvx": "1.55.1",
+    "@duckdb/node-api": "1.5.0-r.1",
+    "@faker-js/faker": "10.3.0",
+    "@flowblade/source-kysely": "^1.3.0",
+    "@httpx/assert": "0.16.8",
     "@mitata/counters": "0.0.8",
-    "@size-limit/esbuild": "12.0.0",
-    "@size-limit/file": "12.0.0",
-    "@testcontainers/mssqlserver": "11.11.0",
+    "@size-limit/esbuild": "12.0.1",
+    "@size-limit/file": "12.0.1",
+    "@testcontainers/mssqlserver": "11.12.0",
     "@total-typescript/ts-reset": "0.6.1",
     "@traversable/zod": "0.0.57",
-    "@types/node": "25.0.10",
-    "@typescript-eslint/eslint-plugin": "8.54.0",
-    "@typescript-eslint/parser": "8.54.0",
-    "@vitest/coverage-v8": "4.0.18",
-    "@vitest/ui": "4.0.18",
+    "@types/node": "25.5.0",
+    "@typescript-eslint/eslint-plugin": "8.57.0",
+    "@typescript-eslint/parser": "8.57.0",
+    "@vitest/coverage-v8": "4.1.0",
+    "@vitest/ui": "4.1.0",
     "ansis": "4.2.0",
     "browserslist-to-esbuild": "2.1.1",
     "core-js": "3.48.0",
     "cross-env": "10.1.0",
-    "es-check": "9.5.4",
-    "esbuild": "0.27.2",
+    "es-check": "9.6.2",
+    "esbuild": "0.27.4",
     "eslint": "8.57.1",
     "execa": "9.6.1",
     "is-in-ci": "2.0.0",
-    "kysely": "0.28.10",
+    "kysely": "0.28.12",
     "mitata": "1.0.34",
     "npm-run-all2": "8.0.4",
     "prettier": "3.8.1",
-    "publint": "0.3.17",
+    "publint": "0.3.18",
     "regexp.escape": "2.0.1",
-    "rimraf": "6.1.2",
-    "size-limit": "12.0.0",
-    "sql-formatter": "15.7.0",
+    "rimraf": "6.1.3",
+    "size-limit": "12.0.1",
+    "sql-formatter": "15.7.2",
     "tarn": "3.0.2",
-    "tedious": "19.2.0",
-    "testcontainers": "11.11.0",
-    "tsdown": "0.20.1",
+    "tedious": "19.2.1",
+    "testcontainers": "11.12.0",
+    "tsdown": "0.21.3",
     "tsx": "4.21.0",
-    "typedoc": "0.28.16",
-    "typedoc-plugin-markdown": "4.9.0",
+    "typedoc": "0.28.17",
+    "typedoc-plugin-markdown": "4.10.0",
     "typescript": "5.9.3",
-    "vite-tsconfig-paths": "6.0.5",
-    "vitest": "4.0.18"
+    "vitest": "4.1.0"
   },
   "files": [
     "dist"