@flowblade/sqlduck 0.0.1

package/README.md

@@ -0,0 +1,20 @@
+## @flowblade/sqlduck
+
+> Currently experimental
+
+### Quick start
+
+### Environment variables
+
+### Schema
+
+### Local scripts
+
+| Name              | Description                    |
+| ----------------- | ------------------------------ |
+| `yarn build`      |                                |
+| `yarn typecheck`  |                                |
+| `yarn lint`       | Check for lint errors          |
+| `yarn lint --fix` | Attempt to run linter auto-fix |
+| `yarn test-unit`  | Run unit tests                 |
+| `yarn test-e2e`   | Run unit tests                 |

package/dist/index.cjs

@@ -0,0 +1,182 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+	if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
+		key = keys[i];
+		if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
+			get: ((k) => from[k]).bind(null, key),
+			enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+		});
+	}
+	return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
+	value: mod,
+	enumerable: true
+}) : target, mod));
+let _duckdb_node_api = require("@duckdb/node-api");
+let zod = require("zod");
+zod = __toESM(zod);
+const toDuckValue = (value) => {
+	if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
+	return value === void 0 ? null : value;
+};
+async function* rowsToColumnsChunks(rows, chunkSize) {
+	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
+	const first = await rows.next();
+	if (first.done) return;
+	const keys = Object.keys(first.value);
+	let columns = keys.map(() => []);
+	let rowsInChunk = 0;
+	keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
+	rowsInChunk++;
+	if (rowsInChunk >= chunkSize) {
+		yield columns;
+		columns = keys.map(() => []);
+		rowsInChunk = 0;
+	}
+	for await (const row of rows) {
+		keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
+		rowsInChunk++;
+		if (rowsInChunk >= chunkSize) {
+			yield columns;
+			columns = keys.map(() => []);
+			rowsInChunk = 0;
+		}
+	}
+	if (rowsInChunk > 0) yield columns;
+}
+const getTableCreateFromZod = (table, schema) => {
+	const fqTable = table.getFullyQualifiedTableName();
+	const json = schema.toJSONSchema({ target: "openapi-3.0" });
+	const columns = [];
+	if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
+	const columnTypes = [];
+	for (const [columnName, def] of Object.entries(json.properties)) {
+		const { type, nullable, format, primaryKey } = def;
+		const c = { name: columnName };
+		switch (type) {
+			case "string":
+				switch (format) {
+					case "date-time":
+						c.duckdbType = _duckdb_node_api.TIMESTAMP;
+						break;
+					case "int64":
+						c.duckdbType = _duckdb_node_api.BIGINT;
+						break;
+					default: c.duckdbType = _duckdb_node_api.VARCHAR;
+				}
+				break;
+			case "number":
+				c.duckdbType = _duckdb_node_api.INTEGER;
+				break;
+			default: throw new Error("Not a supported type");
+		}
+		if (primaryKey === true) c.constraint = "PRIMARY KEY";
+		else if (nullable !== true) c.constraint = "NOT NULL";
+		columnTypes.push([columnName, c.duckdbType]);
+		columns.push(c);
+	}
+	return {
+		ddl: [
+			`CREATE OR REPLACE TABLE ${fqTable} (\n`,
+			columns.map((colDDL) => {
+				const { name, duckdbType, constraint } = colDDL;
+				return `  ${[
+					name,
+					duckdbType.toString(),
+					constraint
+				].filter(Boolean).join(" ")}`;
+			}).join(",\n"),
+			"\n)"
+		].join(""),
+		columnTypes
+	};
+};
+const createTableFromZod = async (params) => {
+	const { conn, table, schema } = params;
+	const { ddl, columnTypes } = getTableCreateFromZod(table, schema);
+	try {
+		await conn.run(ddl);
+	} catch (e) {
+		throw new Error(`Failed to create table '${table.getFullyQualifiedTableName()}': ${e.message}`, { cause: e });
+	}
+	return {
+		ddl,
+		columnTypes
+	};
+};
+var SqlDuck = class {
+	#duck;
+	#logger;
+	constructor(params) {
+		this.#duck = params.conn;
+		this.#logger = params.logger;
+	}
+	toTable = async (params) => {
+		const { table, schema, chunkSize, rowStream } = params;
+		const { columnTypes } = await createTableFromZod({
+			conn: this.#duck,
+			schema,
+			table
+		});
+		const appender = await this.#duck.createAppender(table.fqTable.name, table.fqTable.schema, table.fqTable.database);
+		const chunkTypes = columnTypes.map((v) => v[1]);
+		const columnStream = rowsToColumnsChunks(rowStream, chunkSize ?? 2048);
+		for await (const dataChunk of columnStream) {
+			const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
+			if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
+			chunk.setColumns(dataChunk);
+			appender.appendDataChunk(chunk);
+			appender.flushSync();
+		}
+	};
+};
+var Table = class Table {
+	#fqTable;
+	get fqTable() {
+		return this.#fqTable;
+	}
+	constructor(fqTableOrName) {
+		this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
+	}
+	getFullyQualifiedTableName = (options) => {
+		const { defaultDatabase, defaultSchema } = options ?? {};
+		const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
+		return [
+			database,
+			schema,
+			name
+		].filter(Boolean).join(".");
+	};
+	withDatabase = (database) => {
+		return new Table({
+			...this.#fqTable,
+			database
+		});
+	};
+	withSchema = (schema) => {
+		return new Table({
+			...this.#fqTable,
+			schema
+		});
+	};
+};
+const zodCodecs = {
+	dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
+		decode: (date) => date.toISOString(),
+		encode: (isoString) => new Date(isoString)
+	}),
+	bigintToString: zod.codec(zod.bigint(), zod.string().meta({ format: "int64" }), {
+		decode: (bigint) => bigint.toString(),
+		encode: BigInt
+	})
+};
+exports.SqlDuck = SqlDuck;
+exports.Table = Table;
+exports.getTableCreateFromZod = getTableCreateFromZod;
+exports.zodCodecs = zodCodecs;

package/dist/index.d.cts

@@ -0,0 +1,62 @@
+import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
+import * as z from "zod";
+import { ZodObject } from "zod";
+
+//#region src/table/table.d.ts
+/**
+ * Fully qualified table information
+ */
+type FQTable = {
+  name: string;
+  schema?: string;
+  database?: string;
+};
+declare class Table {
+  #private;
+  get fqTable(): FQTable;
+  constructor(fqTableOrName: FQTable | string);
+  /**
+   * Return fully qualified table name by concatenating
+   * database, schema and table with a 'dot' separator.
+   */
+  getFullyQualifiedTableName: (options?: {
+    defaultDatabase?: string;
+    defaultSchema?: string;
+  }) => string;
+  withDatabase: (database: string) => Table;
+  withSchema: (schema: string) => Table;
+}
+//#endregion
+//#region src/table/table-schema-zod.type.d.ts
+type TableSchemaZod = ZodObject;
+//#endregion
+//#region src/sql-duck.d.ts
+type SqlDuckParams = {
+  conn: DuckDBConnection;
+  logger?: (msg: string) => void;
+};
+type ToTableParams<TSchema extends TableSchemaZod> = {
+  table: Table;
+  schema: TSchema;
+  rowStream: AsyncIterableIterator<z.infer<TSchema>>;
+  chunkSize?: number;
+};
+declare class SqlDuck {
+  #private;
+  constructor(params: SqlDuckParams);
+  toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<undefined>;
+}
+//#endregion
+//#region src/table/get-table-create-from-zod.d.ts
+declare const getTableCreateFromZod: <T extends ZodObject>(table: Table, schema: T) => {
+  ddl: string;
+  columnTypes: [name: string, type: DuckDBType][];
+};
+//#endregion
+//#region src/utils/zod-codecs.d.ts
+declare const zodCodecs: {
+  readonly dateToString: z.ZodCodec<z.ZodDate, z.ZodISODateTime>;
+  readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
+};
+//#endregion
+export { SqlDuck, type SqlDuckParams, Table, getTableCreateFromZod, zodCodecs };

package/dist/index.d.mts

@@ -0,0 +1,62 @@
+import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
+import * as z from "zod";
+import { ZodObject } from "zod";
+
+//#region src/table/table.d.ts
+/**
+ * Fully qualified table information
+ */
+type FQTable = {
+  name: string;
+  schema?: string;
+  database?: string;
+};
+declare class Table {
+  #private;
+  get fqTable(): FQTable;
+  constructor(fqTableOrName: FQTable | string);
+  /**
+   * Return fully qualified table name by concatenating
+   * database, schema and table with a 'dot' separator.
+   */
+  getFullyQualifiedTableName: (options?: {
+    defaultDatabase?: string;
+    defaultSchema?: string;
+  }) => string;
+  withDatabase: (database: string) => Table;
+  withSchema: (schema: string) => Table;
+}
+//#endregion
+//#region src/table/table-schema-zod.type.d.ts
+type TableSchemaZod = ZodObject;
+//#endregion
+//#region src/sql-duck.d.ts
+type SqlDuckParams = {
+  conn: DuckDBConnection;
+  logger?: (msg: string) => void;
+};
+type ToTableParams<TSchema extends TableSchemaZod> = {
+  table: Table;
+  schema: TSchema;
+  rowStream: AsyncIterableIterator<z.infer<TSchema>>;
+  chunkSize?: number;
+};
+declare class SqlDuck {
+  #private;
+  constructor(params: SqlDuckParams);
+  toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<undefined>;
+}
+//#endregion
+//#region src/table/get-table-create-from-zod.d.ts
+declare const getTableCreateFromZod: <T extends ZodObject>(table: Table, schema: T) => {
+  ddl: string;
+  columnTypes: [name: string, type: DuckDBType][];
+};
+//#endregion
+//#region src/utils/zod-codecs.d.ts
+declare const zodCodecs: {
+  readonly dateToString: z.ZodCodec<z.ZodDate, z.ZodISODateTime>;
+  readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
+};
+//#endregion
+export { SqlDuck, type SqlDuckParams, Table, getTableCreateFromZod, zodCodecs };

package/dist/index.mjs

@@ -0,0 +1,158 @@
+import { BIGINT, DuckDBDataChunk, DuckDBTimestampValue, INTEGER, TIMESTAMP, VARCHAR } from "@duckdb/node-api";
+import * as z from "zod";
+const toDuckValue = (value) => {
+	if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
+	return value === void 0 ? null : value;
+};
+async function* rowsToColumnsChunks(rows, chunkSize) {
+	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
+	const first = await rows.next();
+	if (first.done) return;
+	const keys = Object.keys(first.value);
+	let columns = keys.map(() => []);
+	let rowsInChunk = 0;
+	keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
+	rowsInChunk++;
+	if (rowsInChunk >= chunkSize) {
+		yield columns;
+		columns = keys.map(() => []);
+		rowsInChunk = 0;
+	}
+	for await (const row of rows) {
+		keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
+		rowsInChunk++;
+		if (rowsInChunk >= chunkSize) {
+			yield columns;
+			columns = keys.map(() => []);
+			rowsInChunk = 0;
+		}
+	}
+	if (rowsInChunk > 0) yield columns;
+}
+const getTableCreateFromZod = (table, schema) => {
+	const fqTable = table.getFullyQualifiedTableName();
+	const json = schema.toJSONSchema({ target: "openapi-3.0" });
+	const columns = [];
+	if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
+	const columnTypes = [];
+	for (const [columnName, def] of Object.entries(json.properties)) {
+		const { type, nullable, format, primaryKey } = def;
+		const c = { name: columnName };
+		switch (type) {
+			case "string":
+				switch (format) {
+					case "date-time":
+						c.duckdbType = TIMESTAMP;
+						break;
+					case "int64":
+						c.duckdbType = BIGINT;
+						break;
+					default: c.duckdbType = VARCHAR;
+				}
+				break;
+			case "number":
+				c.duckdbType = INTEGER;
+				break;
+			default: throw new Error("Not a supported type");
+		}
+		if (primaryKey === true) c.constraint = "PRIMARY KEY";
+		else if (nullable !== true) c.constraint = "NOT NULL";
+		columnTypes.push([columnName, c.duckdbType]);
+		columns.push(c);
+	}
+	return {
+		ddl: [
+			`CREATE OR REPLACE TABLE ${fqTable} (\n`,
+			columns.map((colDDL) => {
+				const { name, duckdbType, constraint } = colDDL;
+				return `  ${[
+					name,
+					duckdbType.toString(),
+					constraint
+				].filter(Boolean).join(" ")}`;
+			}).join(",\n"),
+			"\n)"
+		].join(""),
+		columnTypes
+	};
+};
+const createTableFromZod = async (params) => {
+	const { conn, table, schema } = params;
+	const { ddl, columnTypes } = getTableCreateFromZod(table, schema);
+	try {
+		await conn.run(ddl);
+	} catch (e) {
+		throw new Error(`Failed to create table '${table.getFullyQualifiedTableName()}': ${e.message}`, { cause: e });
+	}
+	return {
+		ddl,
+		columnTypes
+	};
+};
+var SqlDuck = class {
+	#duck;
+	#logger;
+	constructor(params) {
+		this.#duck = params.conn;
+		this.#logger = params.logger;
+	}
+	toTable = async (params) => {
+		const { table, schema, chunkSize, rowStream } = params;
+		const { columnTypes } = await createTableFromZod({
+			conn: this.#duck,
+			schema,
+			table
+		});
+		const appender = await this.#duck.createAppender(table.fqTable.name, table.fqTable.schema, table.fqTable.database);
+		const chunkTypes = columnTypes.map((v) => v[1]);
+		const columnStream = rowsToColumnsChunks(rowStream, chunkSize ?? 2048);
+		for await (const dataChunk of columnStream) {
+			const chunk = DuckDBDataChunk.create(chunkTypes);
+			if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
+			chunk.setColumns(dataChunk);
+			appender.appendDataChunk(chunk);
+			appender.flushSync();
+		}
+	};
+};
+var Table = class Table {
+	#fqTable;
+	get fqTable() {
+		return this.#fqTable;
+	}
+	constructor(fqTableOrName) {
+		this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
+	}
+	getFullyQualifiedTableName = (options) => {
+		const { defaultDatabase, defaultSchema } = options ?? {};
+		const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
+		return [
+			database,
+			schema,
+			name
+		].filter(Boolean).join(".");
+	};
+	withDatabase = (database) => {
+		return new Table({
+			...this.#fqTable,
+			database
+		});
+	};
+	withSchema = (schema) => {
+		return new Table({
+			...this.#fqTable,
+			schema
+		});
+	};
+};
+const zodCodecs = {
+	dateToString: z.codec(z.date(), z.iso.datetime(), {
+		decode: (date) => date.toISOString(),
+		encode: (isoString) => new Date(isoString)
+	}),
+	bigintToString: z.codec(z.bigint(), z.string().meta({ format: "int64" }), {
+		decode: (bigint) => bigint.toString(),
+		encode: BigInt
+	})
+};
+export { SqlDuck, Table, getTableCreateFromZod, zodCodecs };

package/package.json

@@ -0,0 +1,107 @@
+{
+  "name": "@flowblade/sqlduck",
+  "version": "0.0.1",
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.mts",
+        "default": "./dist/index.mjs"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "author": {
+    "name": "Vanvelthem Sébastien",
+    "url": "https://github.com/belgattitude"
+  },
+  "license": "MIT",
+  "homepage": "https://github.com/belgattitude/flowblade",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/belgattitude/flowblade.git",
+    "directory": "packages/sqlduck"
+  },
+  "scripts": {
+    "clean": "rimraf ./dist ./coverage ./tsconfig.tsbuildinfo",
+    "build": "tsdown",
+    "build-release": "yarn build && rimraf ./_release && yarn pack && mkdir ./_release && tar zxvf ./package.tgz --directory ./_release && rm ./package.tgz",
+    "test": "vitest run",
+    "test-unit": "vitest run",
+    "test-unit-watch": "vitest --ui",
+    "test-e2e": "vitest -c vitest.e2e.config.ts run",
+    "test-e2e-watch": "vitest -c vitest.e2e.config.ts --ui",
+    "typecheck": "tsc --project tsconfig.json --noEmit",
+    "lint": "eslint . --ext .ts,.tsx,.js,.jsx,.mjs,.cjs,.mts,.cts --cache --cache-location ../../../.cache/eslint/sqlduck.eslintcache",
+    "fix-staged": "lint-staged --allow-empty",
+    "check-dist": "es-check --config=.escheckrc.json",
+    "check-pub": "publint",
+    "check-size": "size-limit"
+  },
+  "dependencies": {
+    "@flowblade/core": "workspace:^",
+    "@flowblade/source-duckdb": "workspace:^",
+    "@flowblade/sql-tag": "workspace:^",
+    "@standard-schema/spec": "^1.1.0",
+    "p-mutex": "^1.0.0",
+    "valibot": "^1.2.0",
+    "zod": "^4.2.1"
+  },
+  "peerDependencies": {
+    "@duckdb/node-api": "^1.4.3-r.1"
+  },
+  "devDependencies": {
+    "@belgattitude/eslint-config-bases": "8.8.0",
+    "@dotenvx/dotenvx": "1.51.2",
+    "@duckdb/node-api": "1.4.3-r.1",
+    "@faker-js/faker": "10.1.0",
+    "@flowblade/source-kysely": "workspace:^",
+    "@httpx/assert": "0.16.7",
+    "@size-limit/esbuild": "12.0.0",
+    "@size-limit/file": "12.0.0",
+    "@testcontainers/mssqlserver": "11.10.0",
+    "@total-typescript/ts-reset": "0.6.1",
+    "@traversable/zod": "0.0.57",
+    "@types/node": "25.0.3",
+    "@typescript-eslint/eslint-plugin": "8.50.1",
+    "@typescript-eslint/parser": "8.50.1",
+    "@vitest/coverage-v8": "4.0.16",
+    "@vitest/ui": "4.0.16",
+    "ansis": "4.2.0",
+    "browserslist-to-esbuild": "2.1.1",
+    "core-js": "3.47.0",
+    "cross-env": "10.1.0",
+    "es-check": "9.5.3",
+    "esbuild": "0.27.2",
+    "eslint": "8.57.1",
+    "execa": "9.6.1",
+    "is-in-ci": "2.0.0",
+    "kysely": "0.28.9",
+    "npm-run-all2": "8.0.4",
+    "prettier": "3.7.4",
+    "publint": "0.3.16",
+    "regexp.escape": "2.0.1",
+    "rimraf": "6.1.2",
+    "size-limit": "12.0.0",
+    "sql-formatter": "15.6.12",
+    "tarn": "3.0.2",
+    "tedious": "19.1.3",
+    "testcontainers": "11.10.0",
+    "tsdown": "0.18.2",
+    "tsx": "4.21.0",
+    "typescript": "5.9.3",
+    "vite-tsconfig-paths": "6.0.3",
+    "vitest": "4.0.16"
+  },
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "directory": "_release/package"
+  }
+}