@flowblade/sqlduck 0.9.0 → 0.11.0

package/dist/index.mjs

@@ -1,5 +1,8 @@
 import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
+import { getLogger } from "@logtape/logtape";
 import * as z from "zod";
+import { assertNever } from "@httpx/assert";
+import { isPlainObject } from "@httpx/plain-object";
 //#region src/helpers/duck-exec.ts
 var DuckExec = class {
 	#conn;
@@ -117,6 +120,12 @@ const createOnDataAppendedCollector = () => {
 	};
 };
 //#endregion
+//#region src/config/flowblade-logtape-sqlduck.config.ts
+const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
+//#endregion
+//#region src/logger/sqlduck-default-logtape-logger.ts
+const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
+//#endregion
 //#region src/table/get-duckdb-number-column-type.ts
 const isFloatValue = (value) => {
 	if (!Number.isFinite(value)) return true;
@@ -150,6 +159,16 @@ const createOptions = {
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
 	IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
 };
+const duckDbTypesMap = new Map([
+	["VARCHAR", VARCHAR],
+	["BIGINT", BIGINT],
+	["TIMESTAMP", TIMESTAMP],
+	["UUID", UUID],
+	["BOOLEAN", BOOLEAN],
+	["INTEGER", INTEGER],
+	["DOUBLE", DOUBLE],
+	["FLOAT", FLOAT]
+]);
 const getTableCreateFromZod = (params) => {
 	const { table, schema, options } = params;
 	const { create = "CREATE" } = options ?? {};
@@ -162,9 +181,10 @@ const getTableCreateFromZod = (params) => {
 	if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
 	const columnTypesMap = /* @__PURE__ */ new Map();
 	for (const [columnName, def] of Object.entries(json.properties)) {
-		const { type, nullable, format, primaryKey, minimum, maximum } = def;
+		const { type, duckdbType, nullable, format, primaryKey, minimum, maximum } = def;
 		const c = { name: columnName };
-		switch (type) {
+		if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
+		else switch (type) {
 			case "string":
 				switch (format) {
 					case "date-time":
@@ -220,15 +240,18 @@ const getTableCreateFromZod = (params) => {
 //#endregion
 //#region src/table/create-table-from-zod.ts
 const createTableFromZod = async (params) => {
-	const { conn, table, schema, options } = params;
+	const { conn, table, schema, options, logger = sqlduckDefaultLogtapeLogger } = params;
 	const { ddl, columnTypes } = getTableCreateFromZod({
 		table,
 		schema,
 		options
 	});
+	logger.debug(`Generate DDL for table '${table.getFullName()}'`, { ddl });
 	try {
 		await conn.run(ddl);
+		logger.info(`Table '${table.getFullName()}' successfully created`, { ddl });
 	} catch (e) {
+		logger.error(`Failed to create table '${table.getFullName()}': ${e.message}`, { ddl });
 		throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
 	}
 	return {
@@ -281,11 +304,11 @@ async function* rowsToColumnsChunks(params) {
 //#endregion
 //#region src/sql-duck.ts
 var SqlDuck = class {
-	#duck;
+	#conn;
 	#logger;
 	constructor(params) {
-		this.#duck = params.conn;
-		this.#logger = params.logger;
+		this.#conn = params.conn;
+		this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
 	}
 	/**
 	* Create a table from a Zod schema and fill it with data from a row stream.
@@ -329,12 +352,12 @@ var SqlDuck = class {
 		if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
 		const timeStart = Date.now();
 		const { columnTypes, ddl } = await createTableFromZod({
-			conn: this.#duck,
+			conn: this.#conn,
 			schema,
 			table,
 			options: createOptions
 		});
-		const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
+		const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
 		const chunkTypes = Array.from(columnTypes.values());
 		let totalRows = 0;
 		const dataAppendedCollector = createOnDataAppendedCollector();
@@ -342,29 +365,74 @@ var SqlDuck = class {
 			rows: rowStream,
 			chunkSize
 		});
-		for await (const dataChunk of columnStream) {
-			const chunk = DuckDBDataChunk.create(chunkTypes);
-			if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
-			totalRows += dataChunk?.[0]?.length ?? 0;
-			chunk.setColumns(dataChunk);
-			appender.appendDataChunk(chunk);
-			appender.flushSync();
-			if (onDataAppended !== void 0) {
-				const payload = dataAppendedCollector(totalRows);
-				if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
-				else onDataAppended(payload);
+		try {
+			for await (const dataChunk of columnStream) {
+				const chunk = DuckDBDataChunk.create(chunkTypes);
+				this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
+				totalRows += dataChunk?.[0]?.length ?? 0;
+				chunk.setColumns(dataChunk);
+				appender.appendDataChunk(chunk);
+				appender.flushSync();
+				if (onDataAppended !== void 0) {
+					const payload = dataAppendedCollector(totalRows);
+					if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
+					else onDataAppended(payload);
+				}
 			}
+			appender.closeSync();
+			const timeMs = Math.round(Date.now() - timeStart);
+			this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
+				table: table.getFullName(),
+				timeMs,
+				totalRows
+			});
+			return {
+				timeMs,
+				totalRows,
+				createTableDDL: ddl
+			};
+		} catch (e) {
+			appender.closeSync();
+			const msg = `Failed to append data into table '${table.getFullName()}' - ${e?.message ?? ""}`;
+			this.#logger.error(msg, { table: table.getFullName() });
+			throw new Error(msg, { cause: e });
 		}
-		appender.closeSync();
+	};
+};
+//#endregion
+//#region src/utils/zod-codecs.ts
+const zodCodecs = {
+	dateToString: z.codec(z.date(), z.iso.datetime(), {
+		decode: (date) => date.toISOString(),
+		encode: (isoString) => new Date(isoString)
+	}),
+	bigintToString: z.codec(z.bigint(), z.string().meta({ format: "int64" }), {
+		decode: (bigint) => bigint.toString(),
+		encode: BigInt
+	})
+};
+//#endregion
+//#region src/objects/database.ts
+var Database = class {
+	#params;
+	get alias() {
+		return this.#params.alias;
+	}
+	constructor(params) {
+		this.#params = params;
+	}
+	toJson() {
 		return {
-			timeMs: Math.round(Date.now() - timeStart),
-			totalRows,
-			createTableDDL: ddl
+			type: "database",
+			params: { alias: this.#params.alias }
 		};
-	};
+	}
+	[Symbol.toStringTag]() {
+		return this.alias;
+	}
 };
 //#endregion
-//#region src/table/table.ts
+//#region src/objects/table.ts
 var Table = class Table {
 	#fqTable;
 	get tableName() {
@@ -406,16 +474,261 @@ var Table = class Table {
 	};
 };
 //#endregion
-//#region src/utils/zod-codecs.ts
-const zodCodecs = {
-	dateToString: z.codec(z.date(), z.iso.datetime(), {
-		decode: (date) => date.toISOString(),
-		encode: (isoString) => new Date(isoString)
-	}),
-	bigintToString: z.codec(z.bigint(), z.string().meta({ format: "int64" }), {
-		decode: (bigint) => bigint.toString(),
-		encode: BigInt
-	})
+//#region src/validation/core/duckdb-reserved-keywords.ts
+/**
+* DuckDB reserved keywords that cannot be used as unquoted identifiers.
+* @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
+*/
+const duckdbReservedKeywords = [
+	"ALL",
+	"ANALYSE",
+	"ANALYZE",
+	"AND",
+	"ANY",
+	"ARRAY",
+	"AS",
+	"ASC",
+	"ASYMMETRIC",
+	"BOTH",
+	"CASE",
+	"CAST",
+	"CHECK",
+	"COLLATE",
+	"COLUMN",
+	"CONSTRAINT",
+	"CREATE",
+	"CROSS",
+	"CURRENT_CATALOG",
+	"CURRENT_DATE",
+	"CURRENT_ROLE",
+	"CURRENT_SCHEMA",
+	"CURRENT_TIME",
+	"CURRENT_TIMESTAMP",
+	"CURRENT_USER",
+	"DEFAULT",
+	"DEFERRABLE",
+	"DESC",
+	"DISTINCT",
+	"DO",
+	"ELSE",
+	"END",
+	"EXCEPT",
+	"EXISTS",
+	"EXTRACT",
+	"FALSE",
+	"FETCH",
+	"FOR",
+	"FOREIGN",
+	"FROM",
+	"GRANT",
+	"GROUP",
+	"HAVING",
+	"IF",
+	"ILIKE",
+	"IN",
+	"INITIALLY",
+	"INNER",
+	"INTERSECT",
+	"INTO",
+	"IS",
+	"ISNULL",
+	"JOIN",
+	"LATERAL",
+	"LEADING",
+	"LEFT",
+	"LIKE",
+	"LIMIT",
+	"LOCALTIME",
+	"LOCALTIMESTAMP",
+	"NATURAL",
+	"NOT",
+	"NOTNULL",
+	"NULL",
+	"OFFSET",
+	"ON",
+	"ONLY",
+	"OR",
+	"ORDER",
+	"OUTER",
+	"OVERLAPS",
+	"PLACING",
+	"PRIMARY",
+	"REFERENCES",
+	"RETURNING",
+	"RIGHT",
+	"ROW",
+	"SELECT",
+	"SESSION_USER",
+	"SIMILAR",
+	"SOME",
+	"SYMMETRIC",
+	"TABLE",
+	"THEN",
+	"TO",
+	"TRAILING",
+	"TRUE",
+	"UNION",
+	"UNIQUE",
+	"USING",
+	"VARIADIC",
+	"VERBOSE",
+	"WHEN",
+	"WHERE",
+	"WINDOW",
+	"WITH"
+];
+//#endregion
+//#region src/validation/zod/duckdb-valid-names.schemas.ts
+const duckdbMaximumObjectNameLength = 120;
+const duckDbObjectNameRegex = /^[a-z_]\w*$/i;
+const duckdbReservedKeywordsSet = new Set(duckdbReservedKeywords.map((k) => k.toUpperCase()));
+const duckTableNameSchema = z.string().min(1).max(duckdbMaximumObjectNameLength).regex(duckDbObjectNameRegex, "Table name must start with a letter or underscore, and contain only letters, numbers and underscores").refine((value) => !duckdbReservedKeywordsSet.has(value.toUpperCase()), { error: `Value is a DuckDB reserved keyword and cannot be used as a table name` });
+const duckTableAliasSchema = duckTableNameSchema;
+//#endregion
+//#region src/manager/database/duck-database-manager.schemas.ts
+const duckdbAttachOptionsSchema = z.strictObject({
+	ACCESS_MODE: z.optional(z.enum([
+		"READ_ONLY",
+		"READ_WRITE",
+		"AUTOMATIC"
+	])),
+	COMPRESS: z.optional(z.enum(["true", "false"])),
+	TYPE: z.optional(z.enum(["DUCKDB", "SQLITE"])),
+	BLOCK_SIZE: z.optional(z.int32().min(16384).max(262144)),
+	ROW_GROUP_SIZE: z.optional(z.int32().positive()),
+	STORAGE_VERSION: z.optional(z.string().startsWith("v").regex(/^v?\d{1,4}\.\d{1,4}\.\d{1,4}$/)),
+	ENCRYPTION_KEY: z.optional(z.string().min(8)),
+	ENCRYPTION_CIPHER: z.optional(z.enum([
+		"CBC",
+		"CTR",
+		"GCM"
+	]))
+});
+const duckDatabaseManagerDbParamsSchema = z.discriminatedUnion("type", [z.strictObject({
+	type: z.literal(":memory:"),
+	alias: duckTableAliasSchema,
+	options: z.optional(duckdbAttachOptionsSchema)
+}), z.strictObject({
+	type: z.literal("duckdb"),
+	path: z.string().min(4).endsWith(".db"),
+	alias: duckTableAliasSchema,
+	options: z.optional(duckdbAttachOptionsSchema)
+})]);
+//#endregion
+//#region src/manager/database/commands/duck-database-attach-command.ts
+var DuckDatabaseAttachCommand = class {
+	options;
+	dbParams;
+	constructor(dbParams, options) {
+		this.dbParams = dbParams;
+		this.options = options ?? {};
+	}
+	getRawSql = () => {
+		const dbParams = this.dbParams;
+		const parts = ["ATTACH", this.options.behaviour].filter(Boolean);
+		const { type, alias } = dbParams;
+		switch (type) {
+			case ":memory:":
+				parts.push("':memory:'");
+				break;
+			case "duckdb":
+				parts.push(`'${dbParams.path}'`);
+				break;
+			default: assertNever(type);
+		}
+		if (alias !== null) parts.push("AS", `${alias}`);
+		const options = isPlainObject(dbParams.options) ? Object.entries(dbParams.options).map(([key, value]) => {
+			return key === "ACCESS_MODE" ? value : `${key} '${value}'`;
+		}) : [];
+		if (options.length > 0) parts.push(`(${options.join(", ")})`);
+		return parts.filter(Boolean).join(" ");
+	};
+};
+//#endregion
+//#region src/manager/database/duck-database-manager.ts
+var DuckDatabaseManager = class {
+	#conn;
+	#logger;
+	constructor(conn, params) {
+		this.#conn = conn;
+		this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
+	}
+	/**
+	* Attach a database to the current connection
+	*
+	* @example
+	* ```typescript
+	* const dbManager = new DuckDatabaseManager(conn);
+	* const database = dbManager.attach({
+	*   type: ':memory:', // can be 'duckdb', 's3'...
+	*   alias: 'mydb',
+	*   options: { COMPRESS: 'true' }
+	* });
+	*
+	* console.log(database.alias); // 'mydb'
+	* ```
+	*/
+	attach = async (dbParams, options) => {
+		const params = z.parse(duckDatabaseManagerDbParamsSchema, dbParams);
+		const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
+		await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
+		return new Database({ alias: params.alias });
+	};
+	attachOrReplace = async (dbParams) => {
+		return this.attach(dbParams, { behaviour: "OR REPLACE" });
+	};
+	attachIfNotExists = async (dbParams) => {
+		return this.attach(dbParams, { behaviour: "IF NOT EXISTS" });
+	};
+	showDatabases = async () => {
+		return await this.#executeRawSqlCommand("showDatabases()", `SHOW DATABASES`);
+	};
+	detach = async (dbAlias) => {
+		const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
+		await this.#executeRawSqlCommand(`detach(${safeAlias})`, `DETACH ${safeAlias}`);
+		return true;
+	};
+	detachIfExists = async (dbAlias) => {
+		const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
+		await this.#executeRawSqlCommand(`detachIfExists(${safeAlias})`, `DETACH IF EXISTS ${safeAlias}`);
+		return true;
+	};
+	/**
+	* The statistics recomputed by the ANALYZE statement are only used for join order optimization.
+	*
+	* It is therefore recommended to recompute these statistics for improved join orders,
+	* especially after performing large updates (inserts and/or deletes).
+	*
+	* @link https://duckdb.org/docs/stable/sql/statements/analyze
+	*/
+	analyze = async () => {
+		await this.#executeRawSqlCommand("analyze()", "ANALYZE");
+		return true;
+	};
+	checkpoint = async (dbAlias) => {
+		const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
+		await this.#executeRawSqlCommand(`checkpoint(${safeAlias})`, `CHECKPOINT ${safeAlias}`);
+		return true;
+	};
+	#executeRawSqlCommand = async (name, rawSql) => {
+		const startTime = Date.now();
+		try {
+			const result = await this.#conn.runAndReadAll(rawSql);
+			const timeMs = Math.round(Date.now() - startTime);
+			const data = result.getRowObjectsJS();
+			this.#logger.info(`DuckDatabaseManager.${name} in ${timeMs}ms`, { timeMs });
+			return data;
+		} catch (e) {
+			const msg = `DuckDatabaseManager: failed to run "${name}" - ${e?.message ?? ""}`;
+			const timeMs = Math.round(Date.now() - startTime);
+			this.#logger.error(msg, {
+				name,
+				sql: rawSql,
+				timeMs
+			});
+			throw new Error(msg, { cause: e });
+		}
+	};
 };
 //#endregion
-export { DuckMemory, SqlDuck, Table, getTableCreateFromZod, zodCodecs };
+export { Database, DuckDatabaseManager, DuckMemory, SqlDuck, Table, duckDatabaseManagerDbParamsSchema, duckTableAliasSchema, duckTableNameSchema, duckdbReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -57,8 +57,11 @@
     "@flowblade/core": "^0.2.26",
     "@flowblade/source-duckdb": "^0.20.1",
     "@flowblade/sql-tag": "^0.3.2",
+    "@httpx/assert": "^0.16.8",
+    "@httpx/plain-object": "^2.1.8",
+    "@logtape/logtape": "^2.0.5",
     "@standard-schema/spec": "^1.1.0",
-    "p-mutex": "^1.0.0",
+    "p-queue": "9.1.0",
     "valibot": "^1.3.1",
     "zod": "^4.3.6"
   },
@@ -67,7 +70,7 @@
   },
   "devDependencies": {
     "@belgattitude/eslint-config-bases": "8.10.0",
-    "@dotenvx/dotenvx": "1.55.1",
+    "@dotenvx/dotenvx": "1.57.2",
     "@duckdb/node-api": "1.5.1-r.1",
     "@faker-js/faker": "10.3.0",
     "@flowblade/source-kysely": "^1.3.0",
@@ -81,7 +84,7 @@
     "@types/node": "25.5.0",
     "@typescript-eslint/eslint-plugin": "8.57.2",
     "@typescript-eslint/parser": "8.57.2",
-    "@typescript/native-preview": "7.0.0-dev.20260316.1",
+    "@typescript/native-preview": "7.0.0-dev.20260324.1",
     "@vitest/coverage-v8": "4.1.1",
     "@vitest/ui": "4.1.1",
     "ansis": "4.2.0",
@@ -89,6 +92,7 @@
     "core-js": "3.49.0",
     "cross-env": "10.1.0",
     "es-check": "9.6.3",
+    "es-toolkit": "1.45.1",
     "esbuild": "0.27.4",
     "eslint": "8.57.1",
     "execa": "9.6.1",
@@ -105,11 +109,11 @@
     "tarn": "3.0.2",
     "tedious": "19.2.1",
     "testcontainers": "11.13.0",
-    "tsdown": "0.21.4",
+    "tsdown": "0.21.5",
     "tsx": "4.21.0",
-    "typedoc": "0.28.17",
+    "typedoc": "0.28.18",
     "typedoc-plugin-markdown": "4.11.0",
-    "typescript": "5.9.3",
+    "typescript": "6.0.2",
     "vitest": "4.1.1"
   },
   "files": [