@flowblade/sqlduck 0.9.0 → 0.11.0

package/README.md

@@ -4,6 +4,74 @@
 
 ## Quick start
 
+### Create a database connection
+
+```typescript
+import { DuckDBInstance } from '@duckdb/node-api';
+DuckDBInstance.create(undefined, {
+   access_mode: 'READ_WRITE',
+   max_memory: '512M',
+});
+export const conn = await instance.connect();
+```
+
+### Append data to a database
+
+```typescript
+import { SqlDuck, DuckDatabaseManager } from "@flowblade/sqlduck";
+import * as z from "zod";
+import { conn } from "./db.config.ts";
+
+const dbManager = new DuckDatabaseManager(conn);
+const database = await dbManager.attach({
+    type: ':memory:', // can be 'duckdb', ...
+    alias: 'mydb',
+    options: { COMPRESS: 'false' },
+});
+
+const sqlDuck = new SqlDuck({ conn });
+
+// Define a zod schema, it will be used to create the table
+const userSchema = z.object({
+    id: z.int32().min(1).meta({ primaryKey: true }),
+    name: z.string(),
+});
+
+// Example of a datasource (can be generator, async generator, async iterable)
+async function* getUsers(): AsyncIterableIterator<
+    z.infer<typeof userSchema>
+> {
+    // database or api call
+    yield { id: 1, name: 'John' };
+    yield { id: 2, name: 'Jane' };
+}
+
+// Create a table from the schema and the datasource
+const result = await sqlDuck.toTable({
+    table: new Table({ name: 'user', database: database.alias }),
+    schema: userSchema, // The schema to use to create the table
+    rowStream: getUsers(), // The async iterable that yields rows
+    // 👇Optional:
+    chunkSize: 2048, // Number of rows to append when using duckdb appender. Default is 2048
+    onDataAppended: ({ timeMs, totalRows, rowsPerSecond }) => {
+        console.log(
+            `Appended ${totalRows} in time ${timeMs}ms, est: ${rowsPerSecond} rows/s`
+        );
+    },
+    // Optional table creation options
+    createOptions: {
+        create: 'CREATE_OR_REPLACE',
+    },
+});
+
+console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
+console.log(`Table created with DDL: ${result.createTableDDL}`);
+
+const reader = await conn.runAndReadAll('select * from mydb.user');
+const rows = reader.getRowObjectsJS();
+// [{id: 1, name: 'John'}, {id: 2, name: 'Jane'}]]
+```
+
 ### Create a memory table
 
 ```typescript
@@ -35,7 +103,6 @@ const result = sqlDuck.toTable({
   onDataAppended: ({ total }) => {
     console.log(`Appended ${total} rows so far`);
   },
-  onDataAppendedBatchSize: 4096, // Call onDataAppended every 4096 rows
   // Optional table creation options
   createOptions: {
     create: "CREATE_OR_REPLACE",

package/dist/index.cjs

@@ -22,8 +22,11 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 }) : target, mod));
 //#endregion
 let _duckdb_node_api = require("@duckdb/node-api");
+let _logtape_logtape = require("@logtape/logtape");
 let zod = require("zod");
 zod = __toESM(zod);
+let _httpx_assert = require("@httpx/assert");
+let _httpx_plain_object = require("@httpx/plain-object");
 //#region src/helpers/duck-exec.ts
 var DuckExec = class {
 	#conn;
@@ -141,6 +144,12 @@ const createOnDataAppendedCollector = () => {
 	};
 };
 //#endregion
+//#region src/config/flowblade-logtape-sqlduck.config.ts
+const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
+//#endregion
+//#region src/logger/sqlduck-default-logtape-logger.ts
+const sqlduckDefaultLogtapeLogger = (0, _logtape_logtape.getLogger)(flowbladeLogtapeSqlduckConfig.categories);
+//#endregion
 //#region src/table/get-duckdb-number-column-type.ts
 const isFloatValue = (value) => {
 	if (!Number.isFinite(value)) return true;
@@ -174,6 +183,16 @@ const createOptions = {
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
 	IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
 };
+const duckDbTypesMap = new Map([
+	["VARCHAR", _duckdb_node_api.VARCHAR],
+	["BIGINT", _duckdb_node_api.BIGINT],
+	["TIMESTAMP", _duckdb_node_api.TIMESTAMP],
+	["UUID", _duckdb_node_api.UUID],
+	["BOOLEAN", _duckdb_node_api.BOOLEAN],
+	["INTEGER", _duckdb_node_api.INTEGER],
+	["DOUBLE", _duckdb_node_api.DOUBLE],
+	["FLOAT", _duckdb_node_api.FLOAT]
+]);
 const getTableCreateFromZod = (params) => {
 	const { table, schema, options } = params;
 	const { create = "CREATE" } = options ?? {};
@@ -186,9 +205,10 @@ const getTableCreateFromZod = (params) => {
 	if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
 	const columnTypesMap = /* @__PURE__ */ new Map();
 	for (const [columnName, def] of Object.entries(json.properties)) {
-		const { type, nullable, format, primaryKey, minimum, maximum } = def;
+		const { type, duckdbType, nullable, format, primaryKey, minimum, maximum } = def;
 		const c = { name: columnName };
-		switch (type) {
+		if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
+		else switch (type) {
 			case "string":
 				switch (format) {
 					case "date-time":
@@ -244,15 +264,18 @@ const getTableCreateFromZod = (params) => {
 //#endregion
 //#region src/table/create-table-from-zod.ts
 const createTableFromZod = async (params) => {
-	const { conn, table, schema, options } = params;
+	const { conn, table, schema, options, logger = sqlduckDefaultLogtapeLogger } = params;
 	const { ddl, columnTypes } = getTableCreateFromZod({
 		table,
 		schema,
 		options
 	});
+	logger.debug(`Generate DDL for table '${table.getFullName()}'`, { ddl });
 	try {
 		await conn.run(ddl);
+		logger.info(`Table '${table.getFullName()}' successfully created`, { ddl });
 	} catch (e) {
+		logger.error(`Failed to create table '${table.getFullName()}': ${e.message}`, { ddl });
 		throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
 	}
 	return {
@@ -305,11 +328,11 @@ async function* rowsToColumnsChunks(params) {
 //#endregion
 //#region src/sql-duck.ts
 var SqlDuck = class {
-	#duck;
+	#conn;
 	#logger;
 	constructor(params) {
-		this.#duck = params.conn;
-		this.#logger = params.logger;
+		this.#conn = params.conn;
+		this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
 	}
 	/**
 	* Create a table from a Zod schema and fill it with data from a row stream.
@@ -353,12 +376,12 @@ var SqlDuck = class {
 		if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
 		const timeStart = Date.now();
 		const { columnTypes, ddl } = await createTableFromZod({
-			conn: this.#duck,
+			conn: this.#conn,
 			schema,
 			table,
 			options: createOptions
 		});
-		const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
+		const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
 		const chunkTypes = Array.from(columnTypes.values());
 		let totalRows = 0;
 		const dataAppendedCollector = createOnDataAppendedCollector();
@@ -366,29 +389,74 @@ var SqlDuck = class {
 			rows: rowStream,
 			chunkSize
 		});
-		for await (const dataChunk of columnStream) {
-			const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
-			if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
-			totalRows += dataChunk?.[0]?.length ?? 0;
-			chunk.setColumns(dataChunk);
-			appender.appendDataChunk(chunk);
-			appender.flushSync();
-			if (onDataAppended !== void 0) {
-				const payload = dataAppendedCollector(totalRows);
-				if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
-				else onDataAppended(payload);
+		try {
+			for await (const dataChunk of columnStream) {
+				const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
+				this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
+				totalRows += dataChunk?.[0]?.length ?? 0;
+				chunk.setColumns(dataChunk);
+				appender.appendDataChunk(chunk);
+				appender.flushSync();
+				if (onDataAppended !== void 0) {
+					const payload = dataAppendedCollector(totalRows);
+					if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
+					else onDataAppended(payload);
+				}
 			}
+			appender.closeSync();
+			const timeMs = Math.round(Date.now() - timeStart);
+			this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
+				table: table.getFullName(),
+				timeMs,
+				totalRows
+			});
+			return {
+				timeMs,
+				totalRows,
+				createTableDDL: ddl
+			};
+		} catch (e) {
+			appender.closeSync();
+			const msg = `Failed to append data into table '${table.getFullName()}' - ${e?.message ?? ""}`;
+			this.#logger.error(msg, { table: table.getFullName() });
+			throw new Error(msg, { cause: e });
 		}
-		appender.closeSync();
+	};
+};
+//#endregion
+//#region src/utils/zod-codecs.ts
+const zodCodecs = {
+	dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
+		decode: (date) => date.toISOString(),
+		encode: (isoString) => new Date(isoString)
+	}),
+	bigintToString: zod.codec(zod.bigint(), zod.string().meta({ format: "int64" }), {
+		decode: (bigint) => bigint.toString(),
+		encode: BigInt
+	})
+};
+//#endregion
+//#region src/objects/database.ts
+var Database = class {
+	#params;
+	get alias() {
+		return this.#params.alias;
+	}
+	constructor(params) {
+		this.#params = params;
+	}
+	toJson() {
 		return {
-			timeMs: Math.round(Date.now() - timeStart),
-			totalRows,
-			createTableDDL: ddl
+			type: "database",
+			params: { alias: this.#params.alias }
 		};
-	};
+	}
+	[Symbol.toStringTag]() {
+		return this.alias;
+	}
 };
 //#endregion
-//#region src/table/table.ts
+//#region src/objects/table.ts
 var Table = class Table {
 	#fqTable;
 	get tableName() {
@@ -430,20 +498,273 @@ var Table = class Table {
 	};
 };
 //#endregion
-//#region src/utils/zod-codecs.ts
-const zodCodecs = {
-	dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
-		decode: (date) => date.toISOString(),
-		encode: (isoString) => new Date(isoString)
-	}),
-	bigintToString: zod.codec(zod.bigint(), zod.string().meta({ format: "int64" }), {
-		decode: (bigint) => bigint.toString(),
-		encode: BigInt
-	})
+//#region src/validation/core/duckdb-reserved-keywords.ts
+/**
+* DuckDB reserved keywords that cannot be used as unquoted identifiers.
+* @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
+*/
+const duckdbReservedKeywords = [
+	"ALL",
+	"ANALYSE",
+	"ANALYZE",
+	"AND",
+	"ANY",
+	"ARRAY",
+	"AS",
+	"ASC",
+	"ASYMMETRIC",
+	"BOTH",
+	"CASE",
+	"CAST",
+	"CHECK",
+	"COLLATE",
+	"COLUMN",
+	"CONSTRAINT",
+	"CREATE",
+	"CROSS",
+	"CURRENT_CATALOG",
+	"CURRENT_DATE",
+	"CURRENT_ROLE",
+	"CURRENT_SCHEMA",
+	"CURRENT_TIME",
+	"CURRENT_TIMESTAMP",
+	"CURRENT_USER",
+	"DEFAULT",
+	"DEFERRABLE",
+	"DESC",
+	"DISTINCT",
+	"DO",
+	"ELSE",
+	"END",
+	"EXCEPT",
+	"EXISTS",
+	"EXTRACT",
+	"FALSE",
+	"FETCH",
+	"FOR",
+	"FOREIGN",
+	"FROM",
+	"GRANT",
+	"GROUP",
+	"HAVING",
+	"IF",
+	"ILIKE",
+	"IN",
+	"INITIALLY",
+	"INNER",
+	"INTERSECT",
+	"INTO",
+	"IS",
+	"ISNULL",
+	"JOIN",
+	"LATERAL",
+	"LEADING",
+	"LEFT",
+	"LIKE",
+	"LIMIT",
+	"LOCALTIME",
+	"LOCALTIMESTAMP",
+	"NATURAL",
+	"NOT",
+	"NOTNULL",
+	"NULL",
+	"OFFSET",
+	"ON",
+	"ONLY",
+	"OR",
+	"ORDER",
+	"OUTER",
+	"OVERLAPS",
+	"PLACING",
+	"PRIMARY",
+	"REFERENCES",
+	"RETURNING",
+	"RIGHT",
+	"ROW",
+	"SELECT",
+	"SESSION_USER",
+	"SIMILAR",
+	"SOME",
+	"SYMMETRIC",
+	"TABLE",
+	"THEN",
+	"TO",
+	"TRAILING",
+	"TRUE",
+	"UNION",
+	"UNIQUE",
+	"USING",
+	"VARIADIC",
+	"VERBOSE",
+	"WHEN",
+	"WHERE",
+	"WINDOW",
+	"WITH"
+];
+//#endregion
+//#region src/validation/zod/duckdb-valid-names.schemas.ts
+const duckdbMaximumObjectNameLength = 120;
+const duckDbObjectNameRegex = /^[a-z_]\w*$/i;
+const duckdbReservedKeywordsSet = new Set(duckdbReservedKeywords.map((k) => k.toUpperCase()));
+const duckTableNameSchema = zod.string().min(1).max(duckdbMaximumObjectNameLength).regex(duckDbObjectNameRegex, "Table name must start with a letter or underscore, and contain only letters, numbers and underscores").refine((value) => !duckdbReservedKeywordsSet.has(value.toUpperCase()), { error: `Value is a DuckDB reserved keyword and cannot be used as a table name` });
+const duckTableAliasSchema = duckTableNameSchema;
+//#endregion
+//#region src/manager/database/duck-database-manager.schemas.ts
+const duckdbAttachOptionsSchema = zod.strictObject({
+	ACCESS_MODE: zod.optional(zod.enum([
+		"READ_ONLY",
+		"READ_WRITE",
+		"AUTOMATIC"
+	])),
+	COMPRESS: zod.optional(zod.enum(["true", "false"])),
+	TYPE: zod.optional(zod.enum(["DUCKDB", "SQLITE"])),
+	BLOCK_SIZE: zod.optional(zod.int32().min(16384).max(262144)),
+	ROW_GROUP_SIZE: zod.optional(zod.int32().positive()),
+	STORAGE_VERSION: zod.optional(zod.string().startsWith("v").regex(/^v?\d{1,4}\.\d{1,4}\.\d{1,4}$/)),
+	ENCRYPTION_KEY: zod.optional(zod.string().min(8)),
+	ENCRYPTION_CIPHER: zod.optional(zod.enum([
+		"CBC",
+		"CTR",
+		"GCM"
+	]))
+});
+const duckDatabaseManagerDbParamsSchema = zod.discriminatedUnion("type", [zod.strictObject({
+	type: zod.literal(":memory:"),
+	alias: duckTableAliasSchema,
+	options: zod.optional(duckdbAttachOptionsSchema)
+}), zod.strictObject({
+	type: zod.literal("duckdb"),
+	path: zod.string().min(4).endsWith(".db"),
+	alias: duckTableAliasSchema,
+	options: zod.optional(duckdbAttachOptionsSchema)
+})]);
+//#endregion
+//#region src/manager/database/commands/duck-database-attach-command.ts
+var DuckDatabaseAttachCommand = class {
+	options;
+	dbParams;
+	constructor(dbParams, options) {
+		this.dbParams = dbParams;
+		this.options = options ?? {};
+	}
+	getRawSql = () => {
+		const dbParams = this.dbParams;
+		const parts = ["ATTACH", this.options.behaviour].filter(Boolean);
+		const { type, alias } = dbParams;
+		switch (type) {
+			case ":memory:":
+				parts.push("':memory:'");
+				break;
+			case "duckdb":
+				parts.push(`'${dbParams.path}'`);
+				break;
+			default: (0, _httpx_assert.assertNever)(type);
+		}
+		if (alias !== null) parts.push("AS", `${alias}`);
+		const options = (0, _httpx_plain_object.isPlainObject)(dbParams.options) ? Object.entries(dbParams.options).map(([key, value]) => {
+			return key === "ACCESS_MODE" ? value : `${key} '${value}'`;
+		}) : [];
+		if (options.length > 0) parts.push(`(${options.join(", ")})`);
+		return parts.filter(Boolean).join(" ");
+	};
+};
+//#endregion
+//#region src/manager/database/duck-database-manager.ts
+var DuckDatabaseManager = class {
+	#conn;
+	#logger;
+	constructor(conn, params) {
+		this.#conn = conn;
+		this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
+	}
+	/**
+	* Attach a database to the current connection
+	*
+	* @example
+	* ```typescript
+	* const dbManager = new DuckDatabaseManager(conn);
+	* const database = dbManager.attach({
+	*   type: ':memory:', // can be 'duckdb', 's3'...
+	*   alias: 'mydb',
+	*   options: { COMPRESS: 'true' }
+	* });
+	*
+	* console.log(database.alias); // 'mydb'
+	* ```
+	*/
+	attach = async (dbParams, options) => {
+		const params = zod.parse(duckDatabaseManagerDbParamsSchema, dbParams);
+		const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
+		await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
+		return new Database({ alias: params.alias });
+	};
+	attachOrReplace = async (dbParams) => {
+		return this.attach(dbParams, { behaviour: "OR REPLACE" });
+	};
+	attachIfNotExists = async (dbParams) => {
+		return this.attach(dbParams, { behaviour: "IF NOT EXISTS" });
+	};
+	showDatabases = async () => {
+		return await this.#executeRawSqlCommand("showDatabases()", `SHOW DATABASES`);
+	};
+	detach = async (dbAlias) => {
+		const safeAlias = zod.parse(duckTableAliasSchema, dbAlias);
+		await this.#executeRawSqlCommand(`detach(${safeAlias})`, `DETACH ${safeAlias}`);
+		return true;
+	};
+	detachIfExists = async (dbAlias) => {
+		const safeAlias = zod.parse(duckTableAliasSchema, dbAlias);
+		await this.#executeRawSqlCommand(`detachIfExists(${safeAlias})`, `DETACH IF EXISTS ${safeAlias}`);
+		return true;
+	};
+	/**
+	* The statistics recomputed by the ANALYZE statement are only used for join order optimization.
+	*
+	* It is therefore recommended to recompute these statistics for improved join orders,
+	* especially after performing large updates (inserts and/or deletes).
+	*
+	* @link https://duckdb.org/docs/stable/sql/statements/analyze
+	*/
+	analyze = async () => {
+		await this.#executeRawSqlCommand("analyze()", "ANALYZE");
+		return true;
+	};
+	checkpoint = async (dbAlias) => {
+		const safeAlias = zod.parse(duckTableAliasSchema, dbAlias);
+		await this.#executeRawSqlCommand(`checkpoint(${safeAlias})`, `CHECKPOINT ${safeAlias}`);
+		return true;
+	};
+	#executeRawSqlCommand = async (name, rawSql) => {
+		const startTime = Date.now();
+		try {
+			const result = await this.#conn.runAndReadAll(rawSql);
+			const timeMs = Math.round(Date.now() - startTime);
+			const data = result.getRowObjectsJS();
+			this.#logger.info(`DuckDatabaseManager.${name} in ${timeMs}ms`, { timeMs });
+			return data;
+		} catch (e) {
+			const msg = `DuckDatabaseManager: failed to run "${name}" - ${e?.message ?? ""}`;
+			const timeMs = Math.round(Date.now() - startTime);
+			this.#logger.error(msg, {
+				name,
+				sql: rawSql,
+				timeMs
+			});
+			throw new Error(msg, { cause: e });
+		}
+	};
 };
 //#endregion
+exports.Database = Database;
+exports.DuckDatabaseManager = DuckDatabaseManager;
 exports.DuckMemory = DuckMemory;
 exports.SqlDuck = SqlDuck;
 exports.Table = Table;
+exports.duckDatabaseManagerDbParamsSchema = duckDatabaseManagerDbParamsSchema;
+exports.duckTableAliasSchema = duckTableAliasSchema;
+exports.duckTableNameSchema = duckTableNameSchema;
+exports.duckdbReservedKeywords = duckdbReservedKeywords;
+exports.flowbladeLogtapeSqlduckConfig = flowbladeLogtapeSqlduckConfig;
 exports.getTableCreateFromZod = getTableCreateFromZod;
+exports.sqlduckDefaultLogtapeLogger = sqlduckDefaultLogtapeLogger;
 exports.zodCodecs = zodCodecs;