@flowblade/sqlduck 0.10.0 → 0.12.0

package/README.md

@@ -2,8 +2,81 @@
 
 > Currently experimental
 
+
+- 🛡️ DuckDB table creation from Zod schemas.
+- 🧩 Easily ingest data from generators or async iterables.
+
+
 ## Quick start
 
+### Create a database connection
+
+```typescript
+import { DuckDBInstance } from '@duckdb/node-api';
+DuckDBInstance.create(undefined, {
+   access_mode: 'READ_WRITE',
+   max_memory: '512M',
+});
+export const conn = await instance.connect();
+```
+
+### Append data to a database
+
+```typescript
+import { SqlDuck, DuckDatabaseManager } from "@flowblade/sqlduck";
+import * as z from "zod";
+import { conn } from "./db.config.ts";
+
+const dbManager = new DuckDatabaseManager(conn);
+const database = await dbManager.attach({
+    type: 'memory', // can be 'duckdb', ...
+    alias: 'mydb',
+    options: { COMPRESS: 'false' },
+});
+
+const sqlDuck = new SqlDuck({ conn });
+
+// Define a zod schema, it will be used to create the table
+const userSchema = z.object({
+    id: z.int32().min(1).meta({ primaryKey: true }),
+    name: z.string(),
+});
+
+// Example of a datasource (can be generator, async generator, async iterable)
+async function* getUsers(): AsyncIterableIterator<
+    z.infer<typeof userSchema>
+> {
+    // database or api call
+    yield { id: 1, name: 'John' };
+    yield { id: 2, name: 'Jane' };
+}
+
+// Create a table from the schema and the datasource
+const result = await sqlDuck.toTable({
+    table: new Table({ name: 'user', database: database.alias }),
+    schema: userSchema, // The schema to use to create the table
+    rowStream: getUsers(), // The async iterable that yields rows
+    // 👇Optional:
+    chunkSize: 2048, // Number of rows to append when using duckdb appender. Default is 2048
+    onDataAppended: ({ timeMs, totalRows, rowsPerSecond }) => {
+        console.log(
+            `Appended ${totalRows} in time ${timeMs}ms, est: ${rowsPerSecond} rows/s`
+        );
+    },
+    // Optional table creation options
+    createOptions: {
+        create: 'CREATE_OR_REPLACE',
+    },
+});
+
+console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
+console.log(`Table created with DDL: ${result.createTableDDL}`);
+
+const reader = await conn.runAndReadAll('select * from mydb.user');
+const rows = reader.getRowObjectsJS();
+// [{id: 1, name: 'John'}, {id: 2, name: 'Jane'}]]
+```
+
 ### Create a memory table
 
 ```typescript
@@ -35,7 +108,6 @@ const result = sqlDuck.toTable({
   onDataAppended: ({ total }) => {
     console.log(`Appended ${total} rows so far`);
   },
-  onDataAppendedBatchSize: 4096, // Call onDataAppended every 4096 rows
   // Optional table creation options
   createOptions: {
     create: "CREATE_OR_REPLACE",
@@ -62,30 +134,31 @@ const queryResult = await dbDuckDbMemoryConn.query<{
  RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 66030ms
-     name                                                  hz        min        max       mean        p75        p99       p995       p999     rme  samples
-   · duckdb appender, count: 1000000, chunk size 2048  0.0642  15,577.01  15,577.01  15,577.01  15,577.01  15,577.01  15,577.01  15,577.01  ±0.00%        1
-   · duckdb appender, count: 1000000, chunk size 1024  0.0579  17,263.44  17,263.44  17,263.44  17,263.44  17,263.44  17,263.44  17,263.44  ±0.00%        1
+ ✓ bench/appender.bench.ts > appender benches 4157ms
+     name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
+   · duckdb appender memory, count: 100000, chunk size 2048  2.6950  357.08  385.05  371.06  385.05  385.05  385.05  385.05  ±47.90%        2
+   · duckdb appender file, count: 100000, chunk size 2048    1.4218  703.35  703.35  703.35  703.35  703.35  703.35  703.35   ±0.00%        1
+   · duckdb appender, count: 100000, chunk size 1024         2.5157  391.12  403.89  397.50  403.89  403.89  403.89  403.89  ±20.41%        2
 
- ✓ bench/stream.bench.ts > Bench stream 22923ms
-     name                                                        hz       min       max      mean       p75       p99      p995      p999     rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 1000000)  1.2126    742.59    906.65    824.67    863.91    906.65    906.65    906.65  ±4.26%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 1000000)   0.8049  1,123.18  1,498.68  1,242.43  1,257.91  1,498.68  1,498.68  1,498.68  ±6.40%       10
+ ✓ bench/stream.bench.ts > Bench stream 2809ms
+     name                                                       hz      min     max    mean     p75     p99    p995    p999      rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.2627  87.7271  151.56  107.96  116.92  151.56  151.56  151.56  ±15.98%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.1479   125.04  168.13  139.90  152.97  168.13  168.13  168.13   ±7.78%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 615ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 614ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  16,562.93  0.0242  2.5902  0.0604  0.0734  0.2555  0.3741  0.8135  ±2.32%     8282
+   · getTableCreateFromZod  18,899.24  0.0334  5.3351  0.0529  0.0546  0.1943  0.3087  0.7214  ±2.72%     9450
 
  BENCH  Summary
 
-  duckdb appender, count: 1000000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.11x faster than duckdb appender, count: 1000000, chunk size 1024
+  duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
+    1.07x faster than duckdb appender, count: 100000, chunk size 1024
+    1.90x faster than duckdb appender file, count: 100000, chunk size 2048
 
-  rowToColumnsChunk with chunkSize 2048 (count: 1000000) - bench/stream.bench.ts > Bench stream
-    1.51x faster than mapFakeRowStream with chunkSize 2048 (count: 1000000)
+  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
+    1.30x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
-
 ```
 
 ### Bun 1.3.11
@@ -94,29 +167,31 @@ const queryResult = await dbDuckDbMemoryConn.query<{
  RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 36627ms
-     name                                                  hz       min       max      mean       p75       p99      p995      p999     rme  samples
-   · duckdb appender, count: 1000000, chunk size 2048  0.1177  8,495.41  8,495.41  8,495.41  8,495.41  8,495.41  8,495.41  8,495.41  ±0.00%        1
-   · duckdb appender, count: 1000000, chunk size 1024  0.1064  9,397.97  9,397.97  9,397.97  9,397.97  9,397.97  9,397.97  9,397.97  ±0.00%        1
+ ✓ bench/appender.bench.ts > appender benches 4159ms
+     name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
+   · duckdb appender memory, count: 100000, chunk size 2048  2.6465  375.34  380.38  377.86  380.38  380.38  380.38  380.38   ±8.48%        2
+   · duckdb appender file, count: 100000, chunk size 2048    1.5016  665.98  665.98  665.98  665.98  665.98  665.98  665.98   ±0.00%        1
+   · duckdb appender, count: 100000, chunk size 1024         2.2828  413.11  463.01  438.06  463.01  463.01  463.01  463.01  ±72.39%        2
 
- ✓ bench/stream.bench.ts > Bench stream 23421ms
-     name                                                        hz       min       max      mean       p75       p99      p995      p999     rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 1000000)  1.1378    801.60  1,080.22    878.91    910.91  1,080.22  1,080.22  1,080.22  ±6.85%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 1000000)   0.8118  1,130.36  1,448.99  1,231.78  1,268.45  1,448.99  1,448.99  1,448.99  ±5.34%       10
+ ✓ bench/stream.bench.ts > Bench stream 2690ms
+     name                                                       hz      min     max    mean     p75     p99    p995    p999     rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.5675  95.6610  114.11  104.52  107.75  114.11  114.11  114.11  ±3.46%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.6895   117.83  138.26  130.05  137.51  138.26  138.26  138.26  ±4.05%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 622ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 629ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  22,447.94  0.0210  5.4621  0.0445  0.0442  0.1657  0.2167  2.5852  ±5.37%    11224
+   · getTableCreateFromZod  18,892.06  0.0281  7.5844  0.0529  0.0516  0.1893  0.2477  3.2823  ±6.26%     9447
 
  BENCH  Summary
 
-  rowToColumnsChunk with chunkSize 2048 (count: 1000000) - bench/stream.bench.ts > Bench stream
-    1.40x faster than mapFakeRowStream with chunkSize 2048 (count: 1000000)
+  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
+    1.24x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 
-  duckdb appender, count: 1000000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.11x faster than duckdb appender, count: 1000000, chunk size 1024
+  duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
+    1.16x faster than duckdb appender, count: 100000, chunk size 1024
+    1.76x faster than duckdb appender file, count: 100000, chunk size 2048
 
 ```
 

package/dist/index.d.mts

@@ -1,6 +1,8 @@
-import * as _logtape_logtape0 from "@logtape/logtape";
-import { Logger } from "@logtape/logtape";
+import { n as DuckConnectionParams } from "./types-DCqYqEsa.mjs";
+import * as _$_duckdb_node_api0 from "@duckdb/node-api";
 import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
+import * as _$_logtape_logtape0 from "@logtape/logtape";
+import { Logger } from "@logtape/logtape";
 import * as z from "zod";
 import { ZodObject } from "zod";
 
@@ -23,11 +25,6 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
 type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
 type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
 //#endregion
-//#region src/config/flowblade-logtape-sqlduck.config.d.ts
-declare const flowbladeLogtapeSqlduckConfig: {
-  categories: string[];
-};
-//#endregion
 //#region src/helpers/duck-memory.d.ts
 declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
 type DuckMemoryTag = (typeof duckMemoryTags)[number];
@@ -56,10 +53,7 @@ declare class DuckMemory {
   getSummary: () => Promise<DuckMemorySummary>;
 }
 //#endregion
-//#region src/logger/sqlduck-default-logtape-logger.d.ts
-declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
-//#endregion
-//#region src/table/table.d.ts
+//#region src/objects/table.d.ts
 /**
  * Fully qualified table information
  */
@@ -144,6 +138,11 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
    * Callback called each time a datachunk is appended to the table
    */
   onDataAppended?: OnDataAppendedCb;
+  /**
+   * Automatically checkpoint the table after all chunks have been appended.
+   * @default true
+   */
+  autoCheckpoint?: boolean;
 };
 type ToTableResult = {
   /**
@@ -208,4 +207,83 @@ declare const zodCodecs: {
   readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
 };
 //#endregion
-export { DuckMemory, type DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
+//#region src/objects/database.d.ts
+type DatabaseProperties = {
+  alias: string;
+};
+declare class Database {
+  #private;
+  get alias(): string;
+  constructor(params: DatabaseProperties);
+  toJson(): {
+    type: string;
+    params: {
+      alias: string;
+    };
+  };
+  [Symbol.toStringTag](): string;
+}
+//#endregion
+//#region src/validation/core/duck-reserved-keywords.d.ts
+/**
+ * DuckDB reserved keywords that cannot be used as unquoted identifiers.
+ * @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
+ */
+declare const duckReservedKeywords: readonly ["ALL", "ANALYSE", "ANALYZE", "AND", "ANY", "ARRAY", "AS", "ASC", "ASYMMETRIC", "BOTH", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DEFAULT", "DEFERRABLE", "DESC", "DISTINCT", "DO", "ELSE", "END", "EXCEPT", "EXISTS", "EXTRACT", "FALSE", "FETCH", "FOR", "FOREIGN", "FROM", "GRANT", "GROUP", "HAVING", "IF", "ILIKE", "IN", "INITIALLY", "INNER", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "LATERAL", "LEADING", "LEFT", "LIKE", "LIMIT", "LOCALTIME", "LOCALTIMESTAMP", "NATURAL", "NOT", "NOTNULL", "NULL", "OFFSET", "ON", "ONLY", "OR", "ORDER", "OUTER", "OVERLAPS", "PLACING", "PRIMARY", "REFERENCES", "RETURNING", "RIGHT", "ROW", "SELECT", "SESSION_USER", "SIMILAR", "SOME", "SYMMETRIC", "TABLE", "THEN", "TO", "TRAILING", "TRUE", "UNION", "UNIQUE", "USING", "VARIADIC", "VERBOSE", "WHEN", "WHERE", "WINDOW", "WITH"];
+type DuckdbReservedKeywords = (typeof duckReservedKeywords)[number];
+//#endregion
+//#region src/manager/database/commands/duck-database-attach-command.d.ts
+type Behaviour = 'OR REPLACE' | 'IF NOT EXISTS';
+type DuckDatabaseAttachCommandOptions = {
+  behaviour?: Behaviour;
+};
+//#endregion
+//#region src/manager/database/duck-database-manager.d.ts
+declare class DuckDatabaseManager {
+  #private;
+  constructor(conn: DuckDBConnection, params?: {
+    logger?: Logger;
+  });
+  /**
+   * Attach a database to the current connection
+   *
+   * @example
+   * ```typescript
+   * const dbManager = new DuckDatabaseManager(conn);
+   * const database = dbManager.attach({
+   *   type: 'memory', // can be 'duckdb', 's3'...
+   *   alias: 'mydb',
+   *   options: { COMPRESS: 'true' }
+   * });
+   *
+   * console.log(database.alias); // 'mydb'
+   * ```
+   */
+  attach: (dbParams: DuckConnectionParams, options?: DuckDatabaseAttachCommandOptions) => Promise<Database>;
+  attachOrReplace: (dbParams: DuckConnectionParams) => Promise<Database>;
+  attachIfNotExists: (dbParams: DuckConnectionParams) => Promise<Database>;
+  showDatabases: () => Promise<Record<string, _$_duckdb_node_api0.JS>[]>;
+  detach: (dbAlias: string) => Promise<boolean>;
+  detachIfExists: (dbAlias: string) => Promise<boolean>;
+  /**
+   * The statistics recomputed by the ANALYZE statement are only used for join order optimization.
+   *
+   * It is therefore recommended to recompute these statistics for improved join orders,
+   * especially after performing large updates (inserts and/or deletes).
+   *
+   * @link https://duckdb.org/docs/stable/sql/statements/analyze
+   */
+  analyze: () => Promise<boolean>;
+  checkpoint: (dbAlias: string) => Promise<boolean>;
+  vacuum: () => Promise<boolean>;
+}
+//#endregion
+//#region src/config/flowblade-logtape-sqlduck.config.d.ts
+declare const flowbladeLogtapeSqlduckConfig: {
+  categories: string[];
+};
+//#endregion
+//#region src/logger/sqlduck-default-logtape-logger.d.ts
+declare const sqlduckDefaultLogtapeLogger: _$_logtape_logtape0.Logger;
+//#endregion
+export { Database, type DuckConnectionParams, DuckDatabaseManager, DuckMemory, type DuckMemoryTag, type DuckdbReservedKeywords, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };

package/dist/index.mjs

@@ -1,9 +1,9 @@
-import { getLogger } from "@logtape/logtape";
+import { a as assertValidAliasName, c as duckValidatorsZod, i as duckConnectionParamsZodSchema, l as duckReservedKeywords } from "./zod-CwR_oehs.mjs";
 import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
+import { getLogger } from "@logtape/logtape";
 import * as z from "zod";
-//#region src/config/flowblade-logtape-sqlduck.config.ts
-const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
-//#endregion
+import { assertNever } from "@httpx/assert";
+import { isPlainObject } from "@httpx/plain-object";
 //#region src/helpers/duck-exec.ts
 var DuckExec = class {
 	#conn;
@@ -100,9 +100,6 @@ var DuckMemory = class {
 	};
 };
 //#endregion
-//#region src/logger/sqlduck-default-logtape-logger.ts
-const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
-//#endregion
 //#region src/appender/data-appender-callback.ts
 const isOnDataAppendedAsyncCb = (v) => {
 	return v.constructor.name === "AsyncFunction";
@@ -124,6 +121,177 @@ const createOnDataAppendedCollector = () => {
 	};
 };
 //#endregion
+//#region src/config/flowblade-logtape-sqlduck.config.ts
+const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
+//#endregion
+//#region src/logger/sqlduck-default-logtape-logger.ts
+const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
+//#endregion
+//#region src/objects/database.ts
+var Database = class {
+	#params;
+	get alias() {
+		return this.#params.alias;
+	}
+	constructor(params) {
+		this.#params = params;
+	}
+	toJson() {
+		return {
+			type: "database",
+			params: { alias: this.#params.alias }
+		};
+	}
+	[Symbol.toStringTag]() {
+		return this.alias;
+	}
+};
+//#endregion
+//#region src/manager/database/commands/duck-database-attach-command.ts
+var DuckDatabaseAttachCommand = class {
+	options;
+	dbParams;
+	constructor(dbParams, options) {
+		this.dbParams = dbParams;
+		this.options = options ?? {};
+	}
+	getRawSql = () => {
+		const dbParams = this.dbParams;
+		const parts = ["ATTACH", this.options.behaviour].filter(Boolean);
+		const { type, alias } = dbParams;
+		switch (type) {
+			case "memory":
+				parts.push("':memory:'");
+				break;
+			case "duckdb":
+				parts.push(`'${dbParams.path}'`);
+				break;
+			default: assertNever(type);
+		}
+		if (alias !== null) parts.push("AS", `${alias}`);
+		const options = [];
+		if (isPlainObject(dbParams.options)) for (const [key, value] of Object.entries(dbParams.options)) switch (key) {
+			case "accessMode":
+				options.push(`${value}`);
+				break;
+			case "compress":
+				if (value === true) options.push("COMPRESS");
+				break;
+			case "blockSize":
+				options.push(`BLOCK_SIZE ${value}`);
+				break;
+			case "rowGroupSize":
+				options.push(`ROW_GROUP_SIZE ${value}`);
+				break;
+			case "type":
+				options.push(`TYPE ${value}`);
+				break;
+			case "storageVersion":
+				options.push(`STORAGE_VERSION '${value}'`);
+				break;
+			case "encryptionCipher":
+				options.push(`ENCRYPTION_CIPHER '${value}'`);
+				break;
+			case "encryptionKey":
+				options.push(`ENCRYPTION_KEY '${value}'`);
+				break;
+			default:
+		}
+		if (options.length > 0) parts.push(`(${options.join(", ")})`);
+		return parts.filter(Boolean).join(" ");
+	};
+};
+//#endregion
+//#region src/manager/database/duck-database-manager.ts
+var DuckDatabaseManager = class {
+	#conn;
+	#logger;
+	constructor(conn, params) {
+		this.#conn = conn;
+		this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
+	}
+	/**
+	* Attach a database to the current connection
+	*
+	* @example
+	* ```typescript
+	* const dbManager = new DuckDatabaseManager(conn);
+	* const database = dbManager.attach({
+	*   type: 'memory', // can be 'duckdb', 's3'...
+	*   alias: 'mydb',
+	*   options: { COMPRESS: 'true' }
+	* });
+	*
+	* console.log(database.alias); // 'mydb'
+	* ```
+	*/
+	attach = async (dbParams, options) => {
+		const params = duckConnectionParamsZodSchema.parse(dbParams);
+		const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
+		await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
+		return new Database({ alias: params.alias });
+	};
+	attachOrReplace = async (dbParams) => {
+		return this.attach(dbParams, { behaviour: "OR REPLACE" });
+	};
+	attachIfNotExists = async (dbParams) => {
+		return this.attach(dbParams, { behaviour: "IF NOT EXISTS" });
+	};
+	showDatabases = async () => {
+		return await this.#executeRawSqlCommand("showDatabases()", `SHOW DATABASES`);
+	};
+	detach = async (dbAlias) => {
+		assertValidAliasName(dbAlias);
+		await this.#executeRawSqlCommand(`detach(${dbAlias})`, `DETACH ${dbAlias}`);
+		return true;
+	};
+	detachIfExists = async (dbAlias) => {
+		assertValidAliasName(dbAlias);
+		await this.#executeRawSqlCommand(`detachIfExists(${dbAlias})`, `DETACH IF EXISTS ${dbAlias}`);
+		return true;
+	};
+	/**
+	* The statistics recomputed by the ANALYZE statement are only used for join order optimization.
+	*
+	* It is therefore recommended to recompute these statistics for improved join orders,
+	* especially after performing large updates (inserts and/or deletes).
+	*
+	* @link https://duckdb.org/docs/stable/sql/statements/analyze
+	*/
+	analyze = async () => {
+		await this.#executeRawSqlCommand("analyze()", "ANALYZE");
+		return true;
+	};
+	checkpoint = async (dbAlias) => {
+		const safeAlias = duckValidatorsZod.aliasName.parse(dbAlias);
+		await this.#executeRawSqlCommand(`checkpoint(${safeAlias})`, `CHECKPOINT ${safeAlias}`);
+		return true;
+	};
+	vacuum = async () => {
+		await this.#executeRawSqlCommand("vacuum()", "VACUUM");
+		return true;
+	};
+	#executeRawSqlCommand = async (name, rawSql) => {
+		const startTime = Date.now();
+		try {
+			const result = await this.#conn.runAndReadAll(rawSql);
+			const timeMs = Math.round(Date.now() - startTime);
+			const data = result.getRowObjectsJS();
+			this.#logger.info(`DuckDatabaseManager.${name} in ${timeMs}ms`, { timeMs });
+			return data;
+		} catch (e) {
+			const msg = `DuckDatabaseManager: failed to run "${name}" - ${e?.message ?? ""}`;
+			const timeMs = Math.round(Date.now() - startTime);
+			this.#logger.error(msg, {
+				name,
+				sql: rawSql,
+				timeMs
+			});
+			throw new Error(msg, { cause: e });
+		}
+	};
+};
+//#endregion
 //#region src/table/get-duckdb-number-column-type.ts
 const isFloatValue = (value) => {
 	if (!Number.isFinite(value)) return true;
@@ -157,7 +325,7 @@ const createOptions = {
 	CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
 	IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
 };
-const duckDbTypes = [
+const duckDbTypesMap = new Map([
 	["VARCHAR", VARCHAR],
 	["BIGINT", BIGINT],
 	["TIMESTAMP", TIMESTAMP],
@@ -166,8 +334,7 @@ const duckDbTypes = [
 	["INTEGER", INTEGER],
 	["DOUBLE", DOUBLE],
 	["FLOAT", FLOAT]
-];
-const duckDbTypesMap = new Map(duckDbTypes);
+]);
 const getTableCreateFromZod = (params) => {
 	const { table, schema, options } = params;
 	const { create = "CREATE" } = options ?? {};
@@ -303,10 +470,10 @@ async function* rowsToColumnsChunks(params) {
 //#endregion
 //#region src/sql-duck.ts
 var SqlDuck = class {
-	#duck;
+	#conn;
 	#logger;
 	constructor(params) {
-		this.#duck = params.conn;
+		this.#conn = params.conn;
 		this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
 	}
 	/**
@@ -347,16 +514,16 @@ var SqlDuck = class {
 	* ```
 	*/
 	toTable = async (params) => {
-		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
+		const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended, autoCheckpoint = true } = params;
 		if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
 		const timeStart = Date.now();
 		const { columnTypes, ddl } = await createTableFromZod({
-			conn: this.#duck,
+			conn: this.#conn,
 			schema,
 			table,
 			options: createOptions
 		});
-		const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
+		const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
 		const chunkTypes = Array.from(columnTypes.values());
 		let totalRows = 0;
 		const dataAppendedCollector = createOnDataAppendedCollector();
@@ -379,6 +546,14 @@ var SqlDuck = class {
 				}
 			}
 			appender.closeSync();
+			if (autoCheckpoint && typeof table.databaseName === "string") {
+				const dbManager = new DuckDatabaseManager(this.#conn);
+				try {
+					await dbManager.checkpoint(table.databaseName);
+				} catch (e) {
+					this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending data into table '${table.getFullName()}' - ${e?.message ?? ""}`, { table: table.getFullName() });
+				}
+			}
 			const timeMs = Math.round(Date.now() - timeStart);
 			this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
 				table: table.getFullName(),
@@ -399,7 +574,19 @@ var SqlDuck = class {
 	};
 };
 //#endregion
-//#region src/table/table.ts
+//#region src/utils/zod-codecs.ts
+const zodCodecs = {
+	dateToString: z.codec(z.date(), z.iso.datetime(), {
+		decode: (date) => date.toISOString(),
+		encode: (isoString) => new Date(isoString)
+	}),
+	bigintToString: z.codec(z.bigint(), z.string().meta({ format: "int64" }), {
+		decode: (bigint) => bigint.toString(),
+		encode: BigInt
+	})
+};
+//#endregion
+//#region src/objects/table.ts
 var Table = class Table {
 	#fqTable;
 	get tableName() {
@@ -441,16 +628,4 @@ var Table = class Table {
 	};
 };
 //#endregion
-//#region src/utils/zod-codecs.ts
-const zodCodecs = {
-	dateToString: z.codec(z.date(), z.iso.datetime(), {
-		decode: (date) => date.toISOString(),
-		encode: (isoString) => new Date(isoString)
-	}),
-	bigintToString: z.codec(z.bigint(), z.string().meta({ format: "int64" }), {
-		decode: (bigint) => bigint.toString(),
-		encode: BigInt
-	})
-};
-//#endregion
-export { DuckMemory, SqlDuck, Table, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
+export { Database, DuckDatabaseManager, DuckMemory, SqlDuck, Table, duckReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };