@flowblade/sqlduck 0.11.0 → 0.13.0

package/README.md

@@ -2,6 +2,11 @@
 
 > Currently experimental
 
+
+- 🛡️ DuckDB table creation from Zod schemas.
+- 🧩 Easily ingest data from generators or async iterables.
+
+
 ## Quick start
 
 ### Create a database connection
@@ -24,7 +29,7 @@ import { conn } from "./db.config.ts";
 
 const dbManager = new DuckDatabaseManager(conn);
 const database = await dbManager.attach({
-    type: ':memory:', // can be 'duckdb', ...
+    type: 'memory', // can be 'duckdb', ...
     alias: 'mydb',
     options: { COMPRESS: 'false' },
 });
@@ -129,30 +134,31 @@ const queryResult = await dbDuckDbMemoryConn.query<{
  RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 66030ms
-     name                                                  hz        min        max       mean        p75        p99       p995       p999     rme  samples
-   · duckdb appender, count: 1000000, chunk size 2048  0.0642  15,577.01  15,577.01  15,577.01  15,577.01  15,577.01  15,577.01  15,577.01  ±0.00%        1
-   · duckdb appender, count: 1000000, chunk size 1024  0.0579  17,263.44  17,263.44  17,263.44  17,263.44  17,263.44  17,263.44  17,263.44  ±0.00%        1
+ ✓ bench/appender.bench.ts > appender benches 4157ms
+     name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
+   · duckdb appender memory, count: 100000, chunk size 2048  2.6950  357.08  385.05  371.06  385.05  385.05  385.05  385.05  ±47.90%        2
+   · duckdb appender file, count: 100000, chunk size 2048    1.4218  703.35  703.35  703.35  703.35  703.35  703.35  703.35   ±0.00%        1
+   · duckdb appender, count: 100000, chunk size 1024         2.5157  391.12  403.89  397.50  403.89  403.89  403.89  403.89  ±20.41%        2
 
- ✓ bench/stream.bench.ts > Bench stream 22923ms
-     name                                                        hz       min       max      mean       p75       p99      p995      p999     rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 1000000)  1.2126    742.59    906.65    824.67    863.91    906.65    906.65    906.65  ±4.26%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 1000000)   0.8049  1,123.18  1,498.68  1,242.43  1,257.91  1,498.68  1,498.68  1,498.68  ±6.40%       10
+ ✓ bench/stream.bench.ts > Bench stream 2809ms
+     name                                                       hz      min     max    mean     p75     p99    p995    p999      rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.2627  87.7271  151.56  107.96  116.92  151.56  151.56  151.56  ±15.98%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.1479   125.04  168.13  139.90  152.97  168.13  168.13  168.13   ±7.78%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 615ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 614ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  16,562.93  0.0242  2.5902  0.0604  0.0734  0.2555  0.3741  0.8135  ±2.32%     8282
+   · getTableCreateFromZod  18,899.24  0.0334  5.3351  0.0529  0.0546  0.1943  0.3087  0.7214  ±2.72%     9450
 
  BENCH  Summary
 
-  duckdb appender, count: 1000000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.11x faster than duckdb appender, count: 1000000, chunk size 1024
+  duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
+    1.07x faster than duckdb appender, count: 100000, chunk size 1024
+    1.90x faster than duckdb appender file, count: 100000, chunk size 2048
 
-  rowToColumnsChunk with chunkSize 2048 (count: 1000000) - bench/stream.bench.ts > Bench stream
-    1.51x faster than mapFakeRowStream with chunkSize 2048 (count: 1000000)
+  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
+    1.30x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
-
 ```
 
 ### Bun 1.3.11
@@ -161,29 +167,31 @@ const queryResult = await dbDuckDbMemoryConn.query<{
  RUN  v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
 
 
- ✓ bench/appender.bench.ts > appender benches 36627ms
-     name                                                  hz       min       max      mean       p75       p99      p995      p999     rme  samples
-   · duckdb appender, count: 1000000, chunk size 2048  0.1177  8,495.41  8,495.41  8,495.41  8,495.41  8,495.41  8,495.41  8,495.41  ±0.00%        1
-   · duckdb appender, count: 1000000, chunk size 1024  0.1064  9,397.97  9,397.97  9,397.97  9,397.97  9,397.97  9,397.97  9,397.97  ±0.00%        1
+ ✓ bench/appender.bench.ts > appender benches 4159ms
+     name                                                        hz     min     max    mean     p75     p99    p995    p999      rme  samples
+   · duckdb appender memory, count: 100000, chunk size 2048  2.6465  375.34  380.38  377.86  380.38  380.38  380.38  380.38   ±8.48%        2
+   · duckdb appender file, count: 100000, chunk size 2048    1.5016  665.98  665.98  665.98  665.98  665.98  665.98  665.98   ±0.00%        1
+   · duckdb appender, count: 100000, chunk size 1024         2.2828  413.11  463.01  438.06  463.01  463.01  463.01  463.01  ±72.39%        2
 
- ✓ bench/stream.bench.ts > Bench stream 23421ms
-     name                                                        hz       min       max      mean       p75       p99      p995      p999     rme  samples
-   · rowToColumnsChunk with chunkSize 2048 (count: 1000000)  1.1378    801.60  1,080.22    878.91    910.91  1,080.22  1,080.22  1,080.22  ±6.85%       10
-   · mapFakeRowStream with chunkSize 2048 (count: 1000000)   0.8118  1,130.36  1,448.99  1,231.78  1,268.45  1,448.99  1,448.99  1,448.99  ±5.34%       10
+ ✓ bench/stream.bench.ts > Bench stream 2690ms
+     name                                                       hz      min     max    mean     p75     p99    p995    p999     rme  samples
+   · rowToColumnsChunk with chunkSize 2048 (count: 100000)  9.5675  95.6610  114.11  104.52  107.75  114.11  114.11  114.11  ±3.46%       10
+   · mapFakeRowStream with chunkSize 2048 (count: 100000)   7.6895   117.83  138.26  130.05  137.51  138.26  138.26  138.26  ±4.05%       10
 
- ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 622ms
+ ✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 629ms
      name                          hz     min     max    mean     p75     p99    p995    p999     rme  samples
-   · getTableCreateFromZod  22,447.94  0.0210  5.4621  0.0445  0.0442  0.1657  0.2167  2.5852  ±5.37%    11224
+   · getTableCreateFromZod  18,892.06  0.0281  7.5844  0.0529  0.0516  0.1893  0.2477  3.2823  ±6.26%     9447
 
  BENCH  Summary
 
-  rowToColumnsChunk with chunkSize 2048 (count: 1000000) - bench/stream.bench.ts > Bench stream
-    1.40x faster than mapFakeRowStream with chunkSize 2048 (count: 1000000)
+  rowToColumnsChunk with chunkSize 2048 (count: 100000) - bench/stream.bench.ts > Bench stream
+    1.24x faster than mapFakeRowStream with chunkSize 2048 (count: 100000)
 
   getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
 
-  duckdb appender, count: 1000000, chunk size 2048 - bench/appender.bench.ts > appender benches
-    1.11x faster than duckdb appender, count: 1000000, chunk size 1024
+  duckdb appender memory, count: 100000, chunk size 2048 - bench/appender.bench.ts > appender benches
+    1.16x faster than duckdb appender, count: 100000, chunk size 1024
+    1.76x faster than duckdb appender file, count: 100000, chunk size 2048
 
 ```
 

package/dist/index.d.mts

@@ -1,6 +1,7 @@
-import * as _duckdb_node_api0 from "@duckdb/node-api";
+import { n as DuckConnectionParams } from "./types-DCqYqEsa.mjs";
+import * as _$_duckdb_node_api0 from "@duckdb/node-api";
 import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
-import * as _logtape_logtape0 from "@logtape/logtape";
+import * as _$_logtape_logtape0 from "@logtape/logtape";
 import { Logger } from "@logtape/logtape";
 import * as z from "zod";
 import { ZodObject } from "zod";
@@ -137,6 +138,11 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
    * Callback called each time a datachunk is appended to the table
    */
   onDataAppended?: OnDataAppendedCb;
+  /**
+   * Automatically checkpoint the table after all chunks have been appended.
+   * @default true
+   */
+  autoCheckpoint?: boolean;
 };
 type ToTableResult = {
   /**
@@ -218,64 +224,13 @@ declare class Database {
   [Symbol.toStringTag](): string;
 }
 //#endregion
-//#region src/manager/database/duck-database-manager.schemas.d.ts
-declare const duckDatabaseManagerDbParamsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
-  type: z.ZodLiteral<":memory:">;
-  alias: z.ZodString;
-  options: z.ZodOptional<z.ZodObject<{
-    ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
-      READ_ONLY: "READ_ONLY";
-      READ_WRITE: "READ_WRITE";
-      AUTOMATIC: "AUTOMATIC";
-    }>>;
-    COMPRESS: z.ZodOptional<z.ZodEnum<{
-      true: "true";
-      false: "false";
-    }>>;
-    TYPE: z.ZodOptional<z.ZodEnum<{
-      DUCKDB: "DUCKDB";
-      SQLITE: "SQLITE";
-    }>>;
-    BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
-    ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
-    STORAGE_VERSION: z.ZodOptional<z.ZodString>;
-    ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
-    ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
-      CBC: "CBC";
-      CTR: "CTR";
-      GCM: "GCM";
-    }>>;
-  }, z.core.$strict>>;
-}, z.core.$strict>, z.ZodObject<{
-  type: z.ZodLiteral<"duckdb">;
-  path: z.ZodString;
-  alias: z.ZodString;
-  options: z.ZodOptional<z.ZodObject<{
-    ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
-      READ_ONLY: "READ_ONLY";
-      READ_WRITE: "READ_WRITE";
-      AUTOMATIC: "AUTOMATIC";
-    }>>;
-    COMPRESS: z.ZodOptional<z.ZodEnum<{
-      true: "true";
-      false: "false";
-    }>>;
-    TYPE: z.ZodOptional<z.ZodEnum<{
-      DUCKDB: "DUCKDB";
-      SQLITE: "SQLITE";
-    }>>;
-    BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
-    ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
-    STORAGE_VERSION: z.ZodOptional<z.ZodString>;
-    ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
-    ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
-      CBC: "CBC";
-      CTR: "CTR";
-      GCM: "GCM";
-    }>>;
-  }, z.core.$strict>>;
-}, z.core.$strict>], "type">;
-type DuckDatabaseManagerDbParams = z.infer<typeof duckDatabaseManagerDbParamsSchema>;
+//#region src/validation/core/duck-reserved-keywords.d.ts
+/**
+ * DuckDB reserved keywords that cannot be used as unquoted identifiers.
+ * @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
+ */
+declare const duckReservedKeywords: readonly ["ALL", "ANALYSE", "ANALYZE", "AND", "ANY", "ARRAY", "AS", "ASC", "ASYMMETRIC", "BOTH", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DEFAULT", "DEFERRABLE", "DESC", "DISTINCT", "DO", "ELSE", "END", "EXCEPT", "EXISTS", "EXTRACT", "FALSE", "FETCH", "FOR", "FOREIGN", "FROM", "GRANT", "GROUP", "HAVING", "IF", "ILIKE", "IN", "INITIALLY", "INNER", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "LATERAL", "LEADING", "LEFT", "LIKE", "LIMIT", "LOCALTIME", "LOCALTIMESTAMP", "NATURAL", "NOT", "NOTNULL", "NULL", "OFFSET", "ON", "ONLY", "OR", "ORDER", "OUTER", "OVERLAPS", "PLACING", "PRIMARY", "REFERENCES", "RETURNING", "RIGHT", "ROW", "SELECT", "SESSION_USER", "SIMILAR", "SOME", "SYMMETRIC", "TABLE", "THEN", "TO", "TRAILING", "TRUE", "UNION", "UNIQUE", "USING", "VARIADIC", "VERBOSE", "WHEN", "WHERE", "WINDOW", "WITH"];
+type DuckdbReservedKeywords = (typeof duckReservedKeywords)[number];
 //#endregion
 //#region src/manager/database/commands/duck-database-attach-command.d.ts
 type Behaviour = 'OR REPLACE' | 'IF NOT EXISTS';
@@ -296,7 +251,7 @@ declare class DuckDatabaseManager {
    * ```typescript
    * const dbManager = new DuckDatabaseManager(conn);
    * const database = dbManager.attach({
-   *   type: ':memory:', // can be 'duckdb', 's3'...
+   *   type: 'memory', // can be 'duckdb', 's3'...
    *   alias: 'mydb',
    *   options: { COMPRESS: 'true' }
    * });
@@ -304,10 +259,10 @@ declare class DuckDatabaseManager {
    * console.log(database.alias); // 'mydb'
    * ```
    */
-  attach: (dbParams: DuckDatabaseManagerDbParams, options?: DuckDatabaseAttachCommandOptions) => Promise<Database>;
-  attachOrReplace: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
-  attachIfNotExists: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
-  showDatabases: () => Promise<Record<string, _duckdb_node_api0.JS>[]>;
+  attach: (dbParams: DuckConnectionParams, options?: DuckDatabaseAttachCommandOptions) => Promise<Database>;
+  attachOrReplace: (dbParams: DuckConnectionParams) => Promise<Database>;
+  attachIfNotExists: (dbParams: DuckConnectionParams) => Promise<Database>;
+  showDatabases: () => Promise<Record<string, _$_duckdb_node_api0.JS>[]>;
   detach: (dbAlias: string) => Promise<boolean>;
   detachIfExists: (dbAlias: string) => Promise<boolean>;
   /**
@@ -320,26 +275,15 @@ declare class DuckDatabaseManager {
    */
   analyze: () => Promise<boolean>;
   checkpoint: (dbAlias: string) => Promise<boolean>;
+  vacuum: () => Promise<boolean>;
 }
 //#endregion
-//#region src/validation/core/duckdb-reserved-keywords.d.ts
-/**
- * DuckDB reserved keywords that cannot be used as unquoted identifiers.
- * @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
- */
-declare const duckdbReservedKeywords: readonly ["ALL", "ANALYSE", "ANALYZE", "AND", "ANY", "ARRAY", "AS", "ASC", "ASYMMETRIC", "BOTH", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DEFAULT", "DEFERRABLE", "DESC", "DISTINCT", "DO", "ELSE", "END", "EXCEPT", "EXISTS", "EXTRACT", "FALSE", "FETCH", "FOR", "FOREIGN", "FROM", "GRANT", "GROUP", "HAVING", "IF", "ILIKE", "IN", "INITIALLY", "INNER", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "LATERAL", "LEADING", "LEFT", "LIKE", "LIMIT", "LOCALTIME", "LOCALTIMESTAMP", "NATURAL", "NOT", "NOTNULL", "NULL", "OFFSET", "ON", "ONLY", "OR", "ORDER", "OUTER", "OVERLAPS", "PLACING", "PRIMARY", "REFERENCES", "RETURNING", "RIGHT", "ROW", "SELECT", "SESSION_USER", "SIMILAR", "SOME", "SYMMETRIC", "TABLE", "THEN", "TO", "TRAILING", "TRUE", "UNION", "UNIQUE", "USING", "VARIADIC", "VERBOSE", "WHEN", "WHERE", "WINDOW", "WITH"];
-type DuckdbReservedKeywords = (typeof duckdbReservedKeywords)[number];
-//#endregion
-//#region src/validation/zod/duckdb-valid-names.schemas.d.ts
-declare const duckTableNameSchema: z.ZodString;
-declare const duckTableAliasSchema: z.ZodString;
-//#endregion
 //#region src/config/flowblade-logtape-sqlduck.config.d.ts
 declare const flowbladeLogtapeSqlduckConfig: {
   categories: string[];
 };
 //#endregion
 //#region src/logger/sqlduck-default-logtape-logger.d.ts
-declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
+declare const sqlduckDefaultLogtapeLogger: _$_logtape_logtape0.Logger;
 //#endregion
-export { Database, DuckDatabaseManager, type DuckDatabaseManagerDbParams, DuckMemory, type DuckMemoryTag, type DuckdbReservedKeywords, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckDatabaseManagerDbParamsSchema, duckTableAliasSchema, duckTableNameSchema, duckdbReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
+export { Database, type DuckConnectionParams, DuckDatabaseManager, DuckMemory, type DuckMemoryTag, type DuckdbReservedKeywords, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };