@flowblade/sqlduck 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -4,6 +4,74 @@
4
4
 
5
5
  ## Quick start
6
6
 
7
+ ### Create a database connection
8
+
9
+ ```typescript
10
+ import { DuckDBInstance } from '@duckdb/node-api';
11
+ DuckDBInstance.create(undefined, {
12
+ access_mode: 'READ_WRITE',
13
+ max_memory: '512M',
14
+ });
15
+ export const conn = await instance.connect();
16
+ ```
17
+
18
+ ### Append data to a database
19
+
20
+ ```typescript
21
+ import { SqlDuck, DuckDatabaseManager } from "@flowblade/sqlduck";
22
+ import * as z from "zod";
23
+ import { conn } from "./db.config.ts";
24
+
25
+ const dbManager = new DuckDatabaseManager(conn);
26
+ const database = await dbManager.attach({
27
+ type: ':memory:', // can be 'duckdb', ...
28
+ alias: 'mydb',
29
+ options: { COMPRESS: 'false' },
30
+ });
31
+
32
+ const sqlDuck = new SqlDuck({ conn });
33
+
34
+ // Define a zod schema, it will be used to create the table
35
+ const userSchema = z.object({
36
+ id: z.int32().min(1).meta({ primaryKey: true }),
37
+ name: z.string(),
38
+ });
39
+
40
+ // Example of a datasource (can be generator, async generator, async iterable)
41
+ async function* getUsers(): AsyncIterableIterator<
42
+ z.infer<typeof userSchema>
43
+ > {
44
+ // database or api call
45
+ yield { id: 1, name: 'John' };
46
+ yield { id: 2, name: 'Jane' };
47
+ }
48
+
49
+ // Create a table from the schema and the datasource
50
+ const result = await sqlDuck.toTable({
51
+ table: new Table({ name: 'user', database: database.alias }),
52
+ schema: userSchema, // The schema to use to create the table
53
+ rowStream: getUsers(), // The async iterable that yields rows
54
+ // 👇Optional:
55
+ chunkSize: 2048, // Number of rows to append when using duckdb appender. Default is 2048
56
+ onDataAppended: ({ timeMs, totalRows, rowsPerSecond }) => {
57
+ console.log(
58
+ `Appended ${totalRows} in time ${timeMs}ms, est: ${rowsPerSecond} rows/s`
59
+ );
60
+ },
61
+ // Optional table creation options
62
+ createOptions: {
63
+ create: 'CREATE_OR_REPLACE',
64
+ },
65
+ });
66
+
67
+ console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
68
+ console.log(`Table created with DDL: ${result.createTableDDL}`);
69
+
70
+ const reader = await conn.runAndReadAll('select * from mydb.user');
71
+ const rows = reader.getRowObjectsJS();
72
+ // [{id: 1, name: 'John'}, {id: 2, name: 'Jane'}]]
73
+ ```
74
+
7
75
  ### Create a memory table
8
76
 
9
77
  ```typescript
@@ -35,7 +103,6 @@ const result = sqlDuck.toTable({
35
103
  onDataAppended: ({ total }) => {
36
104
  console.log(`Appended ${total} rows so far`);
37
105
  },
38
- onDataAppendedBatchSize: 4096, // Call onDataAppended every 4096 rows
39
106
  // Optional table creation options
40
107
  createOptions: {
41
108
  create: "CREATE_OR_REPLACE",
package/dist/index.cjs CHANGED
@@ -21,13 +21,12 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
21
21
  enumerable: true
22
22
  }) : target, mod));
23
23
  //#endregion
24
- let _logtape_logtape = require("@logtape/logtape");
25
24
  let _duckdb_node_api = require("@duckdb/node-api");
25
+ let _logtape_logtape = require("@logtape/logtape");
26
26
  let zod = require("zod");
27
27
  zod = __toESM(zod);
28
- //#region src/config/flowblade-logtape-sqlduck.config.ts
29
- const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
30
- //#endregion
28
+ let _httpx_assert = require("@httpx/assert");
29
+ let _httpx_plain_object = require("@httpx/plain-object");
31
30
  //#region src/helpers/duck-exec.ts
32
31
  var DuckExec = class {
33
32
  #conn;
@@ -124,9 +123,6 @@ var DuckMemory = class {
124
123
  };
125
124
  };
126
125
  //#endregion
127
- //#region src/logger/sqlduck-default-logtape-logger.ts
128
- const sqlduckDefaultLogtapeLogger = (0, _logtape_logtape.getLogger)(flowbladeLogtapeSqlduckConfig.categories);
129
- //#endregion
130
126
  //#region src/appender/data-appender-callback.ts
131
127
  const isOnDataAppendedAsyncCb = (v) => {
132
128
  return v.constructor.name === "AsyncFunction";
@@ -148,6 +144,12 @@ const createOnDataAppendedCollector = () => {
148
144
  };
149
145
  };
150
146
  //#endregion
147
+ //#region src/config/flowblade-logtape-sqlduck.config.ts
148
+ const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
149
+ //#endregion
150
+ //#region src/logger/sqlduck-default-logtape-logger.ts
151
+ const sqlduckDefaultLogtapeLogger = (0, _logtape_logtape.getLogger)(flowbladeLogtapeSqlduckConfig.categories);
152
+ //#endregion
151
153
  //#region src/table/get-duckdb-number-column-type.ts
152
154
  const isFloatValue = (value) => {
153
155
  if (!Number.isFinite(value)) return true;
@@ -181,7 +183,7 @@ const createOptions = {
181
183
  CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
182
184
  IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
183
185
  };
184
- const duckDbTypes = [
186
+ const duckDbTypesMap = new Map([
185
187
  ["VARCHAR", _duckdb_node_api.VARCHAR],
186
188
  ["BIGINT", _duckdb_node_api.BIGINT],
187
189
  ["TIMESTAMP", _duckdb_node_api.TIMESTAMP],
@@ -190,8 +192,7 @@ const duckDbTypes = [
190
192
  ["INTEGER", _duckdb_node_api.INTEGER],
191
193
  ["DOUBLE", _duckdb_node_api.DOUBLE],
192
194
  ["FLOAT", _duckdb_node_api.FLOAT]
193
- ];
194
- const duckDbTypesMap = new Map(duckDbTypes);
195
+ ]);
195
196
  const getTableCreateFromZod = (params) => {
196
197
  const { table, schema, options } = params;
197
198
  const { create = "CREATE" } = options ?? {};
@@ -327,10 +328,10 @@ async function* rowsToColumnsChunks(params) {
327
328
  //#endregion
328
329
  //#region src/sql-duck.ts
329
330
  var SqlDuck = class {
330
- #duck;
331
+ #conn;
331
332
  #logger;
332
333
  constructor(params) {
333
- this.#duck = params.conn;
334
+ this.#conn = params.conn;
334
335
  this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
335
336
  }
336
337
  /**
@@ -375,12 +376,12 @@ var SqlDuck = class {
375
376
  if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
376
377
  const timeStart = Date.now();
377
378
  const { columnTypes, ddl } = await createTableFromZod({
378
- conn: this.#duck,
379
+ conn: this.#conn,
379
380
  schema,
380
381
  table,
381
382
  options: createOptions
382
383
  });
383
- const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
384
+ const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
384
385
  const chunkTypes = Array.from(columnTypes.values());
385
386
  let totalRows = 0;
386
387
  const dataAppendedCollector = createOnDataAppendedCollector();
@@ -423,7 +424,39 @@ var SqlDuck = class {
423
424
  };
424
425
  };
425
426
  //#endregion
426
- //#region src/table/table.ts
427
+ //#region src/utils/zod-codecs.ts
428
+ const zodCodecs = {
429
+ dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
430
+ decode: (date) => date.toISOString(),
431
+ encode: (isoString) => new Date(isoString)
432
+ }),
433
+ bigintToString: zod.codec(zod.bigint(), zod.string().meta({ format: "int64" }), {
434
+ decode: (bigint) => bigint.toString(),
435
+ encode: BigInt
436
+ })
437
+ };
438
+ //#endregion
439
+ //#region src/objects/database.ts
440
+ var Database = class {
441
+ #params;
442
+ get alias() {
443
+ return this.#params.alias;
444
+ }
445
+ constructor(params) {
446
+ this.#params = params;
447
+ }
448
+ toJson() {
449
+ return {
450
+ type: "database",
451
+ params: { alias: this.#params.alias }
452
+ };
453
+ }
454
+ [Symbol.toStringTag]() {
455
+ return this.alias;
456
+ }
457
+ };
458
+ //#endregion
459
+ //#region src/objects/table.ts
427
460
  var Table = class Table {
428
461
  #fqTable;
429
462
  get tableName() {
@@ -465,21 +498,272 @@ var Table = class Table {
465
498
  };
466
499
  };
467
500
  //#endregion
468
- //#region src/utils/zod-codecs.ts
469
- const zodCodecs = {
470
- dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
471
- decode: (date) => date.toISOString(),
472
- encode: (isoString) => new Date(isoString)
473
- }),
474
- bigintToString: zod.codec(zod.bigint(), zod.string().meta({ format: "int64" }), {
475
- decode: (bigint) => bigint.toString(),
476
- encode: BigInt
477
- })
501
+ //#region src/validation/core/duckdb-reserved-keywords.ts
502
+ /**
503
+ * DuckDB reserved keywords that cannot be used as unquoted identifiers.
504
+ * @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
505
+ */
506
+ const duckdbReservedKeywords = [
507
+ "ALL",
508
+ "ANALYSE",
509
+ "ANALYZE",
510
+ "AND",
511
+ "ANY",
512
+ "ARRAY",
513
+ "AS",
514
+ "ASC",
515
+ "ASYMMETRIC",
516
+ "BOTH",
517
+ "CASE",
518
+ "CAST",
519
+ "CHECK",
520
+ "COLLATE",
521
+ "COLUMN",
522
+ "CONSTRAINT",
523
+ "CREATE",
524
+ "CROSS",
525
+ "CURRENT_CATALOG",
526
+ "CURRENT_DATE",
527
+ "CURRENT_ROLE",
528
+ "CURRENT_SCHEMA",
529
+ "CURRENT_TIME",
530
+ "CURRENT_TIMESTAMP",
531
+ "CURRENT_USER",
532
+ "DEFAULT",
533
+ "DEFERRABLE",
534
+ "DESC",
535
+ "DISTINCT",
536
+ "DO",
537
+ "ELSE",
538
+ "END",
539
+ "EXCEPT",
540
+ "EXISTS",
541
+ "EXTRACT",
542
+ "FALSE",
543
+ "FETCH",
544
+ "FOR",
545
+ "FOREIGN",
546
+ "FROM",
547
+ "GRANT",
548
+ "GROUP",
549
+ "HAVING",
550
+ "IF",
551
+ "ILIKE",
552
+ "IN",
553
+ "INITIALLY",
554
+ "INNER",
555
+ "INTERSECT",
556
+ "INTO",
557
+ "IS",
558
+ "ISNULL",
559
+ "JOIN",
560
+ "LATERAL",
561
+ "LEADING",
562
+ "LEFT",
563
+ "LIKE",
564
+ "LIMIT",
565
+ "LOCALTIME",
566
+ "LOCALTIMESTAMP",
567
+ "NATURAL",
568
+ "NOT",
569
+ "NOTNULL",
570
+ "NULL",
571
+ "OFFSET",
572
+ "ON",
573
+ "ONLY",
574
+ "OR",
575
+ "ORDER",
576
+ "OUTER",
577
+ "OVERLAPS",
578
+ "PLACING",
579
+ "PRIMARY",
580
+ "REFERENCES",
581
+ "RETURNING",
582
+ "RIGHT",
583
+ "ROW",
584
+ "SELECT",
585
+ "SESSION_USER",
586
+ "SIMILAR",
587
+ "SOME",
588
+ "SYMMETRIC",
589
+ "TABLE",
590
+ "THEN",
591
+ "TO",
592
+ "TRAILING",
593
+ "TRUE",
594
+ "UNION",
595
+ "UNIQUE",
596
+ "USING",
597
+ "VARIADIC",
598
+ "VERBOSE",
599
+ "WHEN",
600
+ "WHERE",
601
+ "WINDOW",
602
+ "WITH"
603
+ ];
604
+ //#endregion
605
+ //#region src/validation/zod/duckdb-valid-names.schemas.ts
606
+ const duckdbMaximumObjectNameLength = 120;
607
+ const duckDbObjectNameRegex = /^[a-z_]\w*$/i;
608
+ const duckdbReservedKeywordsSet = new Set(duckdbReservedKeywords.map((k) => k.toUpperCase()));
609
+ const duckTableNameSchema = zod.string().min(1).max(duckdbMaximumObjectNameLength).regex(duckDbObjectNameRegex, "Table name must start with a letter or underscore, and contain only letters, numbers and underscores").refine((value) => !duckdbReservedKeywordsSet.has(value.toUpperCase()), { error: `Value is a DuckDB reserved keyword and cannot be used as a table name` });
610
+ const duckTableAliasSchema = duckTableNameSchema;
611
+ //#endregion
612
+ //#region src/manager/database/duck-database-manager.schemas.ts
613
+ const duckdbAttachOptionsSchema = zod.strictObject({
614
+ ACCESS_MODE: zod.optional(zod.enum([
615
+ "READ_ONLY",
616
+ "READ_WRITE",
617
+ "AUTOMATIC"
618
+ ])),
619
+ COMPRESS: zod.optional(zod.enum(["true", "false"])),
620
+ TYPE: zod.optional(zod.enum(["DUCKDB", "SQLITE"])),
621
+ BLOCK_SIZE: zod.optional(zod.int32().min(16384).max(262144)),
622
+ ROW_GROUP_SIZE: zod.optional(zod.int32().positive()),
623
+ STORAGE_VERSION: zod.optional(zod.string().startsWith("v").regex(/^v?\d{1,4}\.\d{1,4}\.\d{1,4}$/)),
624
+ ENCRYPTION_KEY: zod.optional(zod.string().min(8)),
625
+ ENCRYPTION_CIPHER: zod.optional(zod.enum([
626
+ "CBC",
627
+ "CTR",
628
+ "GCM"
629
+ ]))
630
+ });
631
+ const duckDatabaseManagerDbParamsSchema = zod.discriminatedUnion("type", [zod.strictObject({
632
+ type: zod.literal(":memory:"),
633
+ alias: duckTableAliasSchema,
634
+ options: zod.optional(duckdbAttachOptionsSchema)
635
+ }), zod.strictObject({
636
+ type: zod.literal("duckdb"),
637
+ path: zod.string().min(4).endsWith(".db"),
638
+ alias: duckTableAliasSchema,
639
+ options: zod.optional(duckdbAttachOptionsSchema)
640
+ })]);
641
+ //#endregion
642
+ //#region src/manager/database/commands/duck-database-attach-command.ts
643
+ var DuckDatabaseAttachCommand = class {
644
+ options;
645
+ dbParams;
646
+ constructor(dbParams, options) {
647
+ this.dbParams = dbParams;
648
+ this.options = options ?? {};
649
+ }
650
+ getRawSql = () => {
651
+ const dbParams = this.dbParams;
652
+ const parts = ["ATTACH", this.options.behaviour].filter(Boolean);
653
+ const { type, alias } = dbParams;
654
+ switch (type) {
655
+ case ":memory:":
656
+ parts.push("':memory:'");
657
+ break;
658
+ case "duckdb":
659
+ parts.push(`'${dbParams.path}'`);
660
+ break;
661
+ default: (0, _httpx_assert.assertNever)(type);
662
+ }
663
+ if (alias !== null) parts.push("AS", `${alias}`);
664
+ const options = (0, _httpx_plain_object.isPlainObject)(dbParams.options) ? Object.entries(dbParams.options).map(([key, value]) => {
665
+ return key === "ACCESS_MODE" ? value : `${key} '${value}'`;
666
+ }) : [];
667
+ if (options.length > 0) parts.push(`(${options.join(", ")})`);
668
+ return parts.filter(Boolean).join(" ");
669
+ };
670
+ };
671
+ //#endregion
672
+ //#region src/manager/database/duck-database-manager.ts
673
+ var DuckDatabaseManager = class {
674
+ #conn;
675
+ #logger;
676
+ constructor(conn, params) {
677
+ this.#conn = conn;
678
+ this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
679
+ }
680
+ /**
681
+ * Attach a database to the current connection
682
+ *
683
+ * @example
684
+ * ```typescript
685
+ * const dbManager = new DuckDatabaseManager(conn);
686
+ * const database = dbManager.attach({
687
+ * type: ':memory:', // can be 'duckdb', 's3'...
688
+ * alias: 'mydb',
689
+ * options: { COMPRESS: 'true' }
690
+ * });
691
+ *
692
+ * console.log(database.alias); // 'mydb'
693
+ * ```
694
+ */
695
+ attach = async (dbParams, options) => {
696
+ const params = zod.parse(duckDatabaseManagerDbParamsSchema, dbParams);
697
+ const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
698
+ await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
699
+ return new Database({ alias: params.alias });
700
+ };
701
+ attachOrReplace = async (dbParams) => {
702
+ return this.attach(dbParams, { behaviour: "OR REPLACE" });
703
+ };
704
+ attachIfNotExists = async (dbParams) => {
705
+ return this.attach(dbParams, { behaviour: "IF NOT EXISTS" });
706
+ };
707
+ showDatabases = async () => {
708
+ return await this.#executeRawSqlCommand("showDatabases()", `SHOW DATABASES`);
709
+ };
710
+ detach = async (dbAlias) => {
711
+ const safeAlias = zod.parse(duckTableAliasSchema, dbAlias);
712
+ await this.#executeRawSqlCommand(`detach(${safeAlias})`, `DETACH ${safeAlias}`);
713
+ return true;
714
+ };
715
+ detachIfExists = async (dbAlias) => {
716
+ const safeAlias = zod.parse(duckTableAliasSchema, dbAlias);
717
+ await this.#executeRawSqlCommand(`detachIfExists(${safeAlias})`, `DETACH IF EXISTS ${safeAlias}`);
718
+ return true;
719
+ };
720
+ /**
721
+ * The statistics recomputed by the ANALYZE statement are only used for join order optimization.
722
+ *
723
+ * It is therefore recommended to recompute these statistics for improved join orders,
724
+ * especially after performing large updates (inserts and/or deletes).
725
+ *
726
+ * @link https://duckdb.org/docs/stable/sql/statements/analyze
727
+ */
728
+ analyze = async () => {
729
+ await this.#executeRawSqlCommand("analyze()", "ANALYZE");
730
+ return true;
731
+ };
732
+ checkpoint = async (dbAlias) => {
733
+ const safeAlias = zod.parse(duckTableAliasSchema, dbAlias);
734
+ await this.#executeRawSqlCommand(`checkpoint(${safeAlias})`, `CHECKPOINT ${safeAlias}`);
735
+ return true;
736
+ };
737
+ #executeRawSqlCommand = async (name, rawSql) => {
738
+ const startTime = Date.now();
739
+ try {
740
+ const result = await this.#conn.runAndReadAll(rawSql);
741
+ const timeMs = Math.round(Date.now() - startTime);
742
+ const data = result.getRowObjectsJS();
743
+ this.#logger.info(`DuckDatabaseManager.${name} in ${timeMs}ms`, { timeMs });
744
+ return data;
745
+ } catch (e) {
746
+ const msg = `DuckDatabaseManager: failed to run "${name}" - ${e?.message ?? ""}`;
747
+ const timeMs = Math.round(Date.now() - startTime);
748
+ this.#logger.error(msg, {
749
+ name,
750
+ sql: rawSql,
751
+ timeMs
752
+ });
753
+ throw new Error(msg, { cause: e });
754
+ }
755
+ };
478
756
  };
479
757
  //#endregion
758
+ exports.Database = Database;
759
+ exports.DuckDatabaseManager = DuckDatabaseManager;
480
760
  exports.DuckMemory = DuckMemory;
481
761
  exports.SqlDuck = SqlDuck;
482
762
  exports.Table = Table;
763
+ exports.duckDatabaseManagerDbParamsSchema = duckDatabaseManagerDbParamsSchema;
764
+ exports.duckTableAliasSchema = duckTableAliasSchema;
765
+ exports.duckTableNameSchema = duckTableNameSchema;
766
+ exports.duckdbReservedKeywords = duckdbReservedKeywords;
483
767
  exports.flowbladeLogtapeSqlduckConfig = flowbladeLogtapeSqlduckConfig;
484
768
  exports.getTableCreateFromZod = getTableCreateFromZod;
485
769
  exports.sqlduckDefaultLogtapeLogger = sqlduckDefaultLogtapeLogger;
package/dist/index.d.cts CHANGED
@@ -1,3 +1,4 @@
1
+ import * as _duckdb_node_api0 from "@duckdb/node-api";
1
2
  import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
2
3
  import * as _logtape_logtape0 from "@logtape/logtape";
3
4
  import { Logger } from "@logtape/logtape";
@@ -23,11 +24,6 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
23
24
  type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
24
25
  type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
25
26
  //#endregion
26
- //#region src/config/flowblade-logtape-sqlduck.config.d.ts
27
- declare const flowbladeLogtapeSqlduckConfig: {
28
- categories: string[];
29
- };
30
- //#endregion
31
27
  //#region src/helpers/duck-memory.d.ts
32
28
  declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
33
29
  type DuckMemoryTag = (typeof duckMemoryTags)[number];
@@ -56,10 +52,7 @@ declare class DuckMemory {
56
52
  getSummary: () => Promise<DuckMemorySummary>;
57
53
  }
58
54
  //#endregion
59
- //#region src/logger/sqlduck-default-logtape-logger.d.ts
60
- declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
61
- //#endregion
62
- //#region src/table/table.d.ts
55
+ //#region src/objects/table.d.ts
63
56
  /**
64
57
  * Fully qualified table information
65
58
  */
@@ -208,4 +201,145 @@ declare const zodCodecs: {
208
201
  readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
209
202
  };
210
203
  //#endregion
211
- export { DuckMemory, DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
204
+ //#region src/objects/database.d.ts
205
+ type DatabaseProperties = {
206
+ alias: string;
207
+ };
208
+ declare class Database {
209
+ #private;
210
+ get alias(): string;
211
+ constructor(params: DatabaseProperties);
212
+ toJson(): {
213
+ type: string;
214
+ params: {
215
+ alias: string;
216
+ };
217
+ };
218
+ [Symbol.toStringTag](): string;
219
+ }
220
+ //#endregion
221
+ //#region src/manager/database/duck-database-manager.schemas.d.ts
222
+ declare const duckDatabaseManagerDbParamsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
223
+ type: z.ZodLiteral<":memory:">;
224
+ alias: z.ZodString;
225
+ options: z.ZodOptional<z.ZodObject<{
226
+ ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
227
+ READ_ONLY: "READ_ONLY";
228
+ READ_WRITE: "READ_WRITE";
229
+ AUTOMATIC: "AUTOMATIC";
230
+ }>>;
231
+ COMPRESS: z.ZodOptional<z.ZodEnum<{
232
+ true: "true";
233
+ false: "false";
234
+ }>>;
235
+ TYPE: z.ZodOptional<z.ZodEnum<{
236
+ DUCKDB: "DUCKDB";
237
+ SQLITE: "SQLITE";
238
+ }>>;
239
+ BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
240
+ ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
241
+ STORAGE_VERSION: z.ZodOptional<z.ZodString>;
242
+ ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
243
+ ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
244
+ CBC: "CBC";
245
+ CTR: "CTR";
246
+ GCM: "GCM";
247
+ }>>;
248
+ }, z.core.$strict>>;
249
+ }, z.core.$strict>, z.ZodObject<{
250
+ type: z.ZodLiteral<"duckdb">;
251
+ path: z.ZodString;
252
+ alias: z.ZodString;
253
+ options: z.ZodOptional<z.ZodObject<{
254
+ ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
255
+ READ_ONLY: "READ_ONLY";
256
+ READ_WRITE: "READ_WRITE";
257
+ AUTOMATIC: "AUTOMATIC";
258
+ }>>;
259
+ COMPRESS: z.ZodOptional<z.ZodEnum<{
260
+ true: "true";
261
+ false: "false";
262
+ }>>;
263
+ TYPE: z.ZodOptional<z.ZodEnum<{
264
+ DUCKDB: "DUCKDB";
265
+ SQLITE: "SQLITE";
266
+ }>>;
267
+ BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
268
+ ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
269
+ STORAGE_VERSION: z.ZodOptional<z.ZodString>;
270
+ ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
271
+ ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
272
+ CBC: "CBC";
273
+ CTR: "CTR";
274
+ GCM: "GCM";
275
+ }>>;
276
+ }, z.core.$strict>>;
277
+ }, z.core.$strict>], "type">;
278
+ type DuckDatabaseManagerDbParams = z.infer<typeof duckDatabaseManagerDbParamsSchema>;
279
+ //#endregion
280
+ //#region src/manager/database/commands/duck-database-attach-command.d.ts
281
+ type Behaviour = 'OR REPLACE' | 'IF NOT EXISTS';
282
+ type DuckDatabaseAttachCommandOptions = {
283
+ behaviour?: Behaviour;
284
+ };
285
+ //#endregion
286
+ //#region src/manager/database/duck-database-manager.d.ts
287
+ declare class DuckDatabaseManager {
288
+ #private;
289
+ constructor(conn: DuckDBConnection, params?: {
290
+ logger?: Logger;
291
+ });
292
+ /**
293
+ * Attach a database to the current connection
294
+ *
295
+ * @example
296
+ * ```typescript
297
+ * const dbManager = new DuckDatabaseManager(conn);
298
+ * const database = dbManager.attach({
299
+ * type: ':memory:', // can be 'duckdb', 's3'...
300
+ * alias: 'mydb',
301
+ * options: { COMPRESS: 'true' }
302
+ * });
303
+ *
304
+ * console.log(database.alias); // 'mydb'
305
+ * ```
306
+ */
307
+ attach: (dbParams: DuckDatabaseManagerDbParams, options?: DuckDatabaseAttachCommandOptions) => Promise<Database>;
308
+ attachOrReplace: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
309
+ attachIfNotExists: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
310
+ showDatabases: () => Promise<Record<string, _duckdb_node_api0.JS>[]>;
311
+ detach: (dbAlias: string) => Promise<boolean>;
312
+ detachIfExists: (dbAlias: string) => Promise<boolean>;
313
+ /**
314
+ * The statistics recomputed by the ANALYZE statement are only used for join order optimization.
315
+ *
316
+ * It is therefore recommended to recompute these statistics for improved join orders,
317
+ * especially after performing large updates (inserts and/or deletes).
318
+ *
319
+ * @link https://duckdb.org/docs/stable/sql/statements/analyze
320
+ */
321
+ analyze: () => Promise<boolean>;
322
+ checkpoint: (dbAlias: string) => Promise<boolean>;
323
+ }
324
+ //#endregion
325
+ //#region src/validation/core/duckdb-reserved-keywords.d.ts
326
+ /**
327
+ * DuckDB reserved keywords that cannot be used as unquoted identifiers.
328
+ * @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
329
+ */
330
+ declare const duckdbReservedKeywords: readonly ["ALL", "ANALYSE", "ANALYZE", "AND", "ANY", "ARRAY", "AS", "ASC", "ASYMMETRIC", "BOTH", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DEFAULT", "DEFERRABLE", "DESC", "DISTINCT", "DO", "ELSE", "END", "EXCEPT", "EXISTS", "EXTRACT", "FALSE", "FETCH", "FOR", "FOREIGN", "FROM", "GRANT", "GROUP", "HAVING", "IF", "ILIKE", "IN", "INITIALLY", "INNER", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "LATERAL", "LEADING", "LEFT", "LIKE", "LIMIT", "LOCALTIME", "LOCALTIMESTAMP", "NATURAL", "NOT", "NOTNULL", "NULL", "OFFSET", "ON", "ONLY", "OR", "ORDER", "OUTER", "OVERLAPS", "PLACING", "PRIMARY", "REFERENCES", "RETURNING", "RIGHT", "ROW", "SELECT", "SESSION_USER", "SIMILAR", "SOME", "SYMMETRIC", "TABLE", "THEN", "TO", "TRAILING", "TRUE", "UNION", "UNIQUE", "USING", "VARIADIC", "VERBOSE", "WHEN", "WHERE", "WINDOW", "WITH"];
331
+ type DuckdbReservedKeywords = (typeof duckdbReservedKeywords)[number];
332
+ //#endregion
333
+ //#region src/validation/zod/duckdb-valid-names.schemas.d.ts
334
+ declare const duckTableNameSchema: z.ZodString;
335
+ declare const duckTableAliasSchema: z.ZodString;
336
+ //#endregion
337
+ //#region src/config/flowblade-logtape-sqlduck.config.d.ts
338
+ declare const flowbladeLogtapeSqlduckConfig: {
339
+ categories: string[];
340
+ };
341
+ //#endregion
342
+ //#region src/logger/sqlduck-default-logtape-logger.d.ts
343
+ declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
344
+ //#endregion
345
+ export { Database, DuckDatabaseManager, type DuckDatabaseManagerDbParams, DuckMemory, DuckMemoryTag, type DuckdbReservedKeywords, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckDatabaseManagerDbParamsSchema, duckTableAliasSchema, duckTableNameSchema, duckdbReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
package/dist/index.d.mts CHANGED
@@ -1,6 +1,7 @@
1
+ import * as _duckdb_node_api0 from "@duckdb/node-api";
2
+ import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
1
3
  import * as _logtape_logtape0 from "@logtape/logtape";
2
4
  import { Logger } from "@logtape/logtape";
3
- import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
4
5
  import * as z from "zod";
5
6
  import { ZodObject } from "zod";
6
7
 
@@ -23,11 +24,6 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
23
24
  type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
24
25
  type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
25
26
  //#endregion
26
- //#region src/config/flowblade-logtape-sqlduck.config.d.ts
27
- declare const flowbladeLogtapeSqlduckConfig: {
28
- categories: string[];
29
- };
30
- //#endregion
31
27
  //#region src/helpers/duck-memory.d.ts
32
28
  declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
33
29
  type DuckMemoryTag = (typeof duckMemoryTags)[number];
@@ -56,10 +52,7 @@ declare class DuckMemory {
56
52
  getSummary: () => Promise<DuckMemorySummary>;
57
53
  }
58
54
  //#endregion
59
- //#region src/logger/sqlduck-default-logtape-logger.d.ts
60
- declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
61
- //#endregion
62
- //#region src/table/table.d.ts
55
+ //#region src/objects/table.d.ts
63
56
  /**
64
57
  * Fully qualified table information
65
58
  */
@@ -208,4 +201,145 @@ declare const zodCodecs: {
208
201
  readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
209
202
  };
210
203
  //#endregion
211
- export { DuckMemory, type DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
204
+ //#region src/objects/database.d.ts
205
+ type DatabaseProperties = {
206
+ alias: string;
207
+ };
208
+ declare class Database {
209
+ #private;
210
+ get alias(): string;
211
+ constructor(params: DatabaseProperties);
212
+ toJson(): {
213
+ type: string;
214
+ params: {
215
+ alias: string;
216
+ };
217
+ };
218
+ [Symbol.toStringTag](): string;
219
+ }
220
+ //#endregion
221
+ //#region src/manager/database/duck-database-manager.schemas.d.ts
222
+ declare const duckDatabaseManagerDbParamsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
223
+ type: z.ZodLiteral<":memory:">;
224
+ alias: z.ZodString;
225
+ options: z.ZodOptional<z.ZodObject<{
226
+ ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
227
+ READ_ONLY: "READ_ONLY";
228
+ READ_WRITE: "READ_WRITE";
229
+ AUTOMATIC: "AUTOMATIC";
230
+ }>>;
231
+ COMPRESS: z.ZodOptional<z.ZodEnum<{
232
+ true: "true";
233
+ false: "false";
234
+ }>>;
235
+ TYPE: z.ZodOptional<z.ZodEnum<{
236
+ DUCKDB: "DUCKDB";
237
+ SQLITE: "SQLITE";
238
+ }>>;
239
+ BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
240
+ ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
241
+ STORAGE_VERSION: z.ZodOptional<z.ZodString>;
242
+ ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
243
+ ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
244
+ CBC: "CBC";
245
+ CTR: "CTR";
246
+ GCM: "GCM";
247
+ }>>;
248
+ }, z.core.$strict>>;
249
+ }, z.core.$strict>, z.ZodObject<{
250
+ type: z.ZodLiteral<"duckdb">;
251
+ path: z.ZodString;
252
+ alias: z.ZodString;
253
+ options: z.ZodOptional<z.ZodObject<{
254
+ ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
255
+ READ_ONLY: "READ_ONLY";
256
+ READ_WRITE: "READ_WRITE";
257
+ AUTOMATIC: "AUTOMATIC";
258
+ }>>;
259
+ COMPRESS: z.ZodOptional<z.ZodEnum<{
260
+ true: "true";
261
+ false: "false";
262
+ }>>;
263
+ TYPE: z.ZodOptional<z.ZodEnum<{
264
+ DUCKDB: "DUCKDB";
265
+ SQLITE: "SQLITE";
266
+ }>>;
267
+ BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
268
+ ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
269
+ STORAGE_VERSION: z.ZodOptional<z.ZodString>;
270
+ ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
271
+ ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
272
+ CBC: "CBC";
273
+ CTR: "CTR";
274
+ GCM: "GCM";
275
+ }>>;
276
+ }, z.core.$strict>>;
277
+ }, z.core.$strict>], "type">;
278
+ type DuckDatabaseManagerDbParams = z.infer<typeof duckDatabaseManagerDbParamsSchema>;
279
+ //#endregion
280
+ //#region src/manager/database/commands/duck-database-attach-command.d.ts
281
+ type Behaviour = 'OR REPLACE' | 'IF NOT EXISTS';
282
+ type DuckDatabaseAttachCommandOptions = {
283
+ behaviour?: Behaviour;
284
+ };
285
+ //#endregion
286
+ //#region src/manager/database/duck-database-manager.d.ts
287
+ declare class DuckDatabaseManager {
288
+ #private;
289
+ constructor(conn: DuckDBConnection, params?: {
290
+ logger?: Logger;
291
+ });
292
+ /**
293
+ * Attach a database to the current connection
294
+ *
295
+ * @example
296
+ * ```typescript
297
+ * const dbManager = new DuckDatabaseManager(conn);
298
+ * const database = dbManager.attach({
299
+ * type: ':memory:', // can be 'duckdb', 's3'...
300
+ * alias: 'mydb',
301
+ * options: { COMPRESS: 'true' }
302
+ * });
303
+ *
304
+ * console.log(database.alias); // 'mydb'
305
+ * ```
306
+ */
307
+ attach: (dbParams: DuckDatabaseManagerDbParams, options?: DuckDatabaseAttachCommandOptions) => Promise<Database>;
308
+ attachOrReplace: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
309
+ attachIfNotExists: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
310
+ showDatabases: () => Promise<Record<string, _duckdb_node_api0.JS>[]>;
311
+ detach: (dbAlias: string) => Promise<boolean>;
312
+ detachIfExists: (dbAlias: string) => Promise<boolean>;
313
+ /**
314
+ * The statistics recomputed by the ANALYZE statement are only used for join order optimization.
315
+ *
316
+ * It is therefore recommended to recompute these statistics for improved join orders,
317
+ * especially after performing large updates (inserts and/or deletes).
318
+ *
319
+ * @link https://duckdb.org/docs/stable/sql/statements/analyze
320
+ */
321
+ analyze: () => Promise<boolean>;
322
+ checkpoint: (dbAlias: string) => Promise<boolean>;
323
+ }
324
+ //#endregion
325
+ //#region src/validation/core/duckdb-reserved-keywords.d.ts
326
+ /**
327
+ * DuckDB reserved keywords that cannot be used as unquoted identifiers.
328
+ * @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
329
+ */
330
+ declare const duckdbReservedKeywords: readonly ["ALL", "ANALYSE", "ANALYZE", "AND", "ANY", "ARRAY", "AS", "ASC", "ASYMMETRIC", "BOTH", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DEFAULT", "DEFERRABLE", "DESC", "DISTINCT", "DO", "ELSE", "END", "EXCEPT", "EXISTS", "EXTRACT", "FALSE", "FETCH", "FOR", "FOREIGN", "FROM", "GRANT", "GROUP", "HAVING", "IF", "ILIKE", "IN", "INITIALLY", "INNER", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "LATERAL", "LEADING", "LEFT", "LIKE", "LIMIT", "LOCALTIME", "LOCALTIMESTAMP", "NATURAL", "NOT", "NOTNULL", "NULL", "OFFSET", "ON", "ONLY", "OR", "ORDER", "OUTER", "OVERLAPS", "PLACING", "PRIMARY", "REFERENCES", "RETURNING", "RIGHT", "ROW", "SELECT", "SESSION_USER", "SIMILAR", "SOME", "SYMMETRIC", "TABLE", "THEN", "TO", "TRAILING", "TRUE", "UNION", "UNIQUE", "USING", "VARIADIC", "VERBOSE", "WHEN", "WHERE", "WINDOW", "WITH"];
331
+ type DuckdbReservedKeywords = (typeof duckdbReservedKeywords)[number];
332
+ //#endregion
333
+ //#region src/validation/zod/duckdb-valid-names.schemas.d.ts
334
+ declare const duckTableNameSchema: z.ZodString;
335
+ declare const duckTableAliasSchema: z.ZodString;
336
+ //#endregion
337
+ //#region src/config/flowblade-logtape-sqlduck.config.d.ts
338
+ declare const flowbladeLogtapeSqlduckConfig: {
339
+ categories: string[];
340
+ };
341
+ //#endregion
342
+ //#region src/logger/sqlduck-default-logtape-logger.d.ts
343
+ declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
344
+ //#endregion
345
+ export { Database, DuckDatabaseManager, type DuckDatabaseManagerDbParams, DuckMemory, type DuckMemoryTag, type DuckdbReservedKeywords, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckDatabaseManagerDbParamsSchema, duckTableAliasSchema, duckTableNameSchema, duckdbReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
package/dist/index.mjs CHANGED
@@ -1,9 +1,8 @@
1
- import { getLogger } from "@logtape/logtape";
2
1
  import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
2
+ import { getLogger } from "@logtape/logtape";
3
3
  import * as z from "zod";
4
- //#region src/config/flowblade-logtape-sqlduck.config.ts
5
- const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
6
- //#endregion
4
+ import { assertNever } from "@httpx/assert";
5
+ import { isPlainObject } from "@httpx/plain-object";
7
6
  //#region src/helpers/duck-exec.ts
8
7
  var DuckExec = class {
9
8
  #conn;
@@ -100,9 +99,6 @@ var DuckMemory = class {
100
99
  };
101
100
  };
102
101
  //#endregion
103
- //#region src/logger/sqlduck-default-logtape-logger.ts
104
- const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
105
- //#endregion
106
102
  //#region src/appender/data-appender-callback.ts
107
103
  const isOnDataAppendedAsyncCb = (v) => {
108
104
  return v.constructor.name === "AsyncFunction";
@@ -124,6 +120,12 @@ const createOnDataAppendedCollector = () => {
124
120
  };
125
121
  };
126
122
  //#endregion
123
+ //#region src/config/flowblade-logtape-sqlduck.config.ts
124
+ const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
125
+ //#endregion
126
+ //#region src/logger/sqlduck-default-logtape-logger.ts
127
+ const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
128
+ //#endregion
127
129
  //#region src/table/get-duckdb-number-column-type.ts
128
130
  const isFloatValue = (value) => {
129
131
  if (!Number.isFinite(value)) return true;
@@ -157,7 +159,7 @@ const createOptions = {
157
159
  CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
158
160
  IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
159
161
  };
160
- const duckDbTypes = [
162
+ const duckDbTypesMap = new Map([
161
163
  ["VARCHAR", VARCHAR],
162
164
  ["BIGINT", BIGINT],
163
165
  ["TIMESTAMP", TIMESTAMP],
@@ -166,8 +168,7 @@ const duckDbTypes = [
166
168
  ["INTEGER", INTEGER],
167
169
  ["DOUBLE", DOUBLE],
168
170
  ["FLOAT", FLOAT]
169
- ];
170
- const duckDbTypesMap = new Map(duckDbTypes);
171
+ ]);
171
172
  const getTableCreateFromZod = (params) => {
172
173
  const { table, schema, options } = params;
173
174
  const { create = "CREATE" } = options ?? {};
@@ -303,10 +304,10 @@ async function* rowsToColumnsChunks(params) {
303
304
  //#endregion
304
305
  //#region src/sql-duck.ts
305
306
  var SqlDuck = class {
306
- #duck;
307
+ #conn;
307
308
  #logger;
308
309
  constructor(params) {
309
- this.#duck = params.conn;
310
+ this.#conn = params.conn;
310
311
  this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
311
312
  }
312
313
  /**
@@ -351,12 +352,12 @@ var SqlDuck = class {
351
352
  if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
352
353
  const timeStart = Date.now();
353
354
  const { columnTypes, ddl } = await createTableFromZod({
354
- conn: this.#duck,
355
+ conn: this.#conn,
355
356
  schema,
356
357
  table,
357
358
  options: createOptions
358
359
  });
359
- const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
360
+ const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
360
361
  const chunkTypes = Array.from(columnTypes.values());
361
362
  let totalRows = 0;
362
363
  const dataAppendedCollector = createOnDataAppendedCollector();
@@ -399,7 +400,39 @@ var SqlDuck = class {
399
400
  };
400
401
  };
401
402
  //#endregion
402
- //#region src/table/table.ts
403
+ //#region src/utils/zod-codecs.ts
404
+ const zodCodecs = {
405
+ dateToString: z.codec(z.date(), z.iso.datetime(), {
406
+ decode: (date) => date.toISOString(),
407
+ encode: (isoString) => new Date(isoString)
408
+ }),
409
+ bigintToString: z.codec(z.bigint(), z.string().meta({ format: "int64" }), {
410
+ decode: (bigint) => bigint.toString(),
411
+ encode: BigInt
412
+ })
413
+ };
414
+ //#endregion
415
+ //#region src/objects/database.ts
416
+ var Database = class {
417
+ #params;
418
+ get alias() {
419
+ return this.#params.alias;
420
+ }
421
+ constructor(params) {
422
+ this.#params = params;
423
+ }
424
+ toJson() {
425
+ return {
426
+ type: "database",
427
+ params: { alias: this.#params.alias }
428
+ };
429
+ }
430
+ [Symbol.toStringTag]() {
431
+ return this.alias;
432
+ }
433
+ };
434
+ //#endregion
435
+ //#region src/objects/table.ts
403
436
  var Table = class Table {
404
437
  #fqTable;
405
438
  get tableName() {
@@ -441,16 +474,261 @@ var Table = class Table {
441
474
  };
442
475
  };
443
476
  //#endregion
444
- //#region src/utils/zod-codecs.ts
445
- const zodCodecs = {
446
- dateToString: z.codec(z.date(), z.iso.datetime(), {
447
- decode: (date) => date.toISOString(),
448
- encode: (isoString) => new Date(isoString)
449
- }),
450
- bigintToString: z.codec(z.bigint(), z.string().meta({ format: "int64" }), {
451
- decode: (bigint) => bigint.toString(),
452
- encode: BigInt
453
- })
477
+ //#region src/validation/core/duckdb-reserved-keywords.ts
478
+ /**
479
+ * DuckDB reserved keywords that cannot be used as unquoted identifiers.
480
+ * @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
481
+ */
482
+ const duckdbReservedKeywords = [
483
+ "ALL",
484
+ "ANALYSE",
485
+ "ANALYZE",
486
+ "AND",
487
+ "ANY",
488
+ "ARRAY",
489
+ "AS",
490
+ "ASC",
491
+ "ASYMMETRIC",
492
+ "BOTH",
493
+ "CASE",
494
+ "CAST",
495
+ "CHECK",
496
+ "COLLATE",
497
+ "COLUMN",
498
+ "CONSTRAINT",
499
+ "CREATE",
500
+ "CROSS",
501
+ "CURRENT_CATALOG",
502
+ "CURRENT_DATE",
503
+ "CURRENT_ROLE",
504
+ "CURRENT_SCHEMA",
505
+ "CURRENT_TIME",
506
+ "CURRENT_TIMESTAMP",
507
+ "CURRENT_USER",
508
+ "DEFAULT",
509
+ "DEFERRABLE",
510
+ "DESC",
511
+ "DISTINCT",
512
+ "DO",
513
+ "ELSE",
514
+ "END",
515
+ "EXCEPT",
516
+ "EXISTS",
517
+ "EXTRACT",
518
+ "FALSE",
519
+ "FETCH",
520
+ "FOR",
521
+ "FOREIGN",
522
+ "FROM",
523
+ "GRANT",
524
+ "GROUP",
525
+ "HAVING",
526
+ "IF",
527
+ "ILIKE",
528
+ "IN",
529
+ "INITIALLY",
530
+ "INNER",
531
+ "INTERSECT",
532
+ "INTO",
533
+ "IS",
534
+ "ISNULL",
535
+ "JOIN",
536
+ "LATERAL",
537
+ "LEADING",
538
+ "LEFT",
539
+ "LIKE",
540
+ "LIMIT",
541
+ "LOCALTIME",
542
+ "LOCALTIMESTAMP",
543
+ "NATURAL",
544
+ "NOT",
545
+ "NOTNULL",
546
+ "NULL",
547
+ "OFFSET",
548
+ "ON",
549
+ "ONLY",
550
+ "OR",
551
+ "ORDER",
552
+ "OUTER",
553
+ "OVERLAPS",
554
+ "PLACING",
555
+ "PRIMARY",
556
+ "REFERENCES",
557
+ "RETURNING",
558
+ "RIGHT",
559
+ "ROW",
560
+ "SELECT",
561
+ "SESSION_USER",
562
+ "SIMILAR",
563
+ "SOME",
564
+ "SYMMETRIC",
565
+ "TABLE",
566
+ "THEN",
567
+ "TO",
568
+ "TRAILING",
569
+ "TRUE",
570
+ "UNION",
571
+ "UNIQUE",
572
+ "USING",
573
+ "VARIADIC",
574
+ "VERBOSE",
575
+ "WHEN",
576
+ "WHERE",
577
+ "WINDOW",
578
+ "WITH"
579
+ ];
580
+ //#endregion
581
+ //#region src/validation/zod/duckdb-valid-names.schemas.ts
582
+ const duckdbMaximumObjectNameLength = 120;
583
+ const duckDbObjectNameRegex = /^[a-z_]\w*$/i;
584
+ const duckdbReservedKeywordsSet = new Set(duckdbReservedKeywords.map((k) => k.toUpperCase()));
585
+ const duckTableNameSchema = z.string().min(1).max(duckdbMaximumObjectNameLength).regex(duckDbObjectNameRegex, "Table name must start with a letter or underscore, and contain only letters, numbers and underscores").refine((value) => !duckdbReservedKeywordsSet.has(value.toUpperCase()), { error: `Value is a DuckDB reserved keyword and cannot be used as a table name` });
586
+ const duckTableAliasSchema = duckTableNameSchema;
587
+ //#endregion
588
+ //#region src/manager/database/duck-database-manager.schemas.ts
589
+ const duckdbAttachOptionsSchema = z.strictObject({
590
+ ACCESS_MODE: z.optional(z.enum([
591
+ "READ_ONLY",
592
+ "READ_WRITE",
593
+ "AUTOMATIC"
594
+ ])),
595
+ COMPRESS: z.optional(z.enum(["true", "false"])),
596
+ TYPE: z.optional(z.enum(["DUCKDB", "SQLITE"])),
597
+ BLOCK_SIZE: z.optional(z.int32().min(16384).max(262144)),
598
+ ROW_GROUP_SIZE: z.optional(z.int32().positive()),
599
+ STORAGE_VERSION: z.optional(z.string().startsWith("v").regex(/^v?\d{1,4}\.\d{1,4}\.\d{1,4}$/)),
600
+ ENCRYPTION_KEY: z.optional(z.string().min(8)),
601
+ ENCRYPTION_CIPHER: z.optional(z.enum([
602
+ "CBC",
603
+ "CTR",
604
+ "GCM"
605
+ ]))
606
+ });
607
+ const duckDatabaseManagerDbParamsSchema = z.discriminatedUnion("type", [z.strictObject({
608
+ type: z.literal(":memory:"),
609
+ alias: duckTableAliasSchema,
610
+ options: z.optional(duckdbAttachOptionsSchema)
611
+ }), z.strictObject({
612
+ type: z.literal("duckdb"),
613
+ path: z.string().min(4).endsWith(".db"),
614
+ alias: duckTableAliasSchema,
615
+ options: z.optional(duckdbAttachOptionsSchema)
616
+ })]);
617
+ //#endregion
618
+ //#region src/manager/database/commands/duck-database-attach-command.ts
619
+ var DuckDatabaseAttachCommand = class {
620
+ options;
621
+ dbParams;
622
+ constructor(dbParams, options) {
623
+ this.dbParams = dbParams;
624
+ this.options = options ?? {};
625
+ }
626
+ getRawSql = () => {
627
+ const dbParams = this.dbParams;
628
+ const parts = ["ATTACH", this.options.behaviour].filter(Boolean);
629
+ const { type, alias } = dbParams;
630
+ switch (type) {
631
+ case ":memory:":
632
+ parts.push("':memory:'");
633
+ break;
634
+ case "duckdb":
635
+ parts.push(`'${dbParams.path}'`);
636
+ break;
637
+ default: assertNever(type);
638
+ }
639
+ if (alias !== null) parts.push("AS", `${alias}`);
640
+ const options = isPlainObject(dbParams.options) ? Object.entries(dbParams.options).map(([key, value]) => {
641
+ return key === "ACCESS_MODE" ? value : `${key} '${value}'`;
642
+ }) : [];
643
+ if (options.length > 0) parts.push(`(${options.join(", ")})`);
644
+ return parts.filter(Boolean).join(" ");
645
+ };
646
+ };
647
+ //#endregion
648
+ //#region src/manager/database/duck-database-manager.ts
649
+ var DuckDatabaseManager = class {
650
+ #conn;
651
+ #logger;
652
+ constructor(conn, params) {
653
+ this.#conn = conn;
654
+ this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
655
+ }
656
+ /**
657
+ * Attach a database to the current connection
658
+ *
659
+ * @example
660
+ * ```typescript
661
+ * const dbManager = new DuckDatabaseManager(conn);
662
+ * const database = dbManager.attach({
663
+ * type: ':memory:', // can be 'duckdb', 's3'...
664
+ * alias: 'mydb',
665
+ * options: { COMPRESS: 'true' }
666
+ * });
667
+ *
668
+ * console.log(database.alias); // 'mydb'
669
+ * ```
670
+ */
671
+ attach = async (dbParams, options) => {
672
+ const params = z.parse(duckDatabaseManagerDbParamsSchema, dbParams);
673
+ const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
674
+ await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
675
+ return new Database({ alias: params.alias });
676
+ };
677
+ attachOrReplace = async (dbParams) => {
678
+ return this.attach(dbParams, { behaviour: "OR REPLACE" });
679
+ };
680
+ attachIfNotExists = async (dbParams) => {
681
+ return this.attach(dbParams, { behaviour: "IF NOT EXISTS" });
682
+ };
683
+ showDatabases = async () => {
684
+ return await this.#executeRawSqlCommand("showDatabases()", `SHOW DATABASES`);
685
+ };
686
+ detach = async (dbAlias) => {
687
+ const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
688
+ await this.#executeRawSqlCommand(`detach(${safeAlias})`, `DETACH ${safeAlias}`);
689
+ return true;
690
+ };
691
+ detachIfExists = async (dbAlias) => {
692
+ const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
693
+ await this.#executeRawSqlCommand(`detachIfExists(${safeAlias})`, `DETACH IF EXISTS ${safeAlias}`);
694
+ return true;
695
+ };
696
+ /**
697
+ * The statistics recomputed by the ANALYZE statement are only used for join order optimization.
698
+ *
699
+ * It is therefore recommended to recompute these statistics for improved join orders,
700
+ * especially after performing large updates (inserts and/or deletes).
701
+ *
702
+ * @link https://duckdb.org/docs/stable/sql/statements/analyze
703
+ */
704
+ analyze = async () => {
705
+ await this.#executeRawSqlCommand("analyze()", "ANALYZE");
706
+ return true;
707
+ };
708
+ checkpoint = async (dbAlias) => {
709
+ const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
710
+ await this.#executeRawSqlCommand(`checkpoint(${safeAlias})`, `CHECKPOINT ${safeAlias}`);
711
+ return true;
712
+ };
713
+ #executeRawSqlCommand = async (name, rawSql) => {
714
+ const startTime = Date.now();
715
+ try {
716
+ const result = await this.#conn.runAndReadAll(rawSql);
717
+ const timeMs = Math.round(Date.now() - startTime);
718
+ const data = result.getRowObjectsJS();
719
+ this.#logger.info(`DuckDatabaseManager.${name} in ${timeMs}ms`, { timeMs });
720
+ return data;
721
+ } catch (e) {
722
+ const msg = `DuckDatabaseManager: failed to run "${name}" - ${e?.message ?? ""}`;
723
+ const timeMs = Math.round(Date.now() - startTime);
724
+ this.#logger.error(msg, {
725
+ name,
726
+ sql: rawSql,
727
+ timeMs
728
+ });
729
+ throw new Error(msg, { cause: e });
730
+ }
731
+ };
454
732
  };
455
733
  //#endregion
456
- export { DuckMemory, SqlDuck, Table, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
734
+ export { Database, DuckDatabaseManager, DuckMemory, SqlDuck, Table, duckDatabaseManagerDbParamsSchema, duckTableAliasSchema, duckTableNameSchema, duckdbReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@flowblade/sqlduck",
3
- "version": "0.10.0",
3
+ "version": "0.11.0",
4
4
  "type": "module",
5
5
  "sideEffects": false,
6
6
  "exports": {
@@ -57,9 +57,11 @@
57
57
  "@flowblade/core": "^0.2.26",
58
58
  "@flowblade/source-duckdb": "^0.20.1",
59
59
  "@flowblade/sql-tag": "^0.3.2",
60
- "@logtape/logtape": "2.0.4",
60
+ "@httpx/assert": "^0.16.8",
61
+ "@httpx/plain-object": "^2.1.8",
62
+ "@logtape/logtape": "^2.0.5",
61
63
  "@standard-schema/spec": "^1.1.0",
62
- "p-mutex": "^1.0.0",
64
+ "p-queue": "9.1.0",
63
65
  "valibot": "^1.3.1",
64
66
  "zod": "^4.3.6"
65
67
  },
@@ -82,7 +84,7 @@
82
84
  "@types/node": "25.5.0",
83
85
  "@typescript-eslint/eslint-plugin": "8.57.2",
84
86
  "@typescript-eslint/parser": "8.57.2",
85
- "@typescript/native-preview": "7.0.0-dev.20260316.1",
87
+ "@typescript/native-preview": "7.0.0-dev.20260324.1",
86
88
  "@vitest/coverage-v8": "4.1.1",
87
89
  "@vitest/ui": "4.1.1",
88
90
  "ansis": "4.2.0",
@@ -90,6 +92,7 @@
90
92
  "core-js": "3.49.0",
91
93
  "cross-env": "10.1.0",
92
94
  "es-check": "9.6.3",
95
+ "es-toolkit": "1.45.1",
93
96
  "esbuild": "0.27.4",
94
97
  "eslint": "8.57.1",
95
98
  "execa": "9.6.1",
@@ -106,11 +109,11 @@
106
109
  "tarn": "3.0.2",
107
110
  "tedious": "19.2.1",
108
111
  "testcontainers": "11.13.0",
109
- "tsdown": "0.21.4",
112
+ "tsdown": "0.21.5",
110
113
  "tsx": "4.21.0",
111
- "typedoc": "0.28.17",
114
+ "typedoc": "0.28.18",
112
115
  "typedoc-plugin-markdown": "4.11.0",
113
- "typescript": "5.9.3",
116
+ "typescript": "6.0.2",
114
117
  "vitest": "4.1.1"
115
118
  },
116
119
  "files": [