@flowblade/sqlduck 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -1
- package/dist/index.cjs +309 -25
- package/dist/index.d.cts +144 -10
- package/dist/index.d.mts +145 -11
- package/dist/index.mjs +304 -26
- package/package.json +10 -7
package/README.md
CHANGED
|
@@ -4,6 +4,74 @@
|
|
|
4
4
|
|
|
5
5
|
## Quick start
|
|
6
6
|
|
|
7
|
+
### Create a database connection
|
|
8
|
+
|
|
9
|
+
```typescript
|
|
10
|
+
import { DuckDBInstance } from '@duckdb/node-api';
|
|
11
|
+
DuckDBInstance.create(undefined, {
|
|
12
|
+
access_mode: 'READ_WRITE',
|
|
13
|
+
max_memory: '512M',
|
|
14
|
+
});
|
|
15
|
+
export const conn = await instance.connect();
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
### Append data to a database
|
|
19
|
+
|
|
20
|
+
```typescript
|
|
21
|
+
import { SqlDuck, DuckDatabaseManager } from "@flowblade/sqlduck";
|
|
22
|
+
import * as z from "zod";
|
|
23
|
+
import { conn } from "./db.config.ts";
|
|
24
|
+
|
|
25
|
+
const dbManager = new DuckDatabaseManager(conn);
|
|
26
|
+
const database = await dbManager.attach({
|
|
27
|
+
type: ':memory:', // can be 'duckdb', ...
|
|
28
|
+
alias: 'mydb',
|
|
29
|
+
options: { COMPRESS: 'false' },
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const sqlDuck = new SqlDuck({ conn });
|
|
33
|
+
|
|
34
|
+
// Define a zod schema, it will be used to create the table
|
|
35
|
+
const userSchema = z.object({
|
|
36
|
+
id: z.int32().min(1).meta({ primaryKey: true }),
|
|
37
|
+
name: z.string(),
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
// Example of a datasource (can be generator, async generator, async iterable)
|
|
41
|
+
async function* getUsers(): AsyncIterableIterator<
|
|
42
|
+
z.infer<typeof userSchema>
|
|
43
|
+
> {
|
|
44
|
+
// database or api call
|
|
45
|
+
yield { id: 1, name: 'John' };
|
|
46
|
+
yield { id: 2, name: 'Jane' };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Create a table from the schema and the datasource
|
|
50
|
+
const result = await sqlDuck.toTable({
|
|
51
|
+
table: new Table({ name: 'user', database: database.alias }),
|
|
52
|
+
schema: userSchema, // The schema to use to create the table
|
|
53
|
+
rowStream: getUsers(), // The async iterable that yields rows
|
|
54
|
+
// 👇Optional:
|
|
55
|
+
chunkSize: 2048, // Number of rows to append when using duckdb appender. Default is 2048
|
|
56
|
+
onDataAppended: ({ timeMs, totalRows, rowsPerSecond }) => {
|
|
57
|
+
console.log(
|
|
58
|
+
`Appended ${totalRows} in time ${timeMs}ms, est: ${rowsPerSecond} rows/s`
|
|
59
|
+
);
|
|
60
|
+
},
|
|
61
|
+
// Optional table creation options
|
|
62
|
+
createOptions: {
|
|
63
|
+
create: 'CREATE_OR_REPLACE',
|
|
64
|
+
},
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
|
|
68
|
+
console.log(`Table created with DDL: ${result.createTableDDL}`);
|
|
69
|
+
|
|
70
|
+
const reader = await conn.runAndReadAll('select * from mydb.user');
|
|
71
|
+
const rows = reader.getRowObjectsJS();
|
|
72
|
+
// [{id: 1, name: 'John'}, {id: 2, name: 'Jane'}]]
|
|
73
|
+
```
|
|
74
|
+
|
|
7
75
|
### Create a memory table
|
|
8
76
|
|
|
9
77
|
```typescript
|
|
@@ -35,7 +103,6 @@ const result = sqlDuck.toTable({
|
|
|
35
103
|
onDataAppended: ({ total }) => {
|
|
36
104
|
console.log(`Appended ${total} rows so far`);
|
|
37
105
|
},
|
|
38
|
-
onDataAppendedBatchSize: 4096, // Call onDataAppended every 4096 rows
|
|
39
106
|
// Optional table creation options
|
|
40
107
|
createOptions: {
|
|
41
108
|
create: "CREATE_OR_REPLACE",
|
package/dist/index.cjs
CHANGED
|
@@ -21,13 +21,12 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
21
21
|
enumerable: true
|
|
22
22
|
}) : target, mod));
|
|
23
23
|
//#endregion
|
|
24
|
-
let _logtape_logtape = require("@logtape/logtape");
|
|
25
24
|
let _duckdb_node_api = require("@duckdb/node-api");
|
|
25
|
+
let _logtape_logtape = require("@logtape/logtape");
|
|
26
26
|
let zod = require("zod");
|
|
27
27
|
zod = __toESM(zod);
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
//#endregion
|
|
28
|
+
let _httpx_assert = require("@httpx/assert");
|
|
29
|
+
let _httpx_plain_object = require("@httpx/plain-object");
|
|
31
30
|
//#region src/helpers/duck-exec.ts
|
|
32
31
|
var DuckExec = class {
|
|
33
32
|
#conn;
|
|
@@ -124,9 +123,6 @@ var DuckMemory = class {
|
|
|
124
123
|
};
|
|
125
124
|
};
|
|
126
125
|
//#endregion
|
|
127
|
-
//#region src/logger/sqlduck-default-logtape-logger.ts
|
|
128
|
-
const sqlduckDefaultLogtapeLogger = (0, _logtape_logtape.getLogger)(flowbladeLogtapeSqlduckConfig.categories);
|
|
129
|
-
//#endregion
|
|
130
126
|
//#region src/appender/data-appender-callback.ts
|
|
131
127
|
const isOnDataAppendedAsyncCb = (v) => {
|
|
132
128
|
return v.constructor.name === "AsyncFunction";
|
|
@@ -148,6 +144,12 @@ const createOnDataAppendedCollector = () => {
|
|
|
148
144
|
};
|
|
149
145
|
};
|
|
150
146
|
//#endregion
|
|
147
|
+
//#region src/config/flowblade-logtape-sqlduck.config.ts
|
|
148
|
+
const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
|
|
149
|
+
//#endregion
|
|
150
|
+
//#region src/logger/sqlduck-default-logtape-logger.ts
|
|
151
|
+
const sqlduckDefaultLogtapeLogger = (0, _logtape_logtape.getLogger)(flowbladeLogtapeSqlduckConfig.categories);
|
|
152
|
+
//#endregion
|
|
151
153
|
//#region src/table/get-duckdb-number-column-type.ts
|
|
152
154
|
const isFloatValue = (value) => {
|
|
153
155
|
if (!Number.isFinite(value)) return true;
|
|
@@ -181,7 +183,7 @@ const createOptions = {
|
|
|
181
183
|
CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
|
|
182
184
|
IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
|
|
183
185
|
};
|
|
184
|
-
const
|
|
186
|
+
const duckDbTypesMap = new Map([
|
|
185
187
|
["VARCHAR", _duckdb_node_api.VARCHAR],
|
|
186
188
|
["BIGINT", _duckdb_node_api.BIGINT],
|
|
187
189
|
["TIMESTAMP", _duckdb_node_api.TIMESTAMP],
|
|
@@ -190,8 +192,7 @@ const duckDbTypes = [
|
|
|
190
192
|
["INTEGER", _duckdb_node_api.INTEGER],
|
|
191
193
|
["DOUBLE", _duckdb_node_api.DOUBLE],
|
|
192
194
|
["FLOAT", _duckdb_node_api.FLOAT]
|
|
193
|
-
];
|
|
194
|
-
const duckDbTypesMap = new Map(duckDbTypes);
|
|
195
|
+
]);
|
|
195
196
|
const getTableCreateFromZod = (params) => {
|
|
196
197
|
const { table, schema, options } = params;
|
|
197
198
|
const { create = "CREATE" } = options ?? {};
|
|
@@ -327,10 +328,10 @@ async function* rowsToColumnsChunks(params) {
|
|
|
327
328
|
//#endregion
|
|
328
329
|
//#region src/sql-duck.ts
|
|
329
330
|
var SqlDuck = class {
|
|
330
|
-
#
|
|
331
|
+
#conn;
|
|
331
332
|
#logger;
|
|
332
333
|
constructor(params) {
|
|
333
|
-
this.#
|
|
334
|
+
this.#conn = params.conn;
|
|
334
335
|
this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
|
|
335
336
|
}
|
|
336
337
|
/**
|
|
@@ -375,12 +376,12 @@ var SqlDuck = class {
|
|
|
375
376
|
if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
|
|
376
377
|
const timeStart = Date.now();
|
|
377
378
|
const { columnTypes, ddl } = await createTableFromZod({
|
|
378
|
-
conn: this.#
|
|
379
|
+
conn: this.#conn,
|
|
379
380
|
schema,
|
|
380
381
|
table,
|
|
381
382
|
options: createOptions
|
|
382
383
|
});
|
|
383
|
-
const appender = await this.#
|
|
384
|
+
const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
|
|
384
385
|
const chunkTypes = Array.from(columnTypes.values());
|
|
385
386
|
let totalRows = 0;
|
|
386
387
|
const dataAppendedCollector = createOnDataAppendedCollector();
|
|
@@ -423,7 +424,39 @@ var SqlDuck = class {
|
|
|
423
424
|
};
|
|
424
425
|
};
|
|
425
426
|
//#endregion
|
|
426
|
-
//#region src/
|
|
427
|
+
//#region src/utils/zod-codecs.ts
|
|
428
|
+
const zodCodecs = {
|
|
429
|
+
dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
|
|
430
|
+
decode: (date) => date.toISOString(),
|
|
431
|
+
encode: (isoString) => new Date(isoString)
|
|
432
|
+
}),
|
|
433
|
+
bigintToString: zod.codec(zod.bigint(), zod.string().meta({ format: "int64" }), {
|
|
434
|
+
decode: (bigint) => bigint.toString(),
|
|
435
|
+
encode: BigInt
|
|
436
|
+
})
|
|
437
|
+
};
|
|
438
|
+
//#endregion
|
|
439
|
+
//#region src/objects/database.ts
|
|
440
|
+
var Database = class {
|
|
441
|
+
#params;
|
|
442
|
+
get alias() {
|
|
443
|
+
return this.#params.alias;
|
|
444
|
+
}
|
|
445
|
+
constructor(params) {
|
|
446
|
+
this.#params = params;
|
|
447
|
+
}
|
|
448
|
+
toJson() {
|
|
449
|
+
return {
|
|
450
|
+
type: "database",
|
|
451
|
+
params: { alias: this.#params.alias }
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
[Symbol.toStringTag]() {
|
|
455
|
+
return this.alias;
|
|
456
|
+
}
|
|
457
|
+
};
|
|
458
|
+
//#endregion
|
|
459
|
+
//#region src/objects/table.ts
|
|
427
460
|
var Table = class Table {
|
|
428
461
|
#fqTable;
|
|
429
462
|
get tableName() {
|
|
@@ -465,21 +498,272 @@ var Table = class Table {
|
|
|
465
498
|
};
|
|
466
499
|
};
|
|
467
500
|
//#endregion
|
|
468
|
-
//#region src/
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
501
|
+
//#region src/validation/core/duckdb-reserved-keywords.ts
|
|
502
|
+
/**
|
|
503
|
+
* DuckDB reserved keywords that cannot be used as unquoted identifiers.
|
|
504
|
+
* @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
|
|
505
|
+
*/
|
|
506
|
+
const duckdbReservedKeywords = [
|
|
507
|
+
"ALL",
|
|
508
|
+
"ANALYSE",
|
|
509
|
+
"ANALYZE",
|
|
510
|
+
"AND",
|
|
511
|
+
"ANY",
|
|
512
|
+
"ARRAY",
|
|
513
|
+
"AS",
|
|
514
|
+
"ASC",
|
|
515
|
+
"ASYMMETRIC",
|
|
516
|
+
"BOTH",
|
|
517
|
+
"CASE",
|
|
518
|
+
"CAST",
|
|
519
|
+
"CHECK",
|
|
520
|
+
"COLLATE",
|
|
521
|
+
"COLUMN",
|
|
522
|
+
"CONSTRAINT",
|
|
523
|
+
"CREATE",
|
|
524
|
+
"CROSS",
|
|
525
|
+
"CURRENT_CATALOG",
|
|
526
|
+
"CURRENT_DATE",
|
|
527
|
+
"CURRENT_ROLE",
|
|
528
|
+
"CURRENT_SCHEMA",
|
|
529
|
+
"CURRENT_TIME",
|
|
530
|
+
"CURRENT_TIMESTAMP",
|
|
531
|
+
"CURRENT_USER",
|
|
532
|
+
"DEFAULT",
|
|
533
|
+
"DEFERRABLE",
|
|
534
|
+
"DESC",
|
|
535
|
+
"DISTINCT",
|
|
536
|
+
"DO",
|
|
537
|
+
"ELSE",
|
|
538
|
+
"END",
|
|
539
|
+
"EXCEPT",
|
|
540
|
+
"EXISTS",
|
|
541
|
+
"EXTRACT",
|
|
542
|
+
"FALSE",
|
|
543
|
+
"FETCH",
|
|
544
|
+
"FOR",
|
|
545
|
+
"FOREIGN",
|
|
546
|
+
"FROM",
|
|
547
|
+
"GRANT",
|
|
548
|
+
"GROUP",
|
|
549
|
+
"HAVING",
|
|
550
|
+
"IF",
|
|
551
|
+
"ILIKE",
|
|
552
|
+
"IN",
|
|
553
|
+
"INITIALLY",
|
|
554
|
+
"INNER",
|
|
555
|
+
"INTERSECT",
|
|
556
|
+
"INTO",
|
|
557
|
+
"IS",
|
|
558
|
+
"ISNULL",
|
|
559
|
+
"JOIN",
|
|
560
|
+
"LATERAL",
|
|
561
|
+
"LEADING",
|
|
562
|
+
"LEFT",
|
|
563
|
+
"LIKE",
|
|
564
|
+
"LIMIT",
|
|
565
|
+
"LOCALTIME",
|
|
566
|
+
"LOCALTIMESTAMP",
|
|
567
|
+
"NATURAL",
|
|
568
|
+
"NOT",
|
|
569
|
+
"NOTNULL",
|
|
570
|
+
"NULL",
|
|
571
|
+
"OFFSET",
|
|
572
|
+
"ON",
|
|
573
|
+
"ONLY",
|
|
574
|
+
"OR",
|
|
575
|
+
"ORDER",
|
|
576
|
+
"OUTER",
|
|
577
|
+
"OVERLAPS",
|
|
578
|
+
"PLACING",
|
|
579
|
+
"PRIMARY",
|
|
580
|
+
"REFERENCES",
|
|
581
|
+
"RETURNING",
|
|
582
|
+
"RIGHT",
|
|
583
|
+
"ROW",
|
|
584
|
+
"SELECT",
|
|
585
|
+
"SESSION_USER",
|
|
586
|
+
"SIMILAR",
|
|
587
|
+
"SOME",
|
|
588
|
+
"SYMMETRIC",
|
|
589
|
+
"TABLE",
|
|
590
|
+
"THEN",
|
|
591
|
+
"TO",
|
|
592
|
+
"TRAILING",
|
|
593
|
+
"TRUE",
|
|
594
|
+
"UNION",
|
|
595
|
+
"UNIQUE",
|
|
596
|
+
"USING",
|
|
597
|
+
"VARIADIC",
|
|
598
|
+
"VERBOSE",
|
|
599
|
+
"WHEN",
|
|
600
|
+
"WHERE",
|
|
601
|
+
"WINDOW",
|
|
602
|
+
"WITH"
|
|
603
|
+
];
|
|
604
|
+
//#endregion
|
|
605
|
+
//#region src/validation/zod/duckdb-valid-names.schemas.ts
|
|
606
|
+
const duckdbMaximumObjectNameLength = 120;
|
|
607
|
+
const duckDbObjectNameRegex = /^[a-z_]\w*$/i;
|
|
608
|
+
const duckdbReservedKeywordsSet = new Set(duckdbReservedKeywords.map((k) => k.toUpperCase()));
|
|
609
|
+
const duckTableNameSchema = zod.string().min(1).max(duckdbMaximumObjectNameLength).regex(duckDbObjectNameRegex, "Table name must start with a letter or underscore, and contain only letters, numbers and underscores").refine((value) => !duckdbReservedKeywordsSet.has(value.toUpperCase()), { error: `Value is a DuckDB reserved keyword and cannot be used as a table name` });
|
|
610
|
+
const duckTableAliasSchema = duckTableNameSchema;
|
|
611
|
+
//#endregion
|
|
612
|
+
//#region src/manager/database/duck-database-manager.schemas.ts
|
|
613
|
+
const duckdbAttachOptionsSchema = zod.strictObject({
|
|
614
|
+
ACCESS_MODE: zod.optional(zod.enum([
|
|
615
|
+
"READ_ONLY",
|
|
616
|
+
"READ_WRITE",
|
|
617
|
+
"AUTOMATIC"
|
|
618
|
+
])),
|
|
619
|
+
COMPRESS: zod.optional(zod.enum(["true", "false"])),
|
|
620
|
+
TYPE: zod.optional(zod.enum(["DUCKDB", "SQLITE"])),
|
|
621
|
+
BLOCK_SIZE: zod.optional(zod.int32().min(16384).max(262144)),
|
|
622
|
+
ROW_GROUP_SIZE: zod.optional(zod.int32().positive()),
|
|
623
|
+
STORAGE_VERSION: zod.optional(zod.string().startsWith("v").regex(/^v?\d{1,4}\.\d{1,4}\.\d{1,4}$/)),
|
|
624
|
+
ENCRYPTION_KEY: zod.optional(zod.string().min(8)),
|
|
625
|
+
ENCRYPTION_CIPHER: zod.optional(zod.enum([
|
|
626
|
+
"CBC",
|
|
627
|
+
"CTR",
|
|
628
|
+
"GCM"
|
|
629
|
+
]))
|
|
630
|
+
});
|
|
631
|
+
const duckDatabaseManagerDbParamsSchema = zod.discriminatedUnion("type", [zod.strictObject({
|
|
632
|
+
type: zod.literal(":memory:"),
|
|
633
|
+
alias: duckTableAliasSchema,
|
|
634
|
+
options: zod.optional(duckdbAttachOptionsSchema)
|
|
635
|
+
}), zod.strictObject({
|
|
636
|
+
type: zod.literal("duckdb"),
|
|
637
|
+
path: zod.string().min(4).endsWith(".db"),
|
|
638
|
+
alias: duckTableAliasSchema,
|
|
639
|
+
options: zod.optional(duckdbAttachOptionsSchema)
|
|
640
|
+
})]);
|
|
641
|
+
//#endregion
|
|
642
|
+
//#region src/manager/database/commands/duck-database-attach-command.ts
|
|
643
|
+
var DuckDatabaseAttachCommand = class {
|
|
644
|
+
options;
|
|
645
|
+
dbParams;
|
|
646
|
+
constructor(dbParams, options) {
|
|
647
|
+
this.dbParams = dbParams;
|
|
648
|
+
this.options = options ?? {};
|
|
649
|
+
}
|
|
650
|
+
getRawSql = () => {
|
|
651
|
+
const dbParams = this.dbParams;
|
|
652
|
+
const parts = ["ATTACH", this.options.behaviour].filter(Boolean);
|
|
653
|
+
const { type, alias } = dbParams;
|
|
654
|
+
switch (type) {
|
|
655
|
+
case ":memory:":
|
|
656
|
+
parts.push("':memory:'");
|
|
657
|
+
break;
|
|
658
|
+
case "duckdb":
|
|
659
|
+
parts.push(`'${dbParams.path}'`);
|
|
660
|
+
break;
|
|
661
|
+
default: (0, _httpx_assert.assertNever)(type);
|
|
662
|
+
}
|
|
663
|
+
if (alias !== null) parts.push("AS", `${alias}`);
|
|
664
|
+
const options = (0, _httpx_plain_object.isPlainObject)(dbParams.options) ? Object.entries(dbParams.options).map(([key, value]) => {
|
|
665
|
+
return key === "ACCESS_MODE" ? value : `${key} '${value}'`;
|
|
666
|
+
}) : [];
|
|
667
|
+
if (options.length > 0) parts.push(`(${options.join(", ")})`);
|
|
668
|
+
return parts.filter(Boolean).join(" ");
|
|
669
|
+
};
|
|
670
|
+
};
|
|
671
|
+
//#endregion
|
|
672
|
+
//#region src/manager/database/duck-database-manager.ts
|
|
673
|
+
var DuckDatabaseManager = class {
|
|
674
|
+
#conn;
|
|
675
|
+
#logger;
|
|
676
|
+
constructor(conn, params) {
|
|
677
|
+
this.#conn = conn;
|
|
678
|
+
this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
|
|
679
|
+
}
|
|
680
|
+
/**
|
|
681
|
+
* Attach a database to the current connection
|
|
682
|
+
*
|
|
683
|
+
* @example
|
|
684
|
+
* ```typescript
|
|
685
|
+
* const dbManager = new DuckDatabaseManager(conn);
|
|
686
|
+
* const database = dbManager.attach({
|
|
687
|
+
* type: ':memory:', // can be 'duckdb', 's3'...
|
|
688
|
+
* alias: 'mydb',
|
|
689
|
+
* options: { COMPRESS: 'true' }
|
|
690
|
+
* });
|
|
691
|
+
*
|
|
692
|
+
* console.log(database.alias); // 'mydb'
|
|
693
|
+
* ```
|
|
694
|
+
*/
|
|
695
|
+
attach = async (dbParams, options) => {
|
|
696
|
+
const params = zod.parse(duckDatabaseManagerDbParamsSchema, dbParams);
|
|
697
|
+
const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
|
|
698
|
+
await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
|
|
699
|
+
return new Database({ alias: params.alias });
|
|
700
|
+
};
|
|
701
|
+
attachOrReplace = async (dbParams) => {
|
|
702
|
+
return this.attach(dbParams, { behaviour: "OR REPLACE" });
|
|
703
|
+
};
|
|
704
|
+
attachIfNotExists = async (dbParams) => {
|
|
705
|
+
return this.attach(dbParams, { behaviour: "IF NOT EXISTS" });
|
|
706
|
+
};
|
|
707
|
+
showDatabases = async () => {
|
|
708
|
+
return await this.#executeRawSqlCommand("showDatabases()", `SHOW DATABASES`);
|
|
709
|
+
};
|
|
710
|
+
detach = async (dbAlias) => {
|
|
711
|
+
const safeAlias = zod.parse(duckTableAliasSchema, dbAlias);
|
|
712
|
+
await this.#executeRawSqlCommand(`detach(${safeAlias})`, `DETACH ${safeAlias}`);
|
|
713
|
+
return true;
|
|
714
|
+
};
|
|
715
|
+
detachIfExists = async (dbAlias) => {
|
|
716
|
+
const safeAlias = zod.parse(duckTableAliasSchema, dbAlias);
|
|
717
|
+
await this.#executeRawSqlCommand(`detachIfExists(${safeAlias})`, `DETACH IF EXISTS ${safeAlias}`);
|
|
718
|
+
return true;
|
|
719
|
+
};
|
|
720
|
+
/**
|
|
721
|
+
* The statistics recomputed by the ANALYZE statement are only used for join order optimization.
|
|
722
|
+
*
|
|
723
|
+
* It is therefore recommended to recompute these statistics for improved join orders,
|
|
724
|
+
* especially after performing large updates (inserts and/or deletes).
|
|
725
|
+
*
|
|
726
|
+
* @link https://duckdb.org/docs/stable/sql/statements/analyze
|
|
727
|
+
*/
|
|
728
|
+
analyze = async () => {
|
|
729
|
+
await this.#executeRawSqlCommand("analyze()", "ANALYZE");
|
|
730
|
+
return true;
|
|
731
|
+
};
|
|
732
|
+
checkpoint = async (dbAlias) => {
|
|
733
|
+
const safeAlias = zod.parse(duckTableAliasSchema, dbAlias);
|
|
734
|
+
await this.#executeRawSqlCommand(`checkpoint(${safeAlias})`, `CHECKPOINT ${safeAlias}`);
|
|
735
|
+
return true;
|
|
736
|
+
};
|
|
737
|
+
#executeRawSqlCommand = async (name, rawSql) => {
|
|
738
|
+
const startTime = Date.now();
|
|
739
|
+
try {
|
|
740
|
+
const result = await this.#conn.runAndReadAll(rawSql);
|
|
741
|
+
const timeMs = Math.round(Date.now() - startTime);
|
|
742
|
+
const data = result.getRowObjectsJS();
|
|
743
|
+
this.#logger.info(`DuckDatabaseManager.${name} in ${timeMs}ms`, { timeMs });
|
|
744
|
+
return data;
|
|
745
|
+
} catch (e) {
|
|
746
|
+
const msg = `DuckDatabaseManager: failed to run "${name}" - ${e?.message ?? ""}`;
|
|
747
|
+
const timeMs = Math.round(Date.now() - startTime);
|
|
748
|
+
this.#logger.error(msg, {
|
|
749
|
+
name,
|
|
750
|
+
sql: rawSql,
|
|
751
|
+
timeMs
|
|
752
|
+
});
|
|
753
|
+
throw new Error(msg, { cause: e });
|
|
754
|
+
}
|
|
755
|
+
};
|
|
478
756
|
};
|
|
479
757
|
//#endregion
|
|
758
|
+
exports.Database = Database;
|
|
759
|
+
exports.DuckDatabaseManager = DuckDatabaseManager;
|
|
480
760
|
exports.DuckMemory = DuckMemory;
|
|
481
761
|
exports.SqlDuck = SqlDuck;
|
|
482
762
|
exports.Table = Table;
|
|
763
|
+
exports.duckDatabaseManagerDbParamsSchema = duckDatabaseManagerDbParamsSchema;
|
|
764
|
+
exports.duckTableAliasSchema = duckTableAliasSchema;
|
|
765
|
+
exports.duckTableNameSchema = duckTableNameSchema;
|
|
766
|
+
exports.duckdbReservedKeywords = duckdbReservedKeywords;
|
|
483
767
|
exports.flowbladeLogtapeSqlduckConfig = flowbladeLogtapeSqlduckConfig;
|
|
484
768
|
exports.getTableCreateFromZod = getTableCreateFromZod;
|
|
485
769
|
exports.sqlduckDefaultLogtapeLogger = sqlduckDefaultLogtapeLogger;
|
package/dist/index.d.cts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as _duckdb_node_api0 from "@duckdb/node-api";
|
|
1
2
|
import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
|
|
2
3
|
import * as _logtape_logtape0 from "@logtape/logtape";
|
|
3
4
|
import { Logger } from "@logtape/logtape";
|
|
@@ -23,11 +24,6 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
|
|
|
23
24
|
type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
|
|
24
25
|
type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
|
|
25
26
|
//#endregion
|
|
26
|
-
//#region src/config/flowblade-logtape-sqlduck.config.d.ts
|
|
27
|
-
declare const flowbladeLogtapeSqlduckConfig: {
|
|
28
|
-
categories: string[];
|
|
29
|
-
};
|
|
30
|
-
//#endregion
|
|
31
27
|
//#region src/helpers/duck-memory.d.ts
|
|
32
28
|
declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
|
|
33
29
|
type DuckMemoryTag = (typeof duckMemoryTags)[number];
|
|
@@ -56,10 +52,7 @@ declare class DuckMemory {
|
|
|
56
52
|
getSummary: () => Promise<DuckMemorySummary>;
|
|
57
53
|
}
|
|
58
54
|
//#endregion
|
|
59
|
-
//#region src/
|
|
60
|
-
declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
|
|
61
|
-
//#endregion
|
|
62
|
-
//#region src/table/table.d.ts
|
|
55
|
+
//#region src/objects/table.d.ts
|
|
63
56
|
/**
|
|
64
57
|
* Fully qualified table information
|
|
65
58
|
*/
|
|
@@ -208,4 +201,145 @@ declare const zodCodecs: {
|
|
|
208
201
|
readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
|
|
209
202
|
};
|
|
210
203
|
//#endregion
|
|
211
|
-
|
|
204
|
+
//#region src/objects/database.d.ts
|
|
205
|
+
type DatabaseProperties = {
|
|
206
|
+
alias: string;
|
|
207
|
+
};
|
|
208
|
+
declare class Database {
|
|
209
|
+
#private;
|
|
210
|
+
get alias(): string;
|
|
211
|
+
constructor(params: DatabaseProperties);
|
|
212
|
+
toJson(): {
|
|
213
|
+
type: string;
|
|
214
|
+
params: {
|
|
215
|
+
alias: string;
|
|
216
|
+
};
|
|
217
|
+
};
|
|
218
|
+
[Symbol.toStringTag](): string;
|
|
219
|
+
}
|
|
220
|
+
//#endregion
|
|
221
|
+
//#region src/manager/database/duck-database-manager.schemas.d.ts
|
|
222
|
+
declare const duckDatabaseManagerDbParamsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
223
|
+
type: z.ZodLiteral<":memory:">;
|
|
224
|
+
alias: z.ZodString;
|
|
225
|
+
options: z.ZodOptional<z.ZodObject<{
|
|
226
|
+
ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
|
|
227
|
+
READ_ONLY: "READ_ONLY";
|
|
228
|
+
READ_WRITE: "READ_WRITE";
|
|
229
|
+
AUTOMATIC: "AUTOMATIC";
|
|
230
|
+
}>>;
|
|
231
|
+
COMPRESS: z.ZodOptional<z.ZodEnum<{
|
|
232
|
+
true: "true";
|
|
233
|
+
false: "false";
|
|
234
|
+
}>>;
|
|
235
|
+
TYPE: z.ZodOptional<z.ZodEnum<{
|
|
236
|
+
DUCKDB: "DUCKDB";
|
|
237
|
+
SQLITE: "SQLITE";
|
|
238
|
+
}>>;
|
|
239
|
+
BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
|
|
240
|
+
ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
|
|
241
|
+
STORAGE_VERSION: z.ZodOptional<z.ZodString>;
|
|
242
|
+
ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
|
|
243
|
+
ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
|
|
244
|
+
CBC: "CBC";
|
|
245
|
+
CTR: "CTR";
|
|
246
|
+
GCM: "GCM";
|
|
247
|
+
}>>;
|
|
248
|
+
}, z.core.$strict>>;
|
|
249
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
250
|
+
type: z.ZodLiteral<"duckdb">;
|
|
251
|
+
path: z.ZodString;
|
|
252
|
+
alias: z.ZodString;
|
|
253
|
+
options: z.ZodOptional<z.ZodObject<{
|
|
254
|
+
ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
|
|
255
|
+
READ_ONLY: "READ_ONLY";
|
|
256
|
+
READ_WRITE: "READ_WRITE";
|
|
257
|
+
AUTOMATIC: "AUTOMATIC";
|
|
258
|
+
}>>;
|
|
259
|
+
COMPRESS: z.ZodOptional<z.ZodEnum<{
|
|
260
|
+
true: "true";
|
|
261
|
+
false: "false";
|
|
262
|
+
}>>;
|
|
263
|
+
TYPE: z.ZodOptional<z.ZodEnum<{
|
|
264
|
+
DUCKDB: "DUCKDB";
|
|
265
|
+
SQLITE: "SQLITE";
|
|
266
|
+
}>>;
|
|
267
|
+
BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
|
|
268
|
+
ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
|
|
269
|
+
STORAGE_VERSION: z.ZodOptional<z.ZodString>;
|
|
270
|
+
ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
|
|
271
|
+
ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
|
|
272
|
+
CBC: "CBC";
|
|
273
|
+
CTR: "CTR";
|
|
274
|
+
GCM: "GCM";
|
|
275
|
+
}>>;
|
|
276
|
+
}, z.core.$strict>>;
|
|
277
|
+
}, z.core.$strict>], "type">;
|
|
278
|
+
type DuckDatabaseManagerDbParams = z.infer<typeof duckDatabaseManagerDbParamsSchema>;
|
|
279
|
+
//#endregion
|
|
280
|
+
//#region src/manager/database/commands/duck-database-attach-command.d.ts
|
|
281
|
+
type Behaviour = 'OR REPLACE' | 'IF NOT EXISTS';
|
|
282
|
+
type DuckDatabaseAttachCommandOptions = {
|
|
283
|
+
behaviour?: Behaviour;
|
|
284
|
+
};
|
|
285
|
+
//#endregion
|
|
286
|
+
//#region src/manager/database/duck-database-manager.d.ts
|
|
287
|
+
declare class DuckDatabaseManager {
|
|
288
|
+
#private;
|
|
289
|
+
constructor(conn: DuckDBConnection, params?: {
|
|
290
|
+
logger?: Logger;
|
|
291
|
+
});
|
|
292
|
+
/**
|
|
293
|
+
* Attach a database to the current connection
|
|
294
|
+
*
|
|
295
|
+
* @example
|
|
296
|
+
* ```typescript
|
|
297
|
+
* const dbManager = new DuckDatabaseManager(conn);
|
|
298
|
+
* const database = dbManager.attach({
|
|
299
|
+
* type: ':memory:', // can be 'duckdb', 's3'...
|
|
300
|
+
* alias: 'mydb',
|
|
301
|
+
* options: { COMPRESS: 'true' }
|
|
302
|
+
* });
|
|
303
|
+
*
|
|
304
|
+
* console.log(database.alias); // 'mydb'
|
|
305
|
+
* ```
|
|
306
|
+
*/
|
|
307
|
+
attach: (dbParams: DuckDatabaseManagerDbParams, options?: DuckDatabaseAttachCommandOptions) => Promise<Database>;
|
|
308
|
+
attachOrReplace: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
|
|
309
|
+
attachIfNotExists: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
|
|
310
|
+
showDatabases: () => Promise<Record<string, _duckdb_node_api0.JS>[]>;
|
|
311
|
+
detach: (dbAlias: string) => Promise<boolean>;
|
|
312
|
+
detachIfExists: (dbAlias: string) => Promise<boolean>;
|
|
313
|
+
/**
|
|
314
|
+
* The statistics recomputed by the ANALYZE statement are only used for join order optimization.
|
|
315
|
+
*
|
|
316
|
+
* It is therefore recommended to recompute these statistics for improved join orders,
|
|
317
|
+
* especially after performing large updates (inserts and/or deletes).
|
|
318
|
+
*
|
|
319
|
+
* @link https://duckdb.org/docs/stable/sql/statements/analyze
|
|
320
|
+
*/
|
|
321
|
+
analyze: () => Promise<boolean>;
|
|
322
|
+
checkpoint: (dbAlias: string) => Promise<boolean>;
|
|
323
|
+
}
|
|
324
|
+
//#endregion
|
|
325
|
+
//#region src/validation/core/duckdb-reserved-keywords.d.ts
|
|
326
|
+
/**
|
|
327
|
+
* DuckDB reserved keywords that cannot be used as unquoted identifiers.
|
|
328
|
+
* @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
|
|
329
|
+
*/
|
|
330
|
+
declare const duckdbReservedKeywords: readonly ["ALL", "ANALYSE", "ANALYZE", "AND", "ANY", "ARRAY", "AS", "ASC", "ASYMMETRIC", "BOTH", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DEFAULT", "DEFERRABLE", "DESC", "DISTINCT", "DO", "ELSE", "END", "EXCEPT", "EXISTS", "EXTRACT", "FALSE", "FETCH", "FOR", "FOREIGN", "FROM", "GRANT", "GROUP", "HAVING", "IF", "ILIKE", "IN", "INITIALLY", "INNER", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "LATERAL", "LEADING", "LEFT", "LIKE", "LIMIT", "LOCALTIME", "LOCALTIMESTAMP", "NATURAL", "NOT", "NOTNULL", "NULL", "OFFSET", "ON", "ONLY", "OR", "ORDER", "OUTER", "OVERLAPS", "PLACING", "PRIMARY", "REFERENCES", "RETURNING", "RIGHT", "ROW", "SELECT", "SESSION_USER", "SIMILAR", "SOME", "SYMMETRIC", "TABLE", "THEN", "TO", "TRAILING", "TRUE", "UNION", "UNIQUE", "USING", "VARIADIC", "VERBOSE", "WHEN", "WHERE", "WINDOW", "WITH"];
|
|
331
|
+
type DuckdbReservedKeywords = (typeof duckdbReservedKeywords)[number];
|
|
332
|
+
//#endregion
|
|
333
|
+
//#region src/validation/zod/duckdb-valid-names.schemas.d.ts
|
|
334
|
+
declare const duckTableNameSchema: z.ZodString;
|
|
335
|
+
declare const duckTableAliasSchema: z.ZodString;
|
|
336
|
+
//#endregion
|
|
337
|
+
//#region src/config/flowblade-logtape-sqlduck.config.d.ts
|
|
338
|
+
declare const flowbladeLogtapeSqlduckConfig: {
|
|
339
|
+
categories: string[];
|
|
340
|
+
};
|
|
341
|
+
//#endregion
|
|
342
|
+
//#region src/logger/sqlduck-default-logtape-logger.d.ts
|
|
343
|
+
declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
|
|
344
|
+
//#endregion
|
|
345
|
+
export { Database, DuckDatabaseManager, type DuckDatabaseManagerDbParams, DuckMemory, DuckMemoryTag, type DuckdbReservedKeywords, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckDatabaseManagerDbParamsSchema, duckTableAliasSchema, duckTableNameSchema, duckdbReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
|
package/dist/index.d.mts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import * as _duckdb_node_api0 from "@duckdb/node-api";
|
|
2
|
+
import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
|
|
1
3
|
import * as _logtape_logtape0 from "@logtape/logtape";
|
|
2
4
|
import { Logger } from "@logtape/logtape";
|
|
3
|
-
import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
|
|
4
5
|
import * as z from "zod";
|
|
5
6
|
import { ZodObject } from "zod";
|
|
6
7
|
|
|
@@ -23,11 +24,6 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
|
|
|
23
24
|
type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
|
|
24
25
|
type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
|
|
25
26
|
//#endregion
|
|
26
|
-
//#region src/config/flowblade-logtape-sqlduck.config.d.ts
|
|
27
|
-
declare const flowbladeLogtapeSqlduckConfig: {
|
|
28
|
-
categories: string[];
|
|
29
|
-
};
|
|
30
|
-
//#endregion
|
|
31
27
|
//#region src/helpers/duck-memory.d.ts
|
|
32
28
|
declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
|
|
33
29
|
type DuckMemoryTag = (typeof duckMemoryTags)[number];
|
|
@@ -56,10 +52,7 @@ declare class DuckMemory {
|
|
|
56
52
|
getSummary: () => Promise<DuckMemorySummary>;
|
|
57
53
|
}
|
|
58
54
|
//#endregion
|
|
59
|
-
//#region src/
|
|
60
|
-
declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
|
|
61
|
-
//#endregion
|
|
62
|
-
//#region src/table/table.d.ts
|
|
55
|
+
//#region src/objects/table.d.ts
|
|
63
56
|
/**
|
|
64
57
|
* Fully qualified table information
|
|
65
58
|
*/
|
|
@@ -208,4 +201,145 @@ declare const zodCodecs: {
|
|
|
208
201
|
readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
|
|
209
202
|
};
|
|
210
203
|
//#endregion
|
|
211
|
-
|
|
204
|
+
//#region src/objects/database.d.ts
|
|
205
|
+
type DatabaseProperties = {
|
|
206
|
+
alias: string;
|
|
207
|
+
};
|
|
208
|
+
declare class Database {
|
|
209
|
+
#private;
|
|
210
|
+
get alias(): string;
|
|
211
|
+
constructor(params: DatabaseProperties);
|
|
212
|
+
toJson(): {
|
|
213
|
+
type: string;
|
|
214
|
+
params: {
|
|
215
|
+
alias: string;
|
|
216
|
+
};
|
|
217
|
+
};
|
|
218
|
+
[Symbol.toStringTag](): string;
|
|
219
|
+
}
|
|
220
|
+
//#endregion
|
|
221
|
+
//#region src/manager/database/duck-database-manager.schemas.d.ts
|
|
222
|
+
declare const duckDatabaseManagerDbParamsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
223
|
+
type: z.ZodLiteral<":memory:">;
|
|
224
|
+
alias: z.ZodString;
|
|
225
|
+
options: z.ZodOptional<z.ZodObject<{
|
|
226
|
+
ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
|
|
227
|
+
READ_ONLY: "READ_ONLY";
|
|
228
|
+
READ_WRITE: "READ_WRITE";
|
|
229
|
+
AUTOMATIC: "AUTOMATIC";
|
|
230
|
+
}>>;
|
|
231
|
+
COMPRESS: z.ZodOptional<z.ZodEnum<{
|
|
232
|
+
true: "true";
|
|
233
|
+
false: "false";
|
|
234
|
+
}>>;
|
|
235
|
+
TYPE: z.ZodOptional<z.ZodEnum<{
|
|
236
|
+
DUCKDB: "DUCKDB";
|
|
237
|
+
SQLITE: "SQLITE";
|
|
238
|
+
}>>;
|
|
239
|
+
BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
|
|
240
|
+
ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
|
|
241
|
+
STORAGE_VERSION: z.ZodOptional<z.ZodString>;
|
|
242
|
+
ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
|
|
243
|
+
ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
|
|
244
|
+
CBC: "CBC";
|
|
245
|
+
CTR: "CTR";
|
|
246
|
+
GCM: "GCM";
|
|
247
|
+
}>>;
|
|
248
|
+
}, z.core.$strict>>;
|
|
249
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
250
|
+
type: z.ZodLiteral<"duckdb">;
|
|
251
|
+
path: z.ZodString;
|
|
252
|
+
alias: z.ZodString;
|
|
253
|
+
options: z.ZodOptional<z.ZodObject<{
|
|
254
|
+
ACCESS_MODE: z.ZodOptional<z.ZodEnum<{
|
|
255
|
+
READ_ONLY: "READ_ONLY";
|
|
256
|
+
READ_WRITE: "READ_WRITE";
|
|
257
|
+
AUTOMATIC: "AUTOMATIC";
|
|
258
|
+
}>>;
|
|
259
|
+
COMPRESS: z.ZodOptional<z.ZodEnum<{
|
|
260
|
+
true: "true";
|
|
261
|
+
false: "false";
|
|
262
|
+
}>>;
|
|
263
|
+
TYPE: z.ZodOptional<z.ZodEnum<{
|
|
264
|
+
DUCKDB: "DUCKDB";
|
|
265
|
+
SQLITE: "SQLITE";
|
|
266
|
+
}>>;
|
|
267
|
+
BLOCK_SIZE: z.ZodOptional<z.ZodInt32>;
|
|
268
|
+
ROW_GROUP_SIZE: z.ZodOptional<z.ZodInt32>;
|
|
269
|
+
STORAGE_VERSION: z.ZodOptional<z.ZodString>;
|
|
270
|
+
ENCRYPTION_KEY: z.ZodOptional<z.ZodString>;
|
|
271
|
+
ENCRYPTION_CIPHER: z.ZodOptional<z.ZodEnum<{
|
|
272
|
+
CBC: "CBC";
|
|
273
|
+
CTR: "CTR";
|
|
274
|
+
GCM: "GCM";
|
|
275
|
+
}>>;
|
|
276
|
+
}, z.core.$strict>>;
|
|
277
|
+
}, z.core.$strict>], "type">;
|
|
278
|
+
type DuckDatabaseManagerDbParams = z.infer<typeof duckDatabaseManagerDbParamsSchema>;
|
|
279
|
+
//#endregion
|
|
280
|
+
//#region src/manager/database/commands/duck-database-attach-command.d.ts
|
|
281
|
+
type Behaviour = 'OR REPLACE' | 'IF NOT EXISTS';
|
|
282
|
+
type DuckDatabaseAttachCommandOptions = {
|
|
283
|
+
behaviour?: Behaviour;
|
|
284
|
+
};
|
|
285
|
+
//#endregion
|
|
286
|
+
//#region src/manager/database/duck-database-manager.d.ts
|
|
287
|
+
declare class DuckDatabaseManager {
|
|
288
|
+
#private;
|
|
289
|
+
constructor(conn: DuckDBConnection, params?: {
|
|
290
|
+
logger?: Logger;
|
|
291
|
+
});
|
|
292
|
+
/**
|
|
293
|
+
* Attach a database to the current connection
|
|
294
|
+
*
|
|
295
|
+
* @example
|
|
296
|
+
* ```typescript
|
|
297
|
+
* const dbManager = new DuckDatabaseManager(conn);
|
|
298
|
+
* const database = dbManager.attach({
|
|
299
|
+
* type: ':memory:', // can be 'duckdb', 's3'...
|
|
300
|
+
* alias: 'mydb',
|
|
301
|
+
* options: { COMPRESS: 'true' }
|
|
302
|
+
* });
|
|
303
|
+
*
|
|
304
|
+
* console.log(database.alias); // 'mydb'
|
|
305
|
+
* ```
|
|
306
|
+
*/
|
|
307
|
+
attach: (dbParams: DuckDatabaseManagerDbParams, options?: DuckDatabaseAttachCommandOptions) => Promise<Database>;
|
|
308
|
+
attachOrReplace: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
|
|
309
|
+
attachIfNotExists: (dbParams: DuckDatabaseManagerDbParams) => Promise<Database>;
|
|
310
|
+
showDatabases: () => Promise<Record<string, _duckdb_node_api0.JS>[]>;
|
|
311
|
+
detach: (dbAlias: string) => Promise<boolean>;
|
|
312
|
+
detachIfExists: (dbAlias: string) => Promise<boolean>;
|
|
313
|
+
/**
|
|
314
|
+
* The statistics recomputed by the ANALYZE statement are only used for join order optimization.
|
|
315
|
+
*
|
|
316
|
+
* It is therefore recommended to recompute these statistics for improved join orders,
|
|
317
|
+
* especially after performing large updates (inserts and/or deletes).
|
|
318
|
+
*
|
|
319
|
+
* @link https://duckdb.org/docs/stable/sql/statements/analyze
|
|
320
|
+
*/
|
|
321
|
+
analyze: () => Promise<boolean>;
|
|
322
|
+
checkpoint: (dbAlias: string) => Promise<boolean>;
|
|
323
|
+
}
|
|
324
|
+
//#endregion
|
|
325
|
+
//#region src/validation/core/duckdb-reserved-keywords.d.ts
|
|
326
|
+
/**
|
|
327
|
+
* DuckDB reserved keywords that cannot be used as unquoted identifiers.
|
|
328
|
+
* @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
|
|
329
|
+
*/
|
|
330
|
+
declare const duckdbReservedKeywords: readonly ["ALL", "ANALYSE", "ANALYZE", "AND", "ANY", "ARRAY", "AS", "ASC", "ASYMMETRIC", "BOTH", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DEFAULT", "DEFERRABLE", "DESC", "DISTINCT", "DO", "ELSE", "END", "EXCEPT", "EXISTS", "EXTRACT", "FALSE", "FETCH", "FOR", "FOREIGN", "FROM", "GRANT", "GROUP", "HAVING", "IF", "ILIKE", "IN", "INITIALLY", "INNER", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "LATERAL", "LEADING", "LEFT", "LIKE", "LIMIT", "LOCALTIME", "LOCALTIMESTAMP", "NATURAL", "NOT", "NOTNULL", "NULL", "OFFSET", "ON", "ONLY", "OR", "ORDER", "OUTER", "OVERLAPS", "PLACING", "PRIMARY", "REFERENCES", "RETURNING", "RIGHT", "ROW", "SELECT", "SESSION_USER", "SIMILAR", "SOME", "SYMMETRIC", "TABLE", "THEN", "TO", "TRAILING", "TRUE", "UNION", "UNIQUE", "USING", "VARIADIC", "VERBOSE", "WHEN", "WHERE", "WINDOW", "WITH"];
|
|
331
|
+
type DuckdbReservedKeywords = (typeof duckdbReservedKeywords)[number];
|
|
332
|
+
//#endregion
|
|
333
|
+
//#region src/validation/zod/duckdb-valid-names.schemas.d.ts
|
|
334
|
+
declare const duckTableNameSchema: z.ZodString;
|
|
335
|
+
declare const duckTableAliasSchema: z.ZodString;
|
|
336
|
+
//#endregion
|
|
337
|
+
//#region src/config/flowblade-logtape-sqlduck.config.d.ts
|
|
338
|
+
declare const flowbladeLogtapeSqlduckConfig: {
|
|
339
|
+
categories: string[];
|
|
340
|
+
};
|
|
341
|
+
//#endregion
|
|
342
|
+
//#region src/logger/sqlduck-default-logtape-logger.d.ts
|
|
343
|
+
declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
|
|
344
|
+
//#endregion
|
|
345
|
+
export { Database, DuckDatabaseManager, type DuckDatabaseManagerDbParams, DuckMemory, type DuckMemoryTag, type DuckdbReservedKeywords, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, duckDatabaseManagerDbParamsSchema, duckTableAliasSchema, duckTableNameSchema, duckdbReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
|
package/dist/index.mjs
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import { getLogger } from "@logtape/logtape";
|
|
2
1
|
import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
|
|
2
|
+
import { getLogger } from "@logtape/logtape";
|
|
3
3
|
import * as z from "zod";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
//#endregion
|
|
4
|
+
import { assertNever } from "@httpx/assert";
|
|
5
|
+
import { isPlainObject } from "@httpx/plain-object";
|
|
7
6
|
//#region src/helpers/duck-exec.ts
|
|
8
7
|
var DuckExec = class {
|
|
9
8
|
#conn;
|
|
@@ -100,9 +99,6 @@ var DuckMemory = class {
|
|
|
100
99
|
};
|
|
101
100
|
};
|
|
102
101
|
//#endregion
|
|
103
|
-
//#region src/logger/sqlduck-default-logtape-logger.ts
|
|
104
|
-
const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
|
|
105
|
-
//#endregion
|
|
106
102
|
//#region src/appender/data-appender-callback.ts
|
|
107
103
|
const isOnDataAppendedAsyncCb = (v) => {
|
|
108
104
|
return v.constructor.name === "AsyncFunction";
|
|
@@ -124,6 +120,12 @@ const createOnDataAppendedCollector = () => {
|
|
|
124
120
|
};
|
|
125
121
|
};
|
|
126
122
|
//#endregion
|
|
123
|
+
//#region src/config/flowblade-logtape-sqlduck.config.ts
|
|
124
|
+
const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
|
|
125
|
+
//#endregion
|
|
126
|
+
//#region src/logger/sqlduck-default-logtape-logger.ts
|
|
127
|
+
const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
|
|
128
|
+
//#endregion
|
|
127
129
|
//#region src/table/get-duckdb-number-column-type.ts
|
|
128
130
|
const isFloatValue = (value) => {
|
|
129
131
|
if (!Number.isFinite(value)) return true;
|
|
@@ -157,7 +159,7 @@ const createOptions = {
|
|
|
157
159
|
CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
|
|
158
160
|
IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
|
|
159
161
|
};
|
|
160
|
-
const
|
|
162
|
+
const duckDbTypesMap = new Map([
|
|
161
163
|
["VARCHAR", VARCHAR],
|
|
162
164
|
["BIGINT", BIGINT],
|
|
163
165
|
["TIMESTAMP", TIMESTAMP],
|
|
@@ -166,8 +168,7 @@ const duckDbTypes = [
|
|
|
166
168
|
["INTEGER", INTEGER],
|
|
167
169
|
["DOUBLE", DOUBLE],
|
|
168
170
|
["FLOAT", FLOAT]
|
|
169
|
-
];
|
|
170
|
-
const duckDbTypesMap = new Map(duckDbTypes);
|
|
171
|
+
]);
|
|
171
172
|
const getTableCreateFromZod = (params) => {
|
|
172
173
|
const { table, schema, options } = params;
|
|
173
174
|
const { create = "CREATE" } = options ?? {};
|
|
@@ -303,10 +304,10 @@ async function* rowsToColumnsChunks(params) {
|
|
|
303
304
|
//#endregion
|
|
304
305
|
//#region src/sql-duck.ts
|
|
305
306
|
var SqlDuck = class {
|
|
306
|
-
#
|
|
307
|
+
#conn;
|
|
307
308
|
#logger;
|
|
308
309
|
constructor(params) {
|
|
309
|
-
this.#
|
|
310
|
+
this.#conn = params.conn;
|
|
310
311
|
this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
|
|
311
312
|
}
|
|
312
313
|
/**
|
|
@@ -351,12 +352,12 @@ var SqlDuck = class {
|
|
|
351
352
|
if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
|
|
352
353
|
const timeStart = Date.now();
|
|
353
354
|
const { columnTypes, ddl } = await createTableFromZod({
|
|
354
|
-
conn: this.#
|
|
355
|
+
conn: this.#conn,
|
|
355
356
|
schema,
|
|
356
357
|
table,
|
|
357
358
|
options: createOptions
|
|
358
359
|
});
|
|
359
|
-
const appender = await this.#
|
|
360
|
+
const appender = await this.#conn.createAppender(table.tableName, table.schemaName, table.databaseName);
|
|
360
361
|
const chunkTypes = Array.from(columnTypes.values());
|
|
361
362
|
let totalRows = 0;
|
|
362
363
|
const dataAppendedCollector = createOnDataAppendedCollector();
|
|
@@ -399,7 +400,39 @@ var SqlDuck = class {
|
|
|
399
400
|
};
|
|
400
401
|
};
|
|
401
402
|
//#endregion
|
|
402
|
-
//#region src/
|
|
403
|
+
//#region src/utils/zod-codecs.ts
|
|
404
|
+
const zodCodecs = {
|
|
405
|
+
dateToString: z.codec(z.date(), z.iso.datetime(), {
|
|
406
|
+
decode: (date) => date.toISOString(),
|
|
407
|
+
encode: (isoString) => new Date(isoString)
|
|
408
|
+
}),
|
|
409
|
+
bigintToString: z.codec(z.bigint(), z.string().meta({ format: "int64" }), {
|
|
410
|
+
decode: (bigint) => bigint.toString(),
|
|
411
|
+
encode: BigInt
|
|
412
|
+
})
|
|
413
|
+
};
|
|
414
|
+
//#endregion
|
|
415
|
+
//#region src/objects/database.ts
|
|
416
|
+
var Database = class {
|
|
417
|
+
#params;
|
|
418
|
+
get alias() {
|
|
419
|
+
return this.#params.alias;
|
|
420
|
+
}
|
|
421
|
+
constructor(params) {
|
|
422
|
+
this.#params = params;
|
|
423
|
+
}
|
|
424
|
+
toJson() {
|
|
425
|
+
return {
|
|
426
|
+
type: "database",
|
|
427
|
+
params: { alias: this.#params.alias }
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
[Symbol.toStringTag]() {
|
|
431
|
+
return this.alias;
|
|
432
|
+
}
|
|
433
|
+
};
|
|
434
|
+
//#endregion
|
|
435
|
+
//#region src/objects/table.ts
|
|
403
436
|
var Table = class Table {
|
|
404
437
|
#fqTable;
|
|
405
438
|
get tableName() {
|
|
@@ -441,16 +474,261 @@ var Table = class Table {
|
|
|
441
474
|
};
|
|
442
475
|
};
|
|
443
476
|
//#endregion
|
|
444
|
-
//#region src/
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
477
|
+
//#region src/validation/core/duckdb-reserved-keywords.ts
|
|
478
|
+
/**
|
|
479
|
+
* DuckDB reserved keywords that cannot be used as unquoted identifiers.
|
|
480
|
+
* @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
|
|
481
|
+
*/
|
|
482
|
+
const duckdbReservedKeywords = [
|
|
483
|
+
"ALL",
|
|
484
|
+
"ANALYSE",
|
|
485
|
+
"ANALYZE",
|
|
486
|
+
"AND",
|
|
487
|
+
"ANY",
|
|
488
|
+
"ARRAY",
|
|
489
|
+
"AS",
|
|
490
|
+
"ASC",
|
|
491
|
+
"ASYMMETRIC",
|
|
492
|
+
"BOTH",
|
|
493
|
+
"CASE",
|
|
494
|
+
"CAST",
|
|
495
|
+
"CHECK",
|
|
496
|
+
"COLLATE",
|
|
497
|
+
"COLUMN",
|
|
498
|
+
"CONSTRAINT",
|
|
499
|
+
"CREATE",
|
|
500
|
+
"CROSS",
|
|
501
|
+
"CURRENT_CATALOG",
|
|
502
|
+
"CURRENT_DATE",
|
|
503
|
+
"CURRENT_ROLE",
|
|
504
|
+
"CURRENT_SCHEMA",
|
|
505
|
+
"CURRENT_TIME",
|
|
506
|
+
"CURRENT_TIMESTAMP",
|
|
507
|
+
"CURRENT_USER",
|
|
508
|
+
"DEFAULT",
|
|
509
|
+
"DEFERRABLE",
|
|
510
|
+
"DESC",
|
|
511
|
+
"DISTINCT",
|
|
512
|
+
"DO",
|
|
513
|
+
"ELSE",
|
|
514
|
+
"END",
|
|
515
|
+
"EXCEPT",
|
|
516
|
+
"EXISTS",
|
|
517
|
+
"EXTRACT",
|
|
518
|
+
"FALSE",
|
|
519
|
+
"FETCH",
|
|
520
|
+
"FOR",
|
|
521
|
+
"FOREIGN",
|
|
522
|
+
"FROM",
|
|
523
|
+
"GRANT",
|
|
524
|
+
"GROUP",
|
|
525
|
+
"HAVING",
|
|
526
|
+
"IF",
|
|
527
|
+
"ILIKE",
|
|
528
|
+
"IN",
|
|
529
|
+
"INITIALLY",
|
|
530
|
+
"INNER",
|
|
531
|
+
"INTERSECT",
|
|
532
|
+
"INTO",
|
|
533
|
+
"IS",
|
|
534
|
+
"ISNULL",
|
|
535
|
+
"JOIN",
|
|
536
|
+
"LATERAL",
|
|
537
|
+
"LEADING",
|
|
538
|
+
"LEFT",
|
|
539
|
+
"LIKE",
|
|
540
|
+
"LIMIT",
|
|
541
|
+
"LOCALTIME",
|
|
542
|
+
"LOCALTIMESTAMP",
|
|
543
|
+
"NATURAL",
|
|
544
|
+
"NOT",
|
|
545
|
+
"NOTNULL",
|
|
546
|
+
"NULL",
|
|
547
|
+
"OFFSET",
|
|
548
|
+
"ON",
|
|
549
|
+
"ONLY",
|
|
550
|
+
"OR",
|
|
551
|
+
"ORDER",
|
|
552
|
+
"OUTER",
|
|
553
|
+
"OVERLAPS",
|
|
554
|
+
"PLACING",
|
|
555
|
+
"PRIMARY",
|
|
556
|
+
"REFERENCES",
|
|
557
|
+
"RETURNING",
|
|
558
|
+
"RIGHT",
|
|
559
|
+
"ROW",
|
|
560
|
+
"SELECT",
|
|
561
|
+
"SESSION_USER",
|
|
562
|
+
"SIMILAR",
|
|
563
|
+
"SOME",
|
|
564
|
+
"SYMMETRIC",
|
|
565
|
+
"TABLE",
|
|
566
|
+
"THEN",
|
|
567
|
+
"TO",
|
|
568
|
+
"TRAILING",
|
|
569
|
+
"TRUE",
|
|
570
|
+
"UNION",
|
|
571
|
+
"UNIQUE",
|
|
572
|
+
"USING",
|
|
573
|
+
"VARIADIC",
|
|
574
|
+
"VERBOSE",
|
|
575
|
+
"WHEN",
|
|
576
|
+
"WHERE",
|
|
577
|
+
"WINDOW",
|
|
578
|
+
"WITH"
|
|
579
|
+
];
|
|
580
|
+
//#endregion
|
|
581
|
+
//#region src/validation/zod/duckdb-valid-names.schemas.ts
|
|
582
|
+
const duckdbMaximumObjectNameLength = 120;
|
|
583
|
+
const duckDbObjectNameRegex = /^[a-z_]\w*$/i;
|
|
584
|
+
const duckdbReservedKeywordsSet = new Set(duckdbReservedKeywords.map((k) => k.toUpperCase()));
|
|
585
|
+
const duckTableNameSchema = z.string().min(1).max(duckdbMaximumObjectNameLength).regex(duckDbObjectNameRegex, "Table name must start with a letter or underscore, and contain only letters, numbers and underscores").refine((value) => !duckdbReservedKeywordsSet.has(value.toUpperCase()), { error: `Value is a DuckDB reserved keyword and cannot be used as a table name` });
|
|
586
|
+
const duckTableAliasSchema = duckTableNameSchema;
|
|
587
|
+
//#endregion
|
|
588
|
+
//#region src/manager/database/duck-database-manager.schemas.ts
|
|
589
|
+
const duckdbAttachOptionsSchema = z.strictObject({
|
|
590
|
+
ACCESS_MODE: z.optional(z.enum([
|
|
591
|
+
"READ_ONLY",
|
|
592
|
+
"READ_WRITE",
|
|
593
|
+
"AUTOMATIC"
|
|
594
|
+
])),
|
|
595
|
+
COMPRESS: z.optional(z.enum(["true", "false"])),
|
|
596
|
+
TYPE: z.optional(z.enum(["DUCKDB", "SQLITE"])),
|
|
597
|
+
BLOCK_SIZE: z.optional(z.int32().min(16384).max(262144)),
|
|
598
|
+
ROW_GROUP_SIZE: z.optional(z.int32().positive()),
|
|
599
|
+
STORAGE_VERSION: z.optional(z.string().startsWith("v").regex(/^v?\d{1,4}\.\d{1,4}\.\d{1,4}$/)),
|
|
600
|
+
ENCRYPTION_KEY: z.optional(z.string().min(8)),
|
|
601
|
+
ENCRYPTION_CIPHER: z.optional(z.enum([
|
|
602
|
+
"CBC",
|
|
603
|
+
"CTR",
|
|
604
|
+
"GCM"
|
|
605
|
+
]))
|
|
606
|
+
});
|
|
607
|
+
const duckDatabaseManagerDbParamsSchema = z.discriminatedUnion("type", [z.strictObject({
|
|
608
|
+
type: z.literal(":memory:"),
|
|
609
|
+
alias: duckTableAliasSchema,
|
|
610
|
+
options: z.optional(duckdbAttachOptionsSchema)
|
|
611
|
+
}), z.strictObject({
|
|
612
|
+
type: z.literal("duckdb"),
|
|
613
|
+
path: z.string().min(4).endsWith(".db"),
|
|
614
|
+
alias: duckTableAliasSchema,
|
|
615
|
+
options: z.optional(duckdbAttachOptionsSchema)
|
|
616
|
+
})]);
|
|
617
|
+
//#endregion
|
|
618
|
+
//#region src/manager/database/commands/duck-database-attach-command.ts
|
|
619
|
+
var DuckDatabaseAttachCommand = class {
|
|
620
|
+
options;
|
|
621
|
+
dbParams;
|
|
622
|
+
constructor(dbParams, options) {
|
|
623
|
+
this.dbParams = dbParams;
|
|
624
|
+
this.options = options ?? {};
|
|
625
|
+
}
|
|
626
|
+
getRawSql = () => {
|
|
627
|
+
const dbParams = this.dbParams;
|
|
628
|
+
const parts = ["ATTACH", this.options.behaviour].filter(Boolean);
|
|
629
|
+
const { type, alias } = dbParams;
|
|
630
|
+
switch (type) {
|
|
631
|
+
case ":memory:":
|
|
632
|
+
parts.push("':memory:'");
|
|
633
|
+
break;
|
|
634
|
+
case "duckdb":
|
|
635
|
+
parts.push(`'${dbParams.path}'`);
|
|
636
|
+
break;
|
|
637
|
+
default: assertNever(type);
|
|
638
|
+
}
|
|
639
|
+
if (alias !== null) parts.push("AS", `${alias}`);
|
|
640
|
+
const options = isPlainObject(dbParams.options) ? Object.entries(dbParams.options).map(([key, value]) => {
|
|
641
|
+
return key === "ACCESS_MODE" ? value : `${key} '${value}'`;
|
|
642
|
+
}) : [];
|
|
643
|
+
if (options.length > 0) parts.push(`(${options.join(", ")})`);
|
|
644
|
+
return parts.filter(Boolean).join(" ");
|
|
645
|
+
};
|
|
646
|
+
};
|
|
647
|
+
//#endregion
|
|
648
|
+
//#region src/manager/database/duck-database-manager.ts
|
|
649
|
+
var DuckDatabaseManager = class {
|
|
650
|
+
#conn;
|
|
651
|
+
#logger;
|
|
652
|
+
constructor(conn, params) {
|
|
653
|
+
this.#conn = conn;
|
|
654
|
+
this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
|
|
655
|
+
}
|
|
656
|
+
/**
|
|
657
|
+
* Attach a database to the current connection
|
|
658
|
+
*
|
|
659
|
+
* @example
|
|
660
|
+
* ```typescript
|
|
661
|
+
* const dbManager = new DuckDatabaseManager(conn);
|
|
662
|
+
* const database = dbManager.attach({
|
|
663
|
+
* type: ':memory:', // can be 'duckdb', 's3'...
|
|
664
|
+
* alias: 'mydb',
|
|
665
|
+
* options: { COMPRESS: 'true' }
|
|
666
|
+
* });
|
|
667
|
+
*
|
|
668
|
+
* console.log(database.alias); // 'mydb'
|
|
669
|
+
* ```
|
|
670
|
+
*/
|
|
671
|
+
attach = async (dbParams, options) => {
|
|
672
|
+
const params = z.parse(duckDatabaseManagerDbParamsSchema, dbParams);
|
|
673
|
+
const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
|
|
674
|
+
await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
|
|
675
|
+
return new Database({ alias: params.alias });
|
|
676
|
+
};
|
|
677
|
+
attachOrReplace = async (dbParams) => {
|
|
678
|
+
return this.attach(dbParams, { behaviour: "OR REPLACE" });
|
|
679
|
+
};
|
|
680
|
+
attachIfNotExists = async (dbParams) => {
|
|
681
|
+
return this.attach(dbParams, { behaviour: "IF NOT EXISTS" });
|
|
682
|
+
};
|
|
683
|
+
showDatabases = async () => {
|
|
684
|
+
return await this.#executeRawSqlCommand("showDatabases()", `SHOW DATABASES`);
|
|
685
|
+
};
|
|
686
|
+
detach = async (dbAlias) => {
|
|
687
|
+
const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
|
|
688
|
+
await this.#executeRawSqlCommand(`detach(${safeAlias})`, `DETACH ${safeAlias}`);
|
|
689
|
+
return true;
|
|
690
|
+
};
|
|
691
|
+
detachIfExists = async (dbAlias) => {
|
|
692
|
+
const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
|
|
693
|
+
await this.#executeRawSqlCommand(`detachIfExists(${safeAlias})`, `DETACH IF EXISTS ${safeAlias}`);
|
|
694
|
+
return true;
|
|
695
|
+
};
|
|
696
|
+
/**
|
|
697
|
+
* The statistics recomputed by the ANALYZE statement are only used for join order optimization.
|
|
698
|
+
*
|
|
699
|
+
* It is therefore recommended to recompute these statistics for improved join orders,
|
|
700
|
+
* especially after performing large updates (inserts and/or deletes).
|
|
701
|
+
*
|
|
702
|
+
* @link https://duckdb.org/docs/stable/sql/statements/analyze
|
|
703
|
+
*/
|
|
704
|
+
analyze = async () => {
|
|
705
|
+
await this.#executeRawSqlCommand("analyze()", "ANALYZE");
|
|
706
|
+
return true;
|
|
707
|
+
};
|
|
708
|
+
checkpoint = async (dbAlias) => {
|
|
709
|
+
const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
|
|
710
|
+
await this.#executeRawSqlCommand(`checkpoint(${safeAlias})`, `CHECKPOINT ${safeAlias}`);
|
|
711
|
+
return true;
|
|
712
|
+
};
|
|
713
|
+
#executeRawSqlCommand = async (name, rawSql) => {
|
|
714
|
+
const startTime = Date.now();
|
|
715
|
+
try {
|
|
716
|
+
const result = await this.#conn.runAndReadAll(rawSql);
|
|
717
|
+
const timeMs = Math.round(Date.now() - startTime);
|
|
718
|
+
const data = result.getRowObjectsJS();
|
|
719
|
+
this.#logger.info(`DuckDatabaseManager.${name} in ${timeMs}ms`, { timeMs });
|
|
720
|
+
return data;
|
|
721
|
+
} catch (e) {
|
|
722
|
+
const msg = `DuckDatabaseManager: failed to run "${name}" - ${e?.message ?? ""}`;
|
|
723
|
+
const timeMs = Math.round(Date.now() - startTime);
|
|
724
|
+
this.#logger.error(msg, {
|
|
725
|
+
name,
|
|
726
|
+
sql: rawSql,
|
|
727
|
+
timeMs
|
|
728
|
+
});
|
|
729
|
+
throw new Error(msg, { cause: e });
|
|
730
|
+
}
|
|
731
|
+
};
|
|
454
732
|
};
|
|
455
733
|
//#endregion
|
|
456
|
-
export { DuckMemory, SqlDuck, Table, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
|
|
734
|
+
export { Database, DuckDatabaseManager, DuckMemory, SqlDuck, Table, duckDatabaseManagerDbParamsSchema, duckTableAliasSchema, duckTableNameSchema, duckdbReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flowblade/sqlduck",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"exports": {
|
|
@@ -57,9 +57,11 @@
|
|
|
57
57
|
"@flowblade/core": "^0.2.26",
|
|
58
58
|
"@flowblade/source-duckdb": "^0.20.1",
|
|
59
59
|
"@flowblade/sql-tag": "^0.3.2",
|
|
60
|
-
"@
|
|
60
|
+
"@httpx/assert": "^0.16.8",
|
|
61
|
+
"@httpx/plain-object": "^2.1.8",
|
|
62
|
+
"@logtape/logtape": "^2.0.5",
|
|
61
63
|
"@standard-schema/spec": "^1.1.0",
|
|
62
|
-
"p-
|
|
64
|
+
"p-queue": "9.1.0",
|
|
63
65
|
"valibot": "^1.3.1",
|
|
64
66
|
"zod": "^4.3.6"
|
|
65
67
|
},
|
|
@@ -82,7 +84,7 @@
|
|
|
82
84
|
"@types/node": "25.5.0",
|
|
83
85
|
"@typescript-eslint/eslint-plugin": "8.57.2",
|
|
84
86
|
"@typescript-eslint/parser": "8.57.2",
|
|
85
|
-
"@typescript/native-preview": "7.0.0-dev.
|
|
87
|
+
"@typescript/native-preview": "7.0.0-dev.20260324.1",
|
|
86
88
|
"@vitest/coverage-v8": "4.1.1",
|
|
87
89
|
"@vitest/ui": "4.1.1",
|
|
88
90
|
"ansis": "4.2.0",
|
|
@@ -90,6 +92,7 @@
|
|
|
90
92
|
"core-js": "3.49.0",
|
|
91
93
|
"cross-env": "10.1.0",
|
|
92
94
|
"es-check": "9.6.3",
|
|
95
|
+
"es-toolkit": "1.45.1",
|
|
93
96
|
"esbuild": "0.27.4",
|
|
94
97
|
"eslint": "8.57.1",
|
|
95
98
|
"execa": "9.6.1",
|
|
@@ -106,11 +109,11 @@
|
|
|
106
109
|
"tarn": "3.0.2",
|
|
107
110
|
"tedious": "19.2.1",
|
|
108
111
|
"testcontainers": "11.13.0",
|
|
109
|
-
"tsdown": "0.21.
|
|
112
|
+
"tsdown": "0.21.5",
|
|
110
113
|
"tsx": "4.21.0",
|
|
111
|
-
"typedoc": "0.28.
|
|
114
|
+
"typedoc": "0.28.18",
|
|
112
115
|
"typedoc-plugin-markdown": "4.11.0",
|
|
113
|
-
"typescript": "
|
|
116
|
+
"typescript": "6.0.2",
|
|
114
117
|
"vitest": "4.1.1"
|
|
115
118
|
},
|
|
116
119
|
"files": [
|