@flowblade/sqlduck 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -30
- package/dist/index.d.mts +23 -79
- package/dist/index.mjs +180 -279
- package/dist/types-DCqYqEsa.d.mts +80 -0
- package/dist/validation/zod/index.d.mts +37 -0
- package/dist/validation/zod/index.mjs +2 -0
- package/dist/zod-DcIc8xQC.mjs +207 -0
- package/package.json +18 -21
- package/dist/index.cjs +0 -770
- package/dist/index.d.cts +0 -345
package/dist/index.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { c as duckValidatorsZod, l as duckReservedKeywords, r as assertValidAliasName, s as duckConnectionParamsZodSchema } from "./zod-DcIc8xQC.mjs";
|
|
1
2
|
import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
|
|
2
3
|
import { getLogger } from "@logtape/logtape";
|
|
3
4
|
import * as z from "zod";
|
|
@@ -126,6 +127,171 @@ const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
|
|
|
126
127
|
//#region src/logger/sqlduck-default-logtape-logger.ts
|
|
127
128
|
const sqlduckDefaultLogtapeLogger = getLogger(flowbladeLogtapeSqlduckConfig.categories);
|
|
128
129
|
//#endregion
|
|
130
|
+
//#region src/objects/database.ts
|
|
131
|
+
var Database = class {
|
|
132
|
+
#params;
|
|
133
|
+
get alias() {
|
|
134
|
+
return this.#params.alias;
|
|
135
|
+
}
|
|
136
|
+
constructor(params) {
|
|
137
|
+
this.#params = params;
|
|
138
|
+
}
|
|
139
|
+
toJson() {
|
|
140
|
+
return {
|
|
141
|
+
type: "database",
|
|
142
|
+
params: { alias: this.#params.alias }
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
[Symbol.toStringTag]() {
|
|
146
|
+
return this.alias;
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
//#endregion
|
|
150
|
+
//#region src/manager/database/commands/duck-database-attach-command.ts
|
|
151
|
+
var DuckDatabaseAttachCommand = class {
|
|
152
|
+
options;
|
|
153
|
+
dbParams;
|
|
154
|
+
constructor(dbParams, options) {
|
|
155
|
+
this.dbParams = dbParams;
|
|
156
|
+
this.options = options ?? {};
|
|
157
|
+
}
|
|
158
|
+
getRawSql = () => {
|
|
159
|
+
const dbParams = this.dbParams;
|
|
160
|
+
const parts = ["ATTACH", this.options.behaviour].filter(Boolean);
|
|
161
|
+
const { type, alias } = dbParams;
|
|
162
|
+
switch (type) {
|
|
163
|
+
case "memory":
|
|
164
|
+
parts.push("':memory:'");
|
|
165
|
+
break;
|
|
166
|
+
case "duckdb":
|
|
167
|
+
parts.push(`'${dbParams.path}'`);
|
|
168
|
+
break;
|
|
169
|
+
default: assertNever(type);
|
|
170
|
+
}
|
|
171
|
+
if (alias !== null) parts.push("AS", `${alias}`);
|
|
172
|
+
const options = [];
|
|
173
|
+
if (isPlainObject(dbParams.options)) for (const [key, value] of Object.entries(dbParams.options)) switch (key) {
|
|
174
|
+
case "accessMode":
|
|
175
|
+
options.push(`${value}`);
|
|
176
|
+
break;
|
|
177
|
+
case "compress":
|
|
178
|
+
if (value === true) options.push("COMPRESS");
|
|
179
|
+
break;
|
|
180
|
+
case "blockSize":
|
|
181
|
+
options.push(`BLOCK_SIZE ${value}`);
|
|
182
|
+
break;
|
|
183
|
+
case "rowGroupSize":
|
|
184
|
+
options.push(`ROW_GROUP_SIZE ${value}`);
|
|
185
|
+
break;
|
|
186
|
+
case "type":
|
|
187
|
+
options.push(`TYPE ${value}`);
|
|
188
|
+
break;
|
|
189
|
+
case "storageVersion":
|
|
190
|
+
options.push(`STORAGE_VERSION '${value}'`);
|
|
191
|
+
break;
|
|
192
|
+
case "encryptionCipher":
|
|
193
|
+
options.push(`ENCRYPTION_CIPHER '${value}'`);
|
|
194
|
+
break;
|
|
195
|
+
case "encryptionKey":
|
|
196
|
+
options.push(`ENCRYPTION_KEY '${value}'`);
|
|
197
|
+
break;
|
|
198
|
+
default:
|
|
199
|
+
}
|
|
200
|
+
if (options.length > 0) parts.push(`(${options.join(", ")})`);
|
|
201
|
+
return parts.filter(Boolean).join(" ");
|
|
202
|
+
};
|
|
203
|
+
};
|
|
204
|
+
//#endregion
|
|
205
|
+
//#region src/manager/database/duck-database-manager.ts
|
|
206
|
+
var DuckDatabaseManager = class {
|
|
207
|
+
#conn;
|
|
208
|
+
#logger;
|
|
209
|
+
constructor(conn, params) {
|
|
210
|
+
this.#conn = conn;
|
|
211
|
+
this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Attach a database to the current connection
|
|
215
|
+
*
|
|
216
|
+
* @example
|
|
217
|
+
* ```typescript
|
|
218
|
+
* const dbManager = new DuckDatabaseManager(conn);
|
|
219
|
+
* const database = dbManager.attach({
|
|
220
|
+
* type: 'memory', // can be 'duckdb', 's3'...
|
|
221
|
+
* alias: 'mydb',
|
|
222
|
+
* options: { COMPRESS: 'true' }
|
|
223
|
+
* });
|
|
224
|
+
*
|
|
225
|
+
* console.log(database.alias); // 'mydb'
|
|
226
|
+
* ```
|
|
227
|
+
*/
|
|
228
|
+
attach = async (dbParams, options) => {
|
|
229
|
+
const params = duckConnectionParamsZodSchema.parse(dbParams);
|
|
230
|
+
const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
|
|
231
|
+
await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
|
|
232
|
+
return new Database({ alias: params.alias });
|
|
233
|
+
};
|
|
234
|
+
attachOrReplace = async (dbParams) => {
|
|
235
|
+
return this.attach(dbParams, { behaviour: "OR REPLACE" });
|
|
236
|
+
};
|
|
237
|
+
attachIfNotExists = async (dbParams) => {
|
|
238
|
+
return this.attach(dbParams, { behaviour: "IF NOT EXISTS" });
|
|
239
|
+
};
|
|
240
|
+
showDatabases = async () => {
|
|
241
|
+
return await this.#executeRawSqlCommand("showDatabases()", `SHOW DATABASES`);
|
|
242
|
+
};
|
|
243
|
+
detach = async (dbAlias) => {
|
|
244
|
+
assertValidAliasName(dbAlias);
|
|
245
|
+
await this.#executeRawSqlCommand(`detach(${dbAlias})`, `DETACH ${dbAlias}`);
|
|
246
|
+
return true;
|
|
247
|
+
};
|
|
248
|
+
detachIfExists = async (dbAlias) => {
|
|
249
|
+
assertValidAliasName(dbAlias);
|
|
250
|
+
await this.#executeRawSqlCommand(`detachIfExists(${dbAlias})`, `DETACH IF EXISTS ${dbAlias}`);
|
|
251
|
+
return true;
|
|
252
|
+
};
|
|
253
|
+
/**
|
|
254
|
+
* The statistics recomputed by the ANALYZE statement are only used for join order optimization.
|
|
255
|
+
*
|
|
256
|
+
* It is therefore recommended to recompute these statistics for improved join orders,
|
|
257
|
+
* especially after performing large updates (inserts and/or deletes).
|
|
258
|
+
*
|
|
259
|
+
* @link https://duckdb.org/docs/stable/sql/statements/analyze
|
|
260
|
+
*/
|
|
261
|
+
analyze = async () => {
|
|
262
|
+
await this.#executeRawSqlCommand("analyze()", "ANALYZE");
|
|
263
|
+
return true;
|
|
264
|
+
};
|
|
265
|
+
checkpoint = async (dbAlias) => {
|
|
266
|
+
const safeAlias = duckValidatorsZod.aliasName.parse(dbAlias);
|
|
267
|
+
await this.#executeRawSqlCommand(`checkpoint(${safeAlias})`, `CHECKPOINT ${safeAlias}`);
|
|
268
|
+
return true;
|
|
269
|
+
};
|
|
270
|
+
vacuum = async () => {
|
|
271
|
+
await this.#executeRawSqlCommand("vacuum()", "VACUUM");
|
|
272
|
+
return true;
|
|
273
|
+
};
|
|
274
|
+
#executeRawSqlCommand = async (name, rawSql) => {
|
|
275
|
+
const startTime = Date.now();
|
|
276
|
+
try {
|
|
277
|
+
const result = await this.#conn.runAndReadAll(rawSql);
|
|
278
|
+
const timeMs = Math.round(Date.now() - startTime);
|
|
279
|
+
const data = result.getRowObjectsJS();
|
|
280
|
+
this.#logger.info(`DuckDatabaseManager.${name} in ${timeMs}ms`, { timeMs });
|
|
281
|
+
return data;
|
|
282
|
+
} catch (e) {
|
|
283
|
+
const msg = `DuckDatabaseManager: failed to run "${name}" - ${e?.message ?? ""}`;
|
|
284
|
+
const timeMs = Math.round(Date.now() - startTime);
|
|
285
|
+
this.#logger.error(msg, {
|
|
286
|
+
name,
|
|
287
|
+
sql: rawSql,
|
|
288
|
+
timeMs
|
|
289
|
+
});
|
|
290
|
+
throw new Error(msg, { cause: e });
|
|
291
|
+
}
|
|
292
|
+
};
|
|
293
|
+
};
|
|
294
|
+
//#endregion
|
|
129
295
|
//#region src/table/get-duckdb-number-column-type.ts
|
|
130
296
|
const isFloatValue = (value) => {
|
|
131
297
|
if (!Number.isFinite(value)) return true;
|
|
@@ -348,8 +514,10 @@ var SqlDuck = class {
|
|
|
348
514
|
* ```
|
|
349
515
|
*/
|
|
350
516
|
toTable = async (params) => {
|
|
351
|
-
const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
|
|
517
|
+
const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended, autoCheckpoint = true } = params;
|
|
352
518
|
if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
|
|
519
|
+
if (autoCheckpoint && typeof table.databaseName !== "string") throw new Error("autoCheckpoint requires table.databaseName to be provided.");
|
|
520
|
+
const dbManager = new DuckDatabaseManager(this.#conn);
|
|
353
521
|
const timeStart = Date.now();
|
|
354
522
|
const { columnTypes, ddl } = await createTableFromZod({
|
|
355
523
|
conn: this.#conn,
|
|
@@ -378,8 +546,18 @@ var SqlDuck = class {
|
|
|
378
546
|
if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
|
|
379
547
|
else onDataAppended(payload);
|
|
380
548
|
}
|
|
549
|
+
if (autoCheckpoint && typeof table.databaseName === "string") try {
|
|
550
|
+
await dbManager.checkpoint(table.databaseName);
|
|
551
|
+
} catch (e) {
|
|
552
|
+
this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending chunk into table '${table.getFullName()}' - ${e?.message ?? ""}`, { table: table.getFullName() });
|
|
553
|
+
}
|
|
381
554
|
}
|
|
382
555
|
appender.closeSync();
|
|
556
|
+
if (autoCheckpoint && typeof table.databaseName === "string") try {
|
|
557
|
+
await dbManager.checkpoint(table.databaseName);
|
|
558
|
+
} catch (e) {
|
|
559
|
+
this.#logger.warning(`Failed to checkpoint database '${table.databaseName}' after appending data into table '${table.getFullName()}' - ${e?.message ?? ""}`, { table: table.getFullName() });
|
|
560
|
+
}
|
|
383
561
|
const timeMs = Math.round(Date.now() - timeStart);
|
|
384
562
|
this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
|
|
385
563
|
table: table.getFullName(),
|
|
@@ -412,26 +590,6 @@ const zodCodecs = {
|
|
|
412
590
|
})
|
|
413
591
|
};
|
|
414
592
|
//#endregion
|
|
415
|
-
//#region src/objects/database.ts
|
|
416
|
-
var Database = class {
|
|
417
|
-
#params;
|
|
418
|
-
get alias() {
|
|
419
|
-
return this.#params.alias;
|
|
420
|
-
}
|
|
421
|
-
constructor(params) {
|
|
422
|
-
this.#params = params;
|
|
423
|
-
}
|
|
424
|
-
toJson() {
|
|
425
|
-
return {
|
|
426
|
-
type: "database",
|
|
427
|
-
params: { alias: this.#params.alias }
|
|
428
|
-
};
|
|
429
|
-
}
|
|
430
|
-
[Symbol.toStringTag]() {
|
|
431
|
-
return this.alias;
|
|
432
|
-
}
|
|
433
|
-
};
|
|
434
|
-
//#endregion
|
|
435
593
|
//#region src/objects/table.ts
|
|
436
594
|
var Table = class Table {
|
|
437
595
|
#fqTable;
|
|
@@ -474,261 +632,4 @@ var Table = class Table {
|
|
|
474
632
|
};
|
|
475
633
|
};
|
|
476
634
|
//#endregion
|
|
477
|
-
|
|
478
|
-
/**
|
|
479
|
-
* DuckDB reserved keywords that cannot be used as unquoted identifiers.
|
|
480
|
-
* @see https://duckdb.org/docs/sql/keywords-and-identifiers.html
|
|
481
|
-
*/
|
|
482
|
-
const duckdbReservedKeywords = [
|
|
483
|
-
"ALL",
|
|
484
|
-
"ANALYSE",
|
|
485
|
-
"ANALYZE",
|
|
486
|
-
"AND",
|
|
487
|
-
"ANY",
|
|
488
|
-
"ARRAY",
|
|
489
|
-
"AS",
|
|
490
|
-
"ASC",
|
|
491
|
-
"ASYMMETRIC",
|
|
492
|
-
"BOTH",
|
|
493
|
-
"CASE",
|
|
494
|
-
"CAST",
|
|
495
|
-
"CHECK",
|
|
496
|
-
"COLLATE",
|
|
497
|
-
"COLUMN",
|
|
498
|
-
"CONSTRAINT",
|
|
499
|
-
"CREATE",
|
|
500
|
-
"CROSS",
|
|
501
|
-
"CURRENT_CATALOG",
|
|
502
|
-
"CURRENT_DATE",
|
|
503
|
-
"CURRENT_ROLE",
|
|
504
|
-
"CURRENT_SCHEMA",
|
|
505
|
-
"CURRENT_TIME",
|
|
506
|
-
"CURRENT_TIMESTAMP",
|
|
507
|
-
"CURRENT_USER",
|
|
508
|
-
"DEFAULT",
|
|
509
|
-
"DEFERRABLE",
|
|
510
|
-
"DESC",
|
|
511
|
-
"DISTINCT",
|
|
512
|
-
"DO",
|
|
513
|
-
"ELSE",
|
|
514
|
-
"END",
|
|
515
|
-
"EXCEPT",
|
|
516
|
-
"EXISTS",
|
|
517
|
-
"EXTRACT",
|
|
518
|
-
"FALSE",
|
|
519
|
-
"FETCH",
|
|
520
|
-
"FOR",
|
|
521
|
-
"FOREIGN",
|
|
522
|
-
"FROM",
|
|
523
|
-
"GRANT",
|
|
524
|
-
"GROUP",
|
|
525
|
-
"HAVING",
|
|
526
|
-
"IF",
|
|
527
|
-
"ILIKE",
|
|
528
|
-
"IN",
|
|
529
|
-
"INITIALLY",
|
|
530
|
-
"INNER",
|
|
531
|
-
"INTERSECT",
|
|
532
|
-
"INTO",
|
|
533
|
-
"IS",
|
|
534
|
-
"ISNULL",
|
|
535
|
-
"JOIN",
|
|
536
|
-
"LATERAL",
|
|
537
|
-
"LEADING",
|
|
538
|
-
"LEFT",
|
|
539
|
-
"LIKE",
|
|
540
|
-
"LIMIT",
|
|
541
|
-
"LOCALTIME",
|
|
542
|
-
"LOCALTIMESTAMP",
|
|
543
|
-
"NATURAL",
|
|
544
|
-
"NOT",
|
|
545
|
-
"NOTNULL",
|
|
546
|
-
"NULL",
|
|
547
|
-
"OFFSET",
|
|
548
|
-
"ON",
|
|
549
|
-
"ONLY",
|
|
550
|
-
"OR",
|
|
551
|
-
"ORDER",
|
|
552
|
-
"OUTER",
|
|
553
|
-
"OVERLAPS",
|
|
554
|
-
"PLACING",
|
|
555
|
-
"PRIMARY",
|
|
556
|
-
"REFERENCES",
|
|
557
|
-
"RETURNING",
|
|
558
|
-
"RIGHT",
|
|
559
|
-
"ROW",
|
|
560
|
-
"SELECT",
|
|
561
|
-
"SESSION_USER",
|
|
562
|
-
"SIMILAR",
|
|
563
|
-
"SOME",
|
|
564
|
-
"SYMMETRIC",
|
|
565
|
-
"TABLE",
|
|
566
|
-
"THEN",
|
|
567
|
-
"TO",
|
|
568
|
-
"TRAILING",
|
|
569
|
-
"TRUE",
|
|
570
|
-
"UNION",
|
|
571
|
-
"UNIQUE",
|
|
572
|
-
"USING",
|
|
573
|
-
"VARIADIC",
|
|
574
|
-
"VERBOSE",
|
|
575
|
-
"WHEN",
|
|
576
|
-
"WHERE",
|
|
577
|
-
"WINDOW",
|
|
578
|
-
"WITH"
|
|
579
|
-
];
|
|
580
|
-
//#endregion
|
|
581
|
-
//#region src/validation/zod/duckdb-valid-names.schemas.ts
|
|
582
|
-
const duckdbMaximumObjectNameLength = 120;
|
|
583
|
-
const duckDbObjectNameRegex = /^[a-z_]\w*$/i;
|
|
584
|
-
const duckdbReservedKeywordsSet = new Set(duckdbReservedKeywords.map((k) => k.toUpperCase()));
|
|
585
|
-
const duckTableNameSchema = z.string().min(1).max(duckdbMaximumObjectNameLength).regex(duckDbObjectNameRegex, "Table name must start with a letter or underscore, and contain only letters, numbers and underscores").refine((value) => !duckdbReservedKeywordsSet.has(value.toUpperCase()), { error: `Value is a DuckDB reserved keyword and cannot be used as a table name` });
|
|
586
|
-
const duckTableAliasSchema = duckTableNameSchema;
|
|
587
|
-
//#endregion
|
|
588
|
-
//#region src/manager/database/duck-database-manager.schemas.ts
|
|
589
|
-
const duckdbAttachOptionsSchema = z.strictObject({
|
|
590
|
-
ACCESS_MODE: z.optional(z.enum([
|
|
591
|
-
"READ_ONLY",
|
|
592
|
-
"READ_WRITE",
|
|
593
|
-
"AUTOMATIC"
|
|
594
|
-
])),
|
|
595
|
-
COMPRESS: z.optional(z.enum(["true", "false"])),
|
|
596
|
-
TYPE: z.optional(z.enum(["DUCKDB", "SQLITE"])),
|
|
597
|
-
BLOCK_SIZE: z.optional(z.int32().min(16384).max(262144)),
|
|
598
|
-
ROW_GROUP_SIZE: z.optional(z.int32().positive()),
|
|
599
|
-
STORAGE_VERSION: z.optional(z.string().startsWith("v").regex(/^v?\d{1,4}\.\d{1,4}\.\d{1,4}$/)),
|
|
600
|
-
ENCRYPTION_KEY: z.optional(z.string().min(8)),
|
|
601
|
-
ENCRYPTION_CIPHER: z.optional(z.enum([
|
|
602
|
-
"CBC",
|
|
603
|
-
"CTR",
|
|
604
|
-
"GCM"
|
|
605
|
-
]))
|
|
606
|
-
});
|
|
607
|
-
const duckDatabaseManagerDbParamsSchema = z.discriminatedUnion("type", [z.strictObject({
|
|
608
|
-
type: z.literal(":memory:"),
|
|
609
|
-
alias: duckTableAliasSchema,
|
|
610
|
-
options: z.optional(duckdbAttachOptionsSchema)
|
|
611
|
-
}), z.strictObject({
|
|
612
|
-
type: z.literal("duckdb"),
|
|
613
|
-
path: z.string().min(4).endsWith(".db"),
|
|
614
|
-
alias: duckTableAliasSchema,
|
|
615
|
-
options: z.optional(duckdbAttachOptionsSchema)
|
|
616
|
-
})]);
|
|
617
|
-
//#endregion
|
|
618
|
-
//#region src/manager/database/commands/duck-database-attach-command.ts
|
|
619
|
-
var DuckDatabaseAttachCommand = class {
|
|
620
|
-
options;
|
|
621
|
-
dbParams;
|
|
622
|
-
constructor(dbParams, options) {
|
|
623
|
-
this.dbParams = dbParams;
|
|
624
|
-
this.options = options ?? {};
|
|
625
|
-
}
|
|
626
|
-
getRawSql = () => {
|
|
627
|
-
const dbParams = this.dbParams;
|
|
628
|
-
const parts = ["ATTACH", this.options.behaviour].filter(Boolean);
|
|
629
|
-
const { type, alias } = dbParams;
|
|
630
|
-
switch (type) {
|
|
631
|
-
case ":memory:":
|
|
632
|
-
parts.push("':memory:'");
|
|
633
|
-
break;
|
|
634
|
-
case "duckdb":
|
|
635
|
-
parts.push(`'${dbParams.path}'`);
|
|
636
|
-
break;
|
|
637
|
-
default: assertNever(type);
|
|
638
|
-
}
|
|
639
|
-
if (alias !== null) parts.push("AS", `${alias}`);
|
|
640
|
-
const options = isPlainObject(dbParams.options) ? Object.entries(dbParams.options).map(([key, value]) => {
|
|
641
|
-
return key === "ACCESS_MODE" ? value : `${key} '${value}'`;
|
|
642
|
-
}) : [];
|
|
643
|
-
if (options.length > 0) parts.push(`(${options.join(", ")})`);
|
|
644
|
-
return parts.filter(Boolean).join(" ");
|
|
645
|
-
};
|
|
646
|
-
};
|
|
647
|
-
//#endregion
|
|
648
|
-
//#region src/manager/database/duck-database-manager.ts
|
|
649
|
-
var DuckDatabaseManager = class {
|
|
650
|
-
#conn;
|
|
651
|
-
#logger;
|
|
652
|
-
constructor(conn, params) {
|
|
653
|
-
this.#conn = conn;
|
|
654
|
-
this.#logger = params?.logger ?? sqlduckDefaultLogtapeLogger.with({ source: "DuckDatabaseManager" });
|
|
655
|
-
}
|
|
656
|
-
/**
|
|
657
|
-
* Attach a database to the current connection
|
|
658
|
-
*
|
|
659
|
-
* @example
|
|
660
|
-
* ```typescript
|
|
661
|
-
* const dbManager = new DuckDatabaseManager(conn);
|
|
662
|
-
* const database = dbManager.attach({
|
|
663
|
-
* type: ':memory:', // can be 'duckdb', 's3'...
|
|
664
|
-
* alias: 'mydb',
|
|
665
|
-
* options: { COMPRESS: 'true' }
|
|
666
|
-
* });
|
|
667
|
-
*
|
|
668
|
-
* console.log(database.alias); // 'mydb'
|
|
669
|
-
* ```
|
|
670
|
-
*/
|
|
671
|
-
attach = async (dbParams, options) => {
|
|
672
|
-
const params = z.parse(duckDatabaseManagerDbParamsSchema, dbParams);
|
|
673
|
-
const rawSql = new DuckDatabaseAttachCommand(params, options).getRawSql();
|
|
674
|
-
await this.#executeRawSqlCommand(`attach(${params.alias})`, rawSql);
|
|
675
|
-
return new Database({ alias: params.alias });
|
|
676
|
-
};
|
|
677
|
-
attachOrReplace = async (dbParams) => {
|
|
678
|
-
return this.attach(dbParams, { behaviour: "OR REPLACE" });
|
|
679
|
-
};
|
|
680
|
-
attachIfNotExists = async (dbParams) => {
|
|
681
|
-
return this.attach(dbParams, { behaviour: "IF NOT EXISTS" });
|
|
682
|
-
};
|
|
683
|
-
showDatabases = async () => {
|
|
684
|
-
return await this.#executeRawSqlCommand("showDatabases()", `SHOW DATABASES`);
|
|
685
|
-
};
|
|
686
|
-
detach = async (dbAlias) => {
|
|
687
|
-
const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
|
|
688
|
-
await this.#executeRawSqlCommand(`detach(${safeAlias})`, `DETACH ${safeAlias}`);
|
|
689
|
-
return true;
|
|
690
|
-
};
|
|
691
|
-
detachIfExists = async (dbAlias) => {
|
|
692
|
-
const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
|
|
693
|
-
await this.#executeRawSqlCommand(`detachIfExists(${safeAlias})`, `DETACH IF EXISTS ${safeAlias}`);
|
|
694
|
-
return true;
|
|
695
|
-
};
|
|
696
|
-
/**
|
|
697
|
-
* The statistics recomputed by the ANALYZE statement are only used for join order optimization.
|
|
698
|
-
*
|
|
699
|
-
* It is therefore recommended to recompute these statistics for improved join orders,
|
|
700
|
-
* especially after performing large updates (inserts and/or deletes).
|
|
701
|
-
*
|
|
702
|
-
* @link https://duckdb.org/docs/stable/sql/statements/analyze
|
|
703
|
-
*/
|
|
704
|
-
analyze = async () => {
|
|
705
|
-
await this.#executeRawSqlCommand("analyze()", "ANALYZE");
|
|
706
|
-
return true;
|
|
707
|
-
};
|
|
708
|
-
checkpoint = async (dbAlias) => {
|
|
709
|
-
const safeAlias = z.parse(duckTableAliasSchema, dbAlias);
|
|
710
|
-
await this.#executeRawSqlCommand(`checkpoint(${safeAlias})`, `CHECKPOINT ${safeAlias}`);
|
|
711
|
-
return true;
|
|
712
|
-
};
|
|
713
|
-
#executeRawSqlCommand = async (name, rawSql) => {
|
|
714
|
-
const startTime = Date.now();
|
|
715
|
-
try {
|
|
716
|
-
const result = await this.#conn.runAndReadAll(rawSql);
|
|
717
|
-
const timeMs = Math.round(Date.now() - startTime);
|
|
718
|
-
const data = result.getRowObjectsJS();
|
|
719
|
-
this.#logger.info(`DuckDatabaseManager.${name} in ${timeMs}ms`, { timeMs });
|
|
720
|
-
return data;
|
|
721
|
-
} catch (e) {
|
|
722
|
-
const msg = `DuckDatabaseManager: failed to run "${name}" - ${e?.message ?? ""}`;
|
|
723
|
-
const timeMs = Math.round(Date.now() - startTime);
|
|
724
|
-
this.#logger.error(msg, {
|
|
725
|
-
name,
|
|
726
|
-
sql: rawSql,
|
|
727
|
-
timeMs
|
|
728
|
-
});
|
|
729
|
-
throw new Error(msg, { cause: e });
|
|
730
|
-
}
|
|
731
|
-
};
|
|
732
|
-
};
|
|
733
|
-
//#endregion
|
|
734
|
-
export { Database, DuckDatabaseManager, DuckMemory, SqlDuck, Table, duckDatabaseManagerDbParamsSchema, duckTableAliasSchema, duckTableNameSchema, duckdbReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
|
|
635
|
+
export { Database, DuckDatabaseManager, DuckMemory, SqlDuck, Table, duckReservedKeywords, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import * as z from "zod";
|
|
2
|
+
|
|
3
|
+
//#region src/validation/zod/duck-connection-params-zod-schema.d.ts
|
|
4
|
+
declare const duckAllConnectionOptionsZodSchema: z.ZodObject<{
|
|
5
|
+
accessMode: z.ZodOptional<z.ZodEnum<{
|
|
6
|
+
READ_ONLY: "READ_ONLY";
|
|
7
|
+
READ_WRITE: "READ_WRITE";
|
|
8
|
+
}>>;
|
|
9
|
+
compress: z.ZodOptional<z.ZodBoolean>;
|
|
10
|
+
type: z.ZodOptional<z.ZodEnum<{
|
|
11
|
+
DUCKDB: "DUCKDB";
|
|
12
|
+
SQLITE: "SQLITE";
|
|
13
|
+
}>>;
|
|
14
|
+
blockSize: z.ZodOptional<z.ZodInt32>;
|
|
15
|
+
rowGroupSize: z.ZodOptional<z.ZodInt32>;
|
|
16
|
+
storageVersion: z.ZodOptional<z.ZodString>;
|
|
17
|
+
encryptionKey: z.ZodOptional<z.ZodString>;
|
|
18
|
+
encryptionCipher: z.ZodOptional<z.ZodEnum<{
|
|
19
|
+
CBC: "CBC";
|
|
20
|
+
CTR: "CTR";
|
|
21
|
+
GCM: "GCM";
|
|
22
|
+
}>>;
|
|
23
|
+
}, z.core.$strict>;
|
|
24
|
+
declare const duckConnectionParamsZodSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
25
|
+
type: z.ZodLiteral<"memory">;
|
|
26
|
+
alias: z.ZodString;
|
|
27
|
+
options: z.ZodOptional<z.ZodObject<{
|
|
28
|
+
accessMode: z.ZodOptional<z.ZodEnum<{
|
|
29
|
+
READ_ONLY: "READ_ONLY";
|
|
30
|
+
READ_WRITE: "READ_WRITE";
|
|
31
|
+
}>>;
|
|
32
|
+
compress: z.ZodOptional<z.ZodBoolean>;
|
|
33
|
+
type: z.ZodOptional<z.ZodEnum<{
|
|
34
|
+
DUCKDB: "DUCKDB";
|
|
35
|
+
SQLITE: "SQLITE";
|
|
36
|
+
}>>;
|
|
37
|
+
blockSize: z.ZodOptional<z.ZodInt32>;
|
|
38
|
+
rowGroupSize: z.ZodOptional<z.ZodInt32>;
|
|
39
|
+
storageVersion: z.ZodOptional<z.ZodString>;
|
|
40
|
+
encryptionKey: z.ZodOptional<z.ZodString>;
|
|
41
|
+
encryptionCipher: z.ZodOptional<z.ZodEnum<{
|
|
42
|
+
CBC: "CBC";
|
|
43
|
+
CTR: "CTR";
|
|
44
|
+
GCM: "GCM";
|
|
45
|
+
}>>;
|
|
46
|
+
}, z.core.$strict>>;
|
|
47
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
48
|
+
type: z.ZodLiteral<"duckdb">;
|
|
49
|
+
path: z.ZodString;
|
|
50
|
+
alias: z.ZodString;
|
|
51
|
+
options: z.ZodOptional<z.ZodObject<{
|
|
52
|
+
accessMode: z.ZodOptional<z.ZodEnum<{
|
|
53
|
+
READ_ONLY: "READ_ONLY";
|
|
54
|
+
READ_WRITE: "READ_WRITE";
|
|
55
|
+
}>>;
|
|
56
|
+
compress: z.ZodOptional<z.ZodBoolean>;
|
|
57
|
+
type: z.ZodOptional<z.ZodEnum<{
|
|
58
|
+
DUCKDB: "DUCKDB";
|
|
59
|
+
SQLITE: "SQLITE";
|
|
60
|
+
}>>;
|
|
61
|
+
blockSize: z.ZodOptional<z.ZodInt32>;
|
|
62
|
+
rowGroupSize: z.ZodOptional<z.ZodInt32>;
|
|
63
|
+
storageVersion: z.ZodOptional<z.ZodString>;
|
|
64
|
+
encryptionKey: z.ZodOptional<z.ZodString>;
|
|
65
|
+
encryptionCipher: z.ZodOptional<z.ZodEnum<{
|
|
66
|
+
CBC: "CBC";
|
|
67
|
+
CTR: "CTR";
|
|
68
|
+
GCM: "GCM";
|
|
69
|
+
}>>;
|
|
70
|
+
}, z.core.$strict>>;
|
|
71
|
+
}, z.core.$strict>], "type">;
|
|
72
|
+
type DuckConnectionParamsZodSchema = z.infer<typeof duckConnectionParamsZodSchema>;
|
|
73
|
+
//#endregion
|
|
74
|
+
//#region src/validation/core/types.d.ts
|
|
75
|
+
type DuckAliasName = string;
|
|
76
|
+
type DuckTableName = string;
|
|
77
|
+
type DuckSchemaName = string;
|
|
78
|
+
type DuckConnectionParams = DuckConnectionParamsZodSchema;
|
|
79
|
+
//#endregion
|
|
80
|
+
export { duckAllConnectionOptionsZodSchema as a, DuckTableName as i, DuckConnectionParams as n, duckConnectionParamsZodSchema as o, DuckSchemaName as r, DuckAliasName as t };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { a as duckAllConnectionOptionsZodSchema, i as DuckTableName, n as DuckConnectionParams, o as duckConnectionParamsZodSchema, r as DuckSchemaName, t as DuckAliasName } from "../../types-DCqYqEsa.mjs";
|
|
2
|
+
import * as _$zod from "zod";
|
|
3
|
+
|
|
4
|
+
//#region src/validation/zod/duck-asserts-zod.d.ts
|
|
5
|
+
declare function assertValidAliasName(aliasName: string): asserts aliasName is DuckAliasName;
|
|
6
|
+
declare function assertValidSchemaName(schemaName: string): asserts schemaName is DuckSchemaName;
|
|
7
|
+
declare function assertValidTableName(tableName: string): asserts tableName is DuckTableName;
|
|
8
|
+
//#endregion
|
|
9
|
+
//#region src/validation/zod/duck-validators-zod.d.ts
|
|
10
|
+
/**
|
|
11
|
+
* Common validators for duckdb parameters, tables...
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* import { duckValidatorsZod } from '@flowblade/sqlduck/zod';
|
|
16
|
+
*
|
|
17
|
+
* duckValidatorsZod.tableName.parse('my_table'); // valid
|
|
18
|
+
* duckValidatorsZod.tableName.parse('my table'); // invalid
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
declare const duckValidatorsZod: {
|
|
22
|
+
/**
|
|
23
|
+
* Validate duckdb objects names like table, alias, and schemas
|
|
24
|
+
* for validity.
|
|
25
|
+
*/
|
|
26
|
+
readonly aliasName: _$zod.ZodString;
|
|
27
|
+
readonly schemaName: _$zod.ZodString;
|
|
28
|
+
readonly tableName: _$zod.ZodString;
|
|
29
|
+
};
|
|
30
|
+
//#endregion
|
|
31
|
+
//#region src/validation/zod/is-parsable-duck-dsn-zod.d.ts
|
|
32
|
+
declare const isParsableDuckDsnZod: (dsn: unknown) => boolean;
|
|
33
|
+
//#endregion
|
|
34
|
+
//#region src/validation/zod/parse-duck-dsn-zod.d.ts
|
|
35
|
+
declare const parseDuckDSNZod: (dsn: string) => DuckConnectionParams;
|
|
36
|
+
//#endregion
|
|
37
|
+
export { assertValidAliasName, assertValidSchemaName, assertValidTableName, duckAllConnectionOptionsZodSchema, duckConnectionParamsZodSchema, duckValidatorsZod, isParsableDuckDsnZod, parseDuckDSNZod };
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
import { a as assertValidTableName, c as duckValidatorsZod, i as assertValidSchemaName, n as parseDuckDSNZod, o as duckAllConnectionOptionsZodSchema, r as assertValidAliasName, s as duckConnectionParamsZodSchema, t as isParsableDuckDsnZod } from "../../zod-DcIc8xQC.mjs";
|
|
2
|
+
export { assertValidAliasName, assertValidSchemaName, assertValidTableName, duckAllConnectionOptionsZodSchema, duckConnectionParamsZodSchema, duckValidatorsZod, isParsableDuckDsnZod, parseDuckDSNZod };
|