@flowblade/sqlduck 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs DELETED
@@ -1,486 +0,0 @@
1
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
- //#region \0rolldown/runtime.js
3
- var __create = Object.create;
4
- var __defProp = Object.defineProperty;
5
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
6
- var __getOwnPropNames = Object.getOwnPropertyNames;
7
- var __getProtoOf = Object.getPrototypeOf;
8
- var __hasOwnProp = Object.prototype.hasOwnProperty;
9
- var __copyProps = (to, from, except, desc) => {
10
- if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
11
- key = keys[i];
12
- if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
13
- get: ((k) => from[k]).bind(null, key),
14
- enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
15
- });
16
- }
17
- return to;
18
- };
19
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
20
- value: mod,
21
- enumerable: true
22
- }) : target, mod));
23
- //#endregion
24
- let _logtape_logtape = require("@logtape/logtape");
25
- let _duckdb_node_api = require("@duckdb/node-api");
26
- let zod = require("zod");
27
- zod = __toESM(zod);
28
- //#region src/config/flowblade-logtape-sqlduck.config.ts
29
- const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
30
- //#endregion
31
- //#region src/helpers/duck-exec.ts
32
- var DuckExec = class {
33
- #conn;
34
- constructor(duckConn) {
35
- this.#conn = duckConn;
36
- }
37
- getRowObjectJS = async (sql) => {
38
- return (await this.#conn.run(sql)).getRowObjectsJS();
39
- };
40
- getRowObjectJson = async (sql) => {
41
- return (await this.#conn.run(sql)).getRowObjectsJson();
42
- };
43
- getOneRowObjectJS = async (sql) => {
44
- const rows = await this.getRowObjectJS(sql);
45
- if (rows.length === 0) return null;
46
- this.#ensureOneRow(rows);
47
- return rows[0];
48
- };
49
- getOneRowObjectJson = async (sql) => {
50
- const rows = await this.getRowObjectJson(sql);
51
- if (rows.length === 0) return null;
52
- this.#ensureOneRow(rows);
53
- return rows[0];
54
- };
55
- #ensureOneRow = (rows) => {
56
- if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
57
- };
58
- };
59
- //#endregion
60
- //#region src/helpers/duck-memory.ts
61
- const duckMemoryTags = [
62
- "BASE_TABLE",
63
- "HASH_TABLE",
64
- "PARQUET_READER",
65
- "CSV_READER",
66
- "ORDER_BY",
67
- "ART_INDEX",
68
- "COLUMN_DATA",
69
- "METADATA",
70
- "OVERFLOW_STRINGS",
71
- "IN_MEMORY_TABLE",
72
- "ALLOCATOR",
73
- "EXTENSION",
74
- "TRANSACTION",
75
- "EXTERNAL_FILE_CACHE",
76
- "WINDOW",
77
- "OBJECT_CACHE"
78
- ];
79
- const orderByParams = {
80
- memory_usage_bytes_desc: "memory_usage_bytes DESC",
81
- tag_desc: "tag DESC",
82
- tag_asc: "tag ASC"
83
- };
84
- var DuckMemory = class {
85
- #conn;
86
- #exec;
87
- constructor(duckdbConn) {
88
- this.#conn = duckdbConn;
89
- this.#exec = new DuckExec(duckdbConn);
90
- }
91
- getAll = async (params) => {
92
- const { orderBy } = params ?? {};
93
- const query = this.#applyOrderBy(`SELECT tag, memory_usage_bytes, temporary_storage_bytes
94
- FROM duckdb_memory() as m`, orderBy);
95
- return (await this.#conn.run(query)).getRowObjectsJS();
96
- };
97
- getByTag = async (tag) => {
98
- if (!duckMemoryTags.includes(tag)) throw new Error(`Invalid DuckDB memory tag: ${tag}`);
99
- const query = `SELECT tag, memory_usage_bytes, temporary_storage_bytes
100
- FROM duckdb_memory() as m
101
- WHERE tag = '${tag}'`;
102
- return this.#exec.getOneRowObjectJS(query);
103
- };
104
- getSummary = async () => {
105
- const rows = await this.getAll();
106
- const summaryInBytes = {
107
- total: 0n,
108
- totalTemp: 0n
109
- };
110
- for (const row of rows) {
111
- summaryInBytes.total += row.memory_usage_bytes;
112
- summaryInBytes.totalTemp += row.temporary_storage_bytes;
113
- }
114
- return {
115
- totalMB: Math.round(Number(summaryInBytes.total / 1048576n)),
116
- totalTempMB: Math.round(Number(summaryInBytes.totalTemp / 1048576n))
117
- };
118
- };
119
- #applyOrderBy = (query, orderBy) => {
120
- if (orderBy === void 0) return query;
121
- const orderByClause = orderByParams[orderBy];
122
- if (orderByClause === void 0) throw new Error(`Invalid orderBy parameter: ${orderBy}`);
123
- return `${query} ORDER BY ${orderByClause}`;
124
- };
125
- };
126
- //#endregion
127
- //#region src/logger/sqlduck-default-logtape-logger.ts
128
- const sqlduckDefaultLogtapeLogger = (0, _logtape_logtape.getLogger)(flowbladeLogtapeSqlduckConfig.categories);
129
- //#endregion
130
- //#region src/appender/data-appender-callback.ts
131
- const isOnDataAppendedAsyncCb = (v) => {
132
- return v.constructor.name === "AsyncFunction";
133
- };
134
- const createOnDataAppendedCollector = () => {
135
- let lastCallbackTimeStart = Date.now();
136
- let appendedTotalRows = 0;
137
- return (currentTotalRows) => {
138
- const cbTimeMs = Math.round(Date.now() - lastCallbackTimeStart);
139
- const cbTotalRows = currentTotalRows - appendedTotalRows;
140
- const stats = {
141
- totalRows: currentTotalRows,
142
- timeMs: cbTimeMs,
143
- rowsPerSecond: Math.round(cbTotalRows / cbTimeMs * 1e3)
144
- };
145
- appendedTotalRows = currentTotalRows;
146
- lastCallbackTimeStart = Date.now();
147
- return stats;
148
- };
149
- };
150
- //#endregion
151
- //#region src/table/get-duckdb-number-column-type.ts
152
- const isFloatValue = (value) => {
153
- if (!Number.isFinite(value)) return true;
154
- if (Math.abs(value) > Number.MAX_SAFE_INTEGER) return true;
155
- return !Number.isInteger(value);
156
- };
157
- const getDuckdbNumberColumnType = (params) => {
158
- const { minimum, maximum } = params;
159
- if (minimum === void 0 || maximum === void 0) return _duckdb_node_api.BIGINT;
160
- if (isFloatValue(minimum) || isFloatValue(maximum)) {
161
- if (minimum >= -34028235e31 && maximum <= 34028235e31) return _duckdb_node_api.FLOAT;
162
- return _duckdb_node_api.DOUBLE;
163
- }
164
- if (minimum >= 0) {
165
- if (maximum <= 255) return _duckdb_node_api.UTINYINT;
166
- if (maximum <= 65535) return _duckdb_node_api.USMALLINT;
167
- if (maximum <= 4294967295) return _duckdb_node_api.UINTEGER;
168
- if (maximum <= 18446744073709551615n) return _duckdb_node_api.UBIGINT;
169
- return _duckdb_node_api.UHUGEINT;
170
- }
171
- if (minimum >= -128 && maximum <= 127) return _duckdb_node_api.TINYINT;
172
- if (minimum >= -32768 && maximum <= 32767) return _duckdb_node_api.SMALLINT;
173
- if (minimum >= -2147483648 && maximum <= 2147483647) return _duckdb_node_api.INTEGER;
174
- if (minimum >= -9223372036854775808n && maximum <= 9223372036854775807n) return _duckdb_node_api.BIGINT;
175
- return _duckdb_node_api.HUGEINT;
176
- };
177
- //#endregion
178
- //#region src/table/get-table-create-from-zod.ts
179
- const createOptions = {
180
- CREATE: "CREATE TABLE",
181
- CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
182
- IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
183
- };
184
- const duckDbTypes = [
185
- ["VARCHAR", _duckdb_node_api.VARCHAR],
186
- ["BIGINT", _duckdb_node_api.BIGINT],
187
- ["TIMESTAMP", _duckdb_node_api.TIMESTAMP],
188
- ["UUID", _duckdb_node_api.UUID],
189
- ["BOOLEAN", _duckdb_node_api.BOOLEAN],
190
- ["INTEGER", _duckdb_node_api.INTEGER],
191
- ["DOUBLE", _duckdb_node_api.DOUBLE],
192
- ["FLOAT", _duckdb_node_api.FLOAT]
193
- ];
194
- const duckDbTypesMap = new Map(duckDbTypes);
195
- const getTableCreateFromZod = (params) => {
196
- const { table, schema, options } = params;
197
- const { create = "CREATE" } = options ?? {};
198
- const fqTable = table.getFullName();
199
- const json = schema.toJSONSchema({
200
- target: "openapi-3.0",
201
- unrepresentable: "throw"
202
- });
203
- const columns = [];
204
- if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
205
- const columnTypesMap = /* @__PURE__ */ new Map();
206
- for (const [columnName, def] of Object.entries(json.properties)) {
207
- const { type, duckdbType, nullable, format, primaryKey, minimum, maximum } = def;
208
- const c = { name: columnName };
209
- if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
210
- else switch (type) {
211
- case "string":
212
- switch (format) {
213
- case "date-time":
214
- c.duckdbType = _duckdb_node_api.TIMESTAMP;
215
- break;
216
- case "int64":
217
- c.duckdbType = _duckdb_node_api.BIGINT;
218
- break;
219
- case "uuid":
220
- c.duckdbType = _duckdb_node_api.UUID;
221
- break;
222
- default: c.duckdbType = _duckdb_node_api.VARCHAR;
223
- }
224
- break;
225
- case "number":
226
- c.duckdbType = getDuckdbNumberColumnType({
227
- minimum,
228
- maximum
229
- });
230
- break;
231
- case "integer":
232
- c.duckdbType = getDuckdbNumberColumnType({
233
- minimum,
234
- maximum
235
- });
236
- break;
237
- case "boolean":
238
- c.duckdbType = _duckdb_node_api.BOOLEAN;
239
- break;
240
- default: throw new Error(`Cannot guess '${columnName}' type - ${JSON.stringify(def)}`);
241
- }
242
- if (primaryKey === true) c.constraint = "PRIMARY KEY";
243
- else if (nullable !== true) c.constraint = "NOT NULL";
244
- columnTypesMap.set(columnName, c.duckdbType);
245
- columns.push(c);
246
- }
247
- return {
248
- ddl: [
249
- `${createOptions[create]} ${fqTable} (\n`,
250
- columns.map((colDDL) => {
251
- const { name, duckdbType, constraint } = colDDL;
252
- return ` ${[
253
- name,
254
- duckdbType.toString(),
255
- constraint
256
- ].filter(Boolean).join(" ")}`;
257
- }).join(",\n"),
258
- "\n)"
259
- ].join(""),
260
- columnTypes: columnTypesMap
261
- };
262
- };
263
- //#endregion
264
- //#region src/table/create-table-from-zod.ts
265
- const createTableFromZod = async (params) => {
266
- const { conn, table, schema, options, logger = sqlduckDefaultLogtapeLogger } = params;
267
- const { ddl, columnTypes } = getTableCreateFromZod({
268
- table,
269
- schema,
270
- options
271
- });
272
- logger.debug(`Generate DDL for table '${table.getFullName()}'`, { ddl });
273
- try {
274
- await conn.run(ddl);
275
- logger.info(`Table '${table.getFullName()}' successfully created`, { ddl });
276
- } catch (e) {
277
- logger.error(`Failed to create table '${table.getFullName()}': ${e.message}`, { ddl });
278
- throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
279
- }
280
- return {
281
- ddl,
282
- columnTypes
283
- };
284
- };
285
- //#endregion
286
- //#region src/utils/rows-to-columns-chunks.ts
287
- const toDuckValue = (value) => {
288
- if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
289
- if (typeof value === "bigint") return value.toString(10);
290
- return value === void 0 ? null : value;
291
- };
292
- /**
293
- * Similar to `rowsToColumns` but yields results in chunks to avoid buffering
294
- * the entire dataset in memory. Each yielded item is a columns array for up to
295
- * `chunkSize` rows.
296
- *
297
- * Example for chunkSize = 2:
298
- * input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
299
- * yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
300
- */
301
- async function* rowsToColumnsChunks(params) {
302
- const { rows, chunkSize } = params;
303
- if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
304
- const first = await rows.next();
305
- if (first.done) return;
306
- const keys = Object.keys(first.value);
307
- let columns = keys.map(() => []);
308
- let rowsInChunk = 0;
309
- keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
310
- rowsInChunk++;
311
- if (rowsInChunk >= chunkSize) {
312
- yield columns;
313
- columns = keys.map(() => []);
314
- rowsInChunk = 0;
315
- }
316
- for await (const row of rows) {
317
- keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
318
- rowsInChunk++;
319
- if (rowsInChunk >= chunkSize) {
320
- yield columns;
321
- columns = keys.map(() => []);
322
- rowsInChunk = 0;
323
- }
324
- }
325
- if (rowsInChunk > 0) yield columns;
326
- }
327
- //#endregion
328
- //#region src/sql-duck.ts
329
- var SqlDuck = class {
330
- #duck;
331
- #logger;
332
- constructor(params) {
333
- this.#duck = params.conn;
334
- this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
335
- }
336
- /**
337
- * Create a table from a Zod schema and fill it with data from a row stream.
338
- *
339
- * @example
340
- * ```typescript
341
- * import * as z from 'zod';
342
- *
343
- * const sqlDuck = new SqlDuck({ conn: duckDbConnection });
344
- *
345
- * // Schema of the table, not that you can use meta to add information
346
- * const userSchema = z.object({
347
- * id: z.number().int().meta({ primaryKey: true }),
348
- * name: z.string(),
349
- * });
350
- *
351
- * // Async generator function that yields rows to insert
352
- * async function* getUserRows(): AsyncIterableIterator<z.infer<typeof userSchema>> {
353
- * // database or api call
354
- * }
355
- *
356
- * const result = sqlDuck.toTable({
357
- * table: new Table({ name: 'user', database: 'mydb' }),
358
- * schema: userSchema,
359
- * rowStream: getUserRows(),
360
- * chunkSize: 2048,
361
- * onDataAppended: ({ total }) => {
362
- * console.log(`Appended ${total} rows so far`);
363
- * },
364
- * createOptions: {
365
- * create: 'CREATE_OR_REPLACE',
366
- * },
367
- * });
368
- *
369
- * console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
370
- * console.log(`Table created with DDL: ${result.createTableDDL}`);
371
- * ```
372
- */
373
- toTable = async (params) => {
374
- const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
375
- if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
376
- const timeStart = Date.now();
377
- const { columnTypes, ddl } = await createTableFromZod({
378
- conn: this.#duck,
379
- schema,
380
- table,
381
- options: createOptions
382
- });
383
- const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
384
- const chunkTypes = Array.from(columnTypes.values());
385
- let totalRows = 0;
386
- const dataAppendedCollector = createOnDataAppendedCollector();
387
- const columnStream = rowsToColumnsChunks({
388
- rows: rowStream,
389
- chunkSize
390
- });
391
- try {
392
- for await (const dataChunk of columnStream) {
393
- const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
394
- this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
395
- totalRows += dataChunk?.[0]?.length ?? 0;
396
- chunk.setColumns(dataChunk);
397
- appender.appendDataChunk(chunk);
398
- appender.flushSync();
399
- if (onDataAppended !== void 0) {
400
- const payload = dataAppendedCollector(totalRows);
401
- if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
402
- else onDataAppended(payload);
403
- }
404
- }
405
- appender.closeSync();
406
- const timeMs = Math.round(Date.now() - timeStart);
407
- this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
408
- table: table.getFullName(),
409
- timeMs,
410
- totalRows
411
- });
412
- return {
413
- timeMs,
414
- totalRows,
415
- createTableDDL: ddl
416
- };
417
- } catch (e) {
418
- appender.closeSync();
419
- const msg = `Failed to append data into table '${table.getFullName()}' - ${e?.message ?? ""}`;
420
- this.#logger.error(msg, { table: table.getFullName() });
421
- throw new Error(msg, { cause: e });
422
- }
423
- };
424
- };
425
- //#endregion
426
- //#region src/table/table.ts
427
- var Table = class Table {
428
- #fqTable;
429
- get tableName() {
430
- return this.#fqTable.name;
431
- }
432
- get schemaName() {
433
- return this.#fqTable.schema;
434
- }
435
- get databaseName() {
436
- return this.#fqTable.database;
437
- }
438
- constructor(fqTableOrName) {
439
- this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
440
- }
441
- /**
442
- * Return fully qualified table name by concatenating
443
- * database, schema and table with a 'dot' separator.
444
- */
445
- getFullName = (options) => {
446
- const { defaultDatabase, defaultSchema } = options ?? {};
447
- const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
448
- return [
449
- database,
450
- schema,
451
- name
452
- ].filter(Boolean).join(".");
453
- };
454
- withDatabase = (database) => {
455
- return new Table({
456
- ...this.#fqTable,
457
- database
458
- });
459
- };
460
- withSchema = (schema) => {
461
- return new Table({
462
- ...this.#fqTable,
463
- schema
464
- });
465
- };
466
- };
467
- //#endregion
468
- //#region src/utils/zod-codecs.ts
469
- const zodCodecs = {
470
- dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
471
- decode: (date) => date.toISOString(),
472
- encode: (isoString) => new Date(isoString)
473
- }),
474
- bigintToString: zod.codec(zod.bigint(), zod.string().meta({ format: "int64" }), {
475
- decode: (bigint) => bigint.toString(),
476
- encode: BigInt
477
- })
478
- };
479
- //#endregion
480
- exports.DuckMemory = DuckMemory;
481
- exports.SqlDuck = SqlDuck;
482
- exports.Table = Table;
483
- exports.flowbladeLogtapeSqlduckConfig = flowbladeLogtapeSqlduckConfig;
484
- exports.getTableCreateFromZod = getTableCreateFromZod;
485
- exports.sqlduckDefaultLogtapeLogger = sqlduckDefaultLogtapeLogger;
486
- exports.zodCodecs = zodCodecs;
package/dist/index.d.cts DELETED
@@ -1,211 +0,0 @@
1
- import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
2
- import * as _logtape_logtape0 from "@logtape/logtape";
3
- import { Logger } from "@logtape/logtape";
4
- import * as z from "zod";
5
- import { ZodObject } from "zod";
6
-
7
- //#region src/appender/data-appender-callback.d.ts
8
- type OnDataAppendedStats = {
9
- /**
10
- * Total number of rows appended so far (all batches included)
11
- */
12
- totalRows: number;
13
- /**
14
- * Time taken to append the last batch in milliseconds
15
- */
16
- timeMs: number;
17
- /**
18
- * Estimated rows per seconds based on the current batch
19
- */
20
- rowsPerSecond: number;
21
- };
22
- type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
23
- type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
24
- type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
25
- //#endregion
26
- //#region src/config/flowblade-logtape-sqlduck.config.d.ts
27
- declare const flowbladeLogtapeSqlduckConfig: {
28
- categories: string[];
29
- };
30
- //#endregion
31
- //#region src/helpers/duck-memory.d.ts
32
- declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
33
- type DuckMemoryTag = (typeof duckMemoryTags)[number];
34
- type DuckMemoryRow = {
35
- tag: DuckMemoryTag;
36
- memory_usage_bytes: bigint;
37
- temporary_storage_bytes: bigint;
38
- };
39
- declare const orderByParams: {
40
- memory_usage_bytes_desc: string;
41
- tag_desc: string;
42
- tag_asc: string;
43
- };
44
- type OrderByParams = keyof typeof orderByParams;
45
- type DuckMemorySummary = {
46
- totalMB: number;
47
- totalTempMB: number;
48
- };
49
- declare class DuckMemory {
50
- #private;
51
- constructor(duckdbConn: DuckDBConnection);
52
- getAll: (params?: {
53
- orderBy?: OrderByParams;
54
- }) => Promise<DuckMemoryRow[]>;
55
- getByTag: (tag: DuckMemoryTag) => Promise<DuckMemoryRow | null>;
56
- getSummary: () => Promise<DuckMemorySummary>;
57
- }
58
- //#endregion
59
- //#region src/logger/sqlduck-default-logtape-logger.d.ts
60
- declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
61
- //#endregion
62
- //#region src/table/table.d.ts
63
- /**
64
- * Fully qualified table information
65
- */
66
- type FQTable = {
67
- name: string;
68
- schema?: string;
69
- database?: string;
70
- };
71
- declare class Table {
72
- #private;
73
- get tableName(): string;
74
- get schemaName(): string | undefined;
75
- get databaseName(): string | undefined;
76
- constructor(fqTableOrName: FQTable | string);
77
- /**
78
- * Return fully qualified table name by concatenating
79
- * database, schema and table with a 'dot' separator.
80
- */
81
- getFullName: (options?: {
82
- defaultDatabase?: string;
83
- defaultSchema?: string;
84
- }) => string;
85
- withDatabase: (database: string) => Table;
86
- withSchema: (schema: string) => Table;
87
- }
88
- //#endregion
89
- //#region src/table/table-schema-zod.type.d.ts
90
- type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID;
91
- type TableSchemaZod = z.ZodObject<Record<string, ZodSchemaSupportedTypes | z.ZodNullable<ZodSchemaSupportedTypes> | z.ZodCodec | z.ZodNullable<z.ZodCodec>>>;
92
- //#endregion
93
- //#region src/table/get-table-create-from-zod.d.ts
94
- type TableCreateOptions = {
95
- create?: 'CREATE' | 'CREATE_OR_REPLACE' | 'IF_NOT_EXISTS';
96
- };
97
- type DuckdbColumnTypeMap<TKeys extends string> = Map<TKeys, DuckDBType>;
98
- type TableCreateFromZodResult<TSchema extends TableSchemaZod> = {
99
- ddl: string;
100
- columnTypes: DuckdbColumnTypeMap<Exclude<keyof TSchema['shape'], symbol | number>>;
101
- };
102
- type GetTableCreateFromZodParams<TSchema extends TableSchemaZod> = {
103
- table: Table;
104
- schema: TSchema;
105
- options?: TableCreateOptions;
106
- };
107
- declare const getTableCreateFromZod: <TSchema extends TableSchemaZod>(params: GetTableCreateFromZodParams<TSchema>) => TableCreateFromZodResult<TSchema>;
108
- //#endregion
109
- //#region src/sql-duck.d.ts
110
- type SqlDuckParams = {
111
- conn: DuckDBConnection;
112
- /**
113
- * Optional logtape/logger to use for logging.
114
- * If not provided, a default logger will be used.
115
- * @see {@link https://github.com/logtape/logtape}
116
- */
117
- logger?: Logger;
118
- };
119
- type RowStream<T> = AsyncIterableIterator<T> | AsyncGenerator<T> | Generator<T>;
120
- type ToTableParams<TSchema extends TableSchemaZod> = {
121
- /**
122
- * Used to create and fill the data into the table
123
- */
124
- table: Table;
125
- /**
126
- * Schema describing the table structure and rowStream content
127
- */
128
- schema: TSchema;
129
- /**
130
- * Stream of rows to insert into the table
131
- */
132
- rowStream: RowStream<z.infer<TSchema>>;
133
- /**
134
- * Chunk size when using appender to insert data.
135
- * Valid numbers between 1 and 2048.
136
- * @default 2048
137
- */
138
- chunkSize?: number;
139
- /**
140
- * Extra options when creating the table
141
- */
142
- createOptions?: TableCreateOptions;
143
- /**
144
- * Callback called each time a datachunk is appended to the table
145
- */
146
- onDataAppended?: OnDataAppendedCb;
147
- };
148
- type ToTableResult = {
149
- /**
150
- * Total time taken to insert the data in milliseconds.
151
- */
152
- timeMs: number;
153
- /**
154
- * Total number of rows inserted into the table.
155
- */
156
- totalRows: number;
157
- /**
158
- * The DDL statement used to create the table.
159
- */
160
- createTableDDL: string;
161
- };
162
- declare class SqlDuck {
163
- #private;
164
- constructor(params: SqlDuckParams);
165
- /**
166
- * Create a table from a Zod schema and fill it with data from a row stream.
167
- *
168
- * @example
169
- * ```typescript
170
- * import * as z from 'zod';
171
- *
172
- * const sqlDuck = new SqlDuck({ conn: duckDbConnection });
173
- *
174
- * // Schema of the table, not that you can use meta to add information
175
- * const userSchema = z.object({
176
- * id: z.number().int().meta({ primaryKey: true }),
177
- * name: z.string(),
178
- * });
179
- *
180
- * // Async generator function that yields rows to insert
181
- * async function* getUserRows(): AsyncIterableIterator<z.infer<typeof userSchema>> {
182
- * // database or api call
183
- * }
184
- *
185
- * const result = sqlDuck.toTable({
186
- * table: new Table({ name: 'user', database: 'mydb' }),
187
- * schema: userSchema,
188
- * rowStream: getUserRows(),
189
- * chunkSize: 2048,
190
- * onDataAppended: ({ total }) => {
191
- * console.log(`Appended ${total} rows so far`);
192
- * },
193
- * createOptions: {
194
- * create: 'CREATE_OR_REPLACE',
195
- * },
196
- * });
197
- *
198
- * console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
199
- * console.log(`Table created with DDL: ${result.createTableDDL}`);
200
- * ```
201
- */
202
- toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<ToTableResult>;
203
- }
204
- //#endregion
205
- //#region src/utils/zod-codecs.d.ts
206
- declare const zodCodecs: {
207
- readonly dateToString: z.ZodCodec<z.ZodDate, z.ZodISODateTime>;
208
- readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
209
- };
210
- //#endregion
211
- export { DuckMemory, DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };