@flowblade/sqlduck 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -30
- package/dist/index.d.mts +90 -12
- package/dist/index.mjs +204 -29
- package/dist/types-DCqYqEsa.d.mts +80 -0
- package/dist/validation/zod/index.d.mts +37 -0
- package/dist/validation/zod/index.mjs +2 -0
- package/dist/zod-CwR_oehs.mjs +207 -0
- package/package.json +26 -26
- package/dist/index.cjs +0 -486
- package/dist/index.d.cts +0 -211
package/dist/index.cjs
DELETED
|
@@ -1,486 +0,0 @@
|
|
|
1
|
-
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
//#region \0rolldown/runtime.js
|
|
3
|
-
var __create = Object.create;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
-
var __copyProps = (to, from, except, desc) => {
|
|
10
|
-
if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
|
|
11
|
-
key = keys[i];
|
|
12
|
-
if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
|
|
13
|
-
get: ((k) => from[k]).bind(null, key),
|
|
14
|
-
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
15
|
-
});
|
|
16
|
-
}
|
|
17
|
-
return to;
|
|
18
|
-
};
|
|
19
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
|
|
20
|
-
value: mod,
|
|
21
|
-
enumerable: true
|
|
22
|
-
}) : target, mod));
|
|
23
|
-
//#endregion
|
|
24
|
-
let _logtape_logtape = require("@logtape/logtape");
|
|
25
|
-
let _duckdb_node_api = require("@duckdb/node-api");
|
|
26
|
-
let zod = require("zod");
|
|
27
|
-
zod = __toESM(zod);
|
|
28
|
-
//#region src/config/flowblade-logtape-sqlduck.config.ts
|
|
29
|
-
const flowbladeLogtapeSqlduckConfig = { categories: ["flowblade", "sqlduck"] };
|
|
30
|
-
//#endregion
|
|
31
|
-
//#region src/helpers/duck-exec.ts
|
|
32
|
-
var DuckExec = class {
|
|
33
|
-
#conn;
|
|
34
|
-
constructor(duckConn) {
|
|
35
|
-
this.#conn = duckConn;
|
|
36
|
-
}
|
|
37
|
-
getRowObjectJS = async (sql) => {
|
|
38
|
-
return (await this.#conn.run(sql)).getRowObjectsJS();
|
|
39
|
-
};
|
|
40
|
-
getRowObjectJson = async (sql) => {
|
|
41
|
-
return (await this.#conn.run(sql)).getRowObjectsJson();
|
|
42
|
-
};
|
|
43
|
-
getOneRowObjectJS = async (sql) => {
|
|
44
|
-
const rows = await this.getRowObjectJS(sql);
|
|
45
|
-
if (rows.length === 0) return null;
|
|
46
|
-
this.#ensureOneRow(rows);
|
|
47
|
-
return rows[0];
|
|
48
|
-
};
|
|
49
|
-
getOneRowObjectJson = async (sql) => {
|
|
50
|
-
const rows = await this.getRowObjectJson(sql);
|
|
51
|
-
if (rows.length === 0) return null;
|
|
52
|
-
this.#ensureOneRow(rows);
|
|
53
|
-
return rows[0];
|
|
54
|
-
};
|
|
55
|
-
#ensureOneRow = (rows) => {
|
|
56
|
-
if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
|
|
57
|
-
};
|
|
58
|
-
};
|
|
59
|
-
//#endregion
|
|
60
|
-
//#region src/helpers/duck-memory.ts
|
|
61
|
-
const duckMemoryTags = [
|
|
62
|
-
"BASE_TABLE",
|
|
63
|
-
"HASH_TABLE",
|
|
64
|
-
"PARQUET_READER",
|
|
65
|
-
"CSV_READER",
|
|
66
|
-
"ORDER_BY",
|
|
67
|
-
"ART_INDEX",
|
|
68
|
-
"COLUMN_DATA",
|
|
69
|
-
"METADATA",
|
|
70
|
-
"OVERFLOW_STRINGS",
|
|
71
|
-
"IN_MEMORY_TABLE",
|
|
72
|
-
"ALLOCATOR",
|
|
73
|
-
"EXTENSION",
|
|
74
|
-
"TRANSACTION",
|
|
75
|
-
"EXTERNAL_FILE_CACHE",
|
|
76
|
-
"WINDOW",
|
|
77
|
-
"OBJECT_CACHE"
|
|
78
|
-
];
|
|
79
|
-
const orderByParams = {
|
|
80
|
-
memory_usage_bytes_desc: "memory_usage_bytes DESC",
|
|
81
|
-
tag_desc: "tag DESC",
|
|
82
|
-
tag_asc: "tag ASC"
|
|
83
|
-
};
|
|
84
|
-
var DuckMemory = class {
|
|
85
|
-
#conn;
|
|
86
|
-
#exec;
|
|
87
|
-
constructor(duckdbConn) {
|
|
88
|
-
this.#conn = duckdbConn;
|
|
89
|
-
this.#exec = new DuckExec(duckdbConn);
|
|
90
|
-
}
|
|
91
|
-
getAll = async (params) => {
|
|
92
|
-
const { orderBy } = params ?? {};
|
|
93
|
-
const query = this.#applyOrderBy(`SELECT tag, memory_usage_bytes, temporary_storage_bytes
|
|
94
|
-
FROM duckdb_memory() as m`, orderBy);
|
|
95
|
-
return (await this.#conn.run(query)).getRowObjectsJS();
|
|
96
|
-
};
|
|
97
|
-
getByTag = async (tag) => {
|
|
98
|
-
if (!duckMemoryTags.includes(tag)) throw new Error(`Invalid DuckDB memory tag: ${tag}`);
|
|
99
|
-
const query = `SELECT tag, memory_usage_bytes, temporary_storage_bytes
|
|
100
|
-
FROM duckdb_memory() as m
|
|
101
|
-
WHERE tag = '${tag}'`;
|
|
102
|
-
return this.#exec.getOneRowObjectJS(query);
|
|
103
|
-
};
|
|
104
|
-
getSummary = async () => {
|
|
105
|
-
const rows = await this.getAll();
|
|
106
|
-
const summaryInBytes = {
|
|
107
|
-
total: 0n,
|
|
108
|
-
totalTemp: 0n
|
|
109
|
-
};
|
|
110
|
-
for (const row of rows) {
|
|
111
|
-
summaryInBytes.total += row.memory_usage_bytes;
|
|
112
|
-
summaryInBytes.totalTemp += row.temporary_storage_bytes;
|
|
113
|
-
}
|
|
114
|
-
return {
|
|
115
|
-
totalMB: Math.round(Number(summaryInBytes.total / 1048576n)),
|
|
116
|
-
totalTempMB: Math.round(Number(summaryInBytes.totalTemp / 1048576n))
|
|
117
|
-
};
|
|
118
|
-
};
|
|
119
|
-
#applyOrderBy = (query, orderBy) => {
|
|
120
|
-
if (orderBy === void 0) return query;
|
|
121
|
-
const orderByClause = orderByParams[orderBy];
|
|
122
|
-
if (orderByClause === void 0) throw new Error(`Invalid orderBy parameter: ${orderBy}`);
|
|
123
|
-
return `${query} ORDER BY ${orderByClause}`;
|
|
124
|
-
};
|
|
125
|
-
};
|
|
126
|
-
//#endregion
|
|
127
|
-
//#region src/logger/sqlduck-default-logtape-logger.ts
|
|
128
|
-
const sqlduckDefaultLogtapeLogger = (0, _logtape_logtape.getLogger)(flowbladeLogtapeSqlduckConfig.categories);
|
|
129
|
-
//#endregion
|
|
130
|
-
//#region src/appender/data-appender-callback.ts
|
|
131
|
-
const isOnDataAppendedAsyncCb = (v) => {
|
|
132
|
-
return v.constructor.name === "AsyncFunction";
|
|
133
|
-
};
|
|
134
|
-
const createOnDataAppendedCollector = () => {
|
|
135
|
-
let lastCallbackTimeStart = Date.now();
|
|
136
|
-
let appendedTotalRows = 0;
|
|
137
|
-
return (currentTotalRows) => {
|
|
138
|
-
const cbTimeMs = Math.round(Date.now() - lastCallbackTimeStart);
|
|
139
|
-
const cbTotalRows = currentTotalRows - appendedTotalRows;
|
|
140
|
-
const stats = {
|
|
141
|
-
totalRows: currentTotalRows,
|
|
142
|
-
timeMs: cbTimeMs,
|
|
143
|
-
rowsPerSecond: Math.round(cbTotalRows / cbTimeMs * 1e3)
|
|
144
|
-
};
|
|
145
|
-
appendedTotalRows = currentTotalRows;
|
|
146
|
-
lastCallbackTimeStart = Date.now();
|
|
147
|
-
return stats;
|
|
148
|
-
};
|
|
149
|
-
};
|
|
150
|
-
//#endregion
|
|
151
|
-
//#region src/table/get-duckdb-number-column-type.ts
|
|
152
|
-
const isFloatValue = (value) => {
|
|
153
|
-
if (!Number.isFinite(value)) return true;
|
|
154
|
-
if (Math.abs(value) > Number.MAX_SAFE_INTEGER) return true;
|
|
155
|
-
return !Number.isInteger(value);
|
|
156
|
-
};
|
|
157
|
-
const getDuckdbNumberColumnType = (params) => {
|
|
158
|
-
const { minimum, maximum } = params;
|
|
159
|
-
if (minimum === void 0 || maximum === void 0) return _duckdb_node_api.BIGINT;
|
|
160
|
-
if (isFloatValue(minimum) || isFloatValue(maximum)) {
|
|
161
|
-
if (minimum >= -34028235e31 && maximum <= 34028235e31) return _duckdb_node_api.FLOAT;
|
|
162
|
-
return _duckdb_node_api.DOUBLE;
|
|
163
|
-
}
|
|
164
|
-
if (minimum >= 0) {
|
|
165
|
-
if (maximum <= 255) return _duckdb_node_api.UTINYINT;
|
|
166
|
-
if (maximum <= 65535) return _duckdb_node_api.USMALLINT;
|
|
167
|
-
if (maximum <= 4294967295) return _duckdb_node_api.UINTEGER;
|
|
168
|
-
if (maximum <= 18446744073709551615n) return _duckdb_node_api.UBIGINT;
|
|
169
|
-
return _duckdb_node_api.UHUGEINT;
|
|
170
|
-
}
|
|
171
|
-
if (minimum >= -128 && maximum <= 127) return _duckdb_node_api.TINYINT;
|
|
172
|
-
if (minimum >= -32768 && maximum <= 32767) return _duckdb_node_api.SMALLINT;
|
|
173
|
-
if (minimum >= -2147483648 && maximum <= 2147483647) return _duckdb_node_api.INTEGER;
|
|
174
|
-
if (minimum >= -9223372036854775808n && maximum <= 9223372036854775807n) return _duckdb_node_api.BIGINT;
|
|
175
|
-
return _duckdb_node_api.HUGEINT;
|
|
176
|
-
};
|
|
177
|
-
//#endregion
|
|
178
|
-
//#region src/table/get-table-create-from-zod.ts
|
|
179
|
-
const createOptions = {
|
|
180
|
-
CREATE: "CREATE TABLE",
|
|
181
|
-
CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
|
|
182
|
-
IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
|
|
183
|
-
};
|
|
184
|
-
const duckDbTypes = [
|
|
185
|
-
["VARCHAR", _duckdb_node_api.VARCHAR],
|
|
186
|
-
["BIGINT", _duckdb_node_api.BIGINT],
|
|
187
|
-
["TIMESTAMP", _duckdb_node_api.TIMESTAMP],
|
|
188
|
-
["UUID", _duckdb_node_api.UUID],
|
|
189
|
-
["BOOLEAN", _duckdb_node_api.BOOLEAN],
|
|
190
|
-
["INTEGER", _duckdb_node_api.INTEGER],
|
|
191
|
-
["DOUBLE", _duckdb_node_api.DOUBLE],
|
|
192
|
-
["FLOAT", _duckdb_node_api.FLOAT]
|
|
193
|
-
];
|
|
194
|
-
const duckDbTypesMap = new Map(duckDbTypes);
|
|
195
|
-
const getTableCreateFromZod = (params) => {
|
|
196
|
-
const { table, schema, options } = params;
|
|
197
|
-
const { create = "CREATE" } = options ?? {};
|
|
198
|
-
const fqTable = table.getFullName();
|
|
199
|
-
const json = schema.toJSONSchema({
|
|
200
|
-
target: "openapi-3.0",
|
|
201
|
-
unrepresentable: "throw"
|
|
202
|
-
});
|
|
203
|
-
const columns = [];
|
|
204
|
-
if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
|
|
205
|
-
const columnTypesMap = /* @__PURE__ */ new Map();
|
|
206
|
-
for (const [columnName, def] of Object.entries(json.properties)) {
|
|
207
|
-
const { type, duckdbType, nullable, format, primaryKey, minimum, maximum } = def;
|
|
208
|
-
const c = { name: columnName };
|
|
209
|
-
if (duckdbType !== void 0 && duckDbTypesMap.has(duckdbType)) c.duckdbType = duckDbTypesMap.get(duckdbType);
|
|
210
|
-
else switch (type) {
|
|
211
|
-
case "string":
|
|
212
|
-
switch (format) {
|
|
213
|
-
case "date-time":
|
|
214
|
-
c.duckdbType = _duckdb_node_api.TIMESTAMP;
|
|
215
|
-
break;
|
|
216
|
-
case "int64":
|
|
217
|
-
c.duckdbType = _duckdb_node_api.BIGINT;
|
|
218
|
-
break;
|
|
219
|
-
case "uuid":
|
|
220
|
-
c.duckdbType = _duckdb_node_api.UUID;
|
|
221
|
-
break;
|
|
222
|
-
default: c.duckdbType = _duckdb_node_api.VARCHAR;
|
|
223
|
-
}
|
|
224
|
-
break;
|
|
225
|
-
case "number":
|
|
226
|
-
c.duckdbType = getDuckdbNumberColumnType({
|
|
227
|
-
minimum,
|
|
228
|
-
maximum
|
|
229
|
-
});
|
|
230
|
-
break;
|
|
231
|
-
case "integer":
|
|
232
|
-
c.duckdbType = getDuckdbNumberColumnType({
|
|
233
|
-
minimum,
|
|
234
|
-
maximum
|
|
235
|
-
});
|
|
236
|
-
break;
|
|
237
|
-
case "boolean":
|
|
238
|
-
c.duckdbType = _duckdb_node_api.BOOLEAN;
|
|
239
|
-
break;
|
|
240
|
-
default: throw new Error(`Cannot guess '${columnName}' type - ${JSON.stringify(def)}`);
|
|
241
|
-
}
|
|
242
|
-
if (primaryKey === true) c.constraint = "PRIMARY KEY";
|
|
243
|
-
else if (nullable !== true) c.constraint = "NOT NULL";
|
|
244
|
-
columnTypesMap.set(columnName, c.duckdbType);
|
|
245
|
-
columns.push(c);
|
|
246
|
-
}
|
|
247
|
-
return {
|
|
248
|
-
ddl: [
|
|
249
|
-
`${createOptions[create]} ${fqTable} (\n`,
|
|
250
|
-
columns.map((colDDL) => {
|
|
251
|
-
const { name, duckdbType, constraint } = colDDL;
|
|
252
|
-
return ` ${[
|
|
253
|
-
name,
|
|
254
|
-
duckdbType.toString(),
|
|
255
|
-
constraint
|
|
256
|
-
].filter(Boolean).join(" ")}`;
|
|
257
|
-
}).join(",\n"),
|
|
258
|
-
"\n)"
|
|
259
|
-
].join(""),
|
|
260
|
-
columnTypes: columnTypesMap
|
|
261
|
-
};
|
|
262
|
-
};
|
|
263
|
-
//#endregion
|
|
264
|
-
//#region src/table/create-table-from-zod.ts
|
|
265
|
-
const createTableFromZod = async (params) => {
|
|
266
|
-
const { conn, table, schema, options, logger = sqlduckDefaultLogtapeLogger } = params;
|
|
267
|
-
const { ddl, columnTypes } = getTableCreateFromZod({
|
|
268
|
-
table,
|
|
269
|
-
schema,
|
|
270
|
-
options
|
|
271
|
-
});
|
|
272
|
-
logger.debug(`Generate DDL for table '${table.getFullName()}'`, { ddl });
|
|
273
|
-
try {
|
|
274
|
-
await conn.run(ddl);
|
|
275
|
-
logger.info(`Table '${table.getFullName()}' successfully created`, { ddl });
|
|
276
|
-
} catch (e) {
|
|
277
|
-
logger.error(`Failed to create table '${table.getFullName()}': ${e.message}`, { ddl });
|
|
278
|
-
throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
|
|
279
|
-
}
|
|
280
|
-
return {
|
|
281
|
-
ddl,
|
|
282
|
-
columnTypes
|
|
283
|
-
};
|
|
284
|
-
};
|
|
285
|
-
//#endregion
|
|
286
|
-
//#region src/utils/rows-to-columns-chunks.ts
|
|
287
|
-
const toDuckValue = (value) => {
|
|
288
|
-
if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
|
|
289
|
-
if (typeof value === "bigint") return value.toString(10);
|
|
290
|
-
return value === void 0 ? null : value;
|
|
291
|
-
};
|
|
292
|
-
/**
|
|
293
|
-
* Similar to `rowsToColumns` but yields results in chunks to avoid buffering
|
|
294
|
-
* the entire dataset in memory. Each yielded item is a columns array for up to
|
|
295
|
-
* `chunkSize` rows.
|
|
296
|
-
*
|
|
297
|
-
* Example for chunkSize = 2:
|
|
298
|
-
* input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
|
|
299
|
-
* yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
|
|
300
|
-
*/
|
|
301
|
-
async function* rowsToColumnsChunks(params) {
|
|
302
|
-
const { rows, chunkSize } = params;
|
|
303
|
-
if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
|
|
304
|
-
const first = await rows.next();
|
|
305
|
-
if (first.done) return;
|
|
306
|
-
const keys = Object.keys(first.value);
|
|
307
|
-
let columns = keys.map(() => []);
|
|
308
|
-
let rowsInChunk = 0;
|
|
309
|
-
keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
|
|
310
|
-
rowsInChunk++;
|
|
311
|
-
if (rowsInChunk >= chunkSize) {
|
|
312
|
-
yield columns;
|
|
313
|
-
columns = keys.map(() => []);
|
|
314
|
-
rowsInChunk = 0;
|
|
315
|
-
}
|
|
316
|
-
for await (const row of rows) {
|
|
317
|
-
keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
|
|
318
|
-
rowsInChunk++;
|
|
319
|
-
if (rowsInChunk >= chunkSize) {
|
|
320
|
-
yield columns;
|
|
321
|
-
columns = keys.map(() => []);
|
|
322
|
-
rowsInChunk = 0;
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
if (rowsInChunk > 0) yield columns;
|
|
326
|
-
}
|
|
327
|
-
//#endregion
|
|
328
|
-
//#region src/sql-duck.ts
|
|
329
|
-
var SqlDuck = class {
|
|
330
|
-
#duck;
|
|
331
|
-
#logger;
|
|
332
|
-
constructor(params) {
|
|
333
|
-
this.#duck = params.conn;
|
|
334
|
-
this.#logger = params.logger ?? sqlduckDefaultLogtapeLogger;
|
|
335
|
-
}
|
|
336
|
-
/**
|
|
337
|
-
* Create a table from a Zod schema and fill it with data from a row stream.
|
|
338
|
-
*
|
|
339
|
-
* @example
|
|
340
|
-
* ```typescript
|
|
341
|
-
* import * as z from 'zod';
|
|
342
|
-
*
|
|
343
|
-
* const sqlDuck = new SqlDuck({ conn: duckDbConnection });
|
|
344
|
-
*
|
|
345
|
-
* // Schema of the table, not that you can use meta to add information
|
|
346
|
-
* const userSchema = z.object({
|
|
347
|
-
* id: z.number().int().meta({ primaryKey: true }),
|
|
348
|
-
* name: z.string(),
|
|
349
|
-
* });
|
|
350
|
-
*
|
|
351
|
-
* // Async generator function that yields rows to insert
|
|
352
|
-
* async function* getUserRows(): AsyncIterableIterator<z.infer<typeof userSchema>> {
|
|
353
|
-
* // database or api call
|
|
354
|
-
* }
|
|
355
|
-
*
|
|
356
|
-
* const result = sqlDuck.toTable({
|
|
357
|
-
* table: new Table({ name: 'user', database: 'mydb' }),
|
|
358
|
-
* schema: userSchema,
|
|
359
|
-
* rowStream: getUserRows(),
|
|
360
|
-
* chunkSize: 2048,
|
|
361
|
-
* onDataAppended: ({ total }) => {
|
|
362
|
-
* console.log(`Appended ${total} rows so far`);
|
|
363
|
-
* },
|
|
364
|
-
* createOptions: {
|
|
365
|
-
* create: 'CREATE_OR_REPLACE',
|
|
366
|
-
* },
|
|
367
|
-
* });
|
|
368
|
-
*
|
|
369
|
-
* console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
|
|
370
|
-
* console.log(`Table created with DDL: ${result.createTableDDL}`);
|
|
371
|
-
* ```
|
|
372
|
-
*/
|
|
373
|
-
toTable = async (params) => {
|
|
374
|
-
const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
|
|
375
|
-
if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
|
|
376
|
-
const timeStart = Date.now();
|
|
377
|
-
const { columnTypes, ddl } = await createTableFromZod({
|
|
378
|
-
conn: this.#duck,
|
|
379
|
-
schema,
|
|
380
|
-
table,
|
|
381
|
-
options: createOptions
|
|
382
|
-
});
|
|
383
|
-
const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
|
|
384
|
-
const chunkTypes = Array.from(columnTypes.values());
|
|
385
|
-
let totalRows = 0;
|
|
386
|
-
const dataAppendedCollector = createOnDataAppendedCollector();
|
|
387
|
-
const columnStream = rowsToColumnsChunks({
|
|
388
|
-
rows: rowStream,
|
|
389
|
-
chunkSize
|
|
390
|
-
});
|
|
391
|
-
try {
|
|
392
|
-
for await (const dataChunk of columnStream) {
|
|
393
|
-
const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
|
|
394
|
-
this.#logger.debug(`Inserting chunk of ${dataChunk.length} rows`, { table: table.getFullName() });
|
|
395
|
-
totalRows += dataChunk?.[0]?.length ?? 0;
|
|
396
|
-
chunk.setColumns(dataChunk);
|
|
397
|
-
appender.appendDataChunk(chunk);
|
|
398
|
-
appender.flushSync();
|
|
399
|
-
if (onDataAppended !== void 0) {
|
|
400
|
-
const payload = dataAppendedCollector(totalRows);
|
|
401
|
-
if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
|
|
402
|
-
else onDataAppended(payload);
|
|
403
|
-
}
|
|
404
|
-
}
|
|
405
|
-
appender.closeSync();
|
|
406
|
-
const timeMs = Math.round(Date.now() - timeStart);
|
|
407
|
-
this.#logger.info(`Successfully appended ${totalRows} rows into '${table.getFullName()}' in ${timeMs}ms`, {
|
|
408
|
-
table: table.getFullName(),
|
|
409
|
-
timeMs,
|
|
410
|
-
totalRows
|
|
411
|
-
});
|
|
412
|
-
return {
|
|
413
|
-
timeMs,
|
|
414
|
-
totalRows,
|
|
415
|
-
createTableDDL: ddl
|
|
416
|
-
};
|
|
417
|
-
} catch (e) {
|
|
418
|
-
appender.closeSync();
|
|
419
|
-
const msg = `Failed to append data into table '${table.getFullName()}' - ${e?.message ?? ""}`;
|
|
420
|
-
this.#logger.error(msg, { table: table.getFullName() });
|
|
421
|
-
throw new Error(msg, { cause: e });
|
|
422
|
-
}
|
|
423
|
-
};
|
|
424
|
-
};
|
|
425
|
-
//#endregion
|
|
426
|
-
//#region src/table/table.ts
|
|
427
|
-
var Table = class Table {
|
|
428
|
-
#fqTable;
|
|
429
|
-
get tableName() {
|
|
430
|
-
return this.#fqTable.name;
|
|
431
|
-
}
|
|
432
|
-
get schemaName() {
|
|
433
|
-
return this.#fqTable.schema;
|
|
434
|
-
}
|
|
435
|
-
get databaseName() {
|
|
436
|
-
return this.#fqTable.database;
|
|
437
|
-
}
|
|
438
|
-
constructor(fqTableOrName) {
|
|
439
|
-
this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
|
|
440
|
-
}
|
|
441
|
-
/**
|
|
442
|
-
* Return fully qualified table name by concatenating
|
|
443
|
-
* database, schema and table with a 'dot' separator.
|
|
444
|
-
*/
|
|
445
|
-
getFullName = (options) => {
|
|
446
|
-
const { defaultDatabase, defaultSchema } = options ?? {};
|
|
447
|
-
const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
|
|
448
|
-
return [
|
|
449
|
-
database,
|
|
450
|
-
schema,
|
|
451
|
-
name
|
|
452
|
-
].filter(Boolean).join(".");
|
|
453
|
-
};
|
|
454
|
-
withDatabase = (database) => {
|
|
455
|
-
return new Table({
|
|
456
|
-
...this.#fqTable,
|
|
457
|
-
database
|
|
458
|
-
});
|
|
459
|
-
};
|
|
460
|
-
withSchema = (schema) => {
|
|
461
|
-
return new Table({
|
|
462
|
-
...this.#fqTable,
|
|
463
|
-
schema
|
|
464
|
-
});
|
|
465
|
-
};
|
|
466
|
-
};
|
|
467
|
-
//#endregion
|
|
468
|
-
//#region src/utils/zod-codecs.ts
|
|
469
|
-
const zodCodecs = {
|
|
470
|
-
dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
|
|
471
|
-
decode: (date) => date.toISOString(),
|
|
472
|
-
encode: (isoString) => new Date(isoString)
|
|
473
|
-
}),
|
|
474
|
-
bigintToString: zod.codec(zod.bigint(), zod.string().meta({ format: "int64" }), {
|
|
475
|
-
decode: (bigint) => bigint.toString(),
|
|
476
|
-
encode: BigInt
|
|
477
|
-
})
|
|
478
|
-
};
|
|
479
|
-
//#endregion
|
|
480
|
-
exports.DuckMemory = DuckMemory;
|
|
481
|
-
exports.SqlDuck = SqlDuck;
|
|
482
|
-
exports.Table = Table;
|
|
483
|
-
exports.flowbladeLogtapeSqlduckConfig = flowbladeLogtapeSqlduckConfig;
|
|
484
|
-
exports.getTableCreateFromZod = getTableCreateFromZod;
|
|
485
|
-
exports.sqlduckDefaultLogtapeLogger = sqlduckDefaultLogtapeLogger;
|
|
486
|
-
exports.zodCodecs = zodCodecs;
|
package/dist/index.d.cts
DELETED
|
@@ -1,211 +0,0 @@
|
|
|
1
|
-
import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
|
|
2
|
-
import * as _logtape_logtape0 from "@logtape/logtape";
|
|
3
|
-
import { Logger } from "@logtape/logtape";
|
|
4
|
-
import * as z from "zod";
|
|
5
|
-
import { ZodObject } from "zod";
|
|
6
|
-
|
|
7
|
-
//#region src/appender/data-appender-callback.d.ts
|
|
8
|
-
type OnDataAppendedStats = {
|
|
9
|
-
/**
|
|
10
|
-
* Total number of rows appended so far (all batches included)
|
|
11
|
-
*/
|
|
12
|
-
totalRows: number;
|
|
13
|
-
/**
|
|
14
|
-
* Time taken to append the last batch in milliseconds
|
|
15
|
-
*/
|
|
16
|
-
timeMs: number;
|
|
17
|
-
/**
|
|
18
|
-
* Estimated rows per seconds based on the current batch
|
|
19
|
-
*/
|
|
20
|
-
rowsPerSecond: number;
|
|
21
|
-
};
|
|
22
|
-
type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
|
|
23
|
-
type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
|
|
24
|
-
type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
|
|
25
|
-
//#endregion
|
|
26
|
-
//#region src/config/flowblade-logtape-sqlduck.config.d.ts
|
|
27
|
-
declare const flowbladeLogtapeSqlduckConfig: {
|
|
28
|
-
categories: string[];
|
|
29
|
-
};
|
|
30
|
-
//#endregion
|
|
31
|
-
//#region src/helpers/duck-memory.d.ts
|
|
32
|
-
declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
|
|
33
|
-
type DuckMemoryTag = (typeof duckMemoryTags)[number];
|
|
34
|
-
type DuckMemoryRow = {
|
|
35
|
-
tag: DuckMemoryTag;
|
|
36
|
-
memory_usage_bytes: bigint;
|
|
37
|
-
temporary_storage_bytes: bigint;
|
|
38
|
-
};
|
|
39
|
-
declare const orderByParams: {
|
|
40
|
-
memory_usage_bytes_desc: string;
|
|
41
|
-
tag_desc: string;
|
|
42
|
-
tag_asc: string;
|
|
43
|
-
};
|
|
44
|
-
type OrderByParams = keyof typeof orderByParams;
|
|
45
|
-
type DuckMemorySummary = {
|
|
46
|
-
totalMB: number;
|
|
47
|
-
totalTempMB: number;
|
|
48
|
-
};
|
|
49
|
-
declare class DuckMemory {
|
|
50
|
-
#private;
|
|
51
|
-
constructor(duckdbConn: DuckDBConnection);
|
|
52
|
-
getAll: (params?: {
|
|
53
|
-
orderBy?: OrderByParams;
|
|
54
|
-
}) => Promise<DuckMemoryRow[]>;
|
|
55
|
-
getByTag: (tag: DuckMemoryTag) => Promise<DuckMemoryRow | null>;
|
|
56
|
-
getSummary: () => Promise<DuckMemorySummary>;
|
|
57
|
-
}
|
|
58
|
-
//#endregion
|
|
59
|
-
//#region src/logger/sqlduck-default-logtape-logger.d.ts
|
|
60
|
-
declare const sqlduckDefaultLogtapeLogger: _logtape_logtape0.Logger;
|
|
61
|
-
//#endregion
|
|
62
|
-
//#region src/table/table.d.ts
|
|
63
|
-
/**
|
|
64
|
-
* Fully qualified table information
|
|
65
|
-
*/
|
|
66
|
-
type FQTable = {
|
|
67
|
-
name: string;
|
|
68
|
-
schema?: string;
|
|
69
|
-
database?: string;
|
|
70
|
-
};
|
|
71
|
-
declare class Table {
|
|
72
|
-
#private;
|
|
73
|
-
get tableName(): string;
|
|
74
|
-
get schemaName(): string | undefined;
|
|
75
|
-
get databaseName(): string | undefined;
|
|
76
|
-
constructor(fqTableOrName: FQTable | string);
|
|
77
|
-
/**
|
|
78
|
-
* Return fully qualified table name by concatenating
|
|
79
|
-
* database, schema and table with a 'dot' separator.
|
|
80
|
-
*/
|
|
81
|
-
getFullName: (options?: {
|
|
82
|
-
defaultDatabase?: string;
|
|
83
|
-
defaultSchema?: string;
|
|
84
|
-
}) => string;
|
|
85
|
-
withDatabase: (database: string) => Table;
|
|
86
|
-
withSchema: (schema: string) => Table;
|
|
87
|
-
}
|
|
88
|
-
//#endregion
|
|
89
|
-
//#region src/table/table-schema-zod.type.d.ts
|
|
90
|
-
type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID;
|
|
91
|
-
type TableSchemaZod = z.ZodObject<Record<string, ZodSchemaSupportedTypes | z.ZodNullable<ZodSchemaSupportedTypes> | z.ZodCodec | z.ZodNullable<z.ZodCodec>>>;
|
|
92
|
-
//#endregion
|
|
93
|
-
//#region src/table/get-table-create-from-zod.d.ts
|
|
94
|
-
type TableCreateOptions = {
|
|
95
|
-
create?: 'CREATE' | 'CREATE_OR_REPLACE' | 'IF_NOT_EXISTS';
|
|
96
|
-
};
|
|
97
|
-
type DuckdbColumnTypeMap<TKeys extends string> = Map<TKeys, DuckDBType>;
|
|
98
|
-
type TableCreateFromZodResult<TSchema extends TableSchemaZod> = {
|
|
99
|
-
ddl: string;
|
|
100
|
-
columnTypes: DuckdbColumnTypeMap<Exclude<keyof TSchema['shape'], symbol | number>>;
|
|
101
|
-
};
|
|
102
|
-
type GetTableCreateFromZodParams<TSchema extends TableSchemaZod> = {
|
|
103
|
-
table: Table;
|
|
104
|
-
schema: TSchema;
|
|
105
|
-
options?: TableCreateOptions;
|
|
106
|
-
};
|
|
107
|
-
declare const getTableCreateFromZod: <TSchema extends TableSchemaZod>(params: GetTableCreateFromZodParams<TSchema>) => TableCreateFromZodResult<TSchema>;
|
|
108
|
-
//#endregion
|
|
109
|
-
//#region src/sql-duck.d.ts
|
|
110
|
-
type SqlDuckParams = {
|
|
111
|
-
conn: DuckDBConnection;
|
|
112
|
-
/**
|
|
113
|
-
* Optional logtape/logger to use for logging.
|
|
114
|
-
* If not provided, a default logger will be used.
|
|
115
|
-
* @see {@link https://github.com/logtape/logtape}
|
|
116
|
-
*/
|
|
117
|
-
logger?: Logger;
|
|
118
|
-
};
|
|
119
|
-
type RowStream<T> = AsyncIterableIterator<T> | AsyncGenerator<T> | Generator<T>;
|
|
120
|
-
type ToTableParams<TSchema extends TableSchemaZod> = {
|
|
121
|
-
/**
|
|
122
|
-
* Used to create and fill the data into the table
|
|
123
|
-
*/
|
|
124
|
-
table: Table;
|
|
125
|
-
/**
|
|
126
|
-
* Schema describing the table structure and rowStream content
|
|
127
|
-
*/
|
|
128
|
-
schema: TSchema;
|
|
129
|
-
/**
|
|
130
|
-
* Stream of rows to insert into the table
|
|
131
|
-
*/
|
|
132
|
-
rowStream: RowStream<z.infer<TSchema>>;
|
|
133
|
-
/**
|
|
134
|
-
* Chunk size when using appender to insert data.
|
|
135
|
-
* Valid numbers between 1 and 2048.
|
|
136
|
-
* @default 2048
|
|
137
|
-
*/
|
|
138
|
-
chunkSize?: number;
|
|
139
|
-
/**
|
|
140
|
-
* Extra options when creating the table
|
|
141
|
-
*/
|
|
142
|
-
createOptions?: TableCreateOptions;
|
|
143
|
-
/**
|
|
144
|
-
* Callback called each time a datachunk is appended to the table
|
|
145
|
-
*/
|
|
146
|
-
onDataAppended?: OnDataAppendedCb;
|
|
147
|
-
};
|
|
148
|
-
type ToTableResult = {
|
|
149
|
-
/**
|
|
150
|
-
* Total time taken to insert the data in milliseconds.
|
|
151
|
-
*/
|
|
152
|
-
timeMs: number;
|
|
153
|
-
/**
|
|
154
|
-
* Total number of rows inserted into the table.
|
|
155
|
-
*/
|
|
156
|
-
totalRows: number;
|
|
157
|
-
/**
|
|
158
|
-
* The DDL statement used to create the table.
|
|
159
|
-
*/
|
|
160
|
-
createTableDDL: string;
|
|
161
|
-
};
|
|
162
|
-
declare class SqlDuck {
|
|
163
|
-
#private;
|
|
164
|
-
constructor(params: SqlDuckParams);
|
|
165
|
-
/**
|
|
166
|
-
* Create a table from a Zod schema and fill it with data from a row stream.
|
|
167
|
-
*
|
|
168
|
-
* @example
|
|
169
|
-
* ```typescript
|
|
170
|
-
* import * as z from 'zod';
|
|
171
|
-
*
|
|
172
|
-
* const sqlDuck = new SqlDuck({ conn: duckDbConnection });
|
|
173
|
-
*
|
|
174
|
-
* // Schema of the table, not that you can use meta to add information
|
|
175
|
-
* const userSchema = z.object({
|
|
176
|
-
* id: z.number().int().meta({ primaryKey: true }),
|
|
177
|
-
* name: z.string(),
|
|
178
|
-
* });
|
|
179
|
-
*
|
|
180
|
-
* // Async generator function that yields rows to insert
|
|
181
|
-
* async function* getUserRows(): AsyncIterableIterator<z.infer<typeof userSchema>> {
|
|
182
|
-
* // database or api call
|
|
183
|
-
* }
|
|
184
|
-
*
|
|
185
|
-
* const result = sqlDuck.toTable({
|
|
186
|
-
* table: new Table({ name: 'user', database: 'mydb' }),
|
|
187
|
-
* schema: userSchema,
|
|
188
|
-
* rowStream: getUserRows(),
|
|
189
|
-
* chunkSize: 2048,
|
|
190
|
-
* onDataAppended: ({ total }) => {
|
|
191
|
-
* console.log(`Appended ${total} rows so far`);
|
|
192
|
-
* },
|
|
193
|
-
* createOptions: {
|
|
194
|
-
* create: 'CREATE_OR_REPLACE',
|
|
195
|
-
* },
|
|
196
|
-
* });
|
|
197
|
-
*
|
|
198
|
-
* console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
|
|
199
|
-
* console.log(`Table created with DDL: ${result.createTableDDL}`);
|
|
200
|
-
* ```
|
|
201
|
-
*/
|
|
202
|
-
toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<ToTableResult>;
|
|
203
|
-
}
|
|
204
|
-
//#endregion
|
|
205
|
-
//#region src/utils/zod-codecs.d.ts
|
|
206
|
-
declare const zodCodecs: {
|
|
207
|
-
readonly dateToString: z.ZodCodec<z.ZodDate, z.ZodISODateTime>;
|
|
208
|
-
readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
|
|
209
|
-
};
|
|
210
|
-
//#endregion
|
|
211
|
-
export { DuckMemory, DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, flowbladeLogtapeSqlduckConfig, getTableCreateFromZod, sqlduckDefaultLogtapeLogger, zodCodecs };
|