@flowblade/sqlduck 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +104 -30
- package/dist/index.d.cts +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.mjs +101 -30
- package/package.json +43 -35
package/dist/index.cjs
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
//#region \0rolldown/runtime.js
|
|
1
3
|
var __create = Object.create;
|
|
2
4
|
var __defProp = Object.defineProperty;
|
|
3
5
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
@@ -18,9 +20,11 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
18
20
|
value: mod,
|
|
19
21
|
enumerable: true
|
|
20
22
|
}) : target, mod));
|
|
23
|
+
//#endregion
|
|
21
24
|
let _duckdb_node_api = require("@duckdb/node-api");
|
|
22
25
|
let zod = require("zod");
|
|
23
26
|
zod = __toESM(zod);
|
|
27
|
+
//#region src/helpers/duck-exec.ts
|
|
24
28
|
var DuckExec = class {
|
|
25
29
|
#conn;
|
|
26
30
|
constructor(duckConn) {
|
|
@@ -48,6 +52,8 @@ var DuckExec = class {
|
|
|
48
52
|
if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
|
|
49
53
|
};
|
|
50
54
|
};
|
|
55
|
+
//#endregion
|
|
56
|
+
//#region src/helpers/duck-memory.ts
|
|
51
57
|
const duckMemoryTags = [
|
|
52
58
|
"BASE_TABLE",
|
|
53
59
|
"HASH_TABLE",
|
|
@@ -62,7 +68,9 @@ const duckMemoryTags = [
|
|
|
62
68
|
"ALLOCATOR",
|
|
63
69
|
"EXTENSION",
|
|
64
70
|
"TRANSACTION",
|
|
65
|
-
"EXTERNAL_FILE_CACHE"
|
|
71
|
+
"EXTERNAL_FILE_CACHE",
|
|
72
|
+
"WINDOW",
|
|
73
|
+
"OBJECT_CACHE"
|
|
66
74
|
];
|
|
67
75
|
const orderByParams = {
|
|
68
76
|
memory_usage_bytes_desc: "memory_usage_bytes DESC",
|
|
@@ -111,35 +119,8 @@ var DuckMemory = class {
|
|
|
111
119
|
return `${query} ORDER BY ${orderByClause}`;
|
|
112
120
|
};
|
|
113
121
|
};
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
return value === void 0 ? null : value;
|
|
117
|
-
};
|
|
118
|
-
async function* rowsToColumnsChunks(rows, chunkSize) {
|
|
119
|
-
if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
|
|
120
|
-
const first = await rows.next();
|
|
121
|
-
if (first.done) return;
|
|
122
|
-
const keys = Object.keys(first.value);
|
|
123
|
-
let columns = keys.map(() => []);
|
|
124
|
-
let rowsInChunk = 0;
|
|
125
|
-
keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
|
|
126
|
-
rowsInChunk++;
|
|
127
|
-
if (rowsInChunk >= chunkSize) {
|
|
128
|
-
yield columns;
|
|
129
|
-
columns = keys.map(() => []);
|
|
130
|
-
rowsInChunk = 0;
|
|
131
|
-
}
|
|
132
|
-
for await (const row of rows) {
|
|
133
|
-
keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
|
|
134
|
-
rowsInChunk++;
|
|
135
|
-
if (rowsInChunk >= chunkSize) {
|
|
136
|
-
yield columns;
|
|
137
|
-
columns = keys.map(() => []);
|
|
138
|
-
rowsInChunk = 0;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
if (rowsInChunk > 0) yield columns;
|
|
142
|
-
}
|
|
122
|
+
//#endregion
|
|
123
|
+
//#region src/appender/data-appender-callback.ts
|
|
143
124
|
const isOnDataAppendedAsyncCb = (v) => {
|
|
144
125
|
return v.constructor.name === "AsyncFunction";
|
|
145
126
|
};
|
|
@@ -159,6 +140,8 @@ const createOnDataAppendedCollector = () => {
|
|
|
159
140
|
return stats;
|
|
160
141
|
};
|
|
161
142
|
};
|
|
143
|
+
//#endregion
|
|
144
|
+
//#region src/table/get-table-create-from-zod.ts
|
|
162
145
|
const createMap = {
|
|
163
146
|
CREATE: "CREATE TABLE",
|
|
164
147
|
CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
|
|
@@ -212,6 +195,8 @@ const getTableCreateFromZod = (table, schema, options) => {
|
|
|
212
195
|
columnTypes
|
|
213
196
|
};
|
|
214
197
|
};
|
|
198
|
+
//#endregion
|
|
199
|
+
//#region src/table/create-table-from-zod.ts
|
|
215
200
|
const createTableFromZod = async (params) => {
|
|
216
201
|
const { conn, table, schema, options } = params;
|
|
217
202
|
const { ddl, columnTypes } = getTableCreateFromZod(table, schema, options);
|
|
@@ -225,6 +210,48 @@ const createTableFromZod = async (params) => {
|
|
|
225
210
|
columnTypes
|
|
226
211
|
};
|
|
227
212
|
};
|
|
213
|
+
//#endregion
|
|
214
|
+
//#region src/utils/rows-to-columns-chunks.ts
|
|
215
|
+
const toDuckValue = (value) => {
|
|
216
|
+
if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
|
|
217
|
+
return value === void 0 ? null : value;
|
|
218
|
+
};
|
|
219
|
+
/**
|
|
220
|
+
* Similar to `rowsToColumns` but yields results in chunks to avoid buffering
|
|
221
|
+
* the entire dataset in memory. Each yielded item is a columns array for up to
|
|
222
|
+
* `chunkSize` rows.
|
|
223
|
+
*
|
|
224
|
+
* Example for chunkSize = 2:
|
|
225
|
+
* input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
|
|
226
|
+
* yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
|
|
227
|
+
*/
|
|
228
|
+
async function* rowsToColumnsChunks(rows, chunkSize) {
|
|
229
|
+
if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
|
|
230
|
+
const first = await rows.next();
|
|
231
|
+
if (first.done) return;
|
|
232
|
+
const keys = Object.keys(first.value);
|
|
233
|
+
let columns = keys.map(() => []);
|
|
234
|
+
let rowsInChunk = 0;
|
|
235
|
+
keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
|
|
236
|
+
rowsInChunk++;
|
|
237
|
+
if (rowsInChunk >= chunkSize) {
|
|
238
|
+
yield columns;
|
|
239
|
+
columns = keys.map(() => []);
|
|
240
|
+
rowsInChunk = 0;
|
|
241
|
+
}
|
|
242
|
+
for await (const row of rows) {
|
|
243
|
+
keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
|
|
244
|
+
rowsInChunk++;
|
|
245
|
+
if (rowsInChunk >= chunkSize) {
|
|
246
|
+
yield columns;
|
|
247
|
+
columns = keys.map(() => []);
|
|
248
|
+
rowsInChunk = 0;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (rowsInChunk > 0) yield columns;
|
|
252
|
+
}
|
|
253
|
+
//#endregion
|
|
254
|
+
//#region src/sql-duck.ts
|
|
228
255
|
var SqlDuck = class {
|
|
229
256
|
#duck;
|
|
230
257
|
#logger;
|
|
@@ -232,6 +259,44 @@ var SqlDuck = class {
|
|
|
232
259
|
this.#duck = params.conn;
|
|
233
260
|
this.#logger = params.logger;
|
|
234
261
|
}
|
|
262
|
+
/**
|
|
263
|
+
* Create a table from a Zod schema and fill it with data from a row stream.
|
|
264
|
+
*
|
|
265
|
+
*
|
|
266
|
+
* @example
|
|
267
|
+
* ```typescript
|
|
268
|
+
* import * as z from 'zod';
|
|
269
|
+
*
|
|
270
|
+
* const sqlDuck = new SqlDuck({ conn: duckDbConnection });
|
|
271
|
+
*
|
|
272
|
+
* // Schema of the table, not that you can use meta to add information
|
|
273
|
+
* const userSchema = z.object({
|
|
274
|
+
* id: z.number().int().meta({ primaryKey: true }),
|
|
275
|
+
* name: z.string(),
|
|
276
|
+
* });
|
|
277
|
+
*
|
|
278
|
+
* // Async generator function that yields rows to insert
|
|
279
|
+
* async function* getUserRows(): AsyncIterableIterator<z.infer<typeof userSchema>> {
|
|
280
|
+
* // database or api call
|
|
281
|
+
* }
|
|
282
|
+
*
|
|
283
|
+
* const result = sqlDuck.toTable({
|
|
284
|
+
* table: new Table({ name: 'user', database: 'mydb' }),
|
|
285
|
+
* schema: userSchema,
|
|
286
|
+
* rowStream: getUserRows(),
|
|
287
|
+
* chunkSize: 2048,
|
|
288
|
+
* onDataAppended: ({ total }) => {
|
|
289
|
+
* console.log(`Appended ${total} rows so far`);
|
|
290
|
+
* },
|
|
291
|
+
* createOptions: {
|
|
292
|
+
* create: 'CREATE_OR_REPLACE',
|
|
293
|
+
* },
|
|
294
|
+
* });
|
|
295
|
+
*
|
|
296
|
+
* console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
|
|
297
|
+
* console.log(`Table created with DDL: ${result.createTableDDL}`);
|
|
298
|
+
* ```
|
|
299
|
+
*/
|
|
235
300
|
toTable = async (params) => {
|
|
236
301
|
const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
|
|
237
302
|
if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
|
|
@@ -267,6 +332,8 @@ var SqlDuck = class {
|
|
|
267
332
|
};
|
|
268
333
|
};
|
|
269
334
|
};
|
|
335
|
+
//#endregion
|
|
336
|
+
//#region src/table/table.ts
|
|
270
337
|
var Table = class Table {
|
|
271
338
|
#fqTable;
|
|
272
339
|
get tableName() {
|
|
@@ -281,6 +348,10 @@ var Table = class Table {
|
|
|
281
348
|
constructor(fqTableOrName) {
|
|
282
349
|
this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
|
|
283
350
|
}
|
|
351
|
+
/**
|
|
352
|
+
* Return fully qualified table name by concatenating
|
|
353
|
+
* database, schema and table with a 'dot' separator.
|
|
354
|
+
*/
|
|
284
355
|
getFullName = (options) => {
|
|
285
356
|
const { defaultDatabase, defaultSchema } = options ?? {};
|
|
286
357
|
const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
|
|
@@ -303,6 +374,8 @@ var Table = class Table {
|
|
|
303
374
|
});
|
|
304
375
|
};
|
|
305
376
|
};
|
|
377
|
+
//#endregion
|
|
378
|
+
//#region src/utils/zod-codecs.ts
|
|
306
379
|
const zodCodecs = {
|
|
307
380
|
dateToString: zod.codec(zod.date(), zod.iso.datetime(), {
|
|
308
381
|
decode: (date) => date.toISOString(),
|
|
@@ -313,6 +386,7 @@ const zodCodecs = {
|
|
|
313
386
|
encode: BigInt
|
|
314
387
|
})
|
|
315
388
|
};
|
|
389
|
+
//#endregion
|
|
316
390
|
exports.DuckMemory = DuckMemory;
|
|
317
391
|
exports.SqlDuck = SqlDuck;
|
|
318
392
|
exports.Table = Table;
|
package/dist/index.d.cts
CHANGED
|
@@ -22,7 +22,7 @@ type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
|
|
|
22
22
|
type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
|
|
23
23
|
//#endregion
|
|
24
24
|
//#region src/helpers/duck-memory.d.ts
|
|
25
|
-
declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE"];
|
|
25
|
+
declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
|
|
26
26
|
type DuckMemoryTag = (typeof duckMemoryTags)[number];
|
|
27
27
|
type DuckMemoryRow = {
|
|
28
28
|
tag: DuckMemoryTag;
|
package/dist/index.d.mts
CHANGED
|
@@ -22,7 +22,7 @@ type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
|
|
|
22
22
|
type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
|
|
23
23
|
//#endregion
|
|
24
24
|
//#region src/helpers/duck-memory.d.ts
|
|
25
|
-
declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE"];
|
|
25
|
+
declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE", "WINDOW", "OBJECT_CACHE"];
|
|
26
26
|
type DuckMemoryTag = (typeof duckMemoryTags)[number];
|
|
27
27
|
type DuckMemoryRow = {
|
|
28
28
|
tag: DuckMemoryTag;
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { BIGINT, DuckDBDataChunk, DuckDBTimestampValue, INTEGER, TIMESTAMP, VARCHAR } from "@duckdb/node-api";
|
|
2
2
|
import * as z from "zod";
|
|
3
|
+
//#region src/helpers/duck-exec.ts
|
|
3
4
|
var DuckExec = class {
|
|
4
5
|
#conn;
|
|
5
6
|
constructor(duckConn) {
|
|
@@ -27,6 +28,8 @@ var DuckExec = class {
|
|
|
27
28
|
if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
|
|
28
29
|
};
|
|
29
30
|
};
|
|
31
|
+
//#endregion
|
|
32
|
+
//#region src/helpers/duck-memory.ts
|
|
30
33
|
const duckMemoryTags = [
|
|
31
34
|
"BASE_TABLE",
|
|
32
35
|
"HASH_TABLE",
|
|
@@ -41,7 +44,9 @@ const duckMemoryTags = [
|
|
|
41
44
|
"ALLOCATOR",
|
|
42
45
|
"EXTENSION",
|
|
43
46
|
"TRANSACTION",
|
|
44
|
-
"EXTERNAL_FILE_CACHE"
|
|
47
|
+
"EXTERNAL_FILE_CACHE",
|
|
48
|
+
"WINDOW",
|
|
49
|
+
"OBJECT_CACHE"
|
|
45
50
|
];
|
|
46
51
|
const orderByParams = {
|
|
47
52
|
memory_usage_bytes_desc: "memory_usage_bytes DESC",
|
|
@@ -90,35 +95,8 @@ var DuckMemory = class {
|
|
|
90
95
|
return `${query} ORDER BY ${orderByClause}`;
|
|
91
96
|
};
|
|
92
97
|
};
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
return value === void 0 ? null : value;
|
|
96
|
-
};
|
|
97
|
-
async function* rowsToColumnsChunks(rows, chunkSize) {
|
|
98
|
-
if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
|
|
99
|
-
const first = await rows.next();
|
|
100
|
-
if (first.done) return;
|
|
101
|
-
const keys = Object.keys(first.value);
|
|
102
|
-
let columns = keys.map(() => []);
|
|
103
|
-
let rowsInChunk = 0;
|
|
104
|
-
keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
|
|
105
|
-
rowsInChunk++;
|
|
106
|
-
if (rowsInChunk >= chunkSize) {
|
|
107
|
-
yield columns;
|
|
108
|
-
columns = keys.map(() => []);
|
|
109
|
-
rowsInChunk = 0;
|
|
110
|
-
}
|
|
111
|
-
for await (const row of rows) {
|
|
112
|
-
keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
|
|
113
|
-
rowsInChunk++;
|
|
114
|
-
if (rowsInChunk >= chunkSize) {
|
|
115
|
-
yield columns;
|
|
116
|
-
columns = keys.map(() => []);
|
|
117
|
-
rowsInChunk = 0;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
if (rowsInChunk > 0) yield columns;
|
|
121
|
-
}
|
|
98
|
+
//#endregion
|
|
99
|
+
//#region src/appender/data-appender-callback.ts
|
|
122
100
|
const isOnDataAppendedAsyncCb = (v) => {
|
|
123
101
|
return v.constructor.name === "AsyncFunction";
|
|
124
102
|
};
|
|
@@ -138,6 +116,8 @@ const createOnDataAppendedCollector = () => {
|
|
|
138
116
|
return stats;
|
|
139
117
|
};
|
|
140
118
|
};
|
|
119
|
+
//#endregion
|
|
120
|
+
//#region src/table/get-table-create-from-zod.ts
|
|
141
121
|
const createMap = {
|
|
142
122
|
CREATE: "CREATE TABLE",
|
|
143
123
|
CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
|
|
@@ -191,6 +171,8 @@ const getTableCreateFromZod = (table, schema, options) => {
|
|
|
191
171
|
columnTypes
|
|
192
172
|
};
|
|
193
173
|
};
|
|
174
|
+
//#endregion
|
|
175
|
+
//#region src/table/create-table-from-zod.ts
|
|
194
176
|
const createTableFromZod = async (params) => {
|
|
195
177
|
const { conn, table, schema, options } = params;
|
|
196
178
|
const { ddl, columnTypes } = getTableCreateFromZod(table, schema, options);
|
|
@@ -204,6 +186,48 @@ const createTableFromZod = async (params) => {
|
|
|
204
186
|
columnTypes
|
|
205
187
|
};
|
|
206
188
|
};
|
|
189
|
+
//#endregion
|
|
190
|
+
//#region src/utils/rows-to-columns-chunks.ts
|
|
191
|
+
const toDuckValue = (value) => {
|
|
192
|
+
if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
|
|
193
|
+
return value === void 0 ? null : value;
|
|
194
|
+
};
|
|
195
|
+
/**
|
|
196
|
+
* Similar to `rowsToColumns` but yields results in chunks to avoid buffering
|
|
197
|
+
* the entire dataset in memory. Each yielded item is a columns array for up to
|
|
198
|
+
* `chunkSize` rows.
|
|
199
|
+
*
|
|
200
|
+
* Example for chunkSize = 2:
|
|
201
|
+
* input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
|
|
202
|
+
* yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
|
|
203
|
+
*/
|
|
204
|
+
async function* rowsToColumnsChunks(rows, chunkSize) {
|
|
205
|
+
if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
|
|
206
|
+
const first = await rows.next();
|
|
207
|
+
if (first.done) return;
|
|
208
|
+
const keys = Object.keys(first.value);
|
|
209
|
+
let columns = keys.map(() => []);
|
|
210
|
+
let rowsInChunk = 0;
|
|
211
|
+
keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
|
|
212
|
+
rowsInChunk++;
|
|
213
|
+
if (rowsInChunk >= chunkSize) {
|
|
214
|
+
yield columns;
|
|
215
|
+
columns = keys.map(() => []);
|
|
216
|
+
rowsInChunk = 0;
|
|
217
|
+
}
|
|
218
|
+
for await (const row of rows) {
|
|
219
|
+
keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
|
|
220
|
+
rowsInChunk++;
|
|
221
|
+
if (rowsInChunk >= chunkSize) {
|
|
222
|
+
yield columns;
|
|
223
|
+
columns = keys.map(() => []);
|
|
224
|
+
rowsInChunk = 0;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
if (rowsInChunk > 0) yield columns;
|
|
228
|
+
}
|
|
229
|
+
//#endregion
|
|
230
|
+
//#region src/sql-duck.ts
|
|
207
231
|
var SqlDuck = class {
|
|
208
232
|
#duck;
|
|
209
233
|
#logger;
|
|
@@ -211,6 +235,44 @@ var SqlDuck = class {
|
|
|
211
235
|
this.#duck = params.conn;
|
|
212
236
|
this.#logger = params.logger;
|
|
213
237
|
}
|
|
238
|
+
/**
|
|
239
|
+
* Create a table from a Zod schema and fill it with data from a row stream.
|
|
240
|
+
*
|
|
241
|
+
*
|
|
242
|
+
* @example
|
|
243
|
+
* ```typescript
|
|
244
|
+
* import * as z from 'zod';
|
|
245
|
+
*
|
|
246
|
+
* const sqlDuck = new SqlDuck({ conn: duckDbConnection });
|
|
247
|
+
*
|
|
248
|
+
* // Schema of the table, not that you can use meta to add information
|
|
249
|
+
* const userSchema = z.object({
|
|
250
|
+
* id: z.number().int().meta({ primaryKey: true }),
|
|
251
|
+
* name: z.string(),
|
|
252
|
+
* });
|
|
253
|
+
*
|
|
254
|
+
* // Async generator function that yields rows to insert
|
|
255
|
+
* async function* getUserRows(): AsyncIterableIterator<z.infer<typeof userSchema>> {
|
|
256
|
+
* // database or api call
|
|
257
|
+
* }
|
|
258
|
+
*
|
|
259
|
+
* const result = sqlDuck.toTable({
|
|
260
|
+
* table: new Table({ name: 'user', database: 'mydb' }),
|
|
261
|
+
* schema: userSchema,
|
|
262
|
+
* rowStream: getUserRows(),
|
|
263
|
+
* chunkSize: 2048,
|
|
264
|
+
* onDataAppended: ({ total }) => {
|
|
265
|
+
* console.log(`Appended ${total} rows so far`);
|
|
266
|
+
* },
|
|
267
|
+
* createOptions: {
|
|
268
|
+
* create: 'CREATE_OR_REPLACE',
|
|
269
|
+
* },
|
|
270
|
+
* });
|
|
271
|
+
*
|
|
272
|
+
* console.log(`Inserted ${result.totalRows} rows in ${result.timeMs}ms`);
|
|
273
|
+
* console.log(`Table created with DDL: ${result.createTableDDL}`);
|
|
274
|
+
* ```
|
|
275
|
+
*/
|
|
214
276
|
toTable = async (params) => {
|
|
215
277
|
const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
|
|
216
278
|
if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
|
|
@@ -246,6 +308,8 @@ var SqlDuck = class {
|
|
|
246
308
|
};
|
|
247
309
|
};
|
|
248
310
|
};
|
|
311
|
+
//#endregion
|
|
312
|
+
//#region src/table/table.ts
|
|
249
313
|
var Table = class Table {
|
|
250
314
|
#fqTable;
|
|
251
315
|
get tableName() {
|
|
@@ -260,6 +324,10 @@ var Table = class Table {
|
|
|
260
324
|
constructor(fqTableOrName) {
|
|
261
325
|
this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
|
|
262
326
|
}
|
|
327
|
+
/**
|
|
328
|
+
* Return fully qualified table name by concatenating
|
|
329
|
+
* database, schema and table with a 'dot' separator.
|
|
330
|
+
*/
|
|
263
331
|
getFullName = (options) => {
|
|
264
332
|
const { defaultDatabase, defaultSchema } = options ?? {};
|
|
265
333
|
const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
|
|
@@ -282,6 +350,8 @@ var Table = class Table {
|
|
|
282
350
|
});
|
|
283
351
|
};
|
|
284
352
|
};
|
|
353
|
+
//#endregion
|
|
354
|
+
//#region src/utils/zod-codecs.ts
|
|
285
355
|
const zodCodecs = {
|
|
286
356
|
dateToString: z.codec(z.date(), z.iso.datetime(), {
|
|
287
357
|
decode: (date) => date.toISOString(),
|
|
@@ -292,4 +362,5 @@ const zodCodecs = {
|
|
|
292
362
|
encode: BigInt
|
|
293
363
|
})
|
|
294
364
|
};
|
|
365
|
+
//#endregion
|
|
295
366
|
export { DuckMemory, SqlDuck, Table, getTableCreateFromZod, zodCodecs };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flowblade/sqlduck",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"exports": {
|
|
@@ -33,6 +33,12 @@
|
|
|
33
33
|
"build-release": "yarn build && rimraf ./_release && yarn pack && mkdir ./_release && tar zxvf ./package.tgz --directory ./_release && rm ./package.tgz",
|
|
34
34
|
"docgen": "run-s docgen-typedoc",
|
|
35
35
|
"docgen-typedoc": "rimraf ./docs/api && typedoc --plugin typedoc-plugin-markdown --out ./docs/api",
|
|
36
|
+
"bench": "vitest bench --run",
|
|
37
|
+
"bench-bun": "bun --bun run vitest bench --run",
|
|
38
|
+
"bench-mitata": "node --experimental-strip-types --expose-gc ./bench/stream.mitata.ts ",
|
|
39
|
+
"bench-mitata-bun": "bun --expose-gc ./bench/stream.mitata.ts ",
|
|
40
|
+
"bench-codspeed": "cross-env CODSPEED=1 vitest bench --run",
|
|
41
|
+
"bench-watch": "vitest bench",
|
|
36
42
|
"test": "vitest run",
|
|
37
43
|
"test-unit": "vitest run",
|
|
38
44
|
"test-unit-bun": "bun --bun run vitest run",
|
|
@@ -48,61 +54,63 @@
|
|
|
48
54
|
"check-size": "size-limit"
|
|
49
55
|
},
|
|
50
56
|
"dependencies": {
|
|
51
|
-
"@flowblade/core": "^0.2.
|
|
52
|
-
"@flowblade/source-duckdb": "^0.
|
|
53
|
-
"@flowblade/sql-tag": "^0.3.
|
|
57
|
+
"@flowblade/core": "^0.2.26",
|
|
58
|
+
"@flowblade/source-duckdb": "^0.19.0",
|
|
59
|
+
"@flowblade/sql-tag": "^0.3.2",
|
|
54
60
|
"@standard-schema/spec": "^1.1.0",
|
|
55
61
|
"p-mutex": "^1.0.0",
|
|
56
62
|
"valibot": "^1.2.0",
|
|
57
|
-
"zod": "^4.3.
|
|
63
|
+
"zod": "^4.3.6"
|
|
58
64
|
},
|
|
59
65
|
"peerDependencies": {
|
|
60
|
-
"@duckdb/node-api": "^1.
|
|
66
|
+
"@duckdb/node-api": "^1.5.0-r.1"
|
|
61
67
|
},
|
|
62
68
|
"devDependencies": {
|
|
63
|
-
"@belgattitude/eslint-config-bases": "8.
|
|
64
|
-
"@dotenvx/dotenvx": "1.
|
|
65
|
-
"@duckdb/node-api": "1.
|
|
66
|
-
"@faker-js/faker": "10.
|
|
67
|
-
"@flowblade/source-kysely": "^1.2.
|
|
69
|
+
"@belgattitude/eslint-config-bases": "8.10.0",
|
|
70
|
+
"@dotenvx/dotenvx": "1.54.1",
|
|
71
|
+
"@duckdb/node-api": "1.5.0-r.1",
|
|
72
|
+
"@faker-js/faker": "10.3.0",
|
|
73
|
+
"@flowblade/source-kysely": "^1.2.3",
|
|
68
74
|
"@httpx/assert": "0.16.7",
|
|
69
|
-
"@
|
|
70
|
-
"@size-limit/
|
|
71
|
-
"@
|
|
75
|
+
"@mitata/counters": "0.0.8",
|
|
76
|
+
"@size-limit/esbuild": "12.0.1",
|
|
77
|
+
"@size-limit/file": "12.0.1",
|
|
78
|
+
"@testcontainers/mssqlserver": "11.12.0",
|
|
72
79
|
"@total-typescript/ts-reset": "0.6.1",
|
|
73
80
|
"@traversable/zod": "0.0.57",
|
|
74
|
-
"@types/node": "25.0.
|
|
75
|
-
"@typescript-eslint/eslint-plugin": "8.
|
|
76
|
-
"@typescript-eslint/parser": "8.
|
|
77
|
-
"@vitest/coverage-v8": "4.0.
|
|
78
|
-
"@vitest/ui": "4.0.
|
|
81
|
+
"@types/node": "25.0.10",
|
|
82
|
+
"@typescript-eslint/eslint-plugin": "8.57.0",
|
|
83
|
+
"@typescript-eslint/parser": "8.57.0",
|
|
84
|
+
"@vitest/coverage-v8": "4.0.18",
|
|
85
|
+
"@vitest/ui": "4.0.18",
|
|
79
86
|
"ansis": "4.2.0",
|
|
80
87
|
"browserslist-to-esbuild": "2.1.1",
|
|
81
|
-
"core-js": "3.
|
|
88
|
+
"core-js": "3.48.0",
|
|
82
89
|
"cross-env": "10.1.0",
|
|
83
|
-
"es-check": "9.
|
|
84
|
-
"esbuild": "0.27.
|
|
90
|
+
"es-check": "9.6.2",
|
|
91
|
+
"esbuild": "0.27.3",
|
|
85
92
|
"eslint": "8.57.1",
|
|
86
93
|
"execa": "9.6.1",
|
|
87
94
|
"is-in-ci": "2.0.0",
|
|
88
|
-
"kysely": "0.28.
|
|
95
|
+
"kysely": "0.28.11",
|
|
96
|
+
"mitata": "1.0.34",
|
|
89
97
|
"npm-run-all2": "8.0.4",
|
|
90
|
-
"prettier": "3.8.
|
|
91
|
-
"publint": "0.3.
|
|
98
|
+
"prettier": "3.8.1",
|
|
99
|
+
"publint": "0.3.18",
|
|
92
100
|
"regexp.escape": "2.0.1",
|
|
93
|
-
"rimraf": "6.1.
|
|
94
|
-
"size-limit": "12.0.
|
|
95
|
-
"sql-formatter": "15.7.
|
|
101
|
+
"rimraf": "6.1.3",
|
|
102
|
+
"size-limit": "12.0.1",
|
|
103
|
+
"sql-formatter": "15.7.2",
|
|
96
104
|
"tarn": "3.0.2",
|
|
97
|
-
"tedious": "19.2.
|
|
98
|
-
"testcontainers": "11.
|
|
99
|
-
"tsdown": "0.
|
|
105
|
+
"tedious": "19.2.1",
|
|
106
|
+
"testcontainers": "11.12.0",
|
|
107
|
+
"tsdown": "0.21.1",
|
|
100
108
|
"tsx": "4.21.0",
|
|
101
|
-
"typedoc": "0.28.
|
|
102
|
-
"typedoc-plugin-markdown": "4.
|
|
109
|
+
"typedoc": "0.28.17",
|
|
110
|
+
"typedoc-plugin-markdown": "4.10.0",
|
|
103
111
|
"typescript": "5.9.3",
|
|
104
|
-
"vite-tsconfig-paths": "6.
|
|
105
|
-
"vitest": "4.0.
|
|
112
|
+
"vite-tsconfig-paths": "6.1.1",
|
|
113
|
+
"vitest": "4.0.18"
|
|
106
114
|
},
|
|
107
115
|
"files": [
|
|
108
116
|
"dist"
|