@flowblade/sqlduck 0.8.3 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -0
- package/dist/index.cjs +70 -15
- package/dist/index.d.cts +13 -6
- package/dist/index.d.mts +13 -6
- package/dist/index.mjs +71 -16
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -54,6 +54,72 @@ const queryResult = await dbDuckDbMemoryConn.query<{
|
|
|
54
54
|
`);
|
|
55
55
|
```
|
|
56
56
|
|
|
57
|
+
## Benchmarks
|
|
58
|
+
|
|
59
|
+
### Node 24
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
RUN v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
✓ bench/appender.bench.ts > appender benches 66030ms
|
|
66
|
+
name hz min max mean p75 p99 p995 p999 rme samples
|
|
67
|
+
· duckdb appender, count: 1000000, chunk size 2048 0.0642 15,577.01 15,577.01 15,577.01 15,577.01 15,577.01 15,577.01 15,577.01 ±0.00% 1
|
|
68
|
+
· duckdb appender, count: 1000000, chunk size 1024 0.0579 17,263.44 17,263.44 17,263.44 17,263.44 17,263.44 17,263.44 17,263.44 ±0.00% 1
|
|
69
|
+
|
|
70
|
+
✓ bench/stream.bench.ts > Bench stream 22923ms
|
|
71
|
+
name hz min max mean p75 p99 p995 p999 rme samples
|
|
72
|
+
· rowToColumnsChunk with chunkSize 2048 (count: 1000000) 1.2126 742.59 906.65 824.67 863.91 906.65 906.65 906.65 ±4.26% 10
|
|
73
|
+
· mapFakeRowStream with chunkSize 2048 (count: 1000000) 0.8049 1,123.18 1,498.68 1,242.43 1,257.91 1,498.68 1,498.68 1,498.68 ±6.40% 10
|
|
74
|
+
|
|
75
|
+
✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 615ms
|
|
76
|
+
name hz min max mean p75 p99 p995 p999 rme samples
|
|
77
|
+
· getTableCreateFromZod 16,562.93 0.0242 2.5902 0.0604 0.0734 0.2555 0.3741 0.8135 ±2.32% 8282
|
|
78
|
+
|
|
79
|
+
BENCH Summary
|
|
80
|
+
|
|
81
|
+
duckdb appender, count: 1000000, chunk size 2048 - bench/appender.bench.ts > appender benches
|
|
82
|
+
1.11x faster than duckdb appender, count: 1000000, chunk size 1024
|
|
83
|
+
|
|
84
|
+
rowToColumnsChunk with chunkSize 2048 (count: 1000000) - bench/stream.bench.ts > Bench stream
|
|
85
|
+
1.51x faster than mapFakeRowStream with chunkSize 2048 (count: 1000000)
|
|
86
|
+
|
|
87
|
+
getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Bun 1.3.11
|
|
92
|
+
|
|
93
|
+
```
|
|
94
|
+
RUN v4.1.1 /home/sebastien/github/flowblade/packages/sqlduck
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
✓ bench/appender.bench.ts > appender benches 36627ms
|
|
98
|
+
name hz min max mean p75 p99 p995 p999 rme samples
|
|
99
|
+
· duckdb appender, count: 1000000, chunk size 2048 0.1177 8,495.41 8,495.41 8,495.41 8,495.41 8,495.41 8,495.41 8,495.41 ±0.00% 1
|
|
100
|
+
· duckdb appender, count: 1000000, chunk size 1024 0.1064 9,397.97 9,397.97 9,397.97 9,397.97 9,397.97 9,397.97 9,397.97 ±0.00% 1
|
|
101
|
+
|
|
102
|
+
✓ bench/stream.bench.ts > Bench stream 23421ms
|
|
103
|
+
name hz min max mean p75 p99 p995 p999 rme samples
|
|
104
|
+
· rowToColumnsChunk with chunkSize 2048 (count: 1000000) 1.1378 801.60 1,080.22 878.91 910.91 1,080.22 1,080.22 1,080.22 ±6.85% 10
|
|
105
|
+
· mapFakeRowStream with chunkSize 2048 (count: 1000000) 0.8118 1,130.36 1,448.99 1,231.78 1,268.45 1,448.99 1,448.99 1,448.99 ±5.34% 10
|
|
106
|
+
|
|
107
|
+
✓ bench/table-create.bench.ts > Bench getTableCreateFromZod 622ms
|
|
108
|
+
name hz min max mean p75 p99 p995 p999 rme samples
|
|
109
|
+
· getTableCreateFromZod 22,447.94 0.0210 5.4621 0.0445 0.0442 0.1657 0.2167 2.5852 ±5.37% 11224
|
|
110
|
+
|
|
111
|
+
BENCH Summary
|
|
112
|
+
|
|
113
|
+
rowToColumnsChunk with chunkSize 2048 (count: 1000000) - bench/stream.bench.ts > Bench stream
|
|
114
|
+
1.40x faster than mapFakeRowStream with chunkSize 2048 (count: 1000000)
|
|
115
|
+
|
|
116
|
+
getTableCreateFromZod - bench/table-create.bench.ts > Bench getTableCreateFromZod
|
|
117
|
+
|
|
118
|
+
duckdb appender, count: 1000000, chunk size 2048 - bench/appender.bench.ts > appender benches
|
|
119
|
+
1.11x faster than duckdb appender, count: 1000000, chunk size 1024
|
|
120
|
+
|
|
121
|
+
```
|
|
122
|
+
|
|
57
123
|
### Local scripts
|
|
58
124
|
|
|
59
125
|
| Name | Description |
|
package/dist/index.cjs
CHANGED
|
@@ -141,21 +141,52 @@ const createOnDataAppendedCollector = () => {
|
|
|
141
141
|
};
|
|
142
142
|
};
|
|
143
143
|
//#endregion
|
|
144
|
+
//#region src/table/get-duckdb-number-column-type.ts
|
|
145
|
+
const isFloatValue = (value) => {
|
|
146
|
+
if (!Number.isFinite(value)) return true;
|
|
147
|
+
if (Math.abs(value) > Number.MAX_SAFE_INTEGER) return true;
|
|
148
|
+
return !Number.isInteger(value);
|
|
149
|
+
};
|
|
150
|
+
const getDuckdbNumberColumnType = (params) => {
|
|
151
|
+
const { minimum, maximum } = params;
|
|
152
|
+
if (minimum === void 0 || maximum === void 0) return _duckdb_node_api.BIGINT;
|
|
153
|
+
if (isFloatValue(minimum) || isFloatValue(maximum)) {
|
|
154
|
+
if (minimum >= -34028235e31 && maximum <= 34028235e31) return _duckdb_node_api.FLOAT;
|
|
155
|
+
return _duckdb_node_api.DOUBLE;
|
|
156
|
+
}
|
|
157
|
+
if (minimum >= 0) {
|
|
158
|
+
if (maximum <= 255) return _duckdb_node_api.UTINYINT;
|
|
159
|
+
if (maximum <= 65535) return _duckdb_node_api.USMALLINT;
|
|
160
|
+
if (maximum <= 4294967295) return _duckdb_node_api.UINTEGER;
|
|
161
|
+
if (maximum <= 18446744073709551615n) return _duckdb_node_api.UBIGINT;
|
|
162
|
+
return _duckdb_node_api.UHUGEINT;
|
|
163
|
+
}
|
|
164
|
+
if (minimum >= -128 && maximum <= 127) return _duckdb_node_api.TINYINT;
|
|
165
|
+
if (minimum >= -32768 && maximum <= 32767) return _duckdb_node_api.SMALLINT;
|
|
166
|
+
if (minimum >= -2147483648 && maximum <= 2147483647) return _duckdb_node_api.INTEGER;
|
|
167
|
+
if (minimum >= -9223372036854775808n && maximum <= 9223372036854775807n) return _duckdb_node_api.BIGINT;
|
|
168
|
+
return _duckdb_node_api.HUGEINT;
|
|
169
|
+
};
|
|
170
|
+
//#endregion
|
|
144
171
|
//#region src/table/get-table-create-from-zod.ts
|
|
145
|
-
const
|
|
172
|
+
const createOptions = {
|
|
146
173
|
CREATE: "CREATE TABLE",
|
|
147
174
|
CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
|
|
148
175
|
IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
|
|
149
176
|
};
|
|
150
|
-
const getTableCreateFromZod = (
|
|
177
|
+
const getTableCreateFromZod = (params) => {
|
|
178
|
+
const { table, schema, options } = params;
|
|
151
179
|
const { create = "CREATE" } = options ?? {};
|
|
152
180
|
const fqTable = table.getFullName();
|
|
153
|
-
const json = schema.toJSONSchema({
|
|
181
|
+
const json = schema.toJSONSchema({
|
|
182
|
+
target: "openapi-3.0",
|
|
183
|
+
unrepresentable: "throw"
|
|
184
|
+
});
|
|
154
185
|
const columns = [];
|
|
155
186
|
if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
|
|
156
|
-
const
|
|
187
|
+
const columnTypesMap = /* @__PURE__ */ new Map();
|
|
157
188
|
for (const [columnName, def] of Object.entries(json.properties)) {
|
|
158
|
-
const { type, nullable, format, primaryKey } = def;
|
|
189
|
+
const { type, nullable, format, primaryKey, minimum, maximum } = def;
|
|
159
190
|
const c = { name: columnName };
|
|
160
191
|
switch (type) {
|
|
161
192
|
case "string":
|
|
@@ -166,22 +197,37 @@ const getTableCreateFromZod = (table, schema, options) => {
|
|
|
166
197
|
case "int64":
|
|
167
198
|
c.duckdbType = _duckdb_node_api.BIGINT;
|
|
168
199
|
break;
|
|
200
|
+
case "uuid":
|
|
201
|
+
c.duckdbType = _duckdb_node_api.UUID;
|
|
202
|
+
break;
|
|
169
203
|
default: c.duckdbType = _duckdb_node_api.VARCHAR;
|
|
170
204
|
}
|
|
171
205
|
break;
|
|
172
206
|
case "number":
|
|
173
|
-
c.duckdbType =
|
|
207
|
+
c.duckdbType = getDuckdbNumberColumnType({
|
|
208
|
+
minimum,
|
|
209
|
+
maximum
|
|
210
|
+
});
|
|
174
211
|
break;
|
|
175
|
-
|
|
212
|
+
case "integer":
|
|
213
|
+
c.duckdbType = getDuckdbNumberColumnType({
|
|
214
|
+
minimum,
|
|
215
|
+
maximum
|
|
216
|
+
});
|
|
217
|
+
break;
|
|
218
|
+
case "boolean":
|
|
219
|
+
c.duckdbType = _duckdb_node_api.BOOLEAN;
|
|
220
|
+
break;
|
|
221
|
+
default: throw new Error(`Cannot guess '${columnName}' type - ${JSON.stringify(def)}`);
|
|
176
222
|
}
|
|
177
223
|
if (primaryKey === true) c.constraint = "PRIMARY KEY";
|
|
178
224
|
else if (nullable !== true) c.constraint = "NOT NULL";
|
|
179
|
-
|
|
225
|
+
columnTypesMap.set(columnName, c.duckdbType);
|
|
180
226
|
columns.push(c);
|
|
181
227
|
}
|
|
182
228
|
return {
|
|
183
229
|
ddl: [
|
|
184
|
-
`${
|
|
230
|
+
`${createOptions[create]} ${fqTable} (\n`,
|
|
185
231
|
columns.map((colDDL) => {
|
|
186
232
|
const { name, duckdbType, constraint } = colDDL;
|
|
187
233
|
return ` ${[
|
|
@@ -192,14 +238,18 @@ const getTableCreateFromZod = (table, schema, options) => {
|
|
|
192
238
|
}).join(",\n"),
|
|
193
239
|
"\n)"
|
|
194
240
|
].join(""),
|
|
195
|
-
columnTypes
|
|
241
|
+
columnTypes: columnTypesMap
|
|
196
242
|
};
|
|
197
243
|
};
|
|
198
244
|
//#endregion
|
|
199
245
|
//#region src/table/create-table-from-zod.ts
|
|
200
246
|
const createTableFromZod = async (params) => {
|
|
201
247
|
const { conn, table, schema, options } = params;
|
|
202
|
-
const { ddl, columnTypes } = getTableCreateFromZod(
|
|
248
|
+
const { ddl, columnTypes } = getTableCreateFromZod({
|
|
249
|
+
table,
|
|
250
|
+
schema,
|
|
251
|
+
options
|
|
252
|
+
});
|
|
203
253
|
try {
|
|
204
254
|
await conn.run(ddl);
|
|
205
255
|
} catch (e) {
|
|
@@ -214,6 +264,7 @@ const createTableFromZod = async (params) => {
|
|
|
214
264
|
//#region src/utils/rows-to-columns-chunks.ts
|
|
215
265
|
const toDuckValue = (value) => {
|
|
216
266
|
if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
|
|
267
|
+
if (typeof value === "bigint") return value.toString(10);
|
|
217
268
|
return value === void 0 ? null : value;
|
|
218
269
|
};
|
|
219
270
|
/**
|
|
@@ -225,7 +276,8 @@ const toDuckValue = (value) => {
|
|
|
225
276
|
* input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
|
|
226
277
|
* yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
|
|
227
278
|
*/
|
|
228
|
-
async function* rowsToColumnsChunks(
|
|
279
|
+
async function* rowsToColumnsChunks(params) {
|
|
280
|
+
const { rows, chunkSize } = params;
|
|
229
281
|
if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
|
|
230
282
|
const first = await rows.next();
|
|
231
283
|
if (first.done) return;
|
|
@@ -262,7 +314,6 @@ var SqlDuck = class {
|
|
|
262
314
|
/**
|
|
263
315
|
* Create a table from a Zod schema and fill it with data from a row stream.
|
|
264
316
|
*
|
|
265
|
-
*
|
|
266
317
|
* @example
|
|
267
318
|
* ```typescript
|
|
268
319
|
* import * as z from 'zod';
|
|
@@ -308,10 +359,13 @@ var SqlDuck = class {
|
|
|
308
359
|
options: createOptions
|
|
309
360
|
});
|
|
310
361
|
const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
|
|
311
|
-
const chunkTypes = columnTypes.
|
|
362
|
+
const chunkTypes = Array.from(columnTypes.values());
|
|
312
363
|
let totalRows = 0;
|
|
313
364
|
const dataAppendedCollector = createOnDataAppendedCollector();
|
|
314
|
-
const columnStream = rowsToColumnsChunks(
|
|
365
|
+
const columnStream = rowsToColumnsChunks({
|
|
366
|
+
rows: rowStream,
|
|
367
|
+
chunkSize
|
|
368
|
+
});
|
|
315
369
|
for await (const dataChunk of columnStream) {
|
|
316
370
|
const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
|
|
317
371
|
if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
|
|
@@ -325,6 +379,7 @@ var SqlDuck = class {
|
|
|
325
379
|
else onDataAppended(payload);
|
|
326
380
|
}
|
|
327
381
|
}
|
|
382
|
+
appender.closeSync();
|
|
328
383
|
return {
|
|
329
384
|
timeMs: Math.round(Date.now() - timeStart),
|
|
330
385
|
totalRows,
|
package/dist/index.d.cts
CHANGED
|
@@ -76,17 +76,25 @@ declare class Table {
|
|
|
76
76
|
withSchema: (schema: string) => Table;
|
|
77
77
|
}
|
|
78
78
|
//#endregion
|
|
79
|
+
//#region src/table/table-schema-zod.type.d.ts
|
|
80
|
+
type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID;
|
|
81
|
+
type TableSchemaZod = z.ZodObject<Record<string, ZodSchemaSupportedTypes | z.ZodNullable<ZodSchemaSupportedTypes> | z.ZodCodec | z.ZodNullable<z.ZodCodec>>>;
|
|
82
|
+
//#endregion
|
|
79
83
|
//#region src/table/get-table-create-from-zod.d.ts
|
|
80
84
|
type TableCreateOptions = {
|
|
81
85
|
create?: 'CREATE' | 'CREATE_OR_REPLACE' | 'IF_NOT_EXISTS';
|
|
82
86
|
};
|
|
83
|
-
|
|
87
|
+
type DuckdbColumnTypeMap<TKeys extends string> = Map<TKeys, DuckDBType>;
|
|
88
|
+
type TableCreateFromZodResult<TSchema extends TableSchemaZod> = {
|
|
84
89
|
ddl: string;
|
|
85
|
-
columnTypes: [
|
|
90
|
+
columnTypes: DuckdbColumnTypeMap<Exclude<keyof TSchema['shape'], symbol | number>>;
|
|
86
91
|
};
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
92
|
+
type GetTableCreateFromZodParams<TSchema extends TableSchemaZod> = {
|
|
93
|
+
table: Table;
|
|
94
|
+
schema: TSchema;
|
|
95
|
+
options?: TableCreateOptions;
|
|
96
|
+
};
|
|
97
|
+
declare const getTableCreateFromZod: <TSchema extends TableSchemaZod>(params: GetTableCreateFromZodParams<TSchema>) => TableCreateFromZodResult<TSchema>;
|
|
90
98
|
//#endregion
|
|
91
99
|
//#region src/sql-duck.d.ts
|
|
92
100
|
type SqlDuckParams = {
|
|
@@ -142,7 +150,6 @@ declare class SqlDuck {
|
|
|
142
150
|
/**
|
|
143
151
|
* Create a table from a Zod schema and fill it with data from a row stream.
|
|
144
152
|
*
|
|
145
|
-
*
|
|
146
153
|
* @example
|
|
147
154
|
* ```typescript
|
|
148
155
|
* import * as z from 'zod';
|
package/dist/index.d.mts
CHANGED
|
@@ -76,17 +76,25 @@ declare class Table {
|
|
|
76
76
|
withSchema: (schema: string) => Table;
|
|
77
77
|
}
|
|
78
78
|
//#endregion
|
|
79
|
+
//#region src/table/table-schema-zod.type.d.ts
|
|
80
|
+
type ZodSchemaSupportedTypes = z.ZodString | z.ZodNumber | z.ZodInt | z.ZodInt32 | z.ZodUInt32 | z.ZodBigInt | z.ZodBoolean | z.ZodDate | z.ZodISODateTime | z.ZodISOTime | z.ZodISODate | z.ZodEmail | z.ZodURL | z.ZodUUID | z.ZodCUID | z.ZodCUID2 | z.ZodULID;
|
|
81
|
+
type TableSchemaZod = z.ZodObject<Record<string, ZodSchemaSupportedTypes | z.ZodNullable<ZodSchemaSupportedTypes> | z.ZodCodec | z.ZodNullable<z.ZodCodec>>>;
|
|
82
|
+
//#endregion
|
|
79
83
|
//#region src/table/get-table-create-from-zod.d.ts
|
|
80
84
|
type TableCreateOptions = {
|
|
81
85
|
create?: 'CREATE' | 'CREATE_OR_REPLACE' | 'IF_NOT_EXISTS';
|
|
82
86
|
};
|
|
83
|
-
|
|
87
|
+
type DuckdbColumnTypeMap<TKeys extends string> = Map<TKeys, DuckDBType>;
|
|
88
|
+
type TableCreateFromZodResult<TSchema extends TableSchemaZod> = {
|
|
84
89
|
ddl: string;
|
|
85
|
-
columnTypes: [
|
|
90
|
+
columnTypes: DuckdbColumnTypeMap<Exclude<keyof TSchema['shape'], symbol | number>>;
|
|
86
91
|
};
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
92
|
+
type GetTableCreateFromZodParams<TSchema extends TableSchemaZod> = {
|
|
93
|
+
table: Table;
|
|
94
|
+
schema: TSchema;
|
|
95
|
+
options?: TableCreateOptions;
|
|
96
|
+
};
|
|
97
|
+
declare const getTableCreateFromZod: <TSchema extends TableSchemaZod>(params: GetTableCreateFromZodParams<TSchema>) => TableCreateFromZodResult<TSchema>;
|
|
90
98
|
//#endregion
|
|
91
99
|
//#region src/sql-duck.d.ts
|
|
92
100
|
type SqlDuckParams = {
|
|
@@ -142,7 +150,6 @@ declare class SqlDuck {
|
|
|
142
150
|
/**
|
|
143
151
|
* Create a table from a Zod schema and fill it with data from a row stream.
|
|
144
152
|
*
|
|
145
|
-
*
|
|
146
153
|
* @example
|
|
147
154
|
* ```typescript
|
|
148
155
|
* import * as z from 'zod';
|
package/dist/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BIGINT, DuckDBDataChunk, DuckDBTimestampValue, INTEGER, TIMESTAMP, VARCHAR } from "@duckdb/node-api";
|
|
1
|
+
import { BIGINT, BOOLEAN, DOUBLE, DuckDBDataChunk, DuckDBTimestampValue, FLOAT, HUGEINT, INTEGER, SMALLINT, TIMESTAMP, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR } from "@duckdb/node-api";
|
|
2
2
|
import * as z from "zod";
|
|
3
3
|
//#region src/helpers/duck-exec.ts
|
|
4
4
|
var DuckExec = class {
|
|
@@ -117,21 +117,52 @@ const createOnDataAppendedCollector = () => {
|
|
|
117
117
|
};
|
|
118
118
|
};
|
|
119
119
|
//#endregion
|
|
120
|
+
//#region src/table/get-duckdb-number-column-type.ts
|
|
121
|
+
const isFloatValue = (value) => {
|
|
122
|
+
if (!Number.isFinite(value)) return true;
|
|
123
|
+
if (Math.abs(value) > Number.MAX_SAFE_INTEGER) return true;
|
|
124
|
+
return !Number.isInteger(value);
|
|
125
|
+
};
|
|
126
|
+
const getDuckdbNumberColumnType = (params) => {
|
|
127
|
+
const { minimum, maximum } = params;
|
|
128
|
+
if (minimum === void 0 || maximum === void 0) return BIGINT;
|
|
129
|
+
if (isFloatValue(minimum) || isFloatValue(maximum)) {
|
|
130
|
+
if (minimum >= -34028235e31 && maximum <= 34028235e31) return FLOAT;
|
|
131
|
+
return DOUBLE;
|
|
132
|
+
}
|
|
133
|
+
if (minimum >= 0) {
|
|
134
|
+
if (maximum <= 255) return UTINYINT;
|
|
135
|
+
if (maximum <= 65535) return USMALLINT;
|
|
136
|
+
if (maximum <= 4294967295) return UINTEGER;
|
|
137
|
+
if (maximum <= 18446744073709551615n) return UBIGINT;
|
|
138
|
+
return UHUGEINT;
|
|
139
|
+
}
|
|
140
|
+
if (minimum >= -128 && maximum <= 127) return TINYINT;
|
|
141
|
+
if (minimum >= -32768 && maximum <= 32767) return SMALLINT;
|
|
142
|
+
if (minimum >= -2147483648 && maximum <= 2147483647) return INTEGER;
|
|
143
|
+
if (minimum >= -9223372036854775808n && maximum <= 9223372036854775807n) return BIGINT;
|
|
144
|
+
return HUGEINT;
|
|
145
|
+
};
|
|
146
|
+
//#endregion
|
|
120
147
|
//#region src/table/get-table-create-from-zod.ts
|
|
121
|
-
const
|
|
148
|
+
const createOptions = {
|
|
122
149
|
CREATE: "CREATE TABLE",
|
|
123
150
|
CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
|
|
124
151
|
IF_NOT_EXISTS: "CREATE TABLE IF NOT EXISTS"
|
|
125
152
|
};
|
|
126
|
-
const getTableCreateFromZod = (
|
|
153
|
+
const getTableCreateFromZod = (params) => {
|
|
154
|
+
const { table, schema, options } = params;
|
|
127
155
|
const { create = "CREATE" } = options ?? {};
|
|
128
156
|
const fqTable = table.getFullName();
|
|
129
|
-
const json = schema.toJSONSchema({
|
|
157
|
+
const json = schema.toJSONSchema({
|
|
158
|
+
target: "openapi-3.0",
|
|
159
|
+
unrepresentable: "throw"
|
|
160
|
+
});
|
|
130
161
|
const columns = [];
|
|
131
162
|
if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
|
|
132
|
-
const
|
|
163
|
+
const columnTypesMap = /* @__PURE__ */ new Map();
|
|
133
164
|
for (const [columnName, def] of Object.entries(json.properties)) {
|
|
134
|
-
const { type, nullable, format, primaryKey } = def;
|
|
165
|
+
const { type, nullable, format, primaryKey, minimum, maximum } = def;
|
|
135
166
|
const c = { name: columnName };
|
|
136
167
|
switch (type) {
|
|
137
168
|
case "string":
|
|
@@ -142,22 +173,37 @@ const getTableCreateFromZod = (table, schema, options) => {
|
|
|
142
173
|
case "int64":
|
|
143
174
|
c.duckdbType = BIGINT;
|
|
144
175
|
break;
|
|
176
|
+
case "uuid":
|
|
177
|
+
c.duckdbType = UUID;
|
|
178
|
+
break;
|
|
145
179
|
default: c.duckdbType = VARCHAR;
|
|
146
180
|
}
|
|
147
181
|
break;
|
|
148
182
|
case "number":
|
|
149
|
-
c.duckdbType =
|
|
183
|
+
c.duckdbType = getDuckdbNumberColumnType({
|
|
184
|
+
minimum,
|
|
185
|
+
maximum
|
|
186
|
+
});
|
|
150
187
|
break;
|
|
151
|
-
|
|
188
|
+
case "integer":
|
|
189
|
+
c.duckdbType = getDuckdbNumberColumnType({
|
|
190
|
+
minimum,
|
|
191
|
+
maximum
|
|
192
|
+
});
|
|
193
|
+
break;
|
|
194
|
+
case "boolean":
|
|
195
|
+
c.duckdbType = BOOLEAN;
|
|
196
|
+
break;
|
|
197
|
+
default: throw new Error(`Cannot guess '${columnName}' type - ${JSON.stringify(def)}`);
|
|
152
198
|
}
|
|
153
199
|
if (primaryKey === true) c.constraint = "PRIMARY KEY";
|
|
154
200
|
else if (nullable !== true) c.constraint = "NOT NULL";
|
|
155
|
-
|
|
201
|
+
columnTypesMap.set(columnName, c.duckdbType);
|
|
156
202
|
columns.push(c);
|
|
157
203
|
}
|
|
158
204
|
return {
|
|
159
205
|
ddl: [
|
|
160
|
-
`${
|
|
206
|
+
`${createOptions[create]} ${fqTable} (\n`,
|
|
161
207
|
columns.map((colDDL) => {
|
|
162
208
|
const { name, duckdbType, constraint } = colDDL;
|
|
163
209
|
return ` ${[
|
|
@@ -168,14 +214,18 @@ const getTableCreateFromZod = (table, schema, options) => {
|
|
|
168
214
|
}).join(",\n"),
|
|
169
215
|
"\n)"
|
|
170
216
|
].join(""),
|
|
171
|
-
columnTypes
|
|
217
|
+
columnTypes: columnTypesMap
|
|
172
218
|
};
|
|
173
219
|
};
|
|
174
220
|
//#endregion
|
|
175
221
|
//#region src/table/create-table-from-zod.ts
|
|
176
222
|
const createTableFromZod = async (params) => {
|
|
177
223
|
const { conn, table, schema, options } = params;
|
|
178
|
-
const { ddl, columnTypes } = getTableCreateFromZod(
|
|
224
|
+
const { ddl, columnTypes } = getTableCreateFromZod({
|
|
225
|
+
table,
|
|
226
|
+
schema,
|
|
227
|
+
options
|
|
228
|
+
});
|
|
179
229
|
try {
|
|
180
230
|
await conn.run(ddl);
|
|
181
231
|
} catch (e) {
|
|
@@ -190,6 +240,7 @@ const createTableFromZod = async (params) => {
|
|
|
190
240
|
//#region src/utils/rows-to-columns-chunks.ts
|
|
191
241
|
const toDuckValue = (value) => {
|
|
192
242
|
if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
|
|
243
|
+
if (typeof value === "bigint") return value.toString(10);
|
|
193
244
|
return value === void 0 ? null : value;
|
|
194
245
|
};
|
|
195
246
|
/**
|
|
@@ -201,7 +252,8 @@ const toDuckValue = (value) => {
|
|
|
201
252
|
* input rows: [{id:'1',name:'A'}, {id:'2',name:'B'}, {id:'3',name:'C'}]
|
|
202
253
|
* yields: [[['1','2'], ['A','B']], [['3'], ['C']]]
|
|
203
254
|
*/
|
|
204
|
-
async function* rowsToColumnsChunks(
|
|
255
|
+
async function* rowsToColumnsChunks(params) {
|
|
256
|
+
const { rows, chunkSize } = params;
|
|
205
257
|
if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
|
|
206
258
|
const first = await rows.next();
|
|
207
259
|
if (first.done) return;
|
|
@@ -238,7 +290,6 @@ var SqlDuck = class {
|
|
|
238
290
|
/**
|
|
239
291
|
* Create a table from a Zod schema and fill it with data from a row stream.
|
|
240
292
|
*
|
|
241
|
-
*
|
|
242
293
|
* @example
|
|
243
294
|
* ```typescript
|
|
244
295
|
* import * as z from 'zod';
|
|
@@ -284,10 +335,13 @@ var SqlDuck = class {
|
|
|
284
335
|
options: createOptions
|
|
285
336
|
});
|
|
286
337
|
const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
|
|
287
|
-
const chunkTypes = columnTypes.
|
|
338
|
+
const chunkTypes = Array.from(columnTypes.values());
|
|
288
339
|
let totalRows = 0;
|
|
289
340
|
const dataAppendedCollector = createOnDataAppendedCollector();
|
|
290
|
-
const columnStream = rowsToColumnsChunks(
|
|
341
|
+
const columnStream = rowsToColumnsChunks({
|
|
342
|
+
rows: rowStream,
|
|
343
|
+
chunkSize
|
|
344
|
+
});
|
|
291
345
|
for await (const dataChunk of columnStream) {
|
|
292
346
|
const chunk = DuckDBDataChunk.create(chunkTypes);
|
|
293
347
|
if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
|
|
@@ -301,6 +355,7 @@ var SqlDuck = class {
|
|
|
301
355
|
else onDataAppended(payload);
|
|
302
356
|
}
|
|
303
357
|
}
|
|
358
|
+
appender.closeSync();
|
|
304
359
|
return {
|
|
305
360
|
timeMs: Math.round(Date.now() - timeStart),
|
|
306
361
|
totalRows,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flowblade/sqlduck",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"exports": {
|
|
@@ -35,8 +35,8 @@
|
|
|
35
35
|
"docgen-typedoc": "rimraf ./docs/api && typedoc --plugin typedoc-plugin-markdown --out ./docs/api",
|
|
36
36
|
"bench": "vitest bench --run",
|
|
37
37
|
"bench-bun": "bun --bun run vitest bench --run",
|
|
38
|
-
"bench-mitata": "node --experimental-strip-types --expose-gc
|
|
39
|
-
"bench-mitata-bun": "bun --expose-gc
|
|
38
|
+
"bench-mitata": "node --experimental-strip-types --expose-gc bench/run_mitata.ts",
|
|
39
|
+
"bench-mitata-bun": "bun --expose-gc bench/run_mitata.ts",
|
|
40
40
|
"bench-codspeed": "cross-env CODSPEED=1 vitest bench --run",
|
|
41
41
|
"bench-watch": "vitest bench",
|
|
42
42
|
"test": "vitest run",
|