@flowblade/sqlduck 0.0.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +21 -7
- package/dist/index.d.cts +9 -3
- package/dist/index.d.mts +9 -3
- package/dist/index.mjs +21 -7
- package/package.json +4 -4
package/dist/index.cjs
CHANGED
|
@@ -57,7 +57,7 @@ const createMap = {
|
|
|
57
57
|
};
|
|
58
58
|
const getTableCreateFromZod = (table, schema, options) => {
|
|
59
59
|
const { create = "CREATE" } = options ?? {};
|
|
60
|
-
const fqTable = table.
|
|
60
|
+
const fqTable = table.getFullName();
|
|
61
61
|
const json = schema.toJSONSchema({ target: "openapi-3.0" });
|
|
62
62
|
const columns = [];
|
|
63
63
|
if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
|
|
@@ -109,7 +109,7 @@ const createTableFromZod = async (params) => {
|
|
|
109
109
|
try {
|
|
110
110
|
await conn.run(ddl);
|
|
111
111
|
} catch (e) {
|
|
112
|
-
throw new Error(`Failed to create table '${table.
|
|
112
|
+
throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
|
|
113
113
|
}
|
|
114
114
|
return {
|
|
115
115
|
ddl,
|
|
@@ -125,33 +125,47 @@ var SqlDuck = class {
|
|
|
125
125
|
}
|
|
126
126
|
toTable = async (params) => {
|
|
127
127
|
const { table, schema, chunkSize, rowStream, createOptions } = params;
|
|
128
|
+
const timeStart = Date.now();
|
|
128
129
|
const { columnTypes } = await createTableFromZod({
|
|
129
130
|
conn: this.#duck,
|
|
130
131
|
schema,
|
|
131
132
|
table,
|
|
132
133
|
options: createOptions
|
|
133
134
|
});
|
|
134
|
-
const appender = await this.#duck.createAppender(table.
|
|
135
|
+
const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
|
|
135
136
|
const chunkTypes = columnTypes.map((v) => v[1]);
|
|
136
|
-
const
|
|
137
|
+
const chunkLimit = chunkSize ?? 2048;
|
|
138
|
+
let totalRows = 0;
|
|
139
|
+
const columnStream = rowsToColumnsChunks(rowStream, chunkLimit);
|
|
137
140
|
for await (const dataChunk of columnStream) {
|
|
138
141
|
const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
|
|
139
142
|
if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
|
|
143
|
+
totalRows += dataChunk?.[0]?.length ?? 0;
|
|
140
144
|
chunk.setColumns(dataChunk);
|
|
141
145
|
appender.appendDataChunk(chunk);
|
|
142
146
|
appender.flushSync();
|
|
143
147
|
}
|
|
148
|
+
return {
|
|
149
|
+
timeMs: Math.round(Date.now() - timeStart),
|
|
150
|
+
totalRows
|
|
151
|
+
};
|
|
144
152
|
};
|
|
145
153
|
};
|
|
146
154
|
var Table = class Table {
|
|
147
155
|
#fqTable;
|
|
148
|
-
get
|
|
149
|
-
return this.#fqTable;
|
|
156
|
+
get tableName() {
|
|
157
|
+
return this.#fqTable.name;
|
|
158
|
+
}
|
|
159
|
+
get schemaName() {
|
|
160
|
+
return this.#fqTable.schema;
|
|
161
|
+
}
|
|
162
|
+
get databaseName() {
|
|
163
|
+
return this.#fqTable.database;
|
|
150
164
|
}
|
|
151
165
|
constructor(fqTableOrName) {
|
|
152
166
|
this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
|
|
153
167
|
}
|
|
154
|
-
|
|
168
|
+
getFullName = (options) => {
|
|
155
169
|
const { defaultDatabase, defaultSchema } = options ?? {};
|
|
156
170
|
const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
|
|
157
171
|
return [
|
package/dist/index.d.cts
CHANGED
|
@@ -13,13 +13,15 @@ type FQTable = {
|
|
|
13
13
|
};
|
|
14
14
|
declare class Table {
|
|
15
15
|
#private;
|
|
16
|
-
get
|
|
16
|
+
get tableName(): string;
|
|
17
|
+
get schemaName(): string | undefined;
|
|
18
|
+
get databaseName(): string | undefined;
|
|
17
19
|
constructor(fqTableOrName: FQTable | string);
|
|
18
20
|
/**
|
|
19
21
|
* Return fully qualified table name by concatenating
|
|
20
22
|
* database, schema and table with a 'dot' separator.
|
|
21
23
|
*/
|
|
22
|
-
|
|
24
|
+
getFullName: (options?: {
|
|
23
25
|
defaultDatabase?: string;
|
|
24
26
|
defaultSchema?: string;
|
|
25
27
|
}) => string;
|
|
@@ -51,10 +53,14 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
|
|
|
51
53
|
chunkSize?: number;
|
|
52
54
|
createOptions?: TableCreateOptions;
|
|
53
55
|
};
|
|
56
|
+
type ToTableStats = {
|
|
57
|
+
timeMs: number;
|
|
58
|
+
totalRows: number;
|
|
59
|
+
};
|
|
54
60
|
declare class SqlDuck {
|
|
55
61
|
#private;
|
|
56
62
|
constructor(params: SqlDuckParams);
|
|
57
|
-
toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<
|
|
63
|
+
toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<ToTableStats>;
|
|
58
64
|
}
|
|
59
65
|
//#endregion
|
|
60
66
|
//#region src/utils/zod-codecs.d.ts
|
package/dist/index.d.mts
CHANGED
|
@@ -13,13 +13,15 @@ type FQTable = {
|
|
|
13
13
|
};
|
|
14
14
|
declare class Table {
|
|
15
15
|
#private;
|
|
16
|
-
get
|
|
16
|
+
get tableName(): string;
|
|
17
|
+
get schemaName(): string | undefined;
|
|
18
|
+
get databaseName(): string | undefined;
|
|
17
19
|
constructor(fqTableOrName: FQTable | string);
|
|
18
20
|
/**
|
|
19
21
|
* Return fully qualified table name by concatenating
|
|
20
22
|
* database, schema and table with a 'dot' separator.
|
|
21
23
|
*/
|
|
22
|
-
|
|
24
|
+
getFullName: (options?: {
|
|
23
25
|
defaultDatabase?: string;
|
|
24
26
|
defaultSchema?: string;
|
|
25
27
|
}) => string;
|
|
@@ -51,10 +53,14 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
|
|
|
51
53
|
chunkSize?: number;
|
|
52
54
|
createOptions?: TableCreateOptions;
|
|
53
55
|
};
|
|
56
|
+
type ToTableStats = {
|
|
57
|
+
timeMs: number;
|
|
58
|
+
totalRows: number;
|
|
59
|
+
};
|
|
54
60
|
declare class SqlDuck {
|
|
55
61
|
#private;
|
|
56
62
|
constructor(params: SqlDuckParams);
|
|
57
|
-
toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<
|
|
63
|
+
toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<ToTableStats>;
|
|
58
64
|
}
|
|
59
65
|
//#endregion
|
|
60
66
|
//#region src/utils/zod-codecs.d.ts
|
package/dist/index.mjs
CHANGED
|
@@ -36,7 +36,7 @@ const createMap = {
|
|
|
36
36
|
};
|
|
37
37
|
const getTableCreateFromZod = (table, schema, options) => {
|
|
38
38
|
const { create = "CREATE" } = options ?? {};
|
|
39
|
-
const fqTable = table.
|
|
39
|
+
const fqTable = table.getFullName();
|
|
40
40
|
const json = schema.toJSONSchema({ target: "openapi-3.0" });
|
|
41
41
|
const columns = [];
|
|
42
42
|
if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
|
|
@@ -88,7 +88,7 @@ const createTableFromZod = async (params) => {
|
|
|
88
88
|
try {
|
|
89
89
|
await conn.run(ddl);
|
|
90
90
|
} catch (e) {
|
|
91
|
-
throw new Error(`Failed to create table '${table.
|
|
91
|
+
throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
|
|
92
92
|
}
|
|
93
93
|
return {
|
|
94
94
|
ddl,
|
|
@@ -104,33 +104,47 @@ var SqlDuck = class {
|
|
|
104
104
|
}
|
|
105
105
|
toTable = async (params) => {
|
|
106
106
|
const { table, schema, chunkSize, rowStream, createOptions } = params;
|
|
107
|
+
const timeStart = Date.now();
|
|
107
108
|
const { columnTypes } = await createTableFromZod({
|
|
108
109
|
conn: this.#duck,
|
|
109
110
|
schema,
|
|
110
111
|
table,
|
|
111
112
|
options: createOptions
|
|
112
113
|
});
|
|
113
|
-
const appender = await this.#duck.createAppender(table.
|
|
114
|
+
const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
|
|
114
115
|
const chunkTypes = columnTypes.map((v) => v[1]);
|
|
115
|
-
const
|
|
116
|
+
const chunkLimit = chunkSize ?? 2048;
|
|
117
|
+
let totalRows = 0;
|
|
118
|
+
const columnStream = rowsToColumnsChunks(rowStream, chunkLimit);
|
|
116
119
|
for await (const dataChunk of columnStream) {
|
|
117
120
|
const chunk = DuckDBDataChunk.create(chunkTypes);
|
|
118
121
|
if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
|
|
122
|
+
totalRows += dataChunk?.[0]?.length ?? 0;
|
|
119
123
|
chunk.setColumns(dataChunk);
|
|
120
124
|
appender.appendDataChunk(chunk);
|
|
121
125
|
appender.flushSync();
|
|
122
126
|
}
|
|
127
|
+
return {
|
|
128
|
+
timeMs: Math.round(Date.now() - timeStart),
|
|
129
|
+
totalRows
|
|
130
|
+
};
|
|
123
131
|
};
|
|
124
132
|
};
|
|
125
133
|
var Table = class Table {
|
|
126
134
|
#fqTable;
|
|
127
|
-
get
|
|
128
|
-
return this.#fqTable;
|
|
135
|
+
get tableName() {
|
|
136
|
+
return this.#fqTable.name;
|
|
137
|
+
}
|
|
138
|
+
get schemaName() {
|
|
139
|
+
return this.#fqTable.schema;
|
|
140
|
+
}
|
|
141
|
+
get databaseName() {
|
|
142
|
+
return this.#fqTable.database;
|
|
129
143
|
}
|
|
130
144
|
constructor(fqTableOrName) {
|
|
131
145
|
this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
|
|
132
146
|
}
|
|
133
|
-
|
|
147
|
+
getFullName = (options) => {
|
|
134
148
|
const { defaultDatabase, defaultSchema } = options ?? {};
|
|
135
149
|
const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
|
|
136
150
|
return [
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flowblade/sqlduck",
|
|
3
|
-
"version": "0.0
|
|
3
|
+
"version": "0.1.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"exports": {
|
|
@@ -46,8 +46,8 @@
|
|
|
46
46
|
"check-size": "size-limit"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@flowblade/core": "^0.2.
|
|
50
|
-
"@flowblade/source-duckdb": "^0.
|
|
49
|
+
"@flowblade/core": "^0.2.25",
|
|
50
|
+
"@flowblade/source-duckdb": "^0.17.1",
|
|
51
51
|
"@flowblade/sql-tag": "^0.2.0",
|
|
52
52
|
"@standard-schema/spec": "^1.1.0",
|
|
53
53
|
"p-mutex": "^1.0.0",
|
|
@@ -62,7 +62,7 @@
|
|
|
62
62
|
"@dotenvx/dotenvx": "1.51.2",
|
|
63
63
|
"@duckdb/node-api": "1.4.3-r.2",
|
|
64
64
|
"@faker-js/faker": "10.1.0",
|
|
65
|
-
"@flowblade/source-kysely": "^1.
|
|
65
|
+
"@flowblade/source-kysely": "^1.2.0",
|
|
66
66
|
"@httpx/assert": "0.16.7",
|
|
67
67
|
"@size-limit/esbuild": "12.0.0",
|
|
68
68
|
"@size-limit/file": "12.0.0",
|