@lightdash/warehouses 0.2597.0 → 0.2598.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -39,6 +39,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreatePos
|
|
|
39
39
|
private withSession;
|
|
40
40
|
private bootstrapSession;
|
|
41
41
|
private getBindValues;
|
|
42
|
+
private static logQueryProfile;
|
|
42
43
|
private static getFieldsFromStreamResult;
|
|
43
44
|
streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void | Promise<void>, options?: {
|
|
44
45
|
values?: AnyType[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA2BhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAED,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAQhD,OAAO,CAAC,kBAAkB;YAQZ,WAAW;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA2BhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAED,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAQhD,OAAO,CAAC,kBAAkB;YAQZ,WAAW;YA2CX,gBAAgB;IAmE9B,OAAO,CAAC,aAAa;mBA0BA,eAAe;IA2CpC,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAalC,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IAyCV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAmBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}
|
|
@@ -106,12 +106,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
106
106
|
const result = await callback(connection);
|
|
107
107
|
const queryMs = performance.now() - queryStart;
|
|
108
108
|
const totalMs = performance.now() - sessionStart;
|
|
109
|
-
this.logger?.info(
|
|
110
|
-
connectMs: Math.round(connectMs),
|
|
111
|
-
bootstrapMs: Math.round(bootstrapMs),
|
|
112
|
-
queryMs: Math.round(queryMs),
|
|
113
|
-
totalMs: Math.round(totalMs),
|
|
114
|
-
});
|
|
109
|
+
this.logger?.info(`DuckDB session timing: connect=${Math.round(connectMs)}ms bootstrap=${Math.round(bootstrapMs)}ms query=${Math.round(queryMs)}ms total=${Math.round(totalMs)}ms`);
|
|
115
110
|
return result;
|
|
116
111
|
}
|
|
117
112
|
finally {
|
|
@@ -136,10 +131,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
136
131
|
await db.run(`SET threads = ${this.resourceLimits.threads};`);
|
|
137
132
|
}
|
|
138
133
|
if (!this.s3Config) {
|
|
139
|
-
this.logger?.info(
|
|
140
|
-
installHttpfsMs: Math.round(installMs),
|
|
141
|
-
loadHttpfsMs: Math.round(loadMs),
|
|
142
|
-
});
|
|
134
|
+
this.logger?.info(`DuckDB bootstrap timing: install_httpfs=${Math.round(installMs)}ms load_httpfs=${Math.round(loadMs)}ms`);
|
|
143
135
|
return;
|
|
144
136
|
}
|
|
145
137
|
const t2 = performance.now();
|
|
@@ -156,11 +148,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
156
148
|
await db.run(`SET s3_use_ssl = ${this.s3Config.useSsl};`);
|
|
157
149
|
await db.run(`SET s3_url_style = '${this.s3Config.forcePathStyle ? 'path' : 'vhost'}';`);
|
|
158
150
|
const s3ConfigMs = performance.now() - t2;
|
|
159
|
-
this.logger?.info(
|
|
160
|
-
installHttpfsMs: Math.round(installMs),
|
|
161
|
-
loadHttpfsMs: Math.round(loadMs),
|
|
162
|
-
s3ConfigMs: Math.round(s3ConfigMs),
|
|
163
|
-
});
|
|
151
|
+
this.logger?.info(`DuckDB bootstrap timing: install_httpfs=${Math.round(installMs)}ms load_httpfs=${Math.round(loadMs)}ms s3_config=${Math.round(s3ConfigMs)}ms`);
|
|
164
152
|
}
|
|
165
153
|
getBindValues(options) {
|
|
166
154
|
const hasValues = !!options?.values && options.values.length > 0;
|
|
@@ -177,6 +165,37 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
177
165
|
}
|
|
178
166
|
return undefined;
|
|
179
167
|
}
|
|
168
|
+
static async logQueryProfile(profilePath, logger) {
|
|
169
|
+
try {
|
|
170
|
+
const raw = await promises_1.default.readFile(profilePath, 'utf-8');
|
|
171
|
+
const profile = JSON.parse(raw);
|
|
172
|
+
const operators = [];
|
|
173
|
+
const walk = (node) => {
|
|
174
|
+
if (node.operator_name) {
|
|
175
|
+
operators.push({
|
|
176
|
+
name: node.operator_name,
|
|
177
|
+
timingMs: Math.round((node.operator_timing ?? 0) * 1000),
|
|
178
|
+
rows: node.operator_cardinality ?? 0,
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
182
|
+
for (const child of node.children ?? []) {
|
|
183
|
+
walk(child);
|
|
184
|
+
}
|
|
185
|
+
};
|
|
186
|
+
walk(profile);
|
|
187
|
+
const operatorStr = operators
|
|
188
|
+
.map((op) => `${op.name}=${op.timingMs}ms(${op.rows}rows)`)
|
|
189
|
+
.join(' ');
|
|
190
|
+
logger.info(`DuckDB query profile: latency=${Math.round((profile.latency ?? 0) * 1000)}ms cpu=${Math.round((profile.cpu_time ?? 0) * 1000)}ms rows=${profile.rows_returned} bytes_read=${profile.total_bytes_read} operators=[${operatorStr}]`);
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
// profiling output not available, skip
|
|
194
|
+
}
|
|
195
|
+
finally {
|
|
196
|
+
await promises_1.default.rm(profilePath, { force: true }).catch(() => { });
|
|
197
|
+
}
|
|
198
|
+
}
|
|
180
199
|
static getFieldsFromStreamResult(result) {
|
|
181
200
|
const columnNames = result.columnNames();
|
|
182
201
|
const fields = {};
|
|
@@ -192,12 +211,22 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
192
211
|
if (options?.timezone) {
|
|
193
212
|
await db.run(`SET TimeZone = '${this.escapeString(options.timezone)}';`);
|
|
194
213
|
}
|
|
214
|
+
const profilePath = this.logger
|
|
215
|
+
? path_1.default.join(os_1.default.tmpdir(), `duckdb-profile-${Date.now()}-${Math.random().toString(36).slice(2)}.json`)
|
|
216
|
+
: undefined;
|
|
217
|
+
if (profilePath) {
|
|
218
|
+
await db.run("PRAGMA enable_profiling='json';");
|
|
219
|
+
await db.run(`PRAGMA profiling_output='${profilePath}';`);
|
|
220
|
+
}
|
|
195
221
|
const result = await db.stream(this.getSQLWithMetadata(sql, options?.tags), this.getBindValues(options));
|
|
196
222
|
const fields = DuckdbWarehouseClient.getFieldsFromStreamResult(result);
|
|
197
223
|
// eslint-disable-next-line no-restricted-syntax
|
|
198
224
|
for await (const rows of result.yieldRowObjectJson()) {
|
|
199
225
|
await streamCallback({ fields, rows });
|
|
200
226
|
}
|
|
227
|
+
if (profilePath) {
|
|
228
|
+
await DuckdbWarehouseClient.logQueryProfile(profilePath, this.logger);
|
|
229
|
+
}
|
|
201
230
|
});
|
|
202
231
|
}
|
|
203
232
|
async executeAsyncQuery(...args) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2598.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"snowflake-sdk": "~2.3.4",
|
|
27
27
|
"ssh2": "^1.14.0",
|
|
28
28
|
"trino-client": "0.2.9",
|
|
29
|
-
"@lightdash/common": "0.
|
|
29
|
+
"@lightdash/common": "0.2598.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node-fetch": "^2.6.13",
|