@lightdash/warehouses 0.2599.0 → 0.2601.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA2BhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAED,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAQhD,OAAO,CAAC,kBAAkB;YAQZ,WAAW;YA2CX,gBAAgB;IAmE9B,OAAO,CAAC,aAAa;mBA0BA,eAAe;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA2BhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,MAAM,CAAC,EAAE,YAAY,CAAC;CACzB,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AAED,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;gBAE3B,IAAI,GAAE,yBAA8B;IAQhD,OAAO,CAAC,kBAAkB;YAQZ,WAAW;YA2CX,gBAAgB;IAmE9B,OAAO,CAAC,aAAa;mBA0BA,eAAe;IAoFpC,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAalC,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA0CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAmBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}
|
|
@@ -165,7 +165,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
165
165
|
}
|
|
166
166
|
return undefined;
|
|
167
167
|
}
|
|
168
|
-
static async logQueryProfile(profilePath, logger) {
|
|
168
|
+
static async logQueryProfile(profilePath, logger, tags) {
|
|
169
169
|
try {
|
|
170
170
|
const raw = await promises_1.default.readFile(profilePath, 'utf-8');
|
|
171
171
|
const profile = JSON.parse(raw);
|
|
@@ -173,7 +173,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
173
173
|
const walk = (node) => {
|
|
174
174
|
if (node.operator_name) {
|
|
175
175
|
operators.push({
|
|
176
|
-
name: node.operator_name,
|
|
176
|
+
name: String(node.operator_name).trim(),
|
|
177
177
|
timingMs: Math.round((node.operator_timing ?? 0) * 1000),
|
|
178
178
|
rows: node.operator_cardinality ?? 0,
|
|
179
179
|
});
|
|
@@ -187,7 +187,36 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
187
187
|
const operatorStr = operators
|
|
188
188
|
.map((op) => `${op.name}=${op.timingMs}ms(${op.rows}rows)`)
|
|
189
189
|
.join(' ');
|
|
190
|
-
|
|
190
|
+
const latencyMs = Math.round((profile.latency ?? 0) * 1000);
|
|
191
|
+
const cpuMs = Math.round((profile.cpu_time ?? 0) * 1000);
|
|
192
|
+
const waitMs = Math.max(latencyMs - cpuMs, 0);
|
|
193
|
+
const readParquetOperators = operators.filter((op) => op.name === 'READ_PARQUET');
|
|
194
|
+
const readParquetMs = readParquetOperators.length > 0
|
|
195
|
+
? readParquetOperators.reduce((sum, op) => sum + op.timingMs, 0)
|
|
196
|
+
: null;
|
|
197
|
+
const rowsScanned = readParquetOperators.length > 0
|
|
198
|
+
? readParquetOperators.reduce((sum, op) => sum + op.rows, 0)
|
|
199
|
+
: null;
|
|
200
|
+
const rowsReturned = typeof profile.rows_returned === 'number'
|
|
201
|
+
? profile.rows_returned
|
|
202
|
+
: null;
|
|
203
|
+
const bytesRead = typeof profile.total_bytes_read === 'number'
|
|
204
|
+
? profile.total_bytes_read
|
|
205
|
+
: null;
|
|
206
|
+
const scanAmplification = rowsScanned !== null && rowsReturned !== null
|
|
207
|
+
? rowsScanned / Math.max(rowsReturned, 1)
|
|
208
|
+
: null;
|
|
209
|
+
logger.info(`DuckDB query profile: latency=${latencyMs}ms cpu=${cpuMs}ms rows=${profile.rows_returned} bytes_read=${profile.total_bytes_read} operators=[${operatorStr}]`, {
|
|
210
|
+
...tags,
|
|
211
|
+
latencyMs,
|
|
212
|
+
cpuMs,
|
|
213
|
+
waitMs,
|
|
214
|
+
readParquetMs,
|
|
215
|
+
bytesRead,
|
|
216
|
+
rowsScanned,
|
|
217
|
+
rowsReturned,
|
|
218
|
+
scanAmplification,
|
|
219
|
+
});
|
|
191
220
|
}
|
|
192
221
|
catch {
|
|
193
222
|
// profiling output not available, skip
|
|
@@ -225,7 +254,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
225
254
|
await streamCallback({ fields, rows });
|
|
226
255
|
}
|
|
227
256
|
if (profilePath) {
|
|
228
|
-
await DuckdbWarehouseClient.logQueryProfile(profilePath, this.logger);
|
|
257
|
+
await DuckdbWarehouseClient.logQueryProfile(profilePath, this.logger, options?.tags);
|
|
229
258
|
}
|
|
230
259
|
});
|
|
231
260
|
}
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
6
|
const common_1 = require("@lightdash/common");
|
|
7
|
+
const promises_1 = __importDefault(require("fs/promises"));
|
|
4
8
|
const DuckdbWarehouseClient_1 = require("./DuckdbWarehouseClient");
|
|
5
9
|
const createInstanceMock = jest.fn();
|
|
6
10
|
// Must provide DuckDBTypeId since mapFieldTypeFromTypeId references it at runtime
|
|
@@ -215,4 +219,49 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
215
219
|
expect(runCalls).toContain("SET TimeZone = 'UTC';");
|
|
216
220
|
expect(streamMock).toHaveBeenCalledTimes(1);
|
|
217
221
|
});
|
|
222
|
+
it('should log structured DuckDB profile metrics with query tags', async () => {
|
|
223
|
+
const runMock = jest.fn(async (sql) => {
|
|
224
|
+
const match = sql.match(/^PRAGMA profiling_output='(.+)';$/);
|
|
225
|
+
if (match) {
|
|
226
|
+
await promises_1.default.writeFile(match[1], JSON.stringify({
|
|
227
|
+
latency: 4.747,
|
|
228
|
+
cpu_time: 4.731,
|
|
229
|
+
rows_returned: 68,
|
|
230
|
+
total_bytes_read: 20225287,
|
|
231
|
+
children: [
|
|
232
|
+
{
|
|
233
|
+
operator_name: 'READ_PARQUET ',
|
|
234
|
+
operator_timing: 4.632,
|
|
235
|
+
operator_cardinality: 9905024,
|
|
236
|
+
children: [],
|
|
237
|
+
},
|
|
238
|
+
],
|
|
239
|
+
}));
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
243
|
+
const logger = { info: jest.fn() };
|
|
244
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
245
|
+
const client = new DuckdbWarehouseClient_1.DuckdbWarehouseClient({ logger });
|
|
246
|
+
await client.runQuery('SELECT 1 AS val', {
|
|
247
|
+
query_uuid: 'query-123',
|
|
248
|
+
chart_uuid: 'chart-123',
|
|
249
|
+
dashboard_uuid: 'dashboard-123',
|
|
250
|
+
query_context: common_1.QueryExecutionContext.DASHBOARD,
|
|
251
|
+
});
|
|
252
|
+
expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('DuckDB query profile: latency=4747ms cpu=4731ms'), expect.objectContaining({
|
|
253
|
+
query_uuid: 'query-123',
|
|
254
|
+
chart_uuid: 'chart-123',
|
|
255
|
+
dashboard_uuid: 'dashboard-123',
|
|
256
|
+
query_context: common_1.QueryExecutionContext.DASHBOARD,
|
|
257
|
+
latencyMs: 4747,
|
|
258
|
+
cpuMs: 4731,
|
|
259
|
+
waitMs: 16,
|
|
260
|
+
readParquetMs: 4632,
|
|
261
|
+
rowsScanned: 9905024,
|
|
262
|
+
rowsReturned: 68,
|
|
263
|
+
bytesRead: 20225287,
|
|
264
|
+
scanAmplification: 9905024 / 68,
|
|
265
|
+
}));
|
|
266
|
+
});
|
|
218
267
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2601.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"snowflake-sdk": "~2.3.4",
|
|
27
27
|
"ssh2": "^1.14.0",
|
|
28
28
|
"trino-client": "0.2.9",
|
|
29
|
-
"@lightdash/common": "0.
|
|
29
|
+
"@lightdash/common": "0.2601.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node-fetch": "^2.6.13",
|