@lightdash/warehouses 0.2629.2 → 0.2630.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -16,12 +16,23 @@ export type DuckdbResourceLimits = {
|
|
|
16
16
|
export type DuckdbLogger = {
|
|
17
17
|
info: (message: string, metadata?: Record<string, unknown>) => void;
|
|
18
18
|
};
|
|
19
|
+
export type DuckdbQueryProfileMetrics = {
|
|
20
|
+
latencyMs: number;
|
|
21
|
+
cpuMs: number;
|
|
22
|
+
waitMs: number;
|
|
23
|
+
readParquetMs: number | null;
|
|
24
|
+
bytesRead: number | null;
|
|
25
|
+
rowsReturned: number | null;
|
|
26
|
+
rowsScanned: number | null;
|
|
27
|
+
scanAmplification: number | null;
|
|
28
|
+
};
|
|
19
29
|
export type DuckdbWarehouseClientArgs = {
|
|
20
30
|
databasePath?: string;
|
|
21
31
|
s3Config?: DuckdbS3SessionConfig;
|
|
22
32
|
resourceLimits?: DuckdbResourceLimits;
|
|
23
33
|
bufferPoolSize?: string;
|
|
24
34
|
logger?: DuckdbLogger;
|
|
35
|
+
onQueryProfile?: (profile: DuckdbQueryProfileMetrics) => void;
|
|
25
36
|
};
|
|
26
37
|
export declare const mapFieldTypeFromTypeId: (typeId: number) => DimensionType;
|
|
27
38
|
export declare class DuckdbSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
@@ -38,6 +49,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreatePos
|
|
|
38
49
|
private readonly resourceLimits?;
|
|
39
50
|
private readonly bufferPoolSize?;
|
|
40
51
|
private readonly logger?;
|
|
52
|
+
private readonly onQueryProfile?;
|
|
41
53
|
constructor(args?: DuckdbWarehouseClientArgs);
|
|
42
54
|
close(): Promise<void>;
|
|
43
55
|
private getSQLWithMetadata;
|
|
@@ -45,7 +57,7 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreatePos
|
|
|
45
57
|
private withSession;
|
|
46
58
|
private bootstrapSession;
|
|
47
59
|
private getBindValues;
|
|
48
|
-
private
|
|
60
|
+
private logQueryProfile;
|
|
49
61
|
private static getFieldsFromStreamResult;
|
|
50
62
|
private static stripSqlComments;
|
|
51
63
|
private static validateSqlFunctions;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAsChE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,aAAa,EACb,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAsChE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACjE,CAAC;AAYF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAEF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AA8CD,mFAAmF;AACnF,wBAAgB,gCAAgC,IAAI,IAAI,CAKvD;AA4BD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACrF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAS;IAEzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;gBAEE,IAAI,GAAE,yBAA8B;IAU1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,kBAAkB;YAQZ,gBAAgB;YAoBhB,WAAW;YAwCX,gBAAgB;IAsG9B,OAAO,CAAC,aAAa;YA0BP,eAAe;IA+F7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAM/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;YAUrB,eAAe;YA8Bf,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA4CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,mBAAmB,CAAC,CACtE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAoBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAC7D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC/D,OAAO,CAAC,gBAAgB,CAAC;IAMtB,YAAY,CACd,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAMK,SAAS,CACX,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAK/B"}
|
|
@@ -142,6 +142,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
142
142
|
this.resourceLimits = args.resourceLimits;
|
|
143
143
|
this.bufferPoolSize = args.bufferPoolSize;
|
|
144
144
|
this.logger = args.logger;
|
|
145
|
+
this.onQueryProfile = args.onQueryProfile;
|
|
145
146
|
}
|
|
146
147
|
async close() {
|
|
147
148
|
clearSharedInstance(this.logger);
|
|
@@ -278,7 +279,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
278
279
|
}
|
|
279
280
|
return undefined;
|
|
280
281
|
}
|
|
281
|
-
|
|
282
|
+
async logQueryProfile(profilePath, logger, tags) {
|
|
282
283
|
try {
|
|
283
284
|
const raw = await promises_1.default.readFile(profilePath, 'utf-8');
|
|
284
285
|
const profile = JSON.parse(raw);
|
|
@@ -330,6 +331,16 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
330
331
|
rowsReturned,
|
|
331
332
|
scanAmplification,
|
|
332
333
|
});
|
|
334
|
+
this.onQueryProfile?.({
|
|
335
|
+
latencyMs,
|
|
336
|
+
cpuMs,
|
|
337
|
+
waitMs,
|
|
338
|
+
readParquetMs,
|
|
339
|
+
bytesRead,
|
|
340
|
+
rowsReturned,
|
|
341
|
+
rowsScanned,
|
|
342
|
+
scanAmplification,
|
|
343
|
+
});
|
|
333
344
|
}
|
|
334
345
|
catch {
|
|
335
346
|
// profiling output not available, skip
|
|
@@ -418,7 +429,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
418
429
|
await streamCallback({ fields, rows });
|
|
419
430
|
}
|
|
420
431
|
if (profilePath) {
|
|
421
|
-
await
|
|
432
|
+
await this.logQueryProfile(profilePath, this.logger, options?.tags);
|
|
422
433
|
}
|
|
423
434
|
});
|
|
424
435
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2630.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"snowflake-sdk": "~2.3.4",
|
|
27
27
|
"ssh2": "^1.14.0",
|
|
28
28
|
"trino-client": "0.2.9",
|
|
29
|
-
"@lightdash/common": "0.
|
|
29
|
+
"@lightdash/common": "0.2630.1"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node-fetch": "^2.6.13",
|