@lightdash/warehouses 1.72.0 → 1.74.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/warehouseClientFromCredentials.d.ts +2 -1
- package/dist/warehouseClientFromCredentials.d.ts.map +1 -1
- package/dist/warehouseClientFromCredentials.js +2 -2
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +11 -7
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +152 -16
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +374 -1
- package/dist/warehouseClients/MotherduckErrorClassifier.d.ts +3 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.js +34 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckErrorClassifier.test.js +31 -0
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.js +52 -0
- package/dist/warehouseClients/MotherduckInstanceCache.d.ts +58 -0
- package/dist/warehouseClients/MotherduckInstanceCache.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.js +172 -0
- package/dist/warehouseClients/MotherduckInstanceCache.js +316 -0
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.js +192 -0
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts +2 -0
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts.map +1 -0
- package/dist/warehouseClients/MotherduckInstanceCache.test.js +302 -0
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ export * from './warehouseClients/AthenaWarehouseClient';
|
|
|
6
6
|
export * from './warehouseClients/BigqueryWarehouseClient';
|
|
7
7
|
export * from './warehouseClients/DatabricksWarehouseClient';
|
|
8
8
|
export * from './warehouseClients/DuckdbWarehouseClient';
|
|
9
|
+
export { MotherduckInstanceCache } from './warehouseClients/MotherduckInstanceCache';
|
|
10
|
+
export type { MotherduckCacheEvent } from './warehouseClients/MotherduckInstanceCache';
|
|
9
11
|
export * from './warehouseClients/PostgresWarehouseClient';
|
|
10
12
|
export * from './warehouseClients/RedshiftWarehouseClient';
|
|
11
13
|
export * from './warehouseClients/redshiftIamCredentials';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,4CAA4C,CAAC;AAC3D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,6CAA6C,CAAC;AAC5D,cAAc,yCAAyC,CAAC;AACxD,cAAc,+BAA+B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,YAAY,EAAE,oBAAoB,EAAE,MAAM,4CAA4C,CAAC;AACvF,cAAc,4CAA4C,CAAC;AAC3D,cAAc,4CAA4C,CAAC;AAC3D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,6CAA6C,CAAC;AAC5D,cAAc,yCAAyC,CAAC;AACxD,cAAc,+BAA+B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -14,6 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.MotherduckInstanceCache = void 0;
|
|
17
18
|
__exportStar(require("./ssh/sshTunnel"), exports);
|
|
18
19
|
__exportStar(require("./types"), exports);
|
|
19
20
|
__exportStar(require("./validateWarehouseColumnReferences"), exports);
|
|
@@ -22,6 +23,8 @@ __exportStar(require("./warehouseClients/AthenaWarehouseClient"), exports);
|
|
|
22
23
|
__exportStar(require("./warehouseClients/BigqueryWarehouseClient"), exports);
|
|
23
24
|
__exportStar(require("./warehouseClients/DatabricksWarehouseClient"), exports);
|
|
24
25
|
__exportStar(require("./warehouseClients/DuckdbWarehouseClient"), exports);
|
|
26
|
+
var MotherduckInstanceCache_1 = require("./warehouseClients/MotherduckInstanceCache");
|
|
27
|
+
Object.defineProperty(exports, "MotherduckInstanceCache", { enumerable: true, get: function () { return MotherduckInstanceCache_1.MotherduckInstanceCache; } });
|
|
25
28
|
__exportStar(require("./warehouseClients/PostgresWarehouseClient"), exports);
|
|
26
29
|
__exportStar(require("./warehouseClients/RedshiftWarehouseClient"), exports);
|
|
27
30
|
__exportStar(require("./warehouseClients/redshiftIamCredentials"), exports);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { CreateWarehouseCredentials } from '@lightdash/common';
|
|
2
2
|
import { WarehouseClient } from './types';
|
|
3
|
-
|
|
3
|
+
import { type DuckdbWarehouseClientOptions } from './warehouseClients/DuckdbWarehouseClient';
|
|
4
|
+
export declare const warehouseClientFromCredentials: (credentials: CreateWarehouseCredentials, options?: DuckdbWarehouseClientOptions) => WarehouseClient;
|
|
4
5
|
//# sourceMappingURL=warehouseClientFromCredentials.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"warehouseClientFromCredentials.d.ts","sourceRoot":"","sources":["../src/warehouseClientFromCredentials.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,0BAA0B,EAG7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"warehouseClientFromCredentials.d.ts","sourceRoot":"","sources":["../src/warehouseClientFromCredentials.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,0BAA0B,EAG7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAK1C,OAAO,EAEH,KAAK,4BAA4B,EACpC,MAAM,0CAA0C,CAAC;AAMlD,eAAO,MAAM,8BAA8B,gBAC1B,0BAA0B,YAC7B,4BAA4B,KACvC,eA4BF,CAAC"}
|
|
@@ -11,7 +11,7 @@ const PostgresWarehouseClient_1 = require("./warehouseClients/PostgresWarehouseC
|
|
|
11
11
|
const RedshiftWarehouseClient_1 = require("./warehouseClients/RedshiftWarehouseClient");
|
|
12
12
|
const SnowflakeWarehouseClient_1 = require("./warehouseClients/SnowflakeWarehouseClient");
|
|
13
13
|
const TrinoWarehouseClient_1 = require("./warehouseClients/TrinoWarehouseClient");
|
|
14
|
-
const warehouseClientFromCredentials = (credentials) => {
|
|
14
|
+
const warehouseClientFromCredentials = (credentials, options) => {
|
|
15
15
|
switch (credentials.type) {
|
|
16
16
|
case common_1.WarehouseTypes.SNOWFLAKE:
|
|
17
17
|
return new SnowflakeWarehouseClient_1.SnowflakeWarehouseClient(credentials);
|
|
@@ -30,7 +30,7 @@ const warehouseClientFromCredentials = (credentials) => {
|
|
|
30
30
|
case common_1.WarehouseTypes.ATHENA:
|
|
31
31
|
return new AthenaWarehouseClient_1.AthenaWarehouseClient(credentials);
|
|
32
32
|
case common_1.WarehouseTypes.DUCKDB:
|
|
33
|
-
return new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials);
|
|
33
|
+
return new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials, options);
|
|
34
34
|
default:
|
|
35
35
|
return (0, common_1.assertUnreachable)(credentials, new common_1.UnexpectedServerError('Warehouse credentials type were not recognised'));
|
|
36
36
|
}
|
|
@@ -15,6 +15,7 @@ export type DuckdbResourceLimits = {
|
|
|
15
15
|
};
|
|
16
16
|
export type DuckdbLogger = {
|
|
17
17
|
info: (message: string, metadata?: Record<string, unknown>) => void;
|
|
18
|
+
warn?: (message: string, metadata?: Record<string, unknown>) => void;
|
|
18
19
|
};
|
|
19
20
|
export type DuckdbQueryProfileMetrics = {
|
|
20
21
|
latencyMs: number;
|
|
@@ -44,8 +45,11 @@ export type DuckdbWarehouseClientOptions = {
|
|
|
44
45
|
*/
|
|
45
46
|
instanceCacheKey?: string;
|
|
46
47
|
logger?: DuckdbLogger;
|
|
48
|
+
enableQueryProfiling?: boolean;
|
|
47
49
|
onQueryProfile?: (profile: DuckdbQueryProfileMetrics) => void;
|
|
48
50
|
embeddedQueryTimeoutMs?: number;
|
|
51
|
+
enableInstanceCache?: boolean;
|
|
52
|
+
projectUuid?: string;
|
|
49
53
|
};
|
|
50
54
|
export declare const mapFieldTypeFromTypeId: (typeId: number) => DimensionType;
|
|
51
55
|
export declare const getDuckdbTimestampDomainFromString: (typeName: string) => TimestampDomain | undefined;
|
|
@@ -55,6 +59,7 @@ export declare class DuckdbSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
|
55
59
|
getMetricSql(sql: string, metric: Metric): string;
|
|
56
60
|
concatString(...args: string[]): string;
|
|
57
61
|
}
|
|
62
|
+
export declare const buildMotherduckConnectionString: ({ database, token, }: Pick<CreateDuckdbMotherduckCredentials, 'database' | 'token'>) => string;
|
|
58
63
|
export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuckdbMotherduckCredentials> {
|
|
59
64
|
private static readonly sharedInstances;
|
|
60
65
|
private static readonly sharedInstanceResourceLimits;
|
|
@@ -70,9 +75,13 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
70
75
|
private readonly sharedResourceLimits?;
|
|
71
76
|
private readonly instanceCacheKey?;
|
|
72
77
|
private readonly logger?;
|
|
78
|
+
private readonly enableQueryProfiling;
|
|
73
79
|
private readonly onQueryProfile?;
|
|
74
80
|
private readonly embeddedQueryTimeoutMs;
|
|
81
|
+
private readonly enableInstanceCache;
|
|
82
|
+
private readonly projectUuid?;
|
|
75
83
|
private allowsPreAggregateFileReads;
|
|
84
|
+
private hasWarnedAboutMotherduckTimezone;
|
|
76
85
|
constructor(credentials?: CreateDuckdbCredentials | DuckdbConnectionCredentials, options?: DuckdbWarehouseClientOptions);
|
|
77
86
|
private static hashDucklakeConfig;
|
|
78
87
|
static createForPreAggregate(credentials: DuckdbS3Credentials, options?: DuckdbWarehouseClientOptions): DuckdbWarehouseClient;
|
|
@@ -113,14 +122,9 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
113
122
|
private withEphemeralQuerySession;
|
|
114
123
|
private withSharedSession;
|
|
115
124
|
private hasResourceLimits;
|
|
116
|
-
|
|
117
|
-
* Dispatches to the appropriate session strategy:
|
|
118
|
-
* - Direct database (non-:memory: databasePath): connects to the MotherDuck path
|
|
119
|
-
* - Resource-limited: isolated ephemeral instance (e.g. parquet conversion)
|
|
120
|
-
* - Shared instance (has instanceCacheKey): warm cached instance for queries
|
|
121
|
-
* - Default: ephemeral query session
|
|
122
|
-
*/
|
|
125
|
+
private isMotherduck;
|
|
123
126
|
private withSession;
|
|
127
|
+
private withMotherduckCachedSession;
|
|
124
128
|
/** Direct connection to the configured MotherDuck database. */
|
|
125
129
|
private withDirectSession;
|
|
126
130
|
private withEmbeddedQueryDeadline;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAQ3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAE9D,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AA+ED,eAAO,MAAM,+BAA+B,yBAGzC,IAAI,CAAC,iCAAiC,EAAE,UAAU,GAAG,OAAO,CAAC,KAAG,MAOlE,CAAC;AA8DF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAE9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IAEtC,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EAiHzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAwC5C,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;mBA6ErB,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAM9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAqC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IA2C/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;YAQN,WAAW;YA8CX,2BAA2B;IA4GzC,+DAA+D;YACjD,iBAAiB;YA8EjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAUnC,OAAO,CAAC,MAAM,CAAC,yBAAyB;YAmB1B,iBAAiB;YA4BjB,eAAe;YASf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAyFf;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN,CAqBA;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -1,9 +1,42 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
37
|
};
|
|
5
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.DuckdbWarehouseClient = exports.DuckdbSqlBuilder = exports.getDuckdbTimestampDomainFromString = exports.mapFieldTypeFromTypeId = void 0;
|
|
39
|
+
exports.DuckdbWarehouseClient = exports.buildMotherduckConnectionString = exports.DuckdbSqlBuilder = exports.getDuckdbTimestampDomainFromString = exports.mapFieldTypeFromTypeId = void 0;
|
|
7
40
|
const node_api_1 = require("@duckdb/node-api");
|
|
8
41
|
const common_1 = require("@lightdash/common");
|
|
9
42
|
const crypto_1 = require("crypto");
|
|
@@ -11,6 +44,8 @@ const fs_1 = __importDefault(require("fs"));
|
|
|
11
44
|
const promises_1 = __importDefault(require("fs/promises"));
|
|
12
45
|
const os_1 = __importDefault(require("os"));
|
|
13
46
|
const path_1 = __importDefault(require("path"));
|
|
47
|
+
const MotherduckErrorClassifier_1 = require("./MotherduckErrorClassifier");
|
|
48
|
+
const MotherduckInstanceCache = __importStar(require("./MotherduckInstanceCache"));
|
|
14
49
|
const WarehouseBaseClient_1 = __importDefault(require("./WarehouseBaseClient"));
|
|
15
50
|
const WarehouseBaseSqlBuilder_1 = __importDefault(require("./WarehouseBaseSqlBuilder"));
|
|
16
51
|
const mapFieldTypeFromTypeId = (typeId) => {
|
|
@@ -180,6 +215,7 @@ const buildMotherduckConnectionString = ({ database, token, }) => {
|
|
|
180
215
|
});
|
|
181
216
|
return `md:${encodeURIComponent(database)}?${params.toString()}`;
|
|
182
217
|
};
|
|
218
|
+
exports.buildMotherduckConnectionString = buildMotherduckConnectionString;
|
|
183
219
|
const EMBEDDED_DATASET_PATTERN = /^[a-z0-9_-]+$/;
|
|
184
220
|
const DEFAULT_EMBEDDED_DATA_DIRECTORY = path_1.default.resolve(__dirname, '../../../backend/assets/playground');
|
|
185
221
|
const EMBEDDED_RESOURCE_LIMITS = {
|
|
@@ -263,6 +299,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
263
299
|
}
|
|
264
300
|
super(effectiveCredentials, new DuckdbSqlBuilder(effectiveCredentials.startOfWeek));
|
|
265
301
|
this.allowsPreAggregateFileReads = false;
|
|
302
|
+
this.hasWarnedAboutMotherduckTimezone = false;
|
|
266
303
|
if (isS3Only) {
|
|
267
304
|
this.s3Config = credentials.s3Config;
|
|
268
305
|
}
|
|
@@ -288,7 +325,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
288
325
|
if (!token) {
|
|
289
326
|
throw new common_1.ParameterError('MotherDuck token is required for DuckDB warehouse connections');
|
|
290
327
|
}
|
|
291
|
-
this.databasePath = buildMotherduckConnectionString({
|
|
328
|
+
this.databasePath = (0, exports.buildMotherduckConnectionString)({
|
|
292
329
|
database: effectiveCredentials.database,
|
|
293
330
|
token,
|
|
294
331
|
});
|
|
@@ -311,9 +348,12 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
311
348
|
this.instanceCacheKey =
|
|
312
349
|
options?.instanceCacheKey ?? ducklakeAutoCacheKey;
|
|
313
350
|
this.logger = options?.logger;
|
|
351
|
+
this.enableQueryProfiling = options?.enableQueryProfiling ?? false;
|
|
314
352
|
this.onQueryProfile = options?.onQueryProfile;
|
|
315
353
|
this.embeddedQueryTimeoutMs =
|
|
316
354
|
options?.embeddedQueryTimeoutMs ?? EMBEDDED_QUERY_TIMEOUT_MS;
|
|
355
|
+
this.enableInstanceCache = options?.enableInstanceCache ?? false;
|
|
356
|
+
this.projectUuid = options?.projectUuid;
|
|
317
357
|
}
|
|
318
358
|
static hashDucklakeConfig(ducklake) {
|
|
319
359
|
return (0, crypto_1.createHash)('sha256')
|
|
@@ -947,16 +987,21 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
947
987
|
hasResourceLimits() {
|
|
948
988
|
return (!!this.resourceLimits && Object.keys(this.resourceLimits).length > 0);
|
|
949
989
|
}
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
990
|
+
isMotherduck() {
|
|
991
|
+
return (!this.embeddedConfig &&
|
|
992
|
+
!this.ducklakeConfig &&
|
|
993
|
+
this.databasePath.startsWith('md:'));
|
|
994
|
+
}
|
|
995
|
+
async withSession(callback, organizationUuid, retryable = () => true, onPhaseTiming) {
|
|
996
|
+
if (this.embeddedConfig) {
|
|
997
|
+
return this.withDirectSession(callback, organizationUuid, onPhaseTiming);
|
|
998
|
+
}
|
|
999
|
+
if (this.isMotherduck()) {
|
|
1000
|
+
if (this.enableInstanceCache &&
|
|
1001
|
+
this.credentials.requireUserCredentials !== true) {
|
|
1002
|
+
return this.withMotherduckCachedSession(callback, retryable, onPhaseTiming);
|
|
1003
|
+
}
|
|
1004
|
+
return this.withDirectSession(callback, organizationUuid, onPhaseTiming);
|
|
960
1005
|
}
|
|
961
1006
|
if (this.hasResourceLimits()) {
|
|
962
1007
|
return this.withIsolatedSession(callback);
|
|
@@ -966,8 +1011,81 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
966
1011
|
}
|
|
967
1012
|
return this.withEphemeralQuerySession(callback);
|
|
968
1013
|
}
|
|
1014
|
+
async withMotherduckCachedSession(callback, retryable, onPhaseTiming) {
|
|
1015
|
+
let activeEntryId;
|
|
1016
|
+
const runAttempt = () => MotherduckInstanceCache.withInstance(this.databasePath, { projectUuid: this.projectUuid }, async (instance, entryId, acquisitionTiming) => {
|
|
1017
|
+
activeEntryId = entryId;
|
|
1018
|
+
const sessionStart = performance.now();
|
|
1019
|
+
const connection = await instance.connect();
|
|
1020
|
+
const { connectMs } = acquisitionTiming;
|
|
1021
|
+
onPhaseTiming?.('connect', acquisitionTiming.waitMs +
|
|
1022
|
+
acquisitionTiming.instanceCreateMs +
|
|
1023
|
+
connectMs);
|
|
1024
|
+
try {
|
|
1025
|
+
const queryStart = performance.now();
|
|
1026
|
+
const result = await callback(connection);
|
|
1027
|
+
const queryMs = performance.now() - queryStart;
|
|
1028
|
+
const totalMs = performance.now() - sessionStart;
|
|
1029
|
+
this.logger?.info(`MotherDuck cached-instance session complete: entry_id=${entryId} connect=${(0, common_1.formatMilliseconds)(connectMs)}ms query=${(0, common_1.formatMilliseconds)(queryMs)}ms total=${(0, common_1.formatMilliseconds)(totalMs)}ms`, {
|
|
1030
|
+
entryId,
|
|
1031
|
+
projectUuid: this.projectUuid,
|
|
1032
|
+
connectMs,
|
|
1033
|
+
queryMs,
|
|
1034
|
+
totalMs,
|
|
1035
|
+
});
|
|
1036
|
+
return result;
|
|
1037
|
+
}
|
|
1038
|
+
finally {
|
|
1039
|
+
connection.closeSync?.();
|
|
1040
|
+
connection.disconnectSync?.();
|
|
1041
|
+
}
|
|
1042
|
+
});
|
|
1043
|
+
try {
|
|
1044
|
+
return await runAttempt();
|
|
1045
|
+
}
|
|
1046
|
+
catch (error) {
|
|
1047
|
+
const errorClass = (0, MotherduckErrorClassifier_1.classifyMotherduckError)(error);
|
|
1048
|
+
const failedEntryId = activeEntryId;
|
|
1049
|
+
if (errorClass === 'auth') {
|
|
1050
|
+
if (failedEntryId) {
|
|
1051
|
+
await MotherduckInstanceCache.invalidate(failedEntryId, 'auth');
|
|
1052
|
+
}
|
|
1053
|
+
throw error;
|
|
1054
|
+
}
|
|
1055
|
+
if (errorClass !== 'stale' || !failedEntryId) {
|
|
1056
|
+
throw error;
|
|
1057
|
+
}
|
|
1058
|
+
await MotherduckInstanceCache.invalidate(failedEntryId, 'stale');
|
|
1059
|
+
if (!retryable()) {
|
|
1060
|
+
throw error;
|
|
1061
|
+
}
|
|
1062
|
+
activeEntryId = undefined;
|
|
1063
|
+
try {
|
|
1064
|
+
const result = await runAttempt();
|
|
1065
|
+
MotherduckInstanceCache.recordRetry(failedEntryId, 'recovered');
|
|
1066
|
+
this.logger?.info('MotherDuck cached-instance session retry recovered', {
|
|
1067
|
+
entryId: failedEntryId,
|
|
1068
|
+
projectUuid: this.projectUuid,
|
|
1069
|
+
});
|
|
1070
|
+
return result;
|
|
1071
|
+
}
|
|
1072
|
+
catch (retryError) {
|
|
1073
|
+
MotherduckInstanceCache.recordRetry(failedEntryId, 'failed');
|
|
1074
|
+
const retryErrorClass = (0, MotherduckErrorClassifier_1.classifyMotherduckError)(retryError);
|
|
1075
|
+
if (activeEntryId &&
|
|
1076
|
+
(retryErrorClass === 'stale' || retryErrorClass === 'auth')) {
|
|
1077
|
+
await MotherduckInstanceCache.invalidate(activeEntryId, retryErrorClass);
|
|
1078
|
+
}
|
|
1079
|
+
this.logger?.info('MotherDuck cached-instance session retry failed', {
|
|
1080
|
+
entryId: failedEntryId,
|
|
1081
|
+
projectUuid: this.projectUuid,
|
|
1082
|
+
});
|
|
1083
|
+
throw retryError;
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
969
1087
|
/** Direct connection to the configured MotherDuck database. */
|
|
970
|
-
async withDirectSession(callback, organizationUuid) {
|
|
1088
|
+
async withDirectSession(callback, organizationUuid, onPhaseTiming) {
|
|
971
1089
|
const sessionStart = performance.now();
|
|
972
1090
|
const releaseEmbeddedConcurrency = this.embeddedConfig
|
|
973
1091
|
? DuckdbWarehouseClient.tryAcquireEmbeddedConcurrency(organizationUuid)
|
|
@@ -992,6 +1110,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
992
1110
|
const connectStart = performance.now();
|
|
993
1111
|
connection = await instance.connect();
|
|
994
1112
|
const connectMs = performance.now() - connectStart;
|
|
1113
|
+
onPhaseTiming?.('connect', instanceCreateMs + connectMs);
|
|
995
1114
|
if (this.embeddedConfig) {
|
|
996
1115
|
await DuckdbWarehouseClient.hardenInstance(connection);
|
|
997
1116
|
}
|
|
@@ -1271,12 +1390,28 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1271
1390
|
}
|
|
1272
1391
|
async streamQuery(sql, streamCallback, options) {
|
|
1273
1392
|
const reportPhase = options?.onPhaseTiming;
|
|
1393
|
+
let hasEmittedRows = false;
|
|
1274
1394
|
await this.withSession(async (db) => {
|
|
1275
1395
|
const sessionStart = performance.now();
|
|
1276
1396
|
if (options?.timezone) {
|
|
1277
|
-
|
|
1397
|
+
if (!this.isMotherduck()) {
|
|
1398
|
+
await db.run(`SET TimeZone = '${this.escapeString(options.timezone)}';`);
|
|
1399
|
+
}
|
|
1400
|
+
else if (!this.hasWarnedAboutMotherduckTimezone) {
|
|
1401
|
+
this.hasWarnedAboutMotherduckTimezone = true;
|
|
1402
|
+
const message = `Requested timezone "${options.timezone}" cannot be applied because MotherDuck locks configuration under saas_mode; timestamps return in the server default zone instead of the configured zone.`;
|
|
1403
|
+
if (this.logger?.warn) {
|
|
1404
|
+
this.logger.warn(message);
|
|
1405
|
+
}
|
|
1406
|
+
else {
|
|
1407
|
+
this.logger?.info(message);
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1278
1410
|
}
|
|
1279
|
-
const profilePath = this.logger &&
|
|
1411
|
+
const profilePath = this.logger &&
|
|
1412
|
+
this.enableQueryProfiling &&
|
|
1413
|
+
!this.embeddedConfig &&
|
|
1414
|
+
!this.isMotherduck()
|
|
1280
1415
|
? path_1.default.join(os_1.default.tmpdir(), `duckdb-profile-${Date.now()}-${Math.random().toString(36).slice(2)}.json`)
|
|
1281
1416
|
: undefined;
|
|
1282
1417
|
if (profilePath) {
|
|
@@ -1303,6 +1438,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1303
1438
|
reportPhase?.('query', performance.now() - queryStart);
|
|
1304
1439
|
fetchStart = performance.now();
|
|
1305
1440
|
}
|
|
1441
|
+
hasEmittedRows = true;
|
|
1306
1442
|
await streamCallback({ fields, rows });
|
|
1307
1443
|
}
|
|
1308
1444
|
if (fetchStart === undefined) {
|
|
@@ -1315,7 +1451,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1315
1451
|
if (profilePath) {
|
|
1316
1452
|
await this.logQueryProfile(profilePath, this.logger, options?.tags);
|
|
1317
1453
|
}
|
|
1318
|
-
}, options?.tags?.organization_uuid);
|
|
1454
|
+
}, options?.tags?.organization_uuid, () => !hasEmittedRows, reportPhase);
|
|
1319
1455
|
}
|
|
1320
1456
|
async executeAsyncQuery(...args) {
|
|
1321
1457
|
const [queryArgs, resultsStreamCallback] = args;
|