@lightdash/warehouses 1.75.1 → 1.76.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/index.d.ts +0 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -3
- package/dist/warehouseClientFromCredentials.d.ts +1 -2
- package/dist/warehouseClientFromCredentials.d.ts.map +1 -1
- package/dist/warehouseClientFromCredentials.js +2 -2
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +7 -6
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +14 -128
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +0 -307
- package/package.json +2 -2
- package/dist/warehouseClients/MotherduckErrorClassifier.d.ts +0 -3
- package/dist/warehouseClients/MotherduckErrorClassifier.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckErrorClassifier.js +0 -34
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckErrorClassifier.test.js +0 -31
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.js +0 -52
- package/dist/warehouseClients/MotherduckInstanceCache.d.ts +0 -58
- package/dist/warehouseClients/MotherduckInstanceCache.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.integration.test.js +0 -172
- package/dist/warehouseClients/MotherduckInstanceCache.js +0 -316
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.security.test.js +0 -192
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts +0 -2
- package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts.map +0 -1
- package/dist/warehouseClients/MotherduckInstanceCache.test.js +0 -302
package/dist/index.d.ts
CHANGED
|
@@ -6,8 +6,6 @@ export * from './warehouseClients/AthenaWarehouseClient';
|
|
|
6
6
|
export * from './warehouseClients/BigqueryWarehouseClient';
|
|
7
7
|
export * from './warehouseClients/DatabricksWarehouseClient';
|
|
8
8
|
export * from './warehouseClients/DuckdbWarehouseClient';
|
|
9
|
-
export { MotherduckInstanceCache } from './warehouseClients/MotherduckInstanceCache';
|
|
10
|
-
export type { MotherduckCacheEvent } from './warehouseClients/MotherduckInstanceCache';
|
|
11
9
|
export * from './warehouseClients/PostgresWarehouseClient';
|
|
12
10
|
export * from './warehouseClients/RedshiftWarehouseClient';
|
|
13
11
|
export * from './warehouseClients/redshiftIamCredentials';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,4CAA4C,CAAC;AAC3D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,6CAA6C,CAAC;AAC5D,cAAc,yCAAyC,CAAC;AACxD,cAAc,+BAA+B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -14,7 +14,6 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.MotherduckInstanceCache = void 0;
|
|
18
17
|
__exportStar(require("./ssh/sshTunnel"), exports);
|
|
19
18
|
__exportStar(require("./types"), exports);
|
|
20
19
|
__exportStar(require("./validateWarehouseColumnReferences"), exports);
|
|
@@ -23,8 +22,6 @@ __exportStar(require("./warehouseClients/AthenaWarehouseClient"), exports);
|
|
|
23
22
|
__exportStar(require("./warehouseClients/BigqueryWarehouseClient"), exports);
|
|
24
23
|
__exportStar(require("./warehouseClients/DatabricksWarehouseClient"), exports);
|
|
25
24
|
__exportStar(require("./warehouseClients/DuckdbWarehouseClient"), exports);
|
|
26
|
-
var MotherduckInstanceCache_1 = require("./warehouseClients/MotherduckInstanceCache");
|
|
27
|
-
Object.defineProperty(exports, "MotherduckInstanceCache", { enumerable: true, get: function () { return MotherduckInstanceCache_1.MotherduckInstanceCache; } });
|
|
28
25
|
__exportStar(require("./warehouseClients/PostgresWarehouseClient"), exports);
|
|
29
26
|
__exportStar(require("./warehouseClients/RedshiftWarehouseClient"), exports);
|
|
30
27
|
__exportStar(require("./warehouseClients/redshiftIamCredentials"), exports);
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { CreateWarehouseCredentials } from '@lightdash/common';
|
|
2
2
|
import { WarehouseClient } from './types';
|
|
3
|
-
|
|
4
|
-
export declare const warehouseClientFromCredentials: (credentials: CreateWarehouseCredentials, options?: DuckdbWarehouseClientOptions) => WarehouseClient;
|
|
3
|
+
export declare const warehouseClientFromCredentials: (credentials: CreateWarehouseCredentials) => WarehouseClient;
|
|
5
4
|
//# sourceMappingURL=warehouseClientFromCredentials.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"warehouseClientFromCredentials.d.ts","sourceRoot":"","sources":["../src/warehouseClientFromCredentials.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,0BAA0B,EAG7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"warehouseClientFromCredentials.d.ts","sourceRoot":"","sources":["../src/warehouseClientFromCredentials.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,0BAA0B,EAG7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAW1C,eAAO,MAAM,8BAA8B,gBAC1B,0BAA0B,KACxC,eA4BF,CAAC"}
|
|
@@ -11,7 +11,7 @@ const PostgresWarehouseClient_1 = require("./warehouseClients/PostgresWarehouseC
|
|
|
11
11
|
const RedshiftWarehouseClient_1 = require("./warehouseClients/RedshiftWarehouseClient");
|
|
12
12
|
const SnowflakeWarehouseClient_1 = require("./warehouseClients/SnowflakeWarehouseClient");
|
|
13
13
|
const TrinoWarehouseClient_1 = require("./warehouseClients/TrinoWarehouseClient");
|
|
14
|
-
const warehouseClientFromCredentials = (credentials
|
|
14
|
+
const warehouseClientFromCredentials = (credentials) => {
|
|
15
15
|
switch (credentials.type) {
|
|
16
16
|
case common_1.WarehouseTypes.SNOWFLAKE:
|
|
17
17
|
return new SnowflakeWarehouseClient_1.SnowflakeWarehouseClient(credentials);
|
|
@@ -30,7 +30,7 @@ const warehouseClientFromCredentials = (credentials, options) => {
|
|
|
30
30
|
case common_1.WarehouseTypes.ATHENA:
|
|
31
31
|
return new AthenaWarehouseClient_1.AthenaWarehouseClient(credentials);
|
|
32
32
|
case common_1.WarehouseTypes.DUCKDB:
|
|
33
|
-
return new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials
|
|
33
|
+
return new DuckdbWarehouseClient_1.DuckdbWarehouseClient(credentials);
|
|
34
34
|
default:
|
|
35
35
|
return (0, common_1.assertUnreachable)(credentials, new common_1.UnexpectedServerError('Warehouse credentials type were not recognised'));
|
|
36
36
|
}
|
|
@@ -48,8 +48,6 @@ export type DuckdbWarehouseClientOptions = {
|
|
|
48
48
|
enableQueryProfiling?: boolean;
|
|
49
49
|
onQueryProfile?: (profile: DuckdbQueryProfileMetrics) => void;
|
|
50
50
|
embeddedQueryTimeoutMs?: number;
|
|
51
|
-
enableInstanceCache?: boolean;
|
|
52
|
-
projectUuid?: string;
|
|
53
51
|
};
|
|
54
52
|
export declare const mapFieldTypeFromTypeId: (typeId: number) => DimensionType;
|
|
55
53
|
export declare const getDuckdbTimestampDomainFromString: (typeName: string) => TimestampDomain | undefined;
|
|
@@ -59,7 +57,6 @@ export declare class DuckdbSqlBuilder extends WarehouseBaseSqlBuilder {
|
|
|
59
57
|
getMetricSql(sql: string, metric: Metric): string;
|
|
60
58
|
concatString(...args: string[]): string;
|
|
61
59
|
}
|
|
62
|
-
export declare const buildMotherduckConnectionString: ({ database, token, }: Pick<CreateDuckdbMotherduckCredentials, 'database' | 'token'>) => string;
|
|
63
60
|
export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuckdbMotherduckCredentials> {
|
|
64
61
|
private static readonly sharedInstances;
|
|
65
62
|
private static readonly sharedInstanceResourceLimits;
|
|
@@ -78,8 +75,6 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
78
75
|
private readonly enableQueryProfiling;
|
|
79
76
|
private readonly onQueryProfile?;
|
|
80
77
|
private readonly embeddedQueryTimeoutMs;
|
|
81
|
-
private readonly enableInstanceCache;
|
|
82
|
-
private readonly projectUuid?;
|
|
83
78
|
private allowsPreAggregateFileReads;
|
|
84
79
|
private hasWarnedAboutMotherduckTimezone;
|
|
85
80
|
constructor(credentials?: CreateDuckdbCredentials | DuckdbConnectionCredentials, options?: DuckdbWarehouseClientOptions);
|
|
@@ -123,8 +118,14 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
123
118
|
private withSharedSession;
|
|
124
119
|
private hasResourceLimits;
|
|
125
120
|
private isMotherduck;
|
|
121
|
+
/**
|
|
122
|
+
* Dispatches to the appropriate session strategy:
|
|
123
|
+
* - Direct database (non-:memory: databasePath): connects to the MotherDuck path
|
|
124
|
+
* - Resource-limited: isolated ephemeral instance (e.g. parquet conversion)
|
|
125
|
+
* - Shared instance (has instanceCacheKey): warm cached instance for queries
|
|
126
|
+
* - Default: ephemeral query session
|
|
127
|
+
*/
|
|
126
128
|
private withSession;
|
|
127
|
-
private withMotherduckCachedSession;
|
|
128
129
|
/** Direct connection to the configured MotherDuck database. */
|
|
129
130
|
private withDirectSession;
|
|
130
131
|
private withEmbeddedQueryDeadline;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,uBAAuB,EAGvB,iCAAiC,EACjC,aAAa,EAMb,MAAM,EAKN,mBAAmB,EACnB,gBAAgB,EAEhB,gBAAgB,EAEhB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAM3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA6DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAE9D,MAAM,MAAM,4BAA4B,GAAG;IACvC,kHAAkH;IAClH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,eAAO,MAAM,sBAAsB,WAAY,MAAM,KAAG,aA+BvD,CAAC;AAEF,eAAO,MAAM,kCAAkC,aACjC,MAAM,KACjB,eAAe,GAAG,SAOpB,CAAC;AAoCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB,CAEpC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAuJD,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,iCAAiC,CAAC;IAC7F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAGhD;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAE/C;IAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sCAAsC,CAG1D;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAkC;IAElE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;IAEV,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,OAAO,CAAC,2BAA2B,CAAS;IAE5C,OAAO,CAAC,gCAAgC,CAAS;IAEjD,YACI,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B,EA+GzC;IAED,OAAO,CAAC,MAAM,CAAC,kBAAkB;IASjC,MAAM,CAAC,qBAAqB,CACxB,WAAW,EAAE,mBAAmB,EAChC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB,CAIvB;IAED,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAwC5C,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,mBAAmB;IAaxC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,uBAAuB;YAsB9B,aAAa;YAoBb,kCAAkC;mBAW3B,qBAAqB;mBA6ErB,uBAAuB;mBAgBvB,yBAAyB;mBAiFzB,mBAAmB;IA6BxC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI,CAM9C;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CACZ;IAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAA+B;IAE3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEjE,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAIjC,OAAO,CAAC,MAAM,CAAC,UAAU;IAIzB,OAAO,CAAC,MAAM,CAAC,6BAA6B;IAuB5C,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAqGzC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAmBrC,OAAO,CAAC,MAAM,CAAC,sBAAsB;IA2CrC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAqC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA+CtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IA2C/B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,YAAY;IAQpB;;;;;;OAMG;YACW,WAAW;IAmBzB,+DAA+D;YACjD,iBAAiB;YAyEjB,yBAAyB;IA2BvC,OAAO,CAAC,4BAA4B;IAOpC,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IA+D/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAUnC,OAAO,CAAC,MAAM,CAAC,yBAAyB;YAmB1B,iBAAiB;YA4BjB,eAAe;YASf,uBAAuB;YAQvB,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAgFf;IAEK,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,mBAAmB,CAAC,CAC9E;;;;;;OA4BJ;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvC;IAEK,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC,CAkBD;IAEK,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,iCAAiC,CAAC,CAAC,UAAU,CAAC,CACrE;;;;;OAGJ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAEK,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CA4C3B;IAEK,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN,CAqBA;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC,CAS3B;CACJ"}
|
|
@@ -1,42 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
4
|
};
|
|
38
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.DuckdbWarehouseClient = exports.
|
|
6
|
+
exports.DuckdbWarehouseClient = exports.DuckdbSqlBuilder = exports.getDuckdbTimestampDomainFromString = exports.mapFieldTypeFromTypeId = void 0;
|
|
40
7
|
const node_api_1 = require("@duckdb/node-api");
|
|
41
8
|
const common_1 = require("@lightdash/common");
|
|
42
9
|
const crypto_1 = require("crypto");
|
|
@@ -44,8 +11,6 @@ const fs_1 = __importDefault(require("fs"));
|
|
|
44
11
|
const promises_1 = __importDefault(require("fs/promises"));
|
|
45
12
|
const os_1 = __importDefault(require("os"));
|
|
46
13
|
const path_1 = __importDefault(require("path"));
|
|
47
|
-
const MotherduckErrorClassifier_1 = require("./MotherduckErrorClassifier");
|
|
48
|
-
const MotherduckInstanceCache = __importStar(require("./MotherduckInstanceCache"));
|
|
49
14
|
const WarehouseBaseClient_1 = __importDefault(require("./WarehouseBaseClient"));
|
|
50
15
|
const WarehouseBaseSqlBuilder_1 = __importDefault(require("./WarehouseBaseSqlBuilder"));
|
|
51
16
|
const mapFieldTypeFromTypeId = (typeId) => {
|
|
@@ -215,7 +180,6 @@ const buildMotherduckConnectionString = ({ database, token, }) => {
|
|
|
215
180
|
});
|
|
216
181
|
return `md:${encodeURIComponent(database)}?${params.toString()}`;
|
|
217
182
|
};
|
|
218
|
-
exports.buildMotherduckConnectionString = buildMotherduckConnectionString;
|
|
219
183
|
const EMBEDDED_DATASET_PATTERN = /^[a-z0-9_-]+$/;
|
|
220
184
|
const DEFAULT_EMBEDDED_DATA_DIRECTORY = path_1.default.resolve(__dirname, '../../../backend/assets/playground');
|
|
221
185
|
const EMBEDDED_RESOURCE_LIMITS = {
|
|
@@ -325,7 +289,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
325
289
|
if (!token) {
|
|
326
290
|
throw new common_1.ParameterError('MotherDuck token is required for DuckDB warehouse connections');
|
|
327
291
|
}
|
|
328
|
-
this.databasePath =
|
|
292
|
+
this.databasePath = buildMotherduckConnectionString({
|
|
329
293
|
database: effectiveCredentials.database,
|
|
330
294
|
token,
|
|
331
295
|
});
|
|
@@ -352,8 +316,6 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
352
316
|
this.onQueryProfile = options?.onQueryProfile;
|
|
353
317
|
this.embeddedQueryTimeoutMs =
|
|
354
318
|
options?.embeddedQueryTimeoutMs ?? EMBEDDED_QUERY_TIMEOUT_MS;
|
|
355
|
-
this.enableInstanceCache = options?.enableInstanceCache ?? false;
|
|
356
|
-
this.projectUuid = options?.projectUuid;
|
|
357
319
|
}
|
|
358
320
|
static hashDucklakeConfig(ducklake) {
|
|
359
321
|
return (0, crypto_1.createHash)('sha256')
|
|
@@ -992,16 +954,16 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
992
954
|
!this.ducklakeConfig &&
|
|
993
955
|
this.databasePath.startsWith('md:'));
|
|
994
956
|
}
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
return this.withDirectSession(callback, organizationUuid
|
|
957
|
+
/**
|
|
958
|
+
* Dispatches to the appropriate session strategy:
|
|
959
|
+
* - Direct database (non-:memory: databasePath): connects to the MotherDuck path
|
|
960
|
+
* - Resource-limited: isolated ephemeral instance (e.g. parquet conversion)
|
|
961
|
+
* - Shared instance (has instanceCacheKey): warm cached instance for queries
|
|
962
|
+
* - Default: ephemeral query session
|
|
963
|
+
*/
|
|
964
|
+
async withSession(callback, organizationUuid) {
|
|
965
|
+
if (this.databasePath !== ':memory:') {
|
|
966
|
+
return this.withDirectSession(callback, organizationUuid);
|
|
1005
967
|
}
|
|
1006
968
|
if (this.hasResourceLimits()) {
|
|
1007
969
|
return this.withIsolatedSession(callback);
|
|
@@ -1011,81 +973,8 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1011
973
|
}
|
|
1012
974
|
return this.withEphemeralQuerySession(callback);
|
|
1013
975
|
}
|
|
1014
|
-
async withMotherduckCachedSession(callback, retryable, onPhaseTiming) {
|
|
1015
|
-
let activeEntryId;
|
|
1016
|
-
const runAttempt = () => MotherduckInstanceCache.withInstance(this.databasePath, { projectUuid: this.projectUuid }, async (instance, entryId, acquisitionTiming) => {
|
|
1017
|
-
activeEntryId = entryId;
|
|
1018
|
-
const sessionStart = performance.now();
|
|
1019
|
-
const connection = await instance.connect();
|
|
1020
|
-
const { connectMs } = acquisitionTiming;
|
|
1021
|
-
onPhaseTiming?.('connect', acquisitionTiming.waitMs +
|
|
1022
|
-
acquisitionTiming.instanceCreateMs +
|
|
1023
|
-
connectMs);
|
|
1024
|
-
try {
|
|
1025
|
-
const queryStart = performance.now();
|
|
1026
|
-
const result = await callback(connection);
|
|
1027
|
-
const queryMs = performance.now() - queryStart;
|
|
1028
|
-
const totalMs = performance.now() - sessionStart;
|
|
1029
|
-
this.logger?.info(`MotherDuck cached-instance session complete: entry_id=${entryId} connect=${(0, common_1.formatMilliseconds)(connectMs)}ms query=${(0, common_1.formatMilliseconds)(queryMs)}ms total=${(0, common_1.formatMilliseconds)(totalMs)}ms`, {
|
|
1030
|
-
entryId,
|
|
1031
|
-
projectUuid: this.projectUuid,
|
|
1032
|
-
connectMs,
|
|
1033
|
-
queryMs,
|
|
1034
|
-
totalMs,
|
|
1035
|
-
});
|
|
1036
|
-
return result;
|
|
1037
|
-
}
|
|
1038
|
-
finally {
|
|
1039
|
-
connection.closeSync?.();
|
|
1040
|
-
connection.disconnectSync?.();
|
|
1041
|
-
}
|
|
1042
|
-
});
|
|
1043
|
-
try {
|
|
1044
|
-
return await runAttempt();
|
|
1045
|
-
}
|
|
1046
|
-
catch (error) {
|
|
1047
|
-
const errorClass = (0, MotherduckErrorClassifier_1.classifyMotherduckError)(error);
|
|
1048
|
-
const failedEntryId = activeEntryId;
|
|
1049
|
-
if (errorClass === 'auth') {
|
|
1050
|
-
if (failedEntryId) {
|
|
1051
|
-
await MotherduckInstanceCache.invalidate(failedEntryId, 'auth');
|
|
1052
|
-
}
|
|
1053
|
-
throw error;
|
|
1054
|
-
}
|
|
1055
|
-
if (errorClass !== 'stale' || !failedEntryId) {
|
|
1056
|
-
throw error;
|
|
1057
|
-
}
|
|
1058
|
-
await MotherduckInstanceCache.invalidate(failedEntryId, 'stale');
|
|
1059
|
-
if (!retryable()) {
|
|
1060
|
-
throw error;
|
|
1061
|
-
}
|
|
1062
|
-
activeEntryId = undefined;
|
|
1063
|
-
try {
|
|
1064
|
-
const result = await runAttempt();
|
|
1065
|
-
MotherduckInstanceCache.recordRetry(failedEntryId, 'recovered');
|
|
1066
|
-
this.logger?.info('MotherDuck cached-instance session retry recovered', {
|
|
1067
|
-
entryId: failedEntryId,
|
|
1068
|
-
projectUuid: this.projectUuid,
|
|
1069
|
-
});
|
|
1070
|
-
return result;
|
|
1071
|
-
}
|
|
1072
|
-
catch (retryError) {
|
|
1073
|
-
MotherduckInstanceCache.recordRetry(failedEntryId, 'failed');
|
|
1074
|
-
const retryErrorClass = (0, MotherduckErrorClassifier_1.classifyMotherduckError)(retryError);
|
|
1075
|
-
if (activeEntryId &&
|
|
1076
|
-
(retryErrorClass === 'stale' || retryErrorClass === 'auth')) {
|
|
1077
|
-
await MotherduckInstanceCache.invalidate(activeEntryId, retryErrorClass);
|
|
1078
|
-
}
|
|
1079
|
-
this.logger?.info('MotherDuck cached-instance session retry failed', {
|
|
1080
|
-
entryId: failedEntryId,
|
|
1081
|
-
projectUuid: this.projectUuid,
|
|
1082
|
-
});
|
|
1083
|
-
throw retryError;
|
|
1084
|
-
}
|
|
1085
|
-
}
|
|
1086
|
-
}
|
|
1087
976
|
/** Direct connection to the configured MotherDuck database. */
|
|
1088
|
-
async withDirectSession(callback, organizationUuid
|
|
977
|
+
async withDirectSession(callback, organizationUuid) {
|
|
1089
978
|
const sessionStart = performance.now();
|
|
1090
979
|
const releaseEmbeddedConcurrency = this.embeddedConfig
|
|
1091
980
|
? DuckdbWarehouseClient.tryAcquireEmbeddedConcurrency(organizationUuid)
|
|
@@ -1110,7 +999,6 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1110
999
|
const connectStart = performance.now();
|
|
1111
1000
|
connection = await instance.connect();
|
|
1112
1001
|
const connectMs = performance.now() - connectStart;
|
|
1113
|
-
onPhaseTiming?.('connect', instanceCreateMs + connectMs);
|
|
1114
1002
|
if (this.embeddedConfig) {
|
|
1115
1003
|
await DuckdbWarehouseClient.hardenInstance(connection);
|
|
1116
1004
|
}
|
|
@@ -1390,7 +1278,6 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1390
1278
|
}
|
|
1391
1279
|
async streamQuery(sql, streamCallback, options) {
|
|
1392
1280
|
const reportPhase = options?.onPhaseTiming;
|
|
1393
|
-
let hasEmittedRows = false;
|
|
1394
1281
|
await this.withSession(async (db) => {
|
|
1395
1282
|
const sessionStart = performance.now();
|
|
1396
1283
|
if (options?.timezone) {
|
|
@@ -1438,7 +1325,6 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1438
1325
|
reportPhase?.('query', performance.now() - queryStart);
|
|
1439
1326
|
fetchStart = performance.now();
|
|
1440
1327
|
}
|
|
1441
|
-
hasEmittedRows = true;
|
|
1442
1328
|
await streamCallback({ fields, rows });
|
|
1443
1329
|
}
|
|
1444
1330
|
if (fetchStart === undefined) {
|
|
@@ -1451,7 +1337,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
1451
1337
|
if (profilePath) {
|
|
1452
1338
|
await this.logQueryProfile(profilePath, this.logger, options?.tags);
|
|
1453
1339
|
}
|
|
1454
|
-
}, options?.tags?.organization_uuid
|
|
1340
|
+
}, options?.tags?.organization_uuid);
|
|
1455
1341
|
}
|
|
1456
1342
|
async executeAsyncQuery(...args) {
|
|
1457
1343
|
const [queryArgs, resultsStreamCallback] = args;
|