@lightdash/warehouses 0.2857.0 → 0.2858.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts +2 -0
- package/dist/warehouseClients/DuckdbWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.js +12 -1
- package/dist/warehouseClients/DuckdbWarehouseClient.test.js +4 -0
- package/package.json +2 -2
|
@@ -75,6 +75,8 @@ export declare class DuckdbWarehouseClient extends WarehouseBaseClient<CreateDuc
|
|
|
75
75
|
private getRequiredInstanceCacheKey;
|
|
76
76
|
private getSQLWithMetadata;
|
|
77
77
|
private static hardenInstance;
|
|
78
|
+
private static usesS3CredentialChain;
|
|
79
|
+
private static loadAwsExtensionForCredentialChain;
|
|
78
80
|
private static bootstrapQuerySession;
|
|
79
81
|
private static bootstrapSharedInstance;
|
|
80
82
|
private static getOrCreateSharedInstance;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,uBAAuB,EACvB,aAAa,EAGb,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA4DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAG9D,MAAM,MAAM,wBAAwB,GAAG,mBAAmB,CAAC;AAE3D,MAAM,MAAM,4BAA4B,GAAG;IACvC,2FAA2F;IAC3F,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACjE,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAmCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AA+CD,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,uBAAuB,CAAC;IACnF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;gBAGN,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B;IA6C1C,MAAM,CAAC,qBAAqB,CACxB,WAAW,CAAC,EAAE,2BAA2B,EACzC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB;IAIxB,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,kBAAkB;mBAQL,cAAc;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,uBAAuB,EACvB,aAAa,EAGb,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA4DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAG9D,MAAM,MAAM,wBAAwB,GAAG,mBAAmB,CAAC;AAE3D,MAAM,MAAM,4BAA4B,GAAG;IACvC,2FAA2F;IAC3F,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACjE,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAmCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AA+CD,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,uBAAuB,CAAC;IACnF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;gBAGN,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B;IA6C1C,MAAM,CAAC,qBAAqB,CACxB,WAAW,CAAC,EAAE,2BAA2B,EACzC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB;IAIxB,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,kBAAkB;mBAQL,cAAc;IAOnC,OAAO,CAAC,MAAM,CAAC,qBAAqB;mBAMf,kCAAkC;mBAUlC,qBAAqB;mBA2DrB,uBAAuB;mBAgBvB,yBAAyB;IA6E9C,OAAO,CAAC,MAAM,CAAC,mBAAmB;IAoBlC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI;IAKzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAqC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IAkCtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IAkC/B,OAAO,CAAC,iBAAiB;IAMzB;;;;;;OAMG;YACW,WAAW;IAkBzB,+DAA+D;YACjD,iBAAiB;IA0C/B,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAM/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;YAUrB,eAAe;YA8Bf,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA4CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,uBAAuB,CAAC,CAAC,mBAAmB,CAAC,CACpE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAoBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,uBAAuB,CAAC,CAAC,UAAU,CAAC,CAC3D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC;IAuCtB,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAuBK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAU/B"}
|
|
@@ -199,11 +199,21 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
199
199
|
await db.run('SET autoload_known_extensions = false;');
|
|
200
200
|
await db.run('SET allow_unredacted_secrets = false;');
|
|
201
201
|
}
|
|
202
|
+
static usesS3CredentialChain(s3Config) {
|
|
203
|
+
return !(s3Config.accessKey && s3Config.secretKey);
|
|
204
|
+
}
|
|
205
|
+
static async loadAwsExtensionForCredentialChain(db, s3Config) {
|
|
206
|
+
if (s3Config && DuckdbWarehouseClient.usesS3CredentialChain(s3Config)) {
|
|
207
|
+
await db.run('INSTALL aws;');
|
|
208
|
+
await db.run('LOAD aws;');
|
|
209
|
+
}
|
|
210
|
+
}
|
|
202
211
|
static async bootstrapQuerySession(db, client) {
|
|
203
212
|
const bootstrapStart = performance.now();
|
|
204
213
|
const httpfsStart = performance.now();
|
|
205
214
|
await db.run('INSTALL httpfs;');
|
|
206
215
|
await db.run('LOAD httpfs;');
|
|
216
|
+
await DuckdbWarehouseClient.loadAwsExtensionForCredentialChain(db, client.s3Config);
|
|
207
217
|
const httpfsMs = performance.now() - httpfsStart;
|
|
208
218
|
await db.run('SET enable_http_metadata_cache = true;');
|
|
209
219
|
await db.run('SET enable_external_file_cache = true;');
|
|
@@ -327,7 +337,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
327
337
|
}
|
|
328
338
|
static buildS3SecretSql(s3Config) {
|
|
329
339
|
const escape = (v) => DuckdbWarehouseClient.sqlBuilder.escapeString(v);
|
|
330
|
-
const usesStaticCredentials =
|
|
340
|
+
const usesStaticCredentials = !DuckdbWarehouseClient.usesS3CredentialChain(s3Config);
|
|
331
341
|
// Static keys may come from dedicated pre-aggregate env vars or fall back
|
|
332
342
|
// to the base S3 config. Without them, let DuckDB resolve AWS credentials
|
|
333
343
|
// through the SDK chain, including IRSA/web identity tokens.
|
|
@@ -418,6 +428,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
418
428
|
async bootstrapIsolatedSession(db, tempDir) {
|
|
419
429
|
await db.run('INSTALL httpfs;');
|
|
420
430
|
await db.run('LOAD httpfs;');
|
|
431
|
+
await DuckdbWarehouseClient.loadAwsExtensionForCredentialChain(db, this.s3Config);
|
|
421
432
|
await DuckdbWarehouseClient.hardenInstance(db);
|
|
422
433
|
await db.run(`SET temp_directory = '${tempDir}';`);
|
|
423
434
|
if (this.resourceLimits?.memoryLimit) {
|
|
@@ -256,6 +256,8 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
256
256
|
},
|
|
257
257
|
});
|
|
258
258
|
await client.runQuery('SELECT 1 AS val');
|
|
259
|
+
expect(runMock).toHaveBeenCalledWith('INSTALL aws;');
|
|
260
|
+
expect(runMock).toHaveBeenCalledWith('LOAD aws;');
|
|
259
261
|
const secretSql = runMock.mock.calls
|
|
260
262
|
.map(([sql]) => sql)
|
|
261
263
|
.find((sql) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'));
|
|
@@ -283,6 +285,8 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
283
285
|
},
|
|
284
286
|
});
|
|
285
287
|
await client.runQuery('SELECT 1 AS val');
|
|
288
|
+
expect(runMock).not.toHaveBeenCalledWith('INSTALL aws;');
|
|
289
|
+
expect(runMock).not.toHaveBeenCalledWith('LOAD aws;');
|
|
286
290
|
const secretSql = runMock.mock.calls
|
|
287
291
|
.map(([sql]) => sql)
|
|
288
292
|
.find((sql) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2858.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"snowflake-sdk": "2.3.4",
|
|
28
28
|
"ssh2": "1.14.0",
|
|
29
29
|
"trino-client": "0.2.9",
|
|
30
|
-
"@lightdash/common": "0.
|
|
30
|
+
"@lightdash/common": "0.2858.0"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"@types/node-fetch": "2.6.13",
|