@lightdash/warehouses 0.2855.1 → 0.2857.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,uBAAuB,EACvB,aAAa,EAGb,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA4DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAG9D,MAAM,MAAM,wBAAwB,GAAG,mBAAmB,CAAC;AAE3D,MAAM,MAAM,4BAA4B,GAAG;IACvC,2FAA2F;IAC3F,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACjE,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAmCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AA+CD,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,uBAAuB,CAAC;IACnF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;gBAGN,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B;IA6C1C,MAAM,CAAC,qBAAqB,CACxB,WAAW,CAAC,EAAE,2BAA2B,EACzC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB;IAIxB,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,qBAAqB;mBAuDrB,uBAAuB;mBAgBvB,yBAAyB;IA6E9C,OAAO,CAAC,MAAM,CAAC,mBAAmB;IAoBlC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI;IAKzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B,OAAO,CAAC,MAAM,CAAC,gBAAgB;
|
|
1
|
+
{"version":3,"file":"DuckdbWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DuckdbWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAO,EACH,OAAO,EACP,uBAAuB,EACvB,aAAa,EAGb,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAEnB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AA4DhE,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACvB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,qBAAqB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,CAAC;AAG9D,MAAM,MAAM,wBAAwB,GAAG,mBAAmB,CAAC;AAE3D,MAAM,MAAM,4BAA4B,GAAG;IACvC,2FAA2F;IAC3F,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,uHAAuH;IACvH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACjE,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,QAAQ,MAAM,KAAG,aA+BvD,CAAC;AAmCF,qBAAa,gBAAiB,SAAQ,uBAAuB;IACzD,cAAc,IAAI,mBAAmB;IAIrC,eAAe,IAAI,MAAM;IAIzB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C;AA+CD,MAAM,MAAM,yBAAyB,GAAG;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,qBAAa,qBAAsB,SAAQ,mBAAmB,CAAC,uBAAuB,CAAC;IACnF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAqC;IAE5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAG5C;IAEJ,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAE5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAwB;IAElD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuB;IAEvD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAuB;IAE7D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAE3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IAEvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAEtB;gBAGN,WAAW,CAAC,EAAE,uBAAuB,GAAG,2BAA2B,EACnE,OAAO,CAAC,EAAE,4BAA4B;IA6C1C,MAAM,CAAC,qBAAqB,CACxB,WAAW,CAAC,EAAE,2BAA2B,EACzC,OAAO,CAAC,EAAE,4BAA4B,GACvC,qBAAqB;IAIxB,OAAO,CAAC,MAAM,CAAC,0BAA0B;IAoBzC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,kBAAkB;mBAQL,cAAc;mBAOd,qBAAqB;mBAuDrB,uBAAuB;mBAgBvB,yBAAyB;IA6E9C,OAAO,CAAC,MAAM,CAAC,mBAAmB;IAoBlC,mFAAmF;IACnF,MAAM,CAAC,gCAAgC,IAAI,IAAI;IAKzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAsC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAK;YAE9C,gBAAgB;IA+E9B,iEAAiE;YACnD,wBAAwB;IA8BtC,gFAAgF;YAClE,mBAAmB;YAkDnB,yBAAyB;YA4CzB,iBAAiB;IAkC/B,OAAO,CAAC,iBAAiB;IAMzB;;;;;;OAMG;YACW,WAAW;IAkBzB,+DAA+D;YACjD,iBAAiB;IA0C/B,OAAO,CAAC,aAAa;YA0BP,eAAe;IAgG7B,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAaxC,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAM/B,OAAO,CAAC,MAAM,CAAC,oBAAoB;YAUrB,eAAe;YA8Bf,mBAAmB;IA6B3B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA4CV,iBAAiB,CACnB,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,uBAAuB,CAAC,CAAC,mBAAmB,CAAC,CACpE;;;;;;IA8BC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IAoBI,QAAQ,CACV,GAAG,IAAI,EAAE,UAAU,CACf,mBAAmB,CAAC,uBAAuB,CAAC,CAAC,UAAU,CAAC,CAC3D;;;;;;IAKC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,UAAU,CACZ,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC;IAuCtB,YAAY,CACd,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CACN;QACI,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,CACN;IAuBK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,gBAAgB,CAAC;CAU/B"}
|
|
@@ -327,6 +327,17 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
327
327
|
}
|
|
328
328
|
static buildS3SecretSql(s3Config) {
|
|
329
329
|
const escape = (v) => DuckdbWarehouseClient.sqlBuilder.escapeString(v);
|
|
330
|
+
const usesStaticCredentials = !!(s3Config.accessKey && s3Config.secretKey);
|
|
331
|
+
// Static keys may come from dedicated pre-aggregate env vars or fall back
|
|
332
|
+
// to the base S3 config. Without them, let DuckDB resolve AWS credentials
|
|
333
|
+
// through the SDK chain, including IRSA/web identity tokens.
|
|
334
|
+
const providerClause = usesStaticCredentials
|
|
335
|
+
? ''
|
|
336
|
+
: `PROVIDER credential_chain,
|
|
337
|
+
REFRESH auto,
|
|
338
|
+
-- Defer credential validation to S3 operations so local bootstrap
|
|
339
|
+
-- does not require AWS credentials when using runtime-provided roles.
|
|
340
|
+
VALIDATION 'none',`;
|
|
330
341
|
const regionClause = s3Config.region
|
|
331
342
|
? `REGION '${escape(s3Config.region)}',`
|
|
332
343
|
: '';
|
|
@@ -338,6 +349,7 @@ class DuckdbWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
338
349
|
: '';
|
|
339
350
|
return `CREATE OR REPLACE SECRET __lightdash_s3 (
|
|
340
351
|
TYPE s3,
|
|
352
|
+
${providerClause}
|
|
341
353
|
${keyIdClause}
|
|
342
354
|
${secretClause}
|
|
343
355
|
ENDPOINT '${escape(s3Config.endpoint)}',
|
|
@@ -242,6 +242,56 @@ describe('DuckdbWarehouseClient', () => {
|
|
|
242
242
|
expect(runMock.mock.calls.filter(([sql]) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'))).toHaveLength(1);
|
|
243
243
|
expect(streamMock).toHaveBeenCalledTimes(2);
|
|
244
244
|
});
|
|
245
|
+
it('should use DuckDB credential chain for S3 config without static credentials', async () => {
|
|
246
|
+
const runMock = jest.fn();
|
|
247
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
248
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
249
|
+
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
250
|
+
type: 'duckdb_s3',
|
|
251
|
+
s3Config: {
|
|
252
|
+
endpoint: 's3.eu-west-1.amazonaws.com',
|
|
253
|
+
region: 'eu-west-1',
|
|
254
|
+
forcePathStyle: false,
|
|
255
|
+
useSsl: true,
|
|
256
|
+
},
|
|
257
|
+
});
|
|
258
|
+
await client.runQuery('SELECT 1 AS val');
|
|
259
|
+
const secretSql = runMock.mock.calls
|
|
260
|
+
.map(([sql]) => sql)
|
|
261
|
+
.find((sql) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'));
|
|
262
|
+
expect(secretSql).toContain('PROVIDER credential_chain');
|
|
263
|
+
expect(secretSql).toContain('REFRESH auto');
|
|
264
|
+
expect(secretSql).toContain("VALIDATION 'none'");
|
|
265
|
+
expect(secretSql).toContain("REGION 'eu-west-1'");
|
|
266
|
+
expect(secretSql).toContain("ENDPOINT 's3.eu-west-1.amazonaws.com'");
|
|
267
|
+
expect(secretSql).not.toContain('KEY_ID');
|
|
268
|
+
expect(secretSql).not.toContain("SECRET '");
|
|
269
|
+
});
|
|
270
|
+
it('should use static DuckDB S3 credentials when configured', async () => {
|
|
271
|
+
const runMock = jest.fn();
|
|
272
|
+
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
|
273
|
+
createInstanceMock.mockResolvedValue(createMockConnection(streamMock, runMock));
|
|
274
|
+
const client = DuckdbWarehouseClient_1.DuckdbWarehouseClient.createForPreAggregate({
|
|
275
|
+
type: 'duckdb_s3',
|
|
276
|
+
s3Config: {
|
|
277
|
+
endpoint: 'localhost:9000',
|
|
278
|
+
region: 'us-east-1',
|
|
279
|
+
accessKey: 'key',
|
|
280
|
+
secretKey: 'secret',
|
|
281
|
+
forcePathStyle: true,
|
|
282
|
+
useSsl: false,
|
|
283
|
+
},
|
|
284
|
+
});
|
|
285
|
+
await client.runQuery('SELECT 1 AS val');
|
|
286
|
+
const secretSql = runMock.mock.calls
|
|
287
|
+
.map(([sql]) => sql)
|
|
288
|
+
.find((sql) => sql.includes('CREATE OR REPLACE SECRET __lightdash_s3'));
|
|
289
|
+
expect(secretSql).not.toContain('PROVIDER credential_chain');
|
|
290
|
+
expect(secretSql).toContain("KEY_ID 'key'");
|
|
291
|
+
expect(secretSql).toContain("SECRET 'secret'");
|
|
292
|
+
expect(secretSql).toContain("URL_STYLE 'path'");
|
|
293
|
+
expect(secretSql).toContain('USE_SSL false');
|
|
294
|
+
});
|
|
245
295
|
it('should treat instanceCacheKey as the shared instance identity', async () => {
|
|
246
296
|
const runMock = jest.fn();
|
|
247
297
|
const streamMock = jest.fn(async () => getMockStreamResult([[{ val: 1 }]], [DUCKDB_TYPE_IDS.INTEGER]));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2857.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"snowflake-sdk": "2.3.4",
|
|
28
28
|
"ssh2": "1.14.0",
|
|
29
29
|
"trino-client": "0.2.9",
|
|
30
|
-
"@lightdash/common": "0.
|
|
30
|
+
"@lightdash/common": "0.2857.0"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"@types/node-fetch": "2.6.13",
|